diff --git a/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php b/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php index 98fe5ecc87..38b7ed7c6e 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php @@ -1,212 +1,215 @@ getRequest(); $user = $request->getUser(); $window_start = (time() - (60 * 15)); // Assume daemons spend about 250ms second in overhead per task acquiring // leases and doing other bookkeeping. This is probably an over-estimation, // but we'd rather show that utilization is too high than too low. $lease_overhead = 0.250; $completed = id(new PhabricatorWorkerArchiveTaskQuery()) ->withDateModifiedSince($window_start) ->execute(); $failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere( 'failureTime > %d', $window_start); $usage_total = 0; $usage_start = PHP_INT_MAX; $completed_info = array(); foreach ($completed as $completed_task) { $class = $completed_task->getTaskClass(); if (empty($completed_info[$class])) { $completed_info[$class] = array( 'n' => 0, 'duration' => 0, ); } $completed_info[$class]['n']++; $duration = $completed_task->getDuration(); $completed_info[$class]['duration'] += $duration; // NOTE: Duration is in microseconds, but we're just using seconds to // compute utilization. $usage_total += $lease_overhead + ($duration / 1000000); $usage_start = min($usage_start, $completed_task->getDateModified()); } $completed_info = isort($completed_info, 'n'); $rows = array(); foreach ($completed_info as $class => $info) { $rows[] = array( $class, number_format($info['n']), number_format((int)($info['duration'] / $info['n'])).' us', ); } if ($failed) { // Add the time it takes to restart the daemons. This includes a guess // about other overhead of 2X. $usage_total += PhutilDaemonOverseer::RESTART_WAIT * count($failed) * 2; foreach ($failed as $failed_task) { $usage_start = min($usage_start, $failed_task->getFailureTime()); } $rows[] = array( phutil_tag('em', array(), pht('Temporary Failures')), count($failed), null, ); } $logs = id(new PhabricatorDaemonLogQuery()) ->setViewer($user) ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE) ->setAllowStatusWrites(true) ->execute(); $taskmasters = 0; foreach ($logs as $log) { if ($log->getDaemon() == 'PhabricatorTaskmasterDaemon') { $taskmasters++; } } if ($taskmasters && $usage_total) { // Total number of wall-time seconds the daemons have been running since // the oldest event. For very short times round up to 15s so we don't // render any ridiculous numbers if you reload the page immediately after // restarting the daemons. $available_time = $taskmasters * max(15, (time() - $usage_start)); // Percentage of those wall-time seconds we can account for, which the // daemons spent doing work: $used_time = ($usage_total / $available_time); $rows[] = array( phutil_tag('em', array(), pht('Queue Utilization (Approximate)')), sprintf('%.1f%%', 100 * $used_time), null, ); } $completed_table = new AphrontTableView($rows); $completed_table->setNoDataString( pht('No tasks have completed in the last 15 minutes.')); $completed_table->setHeaders( array( pht('Class'), pht('Count'), pht('Avg'), )); $completed_table->setColumnClasses( array( 'wide', 'n', 'n', )); $completed_panel = new PHUIObjectBoxView(); $completed_panel->setHeaderText( pht('Recently Completed Tasks (Last 15m)')); $completed_panel->appendChild($completed_table); $daemon_table = new PhabricatorDaemonLogListView(); $daemon_table->setUser($user); $daemon_table->setDaemonLogs($logs); $daemon_panel = new PHUIObjectBoxView(); $daemon_panel->setHeaderText(pht('Active Daemons')); $daemon_panel->appendChild($daemon_table); - $tasks = id(new PhabricatorWorkerActiveTask())->loadAllWhere( - 'leaseOwner IS NOT NULL'); + $tasks = id(new PhabricatorWorkerLeaseQuery()) + ->setSkipLease(true) + ->withLeasedTasks(true) + ->setLimit(100) + ->execute(); $tasks_table = id(new PhabricatorDaemonTasksTableView()) ->setTasks($tasks) ->setNoDataString(pht('No tasks are leased by workers.')); $leased_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Leased Tasks')) ->appendChild($tasks_table); $task_table = new PhabricatorWorkerActiveTask(); $queued = queryfx_all( $task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass ORDER BY N DESC', $task_table->getTableName()); $rows = array(); foreach ($queued as $row) { $rows[] = array( $row['taskClass'], number_format($row['N']), ); } $queued_table = new AphrontTableView($rows); $queued_table->setHeaders( array( pht('Class'), pht('Count'), )); $queued_table->setColumnClasses( array( 'wide', 'n', )); $queued_table->setNoDataString(pht('Task queue is empty.')); $queued_panel = new PHUIObjectBoxView(); $queued_panel->setHeaderText(pht('Queued Tasks')); $queued_panel->appendChild($queued_table); $upcoming = id(new PhabricatorWorkerLeaseQuery()) ->setLimit(10) ->setSkipLease(true) ->execute(); $upcoming_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Next In Queue')) ->appendChild( id(new PhabricatorDaemonTasksTableView()) ->setTasks($upcoming) ->setNoDataString(pht('Task queue is empty.'))); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Console')); $nav = $this->buildSideNavView(); $nav->selectFilter('/'); $nav->appendChild( array( $crumbs, $completed_panel, $daemon_panel, $queued_panel, $leased_panel, $upcoming_panel, )); return $this->buildApplicationPage( $nav, array( 'title' => pht('Console'), 'device' => false, )); } } diff --git a/src/infrastructure/daemon/workers/query/PhabricatorWorkerArchiveTaskQuery.php b/src/infrastructure/daemon/workers/query/PhabricatorWorkerArchiveTaskQuery.php index f026e13456..93e829cc97 100644 --- a/src/infrastructure/daemon/workers/query/PhabricatorWorkerArchiveTaskQuery.php +++ b/src/infrastructure/daemon/workers/query/PhabricatorWorkerArchiveTaskQuery.php @@ -1,82 +1,94 @@ ids = $ids; return $this; } public function withDateModifiedSince($timestamp) { $this->dateModifiedSince = $timestamp; return $this; } public function withDateCreatedBefore($timestamp) { $this->dateCreatedBefore = $timestamp; return $this; } + public function withObjectPHIDs(array $phids) { + $this->objectPHIDs = $phids; + return $this; + } + public function setLimit($limit) { $this->limit = $limit; return $this; } public function execute() { - $task_table = new PhabricatorWorkerArchiveTask(); $conn_r = $task_table->establishConnection('r'); $rows = queryfx_all( $conn_r, - 'SELECT * FROM %T %Q %Q', + 'SELECT * FROM %T %Q ORDER BY id DESC %Q', $task_table->getTableName(), $this->buildWhereClause($conn_r), $this->buildLimitClause($conn_r)); return $task_table->loadAllFromArray($rows); } private function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->ids !== null) { $where[] = qsprintf( $conn_r, 'ids in (%Ld)', $this->ids); } + if ($this->objectPHIDs !== null) { + $where[] = qsprintf( + $conn_r, + 'objectPHID IN (%Ls)', + $this->objectPHIDs); + } + if ($this->dateModifiedSince) { $where[] = qsprintf( $conn_r, 'dateModified > %d', $this->dateModifiedSince); } if ($this->dateCreatedBefore) { $where[] = qsprintf( $conn_r, 'dateCreated < %d', $this->dateCreatedBefore); } return $this->formatWhereClause($where); } private function buildLimitClause(AphrontDatabaseConnection $conn_r) { $clause = ''; if ($this->limit) { $clause = qsprintf($conn_r, 'LIMIT %d', $this->limit); } return $clause; } } diff --git a/src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php b/src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php index b4cded74e3..456a042776 100644 --- a/src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php +++ b/src/infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php @@ -1,276 +1,332 @@ skipLease = $skip; return $this; } public function withIDs(array $ids) { $this->ids = $ids; return $this; } public function withObjectPHIDs(array $phids) { $this->objectPHIDs = $phids; return $this; } + /** + * Select only leased tasks, only unleased tasks, or both types of task. + * + * By default, queries select only unleased tasks (equivalent to passing + * `false` to this method). You can pass `true` to select only leased tasks, + * or `null` to ignore the lease status of tasks. + * + * If your result set potentially includes leased tasks, you must disable + * leasing using @{method:setSkipLease}. These options are intended for use + * when displaying task status information. + * + * @param mixed `true` to select only leased tasks, `false` to select only + * unleased tasks (default), or `null` to select both. + * @return this + */ + public function withLeasedTasks($leased) { + $this->leased = $leased; + return $this; + } + public function setLimit($limit) { $this->limit = $limit; return $this; } public function execute() { if (!$this->limit) { - throw new Exception('You must setLimit() when leasing tasks.'); + throw new Exception( + pht('You must setLimit() when leasing tasks.')); + } + + if ($this->leased !== false) { + if (!$this->skipLease) { + throw new Exception( + pht( + 'If you potentially select leased tasks using withLeasedTasks(), '. + 'you MUST disable lease acquisition by calling setSkipLease().')); + } } $task_table = new PhabricatorWorkerActiveTask(); $taskdata_table = new PhabricatorWorkerTaskData(); $lease_ownership_name = $this->getLeaseOwnershipName(); $conn_w = $task_table->establishConnection('w'); // Try to satisfy the request from new, unleased tasks first. If we don't // find enough tasks, try tasks with expired leases (i.e., tasks which have // previously failed). - $phases = array( - self::PHASE_UNLEASED, - self::PHASE_EXPIRED, - ); + // If we're selecting leased tasks, look for them first. + + $phases = array(); + if ($this->leased !== false) { + $phases[] = self::PHASE_LEASED; + } + if ($this->leased !== true) { + $phases[] = self::PHASE_UNLEASED; + $phases[] = self::PHASE_EXPIRED; + } $limit = $this->limit; $leased = 0; $task_ids = array(); foreach ($phases as $phase) { // NOTE: If we issue `UPDATE ... WHERE ... ORDER BY id ASC`, the query // goes very, very slowly. The `ORDER BY` triggers this, although we get // the same apparent results without it. Without the ORDER BY, binary // read slaves complain that the query isn't repeatable. To avoid both // problems, do a SELECT and then an UPDATE. $rows = queryfx_all( $conn_w, 'SELECT id, leaseOwner FROM %T %Q %Q %Q', $task_table->getTableName(), $this->buildWhereClause($conn_w, $phase), $this->buildOrderClause($conn_w, $phase), $this->buildLimitClause($conn_w, $limit - $leased)); // NOTE: Sometimes, we'll race with another worker and they'll grab // this task before we do. We could reduce how often this happens by // selecting more tasks than we need, then shuffling them and trying // to lock only the number we're actually after. However, the amount // of time workers spend here should be very small relative to their // total runtime, so keep it simple for the moment. if ($rows) { if ($this->skipLease) { $leased += count($rows); $task_ids += array_fuse(ipull($rows, 'id')); } else { queryfx( $conn_w, 'UPDATE %T task SET leaseOwner = %s, leaseExpires = UNIX_TIMESTAMP() + %d %Q', $task_table->getTableName(), $lease_ownership_name, self::getDefaultLeaseDuration(), $this->buildUpdateWhereClause($conn_w, $phase, $rows)); $leased += $conn_w->getAffectedRows(); } if ($leased == $limit) { break; } } } if (!$leased) { return array(); } if ($this->skipLease) { $selection_condition = qsprintf( $conn_w, 'task.id IN (%Ld)', $task_ids); } else { $selection_condition = qsprintf( $conn_w, 'task.leaseOwner = %s AND leaseExpires > UNIX_TIMESTAMP()', $lease_ownership_name); } $data = queryfx_all( $conn_w, 'SELECT task.*, taskdata.data _taskData, UNIX_TIMESTAMP() _serverTime FROM %T task LEFT JOIN %T taskdata ON taskdata.id = task.dataID WHERE %Q %Q %Q', $task_table->getTableName(), $taskdata_table->getTableName(), $selection_condition, $this->buildOrderClause($conn_w, $phase), $this->buildLimitClause($conn_w, $limit)); $tasks = $task_table->loadAllFromArray($data); $tasks = mpull($tasks, null, 'getID'); foreach ($data as $row) { $tasks[$row['id']]->setServerTime($row['_serverTime']); if ($row['_taskData']) { $task_data = json_decode($row['_taskData'], true); } else { $task_data = null; } $tasks[$row['id']]->setData($task_data); } + if ($this->skipLease) { + // Reorder rows into the original phase order if this is a status query. + $tasks = array_select_keys($tasks, $task_ids); + } + return $tasks; } private function buildWhereClause(AphrontDatabaseConnection $conn_w, $phase) { $where = array(); switch ($phase) { + case self::PHASE_LEASED: + $where[] = 'leaseOwner IS NOT NULL'; + $where[] = 'leaseExpires >= UNIX_TIMESTAMP()'; + break; case self::PHASE_UNLEASED: $where[] = 'leaseOwner IS NULL'; break; case self::PHASE_EXPIRED: $where[] = 'leaseExpires < UNIX_TIMESTAMP()'; break; default: throw new Exception("Unknown phase '{$phase}'!"); } - if ($this->ids) { + if ($this->ids !== null) { $where[] = qsprintf($conn_w, 'id IN (%Ld)', $this->ids); } if ($this->objectPHIDs !== null) { $where[] = qsprintf($conn_w, 'objectPHID IN (%Ls)', $this->objectPHIDs); } return $this->formatWhereClause($where); } private function buildUpdateWhereClause( AphrontDatabaseConnection $conn_w, $phase, array $rows) { $where = array(); // NOTE: This is basically working around the MySQL behavior that // `IN (NULL)` doesn't match NULL. switch ($phase) { + case self::PHASE_LEASED: + throw new Exception( + pht( + 'Trying to lease tasks selected in the leased phase! This is '. + 'intended to be imposssible.')); case self::PHASE_UNLEASED: $where[] = qsprintf($conn_w, 'leaseOwner IS NULL'); $where[] = qsprintf($conn_w, 'id IN (%Ld)', ipull($rows, 'id')); break; case self::PHASE_EXPIRED: $in = array(); foreach ($rows as $row) { $in[] = qsprintf( $conn_w, '(id = %d AND leaseOwner = %s)', $row['id'], $row['leaseOwner']); } $where[] = qsprintf($conn_w, '(%Q)', implode(' OR ', $in)); break; default: throw new Exception("Unknown phase '{$phase}'!"); } return $this->formatWhereClause($where); } private function buildOrderClause(AphrontDatabaseConnection $conn_w, $phase) { switch ($phase) { + case self::PHASE_LEASED: + // Ideally we'd probably order these by lease acquisition time, but + // we don't have that handy and this is a good approximation. + return qsprintf($conn_w, 'ORDER BY priority ASC, id ASC'); case self::PHASE_UNLEASED: // When selecting new tasks, we want to consume them in order of // increasing priority (and then FIFO). return qsprintf($conn_w, 'ORDER BY priority ASC, id ASC'); case self::PHASE_EXPIRED: // When selecting failed tasks, we want to consume them in roughly // FIFO order of their failures, which is not necessarily their original // queue order. // Particularly, this is important for tasks which use soft failures to // indicate that they are waiting on other tasks to complete: we need to // push them to the end of the queue after they fail, at least on // average, so we don't deadlock retrying the same blocked task over // and over again. return qsprintf($conn_w, 'ORDER BY leaseExpires ASC'); default: throw new Exception(pht('Unknown phase "%s"!', $phase)); } } private function buildLimitClause(AphrontDatabaseConnection $conn_w, $limit) { return qsprintf($conn_w, 'LIMIT %d', $limit); } private function getLeaseOwnershipName() { static $sequence = 0; // TODO: If the host name is very long, this can overflow the 64-character // column, so we pick just the first part of the host name. It might be // useful to just use a random hash as the identifier instead and put the // pid / time / host (which are somewhat useful diagnostically) elsewhere. // Likely, we could store a daemon ID instead and use that to identify // when and where code executed. See T6742. $host = php_uname('n'); $host = id(new PhutilUTF8StringTruncator()) ->setMaximumBytes(32) ->setTerminator('...') ->truncateString($host); $parts = array( getmypid(), time(), $host, ++$sequence, ); return implode(':', $parts); } } diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php index a254b3eb23..1f108e1b1d 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php @@ -1,212 +1,213 @@ self::IDS_COUNTER, self::CONFIG_TIMESTAMPS => false, self::CONFIG_KEY_SCHEMA => array( 'dataID' => array( 'columns' => array('dataID'), 'unique' => true, ), 'taskClass' => array( 'columns' => array('taskClass'), ), 'leaseExpires' => array( 'columns' => array('leaseExpires'), ), 'leaseOwner' => array( 'columns' => array('leaseOwner(16)'), ), 'key_failuretime' => array( 'columns' => array('failureTime'), ), 'leaseOwner_2' => array( 'columns' => array('leaseOwner', 'priority', 'id'), ), ) + $parent[self::CONFIG_KEY_SCHEMA], ); $config[self::CONFIG_COLUMN_SCHEMA] = array( // T6203/NULLABILITY // This isn't nullable in the archive table, so at a minimum these // should be the same. 'dataID' => 'uint32?', ) + $parent[self::CONFIG_COLUMN_SCHEMA]; return $config + $parent; } public function setServerTime($server_time) { $this->serverTime = $server_time; $this->localTime = time(); return $this; } public function setLeaseDuration($lease_duration) { $this->checkLease(); $server_lease_expires = $this->serverTime + $lease_duration; $this->setLeaseExpires($server_lease_expires); // NOTE: This is primarily to allow unit tests to set negative lease // durations so they don't have to wait around for leases to expire. We // check that the lease is valid above. return $this->forceSaveWithoutLease(); } public function save() { $this->checkLease(); return $this->forceSaveWithoutLease(); } public function forceSaveWithoutLease() { $is_new = !$this->getID(); if ($is_new) { $this->failureCount = 0; } if ($is_new && ($this->getData() !== null)) { $data = new PhabricatorWorkerTaskData(); $data->setData($this->getData()); $data->save(); $this->setDataID($data->getID()); } return parent::save(); } protected function checkLease() { if ($this->leaseOwner) { $current_server_time = $this->serverTime + (time() - $this->localTime); if ($current_server_time >= $this->leaseExpires) { $id = $this->getID(); $class = $this->getTaskClass(); throw new Exception( "Trying to update Task {$id} ({$class}) after lease expiration!"); } } } public function delete() { throw new Exception( 'Active tasks can not be deleted directly. '. 'Use archiveTask() to move tasks to the archive.'); } public function archiveTask($result, $duration) { if ($this->getID() === null) { throw new Exception( "Attempting to archive a task which hasn't been save()d!"); } $this->checkLease(); $archive = id(new PhabricatorWorkerArchiveTask()) ->setID($this->getID()) ->setTaskClass($this->getTaskClass()) ->setLeaseOwner($this->getLeaseOwner()) ->setLeaseExpires($this->getLeaseExpires()) ->setFailureCount($this->getFailureCount()) ->setDataID($this->getDataID()) ->setPriority($this->getPriority()) + ->setObjectPHID($this->getObjectPHID()) ->setResult($result) ->setDuration($duration); // NOTE: This deletes the active task (this object)! $archive->save(); return $archive; } public function executeTask() { // We do this outside of the try .. catch because we don't have permission // to release the lease otherwise. $this->checkLease(); $did_succeed = false; try { $worker = $this->getWorkerInstance(); $maximum_failures = $worker->getMaximumRetryCount(); if ($maximum_failures !== null) { if ($this->getFailureCount() > $maximum_failures) { $id = $this->getID(); throw new PhabricatorWorkerPermanentFailureException( "Task {$id} has exceeded the maximum number of failures ". "({$maximum_failures})."); } } $lease = $worker->getRequiredLeaseTime(); if ($lease !== null) { $this->setLeaseDuration($lease); } $t_start = microtime(true); $worker->executeTask(); $t_end = microtime(true); $duration = (int)(1000000 * ($t_end - $t_start)); $result = $this->archiveTask( PhabricatorWorkerArchiveTask::RESULT_SUCCESS, $duration); $did_succeed = true; } catch (PhabricatorWorkerPermanentFailureException $ex) { $result = $this->archiveTask( PhabricatorWorkerArchiveTask::RESULT_FAILURE, 0); $result->setExecutionException($ex); } catch (PhabricatorWorkerYieldException $ex) { $this->setExecutionException($ex); $retry = $ex->getDuration(); $retry = max($retry, 5); // NOTE: As a side effect, this saves the object. $this->setLeaseDuration($retry); $result = $this; } catch (Exception $ex) { $this->setExecutionException($ex); $this->setFailureCount($this->getFailureCount() + 1); $this->setFailureTime(time()); $retry = $worker->getWaitBeforeRetry($this); $retry = coalesce( $retry, PhabricatorWorkerLeaseQuery::getDefaultWaitBeforeRetry()); // NOTE: As a side effect, this saves the object. $this->setLeaseDuration($retry); $result = $this; } // NOTE: If this throws, we don't want it to cause the task to fail again, // so execute it out here and just let the exception escape. if ($did_succeed) { foreach ($worker->getQueuedTasks() as $task) { list($class, $data) = $task; PhabricatorWorker::scheduleTask( $class, $data, array( 'priority' => $this->getPriority(), )); } } return $result; } } diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php index 3076b36b03..d28f13c8d0 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php @@ -1,96 +1,97 @@ self::IDS_MANUAL, ) + $parent; $config[self::CONFIG_COLUMN_SCHEMA] = array( 'result' => 'uint32', 'duration' => 'uint64', ) + $config[self::CONFIG_COLUMN_SCHEMA]; $config[self::CONFIG_KEY_SCHEMA] = array( 'dateCreated' => array( 'columns' => array('dateCreated'), ), 'leaseOwner' => array( 'columns' => array('leaseOwner', 'priority', 'id'), ), ) + $parent[self::CONFIG_KEY_SCHEMA]; return $config; } public function save() { if ($this->getID() === null) { throw new Exception('Trying to archive a task with no ID.'); } $other = new PhabricatorWorkerActiveTask(); $conn_w = $this->establishConnection('w'); $this->openTransaction(); queryfx( $conn_w, 'DELETE FROM %T WHERE id = %d', $other->getTableName(), $this->getID()); $result = parent::insert(); $this->saveTransaction(); return $result; } public function delete() { $this->openTransaction(); if ($this->getDataID()) { $conn_w = $this->establishConnection('w'); $data_table = new PhabricatorWorkerTaskData(); queryfx( $conn_w, 'DELETE FROM %T WHERE id = %d', $data_table->getTableName(), $this->getDataID()); } $result = parent::delete(); $this->saveTransaction(); return $result; } public function unarchiveTask() { $this->openTransaction(); $active = id(new PhabricatorWorkerActiveTask()) ->setID($this->getID()) ->setTaskClass($this->getTaskClass()) ->setLeaseOwner(null) ->setLeaseExpires(0) ->setFailureCount(0) ->setDataID($this->getDataID()) ->setPriority($this->getPriority()) + ->setObjectPHID($this->getObjectPHID()) ->insert(); $this->setDataID(null); $this->delete(); $this->saveTransaction(); return $active; } }