diff --git a/resources/sql/autopatches/20181024.drydock.01.commandprops.sql b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql new file mode 100644 index 0000000000..e808146b02 --- /dev/null +++ b/resources/sql/autopatches/20181024.drydock.01.commandprops.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_drydock.drydock_command + ADD properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql new file mode 100644 index 0000000000..2c336dc40e --- /dev/null +++ b/resources/sql/autopatches/20181024.drydock.02.commanddefaults.sql @@ -0,0 +1,2 @@ +UPDATE {$NAMESPACE}_drydock.drydock_command + SET properties = '{}' WHERE properties = ''; diff --git a/src/applications/drydock/storage/DrydockCommand.php b/src/applications/drydock/storage/DrydockCommand.php index e7d003bdd6..0f7f253217 100644 --- a/src/applications/drydock/storage/DrydockCommand.php +++ b/src/applications/drydock/storage/DrydockCommand.php @@ -1,70 +1,82 @@ setAuthorPHID($author->getPHID()) ->setIsConsumed(0); } protected function getConfiguration() { return array( + self::CONFIG_SERIALIZATION => array( + 'properties' => self::SERIALIZATION_JSON, + ), self::CONFIG_COLUMN_SCHEMA => array( 'command' => 'text32', 'isConsumed' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_target' => array( 'columns' => array('targetPHID', 'isConsumed'), ), ), ) + parent::getConfiguration(); } public function attachCommandTarget($target) { $this->commandTarget = $target; return $this; } public function getCommandTarget() { return $this->assertAttached($this->commandTarget); } + public function setProperty($key, $value) { + $this->properties[$key] = $value; + return $this; + } + + public function getProperty($key, $default = null) { + return idx($this->properties, $key, $default); + } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return $this->getCommandTarget()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getCommandTarget()->hasAutomaticCapability( $capability, $viewer); } public function describeAutomaticCapability($capability) { return pht('Drydock commands have the same policies as their targets.'); } } diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php index baa80f90d8..817fbaddee 100644 --- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php @@ -1,301 +1,308 @@ getTaskDataValue('resourcePHID'); $hash = PhabricatorHash::digestForIndex($resource_phid); $lock_key = 'drydock.resource:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $resource = $this->loadResource($resource_phid); $this->handleUpdate($resource); } catch (Exception $ex) { $lock->unlock(); $this->flushDrydockTaskQueue(); throw $ex; } $lock->unlock(); } /* -( Updating Resources )------------------------------------------------- */ /** * Update a resource, handling exceptions thrown during the update. * * @param DrydockReosource Resource to update. * @return void * @task update */ private function handleUpdate(DrydockResource $resource) { try { $this->updateResource($resource); } catch (Exception $ex) { if ($this->isTemporaryException($ex)) { $this->yieldResource($resource, $ex); } else { $this->breakResource($resource, $ex); } } } /** * Update a resource. * * @param DrydockResource Resource to update. * @return void * @task update */ private function updateResource(DrydockResource $resource) { $this->processResourceCommands($resource); $resource_status = $resource->getStatus(); switch ($resource_status) { case DrydockResourceStatus::STATUS_PENDING: $this->activateResource($resource); break; case DrydockResourceStatus::STATUS_ACTIVE: // Nothing to do. break; case DrydockResourceStatus::STATUS_RELEASED: case DrydockResourceStatus::STATUS_BROKEN: $this->destroyResource($resource); break; case DrydockResourceStatus::STATUS_DESTROYED: // Nothing to do. break; } $this->yieldIfExpiringResource($resource); } /** * Convert a temporary exception into a yield. * * @param DrydockResource Resource to yield. * @param Exception Temporary exception worker encountered. * @task update */ private function yieldResource(DrydockResource $resource, Exception $ex) { $duration = $this->getYieldDurationFromException($ex); $resource->logEvent( DrydockResourceActivationYieldLogType::LOGCONST, array( 'duration' => $duration, )); throw new PhabricatorWorkerYieldException($duration); } /* -( Processing Commands )------------------------------------------------ */ /** * @task command */ private function processResourceCommands(DrydockResource $resource) { if (!$resource->canReceiveCommands()) { return; } $this->checkResourceExpiration($resource); $commands = $this->loadCommands($resource->getPHID()); foreach ($commands as $command) { if (!$resource->canReceiveCommands()) { break; } $this->processResourceCommand($resource, $command); $command ->setIsConsumed(true) ->save(); } } /** * @task command */ private function processResourceCommand( DrydockResource $resource, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseResource($resource, null); break; case DrydockCommand::COMMAND_RECLAIM: $reclaimer_phid = $command->getAuthorPHID(); $this->releaseResource($resource, $reclaimer_phid); break; } + + // If the command specifies that other worker tasks should be awakened + // after it executes, awaken them now. + $awaken_ids = $command->getProperty('awakenTaskIDs'); + if (is_array($awaken_ids) && $awaken_ids) { + PhabricatorWorker::awakenTaskIDs($awaken_ids); + } } /* -( Activating Resources )----------------------------------------------- */ /** * @task activate */ private function activateResource(DrydockResource $resource) { $blueprint = $resource->getBlueprint(); $blueprint->activateResource($resource); $this->validateActivatedResource($blueprint, $resource); } /** * @task activate */ private function validateActivatedResource( DrydockBlueprint $blueprint, DrydockResource $resource) { if (!$resource->isActivatedResource()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 'must actually allocate the resource it returns.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()')); } } /* -( Releasing Resources )------------------------------------------------ */ /** * @task release */ private function releaseResource( DrydockResource $resource, $reclaimer_phid) { if ($reclaimer_phid) { if (!$this->canReclaimResource($resource)) { return; } $resource->logEvent( DrydockResourceReclaimLogType::LOGCONST, array( 'reclaimerPHID' => $reclaimer_phid, )); } $viewer = $this->getViewer(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $resource ->setStatus(DrydockResourceStatus::STATUS_RELEASED) ->save(); $statuses = array( DrydockLeaseStatus::STATUS_PENDING, DrydockLeaseStatus::STATUS_ACQUIRED, DrydockLeaseStatus::STATUS_ACTIVE, ); $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses($statuses) ->execute(); foreach ($leases as $lease) { $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($lease->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); $lease->scheduleUpdate(); } $this->destroyResource($resource); } /* -( Breaking Resources )------------------------------------------------- */ /** * @task break */ private function breakResource(DrydockResource $resource, Exception $ex) { switch ($resource->getStatus()) { case DrydockResourceStatus::STATUS_BROKEN: case DrydockResourceStatus::STATUS_RELEASED: case DrydockResourceStatus::STATUS_DESTROYED: // If the resource was already broken, just throw a normal exception. // This will retry the task eventually. throw new PhutilProxyException( pht( 'Unexpected failure while destroying resource ("%s").', $resource->getPHID()), $ex); } $resource ->setStatus(DrydockResourceStatus::STATUS_BROKEN) ->save(); $resource->scheduleUpdate(); $resource->logEvent( DrydockResourceActivationFailureLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); throw new PhabricatorWorkerPermanentFailureException( pht( 'Permanent failure while activating resource ("%s"): %s', $resource->getPHID(), $ex->getMessage())); } /* -( Destroying Resources )----------------------------------------------- */ /** * @task destroy */ private function destroyResource(DrydockResource $resource) { $blueprint = $resource->getBlueprint(); $blueprint->destroyResource($resource); DrydockSlotLock::releaseLocks($resource->getPHID()); $resource ->setStatus(DrydockResourceStatus::STATUS_DESTROYED) ->save(); } } diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php index 443780680d..8a81da09b4 100644 --- a/src/applications/drydock/worker/DrydockWorker.php +++ b/src/applications/drydock/worker/DrydockWorker.php @@ -1,255 +1,260 @@ getViewer(); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new PhabricatorWorkerPermanentFailureException( pht('No such lease "%s"!', $lease_phid)); } return $lease; } protected function loadResource($resource_phid) { $viewer = $this->getViewer(); $resource = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withPHIDs(array($resource_phid)) ->executeOne(); if (!$resource) { throw new PhabricatorWorkerPermanentFailureException( pht('No such resource "%s"!', $resource_phid)); } return $resource; } protected function loadOperation($operation_phid) { $viewer = $this->getViewer(); $operation = id(new DrydockRepositoryOperationQuery()) ->setViewer($viewer) ->withPHIDs(array($operation_phid)) ->executeOne(); if (!$operation) { throw new PhabricatorWorkerPermanentFailureException( pht('No such operation "%s"!', $operation_phid)); } return $operation; } protected function loadCommands($target_phid) { $viewer = $this->getViewer(); $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($target_phid)) ->withConsumed(false) ->execute(); $commands = msort($commands, 'getID'); return $commands; } protected function checkLeaseExpiration(DrydockLease $lease) { $this->checkObjectExpiration($lease); } protected function checkResourceExpiration(DrydockResource $resource) { $this->checkObjectExpiration($resource); } private function checkObjectExpiration($object) { // Check if the resource or lease has expired. If it has, we're going to // send it a release command. // This command is sent from within the update worker so it is handled // immediately, but doing this generates a log and improves consistency. $expires = $object->getUntil(); if (!$expires) { return; } $now = PhabricatorTime::getNow(); if ($expires > $now) { return; } $viewer = $this->getViewer(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($object->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); } protected function yieldIfExpiringLease(DrydockLease $lease) { if (!$lease->canReceiveCommands()) { return; } $this->yieldIfExpiring($lease->getUntil()); } protected function yieldIfExpiringResource(DrydockResource $resource) { if (!$resource->canReceiveCommands()) { return; } $this->yieldIfExpiring($resource->getUntil()); } private function yieldIfExpiring($expires) { if (!$expires) { return; } if (!$this->getTaskDataValue('isExpireTask')) { return; } $now = PhabricatorTime::getNow(); throw new PhabricatorWorkerYieldException($expires - $now); } protected function isTemporaryException(Exception $ex) { if ($ex instanceof PhabricatorWorkerYieldException) { return true; } if ($ex instanceof DrydockSlotLockException) { return true; } if ($ex instanceof PhutilAggregateException) { $any_temporary = false; foreach ($ex->getExceptions() as $sub) { if ($this->isTemporaryException($sub)) { $any_temporary = true; break; } } if ($any_temporary) { return true; } } if ($ex instanceof PhutilProxyException) { return $this->isTemporaryException($ex->getPreviousException()); } return false; } protected function getYieldDurationFromException(Exception $ex) { if ($ex instanceof PhabricatorWorkerYieldException) { return $ex->getDuration(); } if ($ex instanceof DrydockSlotLockException) { return 5; } return 15; } protected function flushDrydockTaskQueue() { // NOTE: By default, queued tasks are not scheduled if the current task // fails. This is a good, safe default behavior. For example, it can // protect us from executing side effect tasks too many times, like // sending extra email. // However, it is not the behavior we want in Drydock, because we queue // followup tasks after lease and resource failures and want them to // execute in order to clean things up. // At least for now, we just explicitly flush the queue before exiting // with a failure to make sure tasks get queued up properly. try { $this->flushTaskQueue(); } catch (Exception $ex) { // If this fails, we want to swallow the exception so the caller throws // the original error, since we're more likely to be able to understand // and fix the problem if we have the original error than if we replace // it with this one. phlog($ex); } return $this; } protected function canReclaimResource(DrydockResource $resource) { $viewer = $this->getViewer(); // Don't reclaim a resource if it has been updated recently. If two // leases are fighting, we don't want them to keep reclaiming resources // from one another forever without making progress, so make resources // immune to reclamation for a little while after they activate or update. // TODO: It would be nice to use a more narrow time here, like "last // activation or lease release", but we don't currently store that // anywhere. $updated = $resource->getDateModified(); $now = PhabricatorTime::getNow(); $ago = ($now - $updated); if ($ago < phutil_units('3 minutes in seconds')) { return false; } $statuses = array( DrydockLeaseStatus::STATUS_PENDING, DrydockLeaseStatus::STATUS_ACQUIRED, DrydockLeaseStatus::STATUS_ACTIVE, DrydockLeaseStatus::STATUS_RELEASED, DrydockLeaseStatus::STATUS_BROKEN, ); // Don't reclaim resources that have any active leases. $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses($statuses) ->setLimit(1) ->execute(); if ($leases) { return false; } return true; } protected function reclaimResource( DrydockResource $resource, DrydockLease $lease) { $viewer = $this->getViewer(); + // When the resource releases, we we want to reawaken this task since it + // should be able to start building a new resource right away. + $worker_task_id = $this->getCurrentWorkerTaskID(); + $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($resource->getPHID()) ->setAuthorPHID($lease->getPHID()) ->setCommand(DrydockCommand::COMMAND_RECLAIM) + ->setProperty('awakenTaskIDs', array($worker_task_id)) ->save(); $resource->scheduleUpdate(); return $this; } }