diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php index e3cdff34c5..0f2e0aad44 100644 --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -1,313 +1,380 @@ setAncestorClass(__CLASS__) ->execute(); } public static function getNamedImplementation($class) { return idx(self::getAllBlueprintImplementations(), $class); } protected function newResourceTemplate(DrydockBlueprint $blueprint) { $resource = id(new DrydockResource()) ->setBlueprintPHID($blueprint->getPHID()) ->attachBlueprint($blueprint) ->setType($this->getType()) ->setStatus(DrydockResourceStatus::STATUS_PENDING); // Pre-allocate the resource PHID. $resource->setPHID($resource->generatePHID()); return $resource; } protected function newLease(DrydockBlueprint $blueprint) { return DrydockLease::initializeNewLease(); } protected function requireActiveLease(DrydockLease $lease) { $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: throw new PhabricatorWorkerYieldException(15); case DrydockLeaseStatus::STATUS_ACTIVE: return; default: throw new Exception( pht( 'Lease ("%s") is in bad state ("%s"), expected "%s".', $lease->getPHID(), $lease_status, DrydockLeaseStatus::STATUS_ACTIVE)); } } + + /** + * Apply standard limits on resource allocation rate. + * + * @param DrydockBlueprint The blueprint requesting an allocation. + * @return bool True if further allocations should be limited. + */ + protected function shouldLimitAllocatingPoolSize( + DrydockBlueprint $blueprint) { + + // TODO: If this mechanism sticks around, these values should be + // configurable by the blueprint implementation. + + // Limit on total number of active resources. + $total_limit = 1; + + // Always allow at least this many allocations to be in flight at once. + $min_allowed = 1; + + // Allow this fraction of allocating resources as a fraction of active + // resources. + $growth_factor = 0.25; + + $resource = new DrydockResource(); + $conn_r = $resource->establishConnection('r'); + + $counts = queryfx_all( + $conn_r, + 'SELECT status, COUNT(*) N FROM %T WHERE blueprintPHID = %s', + $resource->getTableName(), + $blueprint->getPHID()); + $counts = ipull($counts, 'N', 'status'); + + $n_alloc = idx($counts, DrydockResourceStatus::STATUS_PENDING, 0); + $n_active = idx($counts, DrydockResourceStatus::STATUS_ACTIVE, 0); + $n_broken = idx($counts, DrydockResourceStatus::STATUS_BROKEN, 0); + $n_released = idx($counts, DrydockResourceStatus::STATUS_RELEASED, 0); + + // If we're at the limit on total active resources, limit additional + // allocations. + $n_total = ($n_alloc + $n_active + $n_broken + $n_released); + if ($n_total >= $total_limit) { + return true; + } + + // If the number of in-flight allocations is fewer than the minimum number + // of allowed allocations, don't impose a limit. + if ($n_alloc < $min_allowed) { + return false; + } + + $allowed_alloc = (int)ceil($n_active * $growth_factor); + + // If the number of in-flight allocation is fewer than the number of + // allowed allocations according to the pool growth factor, don't impose + // a limit. + if ($n_alloc < $allowed_alloc) { + return false; + } + + return true; + } + } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php index e086294c4d..30dd6fe4c5 100644 --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -1,375 +1,394 @@ getViewer(); + + if ($this->shouldLimitAllocatingPoolSize($blueprint)) { + return false; + } + + // TODO: If we have a pending resource which is compatible with the + // configuration for this lease, prevent a new allocation? Otherwise the + // queue can fill up with copies of requests from the same lease. But + // maybe we can deal with this with "pre-leasing"? + return true; } public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { + // Don't hand out leases on working copies which have not activated, since + // it may take an arbitrarily long time for them to acquire a host. + if (!$resource->isActive()) { + return false; + } + $need_map = $lease->getAttribute('repositories.map'); if (!is_array($need_map)) { return false; } $have_map = $resource->getAttribute('repositories.map'); if (!is_array($have_map)) { return false; } $have_as = ipull($have_map, 'phid'); $need_as = ipull($need_map, 'phid'); foreach ($need_as as $need_directory => $need_phid) { if (empty($have_as[$need_directory])) { // This resource is missing a required working copy. return false; } if ($have_as[$need_directory] != $need_phid) { // This resource has a required working copy, but it contains // the wrong repository. return false; } unset($have_as[$need_directory]); } if ($have_as && $lease->getAttribute('repositories.strict')) { // This resource has extra repositories, but the lease is strict about // which repositories are allowed to exist. return false; } if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { return false; } return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->needSlotLock($this->getLeaseSlotLock($resource)) ->acquireOnResource($resource); } private function getLeaseSlotLock(DrydockResource $resource) { $resource_phid = $resource->getPHID(); return "workingcopy.lease({$resource_phid})"; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $this->newResourceTemplate($blueprint); $resource_phid = $resource->getPHID(); $host_lease = $this->newLease($blueprint) ->setResourceType('host') ->setOwnerPHID($resource_phid) ->setAttribute('workingcopy.resourcePHID', $resource_phid); $resource ->setAttribute('host.leasePHID', $host_lease->getPHID()) ->save(); $host_lease->queueForActivation(); // TODO: Add some limits to the number of working copies we can have at // once? $map = $lease->getAttribute('repositories.map'); foreach ($map as $key => $value) { $map[$key] = array_select_keys( $value, array( 'phid', )); } return $resource ->setAttribute('repositories.map', $map) ->allocateResource(); } public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { $lease = $this->loadHostLease($resource); $this->requireActiveLease($lease); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); // TODO: Make this configurable. $resource_id = $resource->getID(); $root = "/var/drydock/workingcopy-{$resource_id}"; $map = $resource->getAttribute('repositories.map'); $repositories = $this->loadRepositories(ipull($map, 'phid')); foreach ($map as $directory => $spec) { // TODO: Validate directory isn't goofy like "/etc" or "../../lol" // somewhere? $repository = $repositories[$spec['phid']]; $path = "{$root}/repo/{$directory}/"; // TODO: Run these in parallel? $interface->execx( 'git clone -- %s %s', (string)$repository->getCloneURIObject(), $path); } $resource ->setAttribute('workingcopy.root', $root) ->activateResource(); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { try { $lease = $this->loadHostLease($resource); } catch (Exception $ex) { // If we can't load the lease, assume we don't need to take any actions // to destroy it. return; } // Destroy the lease on the host. $lease->releaseOnDestruction(); if ($lease->isActive()) { // Destroy the working copy on disk. $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); $root_key = 'workingcopy.root'; $root = $resource->getAttribute($root_key); if (strlen($root)) { $interface->execx('rm -rf -- %s', $root); } } } public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource) { return pht('Working Copy'); } public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $host_lease = $this->loadHostLease($resource); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $host_lease->getInterface($command_type); $map = $lease->getAttribute('repositories.map'); $root = $resource->getAttribute('workingcopy.root'); $default = null; foreach ($map as $directory => $spec) { $cmd = array(); $arg = array(); $cmd[] = 'cd %s'; $arg[] = "{$root}/repo/{$directory}/"; $cmd[] = 'git clean -d --force'; $cmd[] = 'git fetch'; $commit = idx($spec, 'commit'); $branch = idx($spec, 'branch'); $ref = idx($spec, 'ref'); if ($commit !== null) { $cmd[] = 'git reset --hard %s'; $arg[] = $commit; } else if ($branch !== null) { $cmd[] = 'git checkout %s'; $arg[] = $branch; $cmd[] = 'git reset --hard origin/%s'; $arg[] = $branch; } else if ($ref) { $ref_uri = $ref['uri']; $ref_ref = $ref['ref']; $cmd[] = 'git fetch --no-tags -- %s +%s:%s'; $arg[] = $ref_uri; $arg[] = $ref_ref; $arg[] = $ref_ref; $cmd[] = 'git checkout %s'; $arg[] = $ref_ref; $cmd[] = 'git reset --hard %s'; $arg[] = $ref_ref; } else { $cmd[] = 'git reset --hard HEAD'; } $cmd = implode(' && ', $cmd); $argv = array_merge(array($cmd), $arg); $result = call_user_func_array( array($interface, 'execx'), $argv); if (idx($spec, 'default')) { $default = $directory; } } if ($default === null) { $default = head_key($map); } // TODO: Use working storage? $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/"); $lease->activateOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We leave working copies around even if there are no leases on them, // since the cost to maintain them is nearly zero but rebuilding them is // moderately expensive and it's likely that they'll be reused. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // When we activate a lease we just reset the working copy state and do // not create any new state, so we don't need to do anything special when // destroying a lease. return; } public function getType() { return 'working-copy'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $host_lease = $this->loadHostLease($resource); $command_interface = $host_lease->getInterface($type); $path = $lease->getAttribute('workingcopy.default'); $command_interface->setWorkingDirectory($path); return $command_interface; } } private function loadRepositories(array $phids) { + $viewer = $this->getViewer(); + $repositories = id(new PhabricatorRepositoryQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->setViewer($viewer) ->withPHIDs($phids) ->execute(); $repositories = mpull($repositories, null, 'getPHID'); foreach ($phids as $phid) { if (empty($repositories[$phid])) { throw new Exception( pht( 'Repository PHID "%s" does not exist.', $phid)); } } foreach ($repositories as $repository) { $repository_vcs = $repository->getVersionControlSystem(); switch ($repository_vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: break; default: throw new Exception( pht( 'Repository ("%s") has unsupported VCS ("%s").', $repository->getPHID(), $repository_vcs)); } } return $repositories; } private function loadHostLease(DrydockResource $resource) { - $viewer = PhabricatorUser::getOmnipotentUser(); + $viewer = $this->getViewer(); $lease_phid = $resource->getAttribute('host.leasePHID'); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new Exception( pht( 'Unable to load lease ("%s").', $lease_phid)); } return $lease; } } diff --git a/src/applications/drydock/storage/DrydockResource.php b/src/applications/drydock/storage/DrydockResource.php index 1ee663fa3c..3eceec8b77 100644 --- a/src/applications/drydock/storage/DrydockResource.php +++ b/src/applications/drydock/storage/DrydockResource.php @@ -1,332 +1,341 @@ true, self::CONFIG_SERIALIZATION => array( 'attributes' => self::SERIALIZATION_JSON, 'capabilities' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'ownerPHID' => 'phid?', 'status' => 'text32', 'type' => 'text64', 'until' => 'epoch?', ), self::CONFIG_KEY_SCHEMA => array( 'key_type' => array( 'columns' => array('type', 'status'), ), 'key_blueprint' => array( 'columns' => array('blueprintPHID', 'status'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID(DrydockResourcePHIDType::TYPECONST); } public function getResourceName() { return $this->getBlueprint()->getResourceName($this); } public function getAttribute($key, $default = null) { return idx($this->attributes, $key, $default); } public function getAttributesForTypeSpec(array $attribute_names) { return array_select_keys($this->attributes, $attribute_names); } public function setAttribute($key, $value) { $this->attributes[$key] = $value; return $this; } public function getCapability($key, $default = null) { return idx($this->capbilities, $key, $default); } public function getInterface(DrydockLease $lease, $type) { return $this->getBlueprint()->getInterface($this, $lease, $type); } public function getBlueprint() { return $this->assertAttached($this->blueprint); } public function attachBlueprint(DrydockBlueprint $blueprint) { $this->blueprint = $blueprint; return $this; } public function getUnconsumedCommands() { return $this->assertAttached($this->unconsumedCommands); } public function attachUnconsumedCommands(array $commands) { $this->unconsumedCommands = $commands; return $this; } public function isReleasing() { foreach ($this->getUnconsumedCommands() as $command) { if ($command->getCommand() == DrydockCommand::COMMAND_RELEASE) { return true; } } return false; } public function setActivateWhenAllocated($activate) { $this->activateWhenAllocated = $activate; return $this; } public function needSlotLock($key) { $this->slotLocks[] = $key; return $this; } public function allocateResource() { // We expect resources to have a pregenerated PHID, as they should have // been created by a call to DrydockBlueprint->newResourceTemplate(). if (!$this->getPHID()) { throw new Exception( pht( 'Trying to allocate a resource with no generated PHID. Use "%s" to '. 'create new resource templates.', 'newResourceTemplate()')); } $expect_status = DrydockResourceStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to allocate a resource from the wrong status. Status must '. 'be "%s", actually "%s".', $expect_status, $actual_status)); } if ($this->activateWhenAllocated) { $new_status = DrydockResourceStatus::STATUS_ACTIVE; } else { $new_status = DrydockResourceStatus::STATUS_PENDING; } $this->openTransaction(); try { DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); } catch (DrydockSlotLockException $ex) { $this->killTransaction(); if ($this->getID()) { $log_target = $this; } else { // If we don't have an ID, we have to log this on the blueprint, as the // resource is not going to be saved so the PHID will vanish. $log_target = $this->getBlueprint(); } $log_target->logEvent( DrydockSlotLockFailureLogType::LOGCONST, array( 'locks' => $ex->getLockMap(), )); throw $ex; } $this ->setStatus($new_status) ->save(); $this->saveTransaction(); $this->isAllocated = true; if ($new_status == DrydockResourceStatus::STATUS_ACTIVE) { $this->didActivate(); } return $this; } public function isAllocatedResource() { return $this->isAllocated; } public function activateResource() { if (!$this->getID()) { throw new Exception( pht( 'Trying to activate a resource which has not yet been persisted.')); } $expect_status = DrydockResourceStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to activate a resource from the wrong status. Status must '. 'be "%s", actually "%s".', $expect_status, $actual_status)); } $this->openTransaction(); try { DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); } catch (DrydockSlotLockException $ex) { $this->killTransaction(); $this->logEvent( DrydockSlotLockFailureLogType::LOGCONST, array( 'locks' => $ex->getLockMap(), )); throw $ex; } $this ->setStatus(DrydockResourceStatus::STATUS_ACTIVE) ->save(); $this->saveTransaction(); $this->isActivated = true; $this->didActivate(); return $this; } public function isActivatedResource() { return $this->isActivated; } public function canRelease() { switch ($this->getStatus()) { case DrydockResourceStatus::STATUS_RELEASED: case DrydockResourceStatus::STATUS_DESTROYED: return false; default: return true; } } public function scheduleUpdate($epoch = null) { PhabricatorWorker::scheduleTask( 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $this->getPHID(), 'isExpireTask' => ($epoch !== null), ), array( 'objectPHID' => $this->getPHID(), 'delayUntil' => ($epoch ? (int)$epoch : null), )); } private function didActivate() { $viewer = PhabricatorUser::getOmnipotentUser(); $need_update = false; $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($this->getPHID())) ->withConsumed(false) ->execute(); if ($commands) { $need_update = true; } if ($need_update) { $this->scheduleUpdate(); } $expires = $this->getUntil(); if ($expires) { $this->scheduleUpdate($expires); } } public function canReceiveCommands() { switch ($this->getStatus()) { case DrydockResourceStatus::STATUS_RELEASED: case DrydockResourceStatus::STATUS_BROKEN: case DrydockResourceStatus::STATUS_DESTROYED: return false; default: return true; } } + public function isActive() { + switch ($this->getStatus()) { + case DrydockResourceStatus::STATUS_ACTIVE: + return true; + } + + return false; + } + public function logEvent($type, array $data = array()) { $log = id(new DrydockLog()) ->setEpoch(PhabricatorTime::getNow()) ->setType($type) ->setData($data); $log->setResourcePHID($this->getPHID()); $log->setBlueprintPHID($this->getBlueprintPHID()); return $log->save(); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { return $this->getBlueprint()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getBlueprint()->hasAutomaticCapability( $capability, $viewer); } public function describeAutomaticCapability($capability) { return pht('Resources inherit the policies of their blueprints.'); } } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index b6ac6cec52..a93997259d 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,734 +1,741 @@ getTaskDataValue('leasePHID'); $hash = PhabricatorHash::digestForIndex($lease_phid); $lock_key = 'drydock.lease:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $lease = $this->loadLease($lease_phid); $this->handleUpdate($lease); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); } /* -( Updating Leases )---------------------------------------------------- */ /** * @task update */ private function handleUpdate(DrydockLease $lease) { try { $this->updateLease($lease); } catch (Exception $ex) { if ($this->isTemporaryException($ex)) { $this->yieldLease($lease, $ex); } else { $this->breakLease($lease, $ex); } } } /** * @task update */ private function updateLease(DrydockLease $lease) { $this->processLeaseCommands($lease); $lease_status = $lease->getStatus(); switch ($lease_status) { case DrydockLeaseStatus::STATUS_PENDING: $this->executeAllocator($lease); break; case DrydockLeaseStatus::STATUS_ACQUIRED: $this->activateLease($lease); break; case DrydockLeaseStatus::STATUS_ACTIVE: // Nothing to do. break; case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_BROKEN: $this->destroyLease($lease); break; case DrydockLeaseStatus::STATUS_DESTROYED: break; } $this->yieldIfExpiringLease($lease); } /** * @task update */ private function yieldLease(DrydockLease $lease, Exception $ex) { $duration = $this->getYieldDurationFromException($ex); $lease->logEvent( DrydockLeaseActivationYieldLogType::LOGCONST, array( 'duration' => $duration, )); throw new PhabricatorWorkerYieldException($duration); } /* -( Processing Commands )------------------------------------------------ */ /** * @task command */ private function processLeaseCommands(DrydockLease $lease) { if (!$lease->canReceiveCommands()) { return; } $this->checkLeaseExpiration($lease); $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { if (!$lease->canReceiveCommands()) { break; } $this->processLeaseCommand($lease, $command); $command ->setIsConsumed(true) ->save(); } } /** * @task command */ private function processLeaseCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseLease($lease); break; } } /* -( Drydock Allocator )-------------------------------------------------- */ /** * Find or build a resource which can satisfy a given lease request, then * acquire the lease. * * @param DrydockLease Requested lease. * @return void * @task allocator */ private function executeAllocator(DrydockLease $lease) { $blueprints = $this->loadBlueprintsForAllocatingLease($lease); // If we get nothing back, that means no blueprint is defined which can // ever build the requested resource. This is a permanent failure, since // we don't expect to succeed no matter how many times we try. if (!$blueprints) { throw new PhabricatorWorkerPermanentFailureException( pht( 'No active Drydock blueprint exists which can ever allocate a '. 'resource for lease "%s".', $lease->getPHID())); } // First, try to find a suitable open resource which we can acquire a new // lease on. $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); // If no resources exist yet, see if we can build one. if (!$resources) { $usable_blueprints = $this->removeOverallocatedBlueprints( $blueprints, $lease); // If we get nothing back here, some blueprint claims it can eventually // satisfy the lease, just not right now. This is a temporary failure, // and we expect allocation to succeed eventually. if (!$usable_blueprints) { $lease->logEvent( DrydockLeaseWaitingForResourcesLogType::LOGCONST, array( 'blueprintPHIDs' => mpull($blueprints, 'getPHID'), )); throw new PhabricatorWorkerYieldException(15); } $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease); $exceptions = array(); foreach ($usable_blueprints as $blueprint) { try { $resources[] = $this->allocateResource($blueprint, $lease); // Bail after allocating one resource, we don't need any more than // this. break; } catch (Exception $ex) { $exceptions[] = $ex; } } if (!$resources) { throw new PhutilAggregateException( pht( 'All blueprints failed to allocate a suitable new resource when '. 'trying to allocate lease "%s".', $lease->getPHID()), $exceptions); } // NOTE: We have not acquired the lease yet, so it is possible that the // resource we just built will be snatched up by some other lease before // we can. This is not problematic: we'll retry a little later and should // suceed eventually. } $resources = $this->rankResources($resources, $lease); $exceptions = array(); $allocated = false; foreach ($resources as $resource) { try { $this->acquireLease($resource, $lease); $allocated = true; break; } catch (Exception $ex) { $exceptions[] = $ex; } } if (!$allocated) { throw new PhutilAggregateException( pht( 'Unable to acquire lease "%s" on any resouce.', $lease->getPHID()), $exceptions); } } /** * Get all the @{class:DrydockBlueprintImplementation}s which can possibly * build a resource to satisfy a lease. * * This method returns blueprints which might, at some time, be able to * build a resource which can satisfy the lease. They may not be able to * build that resource right now. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprint * implementations. * @task allocator */ private function loadBlueprintImplementationsForAllocatingLease( DrydockLease $lease) { $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); $keep = array(); foreach ($impls as $key => $impl) { // Don't use disabled blueprint types. if (!$impl->isEnabled()) { continue; } // Don't use blueprint types which can't allocate the correct kind of // resource. if ($impl->getType() != $lease->getResourceType()) { continue; } if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $impl; } return $keep; } /** * Get all the concrete @{class:DrydockBlueprint}s which can possibly * build a resource to satisfy a lease. * * @param DrydockLease Requested lease. * @return list List of qualifying blueprints. * @task allocator */ private function loadBlueprintsForAllocatingLease( DrydockLease $lease) { $viewer = $this->getViewer(); $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); if (!$impls) { return array(); } $blueprints = id(new DrydockBlueprintQuery()) ->setViewer($viewer) ->withBlueprintClasses(array_keys($impls)) ->withDisabled(false) ->execute(); $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canEverAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Load a list of all resources which a given lease can possibly be * allocated against. * * @param list Blueprints which may produce suitable * resources. * @param DrydockLease Requested lease. * @return list Resources which may be able to allocate * the lease. * @task allocator */ private function loadResourcesForAllocatingLease( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $viewer = $this->getViewer(); $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) ->withTypes(array($lease->getResourceType())) ->withStatuses( array( DrydockResourceStatus::STATUS_PENDING, DrydockResourceStatus::STATUS_ACTIVE, )) ->execute(); $keep = array(); foreach ($resources as $key => $resource) { $blueprint = $resource->getBlueprint(); if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { continue; } $keep[$key] = $resource; } return $keep; } /** * Remove blueprints which are too heavily allocated to build a resource for * a lease from a list of blueprints. * * @param list List of blueprints. * @return list List with blueprints that can not allocate * a resource for the lease right now removed. * @task allocator */ private function removeOverallocatedBlueprints( array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); $keep = array(); foreach ($blueprints as $key => $blueprint) { if (!$blueprint->canAllocateResourceForLease($lease)) { continue; } $keep[$key] = $blueprint; } return $keep; } /** * Rank blueprints by suitability for building a new resource for a * particular lease. * * @param list List of blueprints. * @param DrydockLease Requested lease. * @return list Ranked list of blueprints. * @task allocator */ private function rankBlueprints(array $blueprints, DrydockLease $lease) { assert_instances_of($blueprints, 'DrydockBlueprint'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($blueprints); return $blueprints; } /** * Rank resources by suitability for allocating a particular lease. * * @param list List of resources. * @param DrydockLease Requested lease. * @return list Ranked list of resources. * @task allocator */ private function rankResources(array $resources, DrydockLease $lease) { assert_instances_of($resources, 'DrydockResource'); // TODO: Implement improvements to this ranking algorithm if they become // available. shuffle($resources); return $resources; } /** * Perform an actual resource allocation with a particular blueprint. * * @param DrydockBlueprint The blueprint to allocate a resource from. * @param DrydockLease Requested lease. * @return DrydockResource Allocated resource. * @task allocator */ private function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $blueprint->allocateResource($lease); $this->validateAllocatedResource($blueprint, $resource, $lease); // If this resource was allocated as a pending resource, queue a task to // activate it. if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { PhabricatorWorker::scheduleTask( 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $resource->getPHID(), ), array( 'objectPHID' => $resource->getPHID(), )); } return $resource; } /** * Check that the resource a blueprint allocated is roughly the sort of * object we expect. * * @param DrydockBlueprint Blueprint which built the resource. * @param wild Thing which the blueprint claims is a valid resource. * @param DrydockLease Lease the resource was allocated for. * @return void * @task allocator */ private function validateAllocatedResource( DrydockBlueprint $blueprint, $resource, DrydockLease $lease) { if (!($resource instanceof DrydockResource)) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. 'return an object of type %s or throw, but returned something else.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()', 'DrydockResource')); } if (!$resource->isAllocatedResource()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 'must actually allocate the resource it returns.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'allocateResource()')); } $resource_type = $resource->getType(); $lease_type = $lease->getResourceType(); if ($resource_type !== $lease_type) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'built a resource of type "%s" to satisfy a lease requesting a '. 'resource of type "%s".', $blueprint->getBlueprintName(), $blueprint->getClassName(), $resource_type, $lease_type)); } } /* -( Acquiring Leases )--------------------------------------------------- */ /** * Perform an actual lease acquisition on a particular resource. * * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Lease to acquire. * @return void * @task acquire */ private function acquireLease( DrydockResource $resource, DrydockLease $lease) { $blueprint = $resource->getBlueprint(); $blueprint->acquireLease($resource, $lease); $this->validateAcquiredLease($blueprint, $resource, $lease); // If this lease has been acquired but not activated, queue a task to // activate it. if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { - PhabricatorWorker::scheduleTask( + $this->queueTask( __CLASS__, array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); } } /** * Make sure that a lease was really acquired properly. * * @param DrydockBlueprint Blueprint which created the resource. * @param DrydockResource Resource which was acquired. * @param DrydockLease The lease which was supposedly acquired. * @return void * @task acquire */ private function validateAcquiredLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isAcquiredLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without acquiring a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } $lease_phid = $lease->getResourcePHID(); $resource_phid = $resource->getPHID(); if ($lease_phid !== $resource_phid) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" with a lease acquired on the wrong resource.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } /* -( Activating Leases )-------------------------------------------------- */ /** * @task activate */ private function activateLease(DrydockLease $lease) { $resource = $lease->getResource(); if (!$resource) { throw new Exception( pht('Trying to activate lease with no resource.')); } $resource_status = $resource->getStatus(); if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { throw new PhabricatorWorkerYieldException(15); } if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { throw new Exception( pht( 'Trying to activate lease on a dead resource (in status "%s").', $resource_status)); } // NOTE: We can race resource destruction here. Between the time we // performed the read above and now, the resource might have closed, so // we may activate leases on dead resources. At least for now, this seems // fine: a resource dying right before we activate a lease on it should not // be distinguisahble from a resource dying right after we activate a lease // on it. We end up with an active lease on a dead resource either way, and // can not prevent resources dying from lightning strikes. $blueprint = $resource->getBlueprint(); $blueprint->activateLease($resource, $lease); $this->validateActivatedLease($blueprint, $resource, $lease); } /** * @task activate */ private function validateActivatedLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { if (!$lease->isActivatedLease()) { throw new Exception( pht( 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 'returned from "%s" without activating a lease.', $blueprint->getBlueprintName(), $blueprint->getClassName(), 'acquireLease()')); } } /* -( Releasing Leases )--------------------------------------------------- */ /** * @task release */ private function releaseLease(DrydockLease $lease) { $lease ->setStatus(DrydockLeaseStatus::STATUS_RELEASED) ->save(); $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST); $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); } $this->destroyLease($lease); } /* -( Breaking Leases )---------------------------------------------------- */ /** * @task break */ protected function breakLease(DrydockLease $lease, Exception $ex) { switch ($lease->getStatus()) { case DrydockLeaseStatus::STATUS_BROKEN: case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: throw new PhutilProxyException( pht( 'Unexpected failure while destroying lease ("%s").', $lease->getPHID()), $ex); } $lease ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) ->save(); - $lease->scheduleUpdate(); + $this->queueTask( + __CLASS__, + array( + 'leasePHID' => $lease->getPHID(), + ), + array( + 'objectPHID' => $lease->getPHID(), + )); $lease->logEvent( DrydockLeaseActivationFailureLogType::LOGCONST, array( 'class' => get_class($ex), 'message' => $ex->getMessage(), )); throw new PhabricatorWorkerPermanentFailureException( pht( 'Permanent failure while activating lease ("%s"): %s', $lease->getPHID(), $ex->getMessage())); } /* -( Destroying Leases )-------------------------------------------------- */ /** * @task destroy */ private function destroyLease(DrydockLease $lease) { $resource = $lease->getResource(); if ($resource) { $blueprint = $resource->getBlueprint(); $blueprint->destroyLease($resource, $lease); } DrydockSlotLock::releaseLocks($lease->getPHID()); $lease ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) ->save(); $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); } }