diff --git a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php index 6a6e146cf1..f20b1c6f9b 100644 --- a/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockBlueprintImplementation.php @@ -1,464 +1,529 @@ setViewer(PhabricatorUser::getOmnipotentUser()) ->withIDs(array($lease_id)) ->execute(); $lease = idx($query, $lease_id); if (!$lease) { throw new Exception(pht("No such lease '%d'!", $lease_id)); } return $lease; } protected function getInstance() { if (!$this->instance) { throw new Exception( pht('Attach the blueprint instance to the implementation.')); } return $this->instance; } public function attachInstance(DrydockBlueprint $instance) { $this->instance = $instance; return $this; } public function getFieldSpecifications() { return array(); } public function getDetail($key, $default = null) { return $this->getInstance()->getDetail($key, $default); } /* -( Lease Acquisition )-------------------------------------------------- */ - /** - * @task lease - */ - final public function filterResource( - DrydockResource $resource, - DrydockLease $lease) { - - $scope = $this->pushActiveScope($resource, $lease); - - return $this->canAllocateLease($resource, $lease); - } - - /** * Enforce basic checks on lease/resource compatibility. Allows resources to * reject leases if they are incompatible, even if the resource types match. * * For example, if a resource represents a 32-bit host, this method might - * reject leases that need a 64-bit host. If a resource represents a working - * copy of repository "X", this method might reject leases which need a - * working copy of repository "Y". Generally, although the main types of - * a lease and resource may match (e.g., both "host"), it may not actually be - * possible to satisfy the lease with a specific resource. + * reject leases that need a 64-bit host. The blueprint might also reject + * a resource if the lease needs 8GB of RAM and the resource only has 6GB + * free. * - * This method generally should not enforce limits or perform capacity - * checks. Perform those in @{method:shouldAllocateLease} instead. It also - * should not perform actual acquisition of the lease; perform that in - * @{method:executeAcquireLease} instead. + * This method should not acquire locks or expect anything to be locked. This + * is a coarse compatibility check between a lease and a resource. * - * @param DrydockResource Candidiate resource to allocate the lease on. - * @param DrydockLease Pending lease that wants to allocate here. - * @return bool True if the resource and lease are compatible. + * @param DrydockBlueprint Concrete blueprint to allocate for. + * @param DrydockResource Candidiate resource to allocate the lease on. + * @param DrydockLease Pending lease that wants to allocate here. + * @return bool True if the resource and lease are compatible. * @task lease */ - abstract protected function canAllocateLease( + abstract public function canAllocateLeaseOnResource( + DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease); /** * @task lease */ final public function allocateLease( DrydockResource $resource, DrydockLease $lease) { $scope = $this->pushActiveScope($resource, $lease); $this->log(pht('Trying to Allocate Lease')); $lease->setStatus(DrydockLeaseStatus::STATUS_ACQUIRING); $lease->setResourceID($resource->getID()); $lease->attachResource($resource); $ephemeral_lease = id(clone $lease)->makeEphemeral(); $allocated = false; $allocation_exception = null; $resource->openTransaction(); $resource->beginReadLocking(); $resource->reload(); // TODO: Policy stuff. $other_leases = id(new DrydockLease())->loadAllWhere( 'status IN (%Ld) AND resourceID = %d', array( DrydockLeaseStatus::STATUS_ACQUIRING, DrydockLeaseStatus::STATUS_ACTIVE, ), $resource->getID()); try { $allocated = $this->shouldAllocateLease( $resource, $ephemeral_lease, $other_leases); } catch (Exception $ex) { $allocation_exception = $ex; } if ($allocated) { $lease->save(); } $resource->endReadLocking(); if ($allocated) { $resource->saveTransaction(); $this->log(pht('Allocated Lease')); } else { $resource->killTransaction(); $this->log(pht('Failed to Allocate Lease')); } if ($allocation_exception) { $this->logException($allocation_exception); } return $allocated; } /** * Enforce lease limits on resources. Allows resources to reject leases if * they would become over-allocated by accepting them. * * For example, if a resource represents disk space, this method might check * how much space the lease is asking for (say, 200MB) and how much space is * left unallocated on the resource. It could grant the lease (return true) * if it has enough remaining space (more than 200MB), and reject the lease * (return false) if it does not (less than 200MB). * * A resource might also allow only exclusive leases. In this case it could * accept a new lease (return true) if there are no active leases, or reject * the new lease (return false) if there any other leases. * * A lock is held on the resource while this method executes to prevent * multiple processes from allocating leases on the resource simultaneously. * However, this means you should implement the method as cheaply as possible. * In particular, do not perform any actual acquisition or setup in this * method. * * If allocation is permitted, the lease will be moved to `ACQUIRING` status * and @{method:executeAcquireLease} will be called to actually perform * acquisition. * * General compatibility checks unrelated to resource limits and capacity are * better implemented in @{method:canAllocateLease}, which serves as a * cheap filter before lock acquisition. * * @param DrydockResource Candidate resource to allocate the lease on. * @param DrydockLease Pending lease that wants to allocate here. * @param list Other allocated and acquired leases on the * resource. The implementation can inspect them * to verify it can safely add the new lease. * @return bool True to allocate the lease on the resource; * false to reject it. * @task lease */ abstract protected function shouldAllocateLease( DrydockResource $resource, DrydockLease $lease, array $other_leases); /** * @task lease */ final public function acquireLease( DrydockResource $resource, DrydockLease $lease) { $scope = $this->pushActiveScope($resource, $lease); $this->log(pht('Acquiring Lease')); $lease->setStatus(DrydockLeaseStatus::STATUS_ACTIVE); $lease->setResourceID($resource->getID()); $lease->attachResource($resource); $ephemeral_lease = id(clone $lease)->makeEphemeral(); try { $this->executeAcquireLease($resource, $ephemeral_lease); } catch (Exception $ex) { $this->logException($ex); throw $ex; } $lease->setAttributes($ephemeral_lease->getAttributes()); $lease->save(); $this->log(pht('Acquired Lease')); } /** * Acquire and activate an allocated lease. Allows resources to peform setup * as leases are brought online. * * Following a successful call to @{method:canAllocateLease}, a lease is moved * to `ACQUIRING` status and this method is called after resource locks are * released. Nothing is locked while this method executes; the implementation * is free to perform expensive operations like writing files and directories, * executing commands, etc. * * After this method executes, the lease status is moved to `ACTIVE` and the * original leasee may access it. * * If acquisition fails, throw an exception. * * @param DrydockResource Resource to acquire a lease on. * @param DrydockLease Lease to acquire. * @return void */ abstract protected function executeAcquireLease( DrydockResource $resource, DrydockLease $lease); final public function releaseLease( DrydockResource $resource, DrydockLease $lease) { $scope = $this->pushActiveScope(null, $lease); $released = false; $lease->openTransaction(); $lease->beginReadLocking(); $lease->reload(); if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACTIVE) { $lease->setStatus(DrydockLeaseStatus::STATUS_RELEASED); $lease->save(); $released = true; } $lease->endReadLocking(); $lease->saveTransaction(); if (!$released) { throw new Exception(pht('Unable to release lease: lease not active!')); } } /* -( Resource Allocation )------------------------------------------------ */ - public function canAllocateMoreResources(array $pool) { - return true; - } + /** + * Enforce fundamental implementation/lease checks. Allows implementations to + * reject a lease which no concrete blueprint can ever satisfy. + * + * For example, if a lease only builds ARM hosts and the lease needs a + * PowerPC host, it may be rejected here. + * + * This is the earliest rejection phase, and followed by + * @{method:canEverAllocateResourceForLease}. + * + * This method should not actually check if a resource can be allocated + * right now, or even if a blueprint which can allocate a suitable resource + * really exists, only if some blueprint may conceivably exist which could + * plausibly be able to build a suitable resource. + * + * @param DrydockLease Requested lease. + * @return bool True if some concrete blueprint of this implementation's + * type might ever be able to build a resource for the lease. + * @task resource + */ + abstract public function canAnyBlueprintEverAllocateResourceForLease( + DrydockLease $lease); + + + /** + * Enforce basic blueprint/lease checks. Allows blueprints to reject a lease + * which they can not build a resource for. + * + * This is the second rejection phase. It follows + * @{method:canAnyBlueprintEverAllocateResourceForLease} and is followed by + * @{method:canAllocateResourceForLease}. + * + * This method should not check if a resource can be built right now, only + * if the blueprint as configured may, at some time, be able to build a + * suitable resource. + * + * @param DrydockBlueprint Blueprint which may be asked to allocate a + * resource. + * @param DrydockLease Requested lease. + * @return bool True if this blueprint can eventually build a suitable + * resource for the lease, as currently configured. + * @task resource + */ + abstract public function canEverAllocateResourceForLease( + DrydockBlueprint $blueprint, + DrydockLease $lease); + - abstract protected function executeAllocateResource(DrydockLease $lease); + /** + * Enforce basic availability limits. Allows blueprints to reject resource + * allocation if they are currently overallocated. + * + * This method should perform basic capacity/limit checks. For example, if + * it has a limit of 6 resources and currently has 6 resources allocated, + * it might reject new leases. + * + * This method should not acquire locks or expect locks to be acquired. This + * is a coarse check to determine if the operation is likely to succeed + * right now without needing to acquire locks. + * + * It is expected that this method will sometimes return `true` (indicating + * that a resource can be allocated) but find that another allocator has + * eaten up free capacity by the time it actually tries to build a resource. + * This is normal and the allocator will recover from it. + * + * @param DrydockBlueprint The blueprint which may be asked to allocate a + * resource. + * @param DrydockLease Requested lease. + * @return bool True if this blueprint appears likely to be able to allocate + * a suitable resource. + */ + abstract public function canAllocateResourceForLease( + DrydockBlueprint $blueprint, + DrydockLease $lease); - final public function allocateResource(DrydockLease $lease) { + /** + * Allocate a suitable resource for a lease. + * + * This method MUST acquire, hold, and manage locks to prevent multiple + * allocations from racing. World state is not locked before this method is + * called. Blueprints are entirely responsible for any lock handling they + * need to perform. + * + * @param DrydockBlueprint The blueprint which should allocate a resource. + * @param DrydockLease Requested lease. + * @return DrydockResource Allocated resource. + */ + abstract protected function executeAllocateResource( + DrydockBlueprint $blueprint, + DrydockLease $lease); + + final public function allocateResource( + DrydockBlueprint $blueprint, + DrydockLease $lease) { + $scope = $this->pushActiveScope(null, $lease); $this->log( pht( "Blueprint '%s': Allocating Resource for '%s'", $this->getBlueprintClass(), $lease->getLeaseName())); try { - $resource = $this->executeAllocateResource($lease); + $resource = $this->executeAllocateResource($blueprint, $lease); $this->validateAllocatedResource($resource); } catch (Exception $ex) { $this->logException($ex); throw $ex; } return $resource; } /* -( Logging )------------------------------------------------------------ */ /** * @task log */ protected function logException(Exception $ex) { $this->log($ex->getMessage()); } /** * @task log */ protected function log($message) { self::writeLog( $this->activeResource, $this->activeLease, $message); } /** * @task log */ public static function writeLog( DrydockResource $resource = null, DrydockLease $lease = null, $message = null) { $log = id(new DrydockLog()) ->setEpoch(time()) ->setMessage($message); if ($resource) { $log->setResourceID($resource->getID()); } if ($lease) { $log->setLeaseID($lease->getID()); } $log->save(); } public static function getAllBlueprintImplementations() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) ->execute(); } - public static function getAllBlueprintImplementationsForResource($type) { - static $groups = null; - if ($groups === null) { - $groups = mgroup(self::getAllBlueprintImplementations(), 'getType'); - } - return idx($groups, $type, array()); - } - public static function getNamedImplementation($class) { return idx(self::getAllBlueprintImplementations(), $class); } protected function newResourceTemplate($name) { $resource = id(new DrydockResource()) ->setBlueprintPHID($this->getInstance()->getPHID()) ->setBlueprintClass($this->getBlueprintClass()) ->setType($this->getType()) ->setStatus(DrydockResourceStatus::STATUS_PENDING) ->setName($name) ->save(); $this->activeResource = $resource; $this->log( pht( "Blueprint '%s': Created New Template", $this->getBlueprintClass())); return $resource; } /** * Sanity checks that the blueprint is implemented properly. */ private function validateAllocatedResource($resource) { $blueprint = $this->getBlueprintClass(); if (!($resource instanceof DrydockResource)) { throw new Exception( pht( "Blueprint '%s' is not properly implemented: %s must return an ". "object of type %s or throw, but returned something else.", $blueprint, 'executeAllocateResource()', 'DrydockResource')); } $current_status = $resource->getStatus(); $req_status = DrydockResourceStatus::STATUS_OPEN; if ($current_status != $req_status) { $current_name = DrydockResourceStatus::getNameForStatus($current_status); $req_name = DrydockResourceStatus::getNameForStatus($req_status); throw new Exception( pht( "Blueprint '%s' is not properly implemented: %s must return a %s ". "with status '%s', but returned one with status '%s'.", $blueprint, 'executeAllocateResource()', 'DrydockResource', $req_name, $current_name)); } } private function pushActiveScope( DrydockResource $resource = null, DrydockLease $lease = null) { if (($this->activeResource !== null) || ($this->activeLease !== null)) { throw new Exception(pht('There is already an active resource or lease!')); } $this->activeResource = $resource; $this->activeLease = $lease; return new DrydockBlueprintScopeGuard($this); } public function popActiveScope() { $this->activeResource = null; $this->activeLease = null; } } diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php index 264394f8ac..b85a5dde9f 100644 --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -1,120 +1,145 @@ getAttribute('repositoryID'); $lease_repo = $lease->getAttribute('repositoryID'); return ($resource_repo && $lease_repo && ($resource_repo == $lease_repo)); } protected function shouldAllocateLease( DrydockResource $resource, DrydockLease $lease, array $other_leases) { - + // TODO: These checks are out of date. return !$other_leases; } - protected function executeAllocateResource(DrydockLease $lease) { + protected function executeAllocateResource( + DrydockBlueprint $blueprint, + DrydockLease $lease) { + $repository_id = $lease->getAttribute('repositoryID'); if (!$repository_id) { throw new Exception( pht( "Lease is missing required '%s' attribute.", 'repositoryID')); } $repository = id(new PhabricatorRepositoryQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withIDs(array($repository_id)) ->executeOne(); if (!$repository) { throw new Exception( pht( "Repository '%s' does not exist!", $repository_id)); } switch ($repository->getVersionControlSystem()) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: break; default: throw new Exception(pht('Unsupported VCS!')); } // TODO: Policy stuff here too. $host_lease = id(new DrydockLease()) ->setResourceType('host') ->waitUntilActive(); $path = $host_lease->getAttribute('path').$repository->getCallsign(); $this->log( pht('Cloning %s into %s....', $repository->getCallsign(), $path)); $cmd = $host_lease->getInterface('command'); $cmd->execx( 'git clone --origin origin %P %s', $repository->getRemoteURIEnvelope(), $path); $this->log(pht('Complete.')); $resource = $this->newResourceTemplate( pht( 'Working Copy (%s)', $repository->getCallsign())); $resource->setStatus(DrydockResourceStatus::STATUS_OPEN); $resource->setAttribute('lease.host', $host_lease->getID()); $resource->setAttribute('path', $path); $resource->setAttribute('repositoryID', $repository->getID()); $resource->save(); return $resource; } protected function executeAcquireLease( DrydockResource $resource, DrydockLease $lease) { return; } public function getType() { return 'working-copy'; } public function getInterface( DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case 'command': return $this ->loadLease($resource->getAttribute('lease.host')) ->getInterface($type); } throw new Exception(pht("No interface of type '%s'.", $type)); } } diff --git a/src/applications/drydock/storage/DrydockBlueprint.php b/src/applications/drydock/storage/DrydockBlueprint.php index 8185d35388..132fb7fdc0 100644 --- a/src/applications/drydock/storage/DrydockBlueprint.php +++ b/src/applications/drydock/storage/DrydockBlueprint.php @@ -1,153 +1,164 @@ setViewer($actor) ->withClasses(array('PhabricatorDrydockApplication')) ->executeOne(); $view_policy = $app->getPolicy( DrydockDefaultViewCapability::CAPABILITY); $edit_policy = $app->getPolicy( DrydockDefaultEditCapability::CAPABILITY); return id(new DrydockBlueprint()) ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setBlueprintName(''); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'details' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'className' => 'text255', 'blueprintName' => 'sort255', ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID( DrydockBlueprintPHIDType::TYPECONST); } public function getImplementation() { - $class = $this->className; - $implementations = - DrydockBlueprintImplementation::getAllBlueprintImplementations(); - if (!isset($implementations[$class])) { - throw new Exception( - pht( - "Invalid class name for blueprint (got '%s')", - $class)); - } - return id(new $class())->attachInstance($this); + return $this->assertAttached($this->implementation); } public function attachImplementation(DrydockBlueprintImplementation $impl) { $this->implementation = $impl; return $this; } public function getDetail($key, $default = null) { return idx($this->details, $key, $default); } public function setDetail($key, $value) { $this->details[$key] = $value; return $this; } + public function canEverAllocateResourceForLease(DrydockLease $lease) { + return $this->getImplementation()->canEverAllocateResourceForLease( + $this, + $lease); + } + + public function canAllocateResourceForLease(DrydockLease $lease) { + return $this->getImplementation()->canAllocateResourceForLease( + $this, + $lease); + } + + public function canAllocateLeaseOnResource( + DrydockResource $resource, + DrydockLease $lease) { + return $this->getImplementation()->canAllocateLeaseOnResource( + $this, + $resource, + $lease); + } /* -( PhabricatorApplicationTransactionInterface )------------------------- */ public function getApplicationTransactionEditor() { return new DrydockBlueprintEditor(); } public function getApplicationTransactionObject() { return $this; } public function getApplicationTransactionTemplate() { return new DrydockBlueprintTransaction(); } public function willRenderTimeline( PhabricatorApplicationTransactionView $timeline, AphrontRequest $request) { return $timeline; } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: return $this->getViewPolicy(); case PhabricatorPolicyCapability::CAN_EDIT: return $this->getEditPolicy(); } } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return false; } public function describeAutomaticCapability($capability) { return null; } /* -( PhabricatorCustomFieldInterface )------------------------------------ */ public function getCustomFieldSpecificationForRole($role) { return array(); } public function getCustomFieldBaseClass() { return 'DrydockBlueprintCustomField'; } public function getCustomFields() { return $this->assertAttached($this->customFields); } public function attachCustomFields(PhabricatorCustomFieldAttachment $fields) { $this->customFields = $fields; return $this; } } diff --git a/src/applications/drydock/worker/DrydockAllocatorWorker.php b/src/applications/drydock/worker/DrydockAllocatorWorker.php index f9a647a3a8..1898bcaf66 100644 --- a/src/applications/drydock/worker/DrydockAllocatorWorker.php +++ b/src/applications/drydock/worker/DrydockAllocatorWorker.php @@ -1,187 +1,311 @@ lease)) { - $lease = id(new DrydockLeaseQuery()) - ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withIDs(array($this->getTaskData())) - ->executeOne(); - if (!$lease) { - throw new PhabricatorWorkerPermanentFailureException( - pht('No such lease %d!', $this->getTaskData())); - } - $this->lease = $lease; + $viewer = $this->getViewer(); + + // TODO: Make the task data a dictionary like every other worker, and + // probably make this a PHID. + $lease_id = $this->getTaskData(); + + $lease = id(new DrydockLeaseQuery()) + ->setViewer($viewer) + ->withIDs(array($lease_id)) + ->executeOne(); + if (!$lease) { + throw new PhabricatorWorkerPermanentFailureException( + pht('No such lease "%s"!', $lease_id)); } - return $this->lease; - } - private function logToDrydock($message) { - DrydockBlueprintImplementation::writeLog( - null, - $this->loadLease(), - $message); + return $lease; } protected function doWork() { $lease = $this->loadLease(); - $this->logToDrydock(pht('Allocating Lease')); - - try { - $this->allocateLease($lease); - } catch (Exception $ex) { - - // TODO: We should really do this when archiving the task, if we've - // suffered a permanent failure. But we don't have hooks for that yet - // and always fail after the first retry right now, so this is - // functionally equivalent. - $lease->reload(); - if ($lease->getStatus() == DrydockLeaseStatus::STATUS_PENDING) { - $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); - $lease->save(); + $this->allocateLease($lease); + } + + private function allocateLease(DrydockLease $lease) { + $blueprints = $this->loadBlueprintsForAllocatingLease($lease); + + // If we get nothing back, that means no blueprint is defined which can + // ever build the requested resource. This is a permanent failure, since + // we don't expect to succeed no matter how many times we try. + if (!$blueprints) { + $lease + ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) + ->save(); + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'No active Drydock blueprint exists which can ever allocate a '. + 'resource for lease "%s".', + $lease->getPHID())); + } + + // First, try to find a suitable open resource which we can acquire a new + // lease on. + $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); + + // If no resources exist yet, see if we can build one. + if (!$resources) { + $usable_blueprints = $this->removeOverallocatedBlueprints( + $blueprints, + $lease); + + // If we get nothing back here, some blueprint claims it can eventually + // satisfy the lease, just not right now. This is a temporary failure, + // and we expect allocation to succeed eventually. + if (!$blueprints) { + // TODO: More formal temporary failure here. We should retry this + // "soon" but not "immediately". + throw new Exception( + pht('No blueprints have space to allocate a resource right now.')); + } + + $usable_blueprints = $this->rankBlueprints($blueprints, $lease); + + $exceptions = array(); + foreach ($usable_blueprints as $blueprint) { + try { + $resources[] = $blueprint->allocateResource($lease); + // Bail after allocating one resource, we don't need any more than + // this. + break; + } catch (Exception $ex) { + $exceptions[] = $ex; + } } - throw $ex; + if (!$resources) { + // TODO: We should distinguish between temporary and permament failures + // here. If any blueprint failed temporarily, retry "soon". If none + // of these failures were temporary, maybe this should be a permanent + // failure? + throw new PhutilAggregateException( + pht( + 'All blueprints failed to allocate a suitable new resource when '. + 'trying to allocate lease "%s".', + $lease->getPHID()), + $exceptions); + } + + // NOTE: We have not acquired the lease yet, so it is possible that the + // resource we just built will be snatched up by some other lease before + // we can. This is not problematic: we'll retry a little later and should + // suceed eventually. + } + + $resources = $this->rankResources($resources, $lease); + + $exceptions = array(); + $allocated = false; + foreach ($resources as $resource) { + try { + $blueprint->allocateLease($resource, $lease); + $allocated = true; + break; + } catch (Exception $ex) { + $exceptions[] = $ex; + } + } + + if (!$allocated) { + // TODO: We should distinguish between temporary and permanent failures + // here. If any failures were temporary (specifically, failed to acquire + // locks) + + throw new PhutilAggregateException( + pht( + 'Unable to acquire lease "%s" on any resouce.', + $lease->getPHID()), + $exceptions); } } - private function loadAllBlueprints() { - $viewer = PhabricatorUser::getOmnipotentUser(); - $instances = id(new DrydockBlueprintQuery()) + + /** + * Load a list of all resources which a given lease can possibly be + * allocated against. + * + * @param list Blueprints which may produce suitable + * resources. + * @param DrydockLease Requested lease. + * @return list Resources which may be able to allocate + * the lease. + */ + private function loadResourcesForAllocatingLease( + array $blueprints, + DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); + $viewer = $this->getViewer(); + + $resources = id(new DrydockResourceQuery()) ->setViewer($viewer) + ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) + ->withTypes(array($lease->getResourceType())) + ->withStatuses( + array( + DrydockResourceStatus::STATUS_PENDING, + DrydockResourceStatus::STATUS_OPEN, + )) ->execute(); - $blueprints = array(); - foreach ($instances as $instance) { - $blueprints[$instance->getPHID()] = $instance; + + $keep = array(); + foreach ($resources as $key => $resource) { + if (!$resource->canAllocateLease($lease)) { + continue; + } + + $keep[$key] = $resource; } - return $blueprints; + + return $keep; } - private function allocateLease(DrydockLease $lease) { - $type = $lease->getResourceType(); - $blueprints = $this->loadAllBlueprints(); + /** + * Rank blueprints by suitability for building a new resource for a + * particular lease. + * + * @param list List of blueprints. + * @param DrydockLease Requested lease. + * @return list Ranked list of blueprints. + */ + private function rankBlueprints(array $blueprints, DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); - // TODO: Policy stuff. - $pool = id(new DrydockResource())->loadAllWhere( - 'type = %s AND status = %s', - $lease->getResourceType(), - DrydockResourceStatus::STATUS_OPEN); + // TODO: Implement improvements to this ranking algorithm if they become + // available. + shuffle($blueprints); - $this->logToDrydock( - pht('Found %d Open Resource(s)', count($pool))); + return $blueprints; + } - $candidates = array(); - foreach ($pool as $key => $candidate) { - if (!isset($blueprints[$candidate->getBlueprintPHID()])) { - unset($pool[$key]); - continue; - } - $blueprint = $blueprints[$candidate->getBlueprintPHID()]; - $implementation = $blueprint->getImplementation(); + /** + * Rank resources by suitability for allocating a particular lease. + * + * @param list List of resources. + * @param DrydockLease Requested lease. + * @return list Ranked list of resources. + */ + private function rankResources(array $resources, DrydockLease $lease) { + assert_instances_of($resources, 'DrydockResource'); - if ($implementation->filterResource($candidate, $lease)) { - $candidates[] = $candidate; - } + // TODO: Implement improvements to this ranking algorithm if they become + // available. + shuffle($resources); + + return $resources; + } + + + /** + * Get all the concrete @{class:DrydockBlueprint}s which can possibly + * build a resource to satisfy a lease. + * + * @param DrydockLease Requested lease. + * @return list List of qualifying blueprints. + */ + private function loadBlueprintsForAllocatingLease( + DrydockLease $lease) { + $viewer = $this->getViewer(); + + $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); + if (!$impls) { + return array(); } - $this->logToDrydock(pht('%d Open Resource(s) Remain', count($candidates))); + // TODO: When blueprints can be disabled, this query should ignore disabled + // blueprints. - $resource = null; - if ($candidates) { - shuffle($candidates); - foreach ($candidates as $candidate_resource) { - $blueprint = $blueprints[$candidate_resource->getBlueprintPHID()] - ->getImplementation(); - if ($blueprint->allocateLease($candidate_resource, $lease)) { - $resource = $candidate_resource; - break; - } + $blueprints = id(new DrydockBlueprintQuery()) + ->setViewer($viewer) + ->withBlueprintClasses(array_keys($impls)) + ->execute(); + + $keep = array(); + foreach ($blueprints as $key => $blueprint) { + if (!$blueprint->canEverAllocateResourceForLease($lease)) { + continue; } + + $keep[$key] = $blueprint; } - if (!$resource) { - $blueprints = DrydockBlueprintImplementation - ::getAllBlueprintImplementationsForResource($type); + return $keep; + } - $this->logToDrydock( - pht('Found %d Blueprints', count($blueprints))); - foreach ($blueprints as $key => $candidate_blueprint) { - if (!$candidate_blueprint->isEnabled()) { - unset($blueprints[$key]); - continue; - } + /** + * Get all the @{class:DrydockBlueprintImplementation}s which can possibly + * build a resource to satisfy a lease. + * + * This method returns blueprints which might, at some time, be able to + * build a resource which can satisfy the lease. They may not be able to + * build that resource right now. + * + * @param DrydockLease Requested lease. + * @return list List of qualifying blueprint + * implementations. + */ + private function loadBlueprintImplementationsForAllocatingLease( + DrydockLease $lease) { + + $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); + + $keep = array(); + foreach ($impls as $key => $impl) { + // Don't use disabled blueprint types. + if (!$impl->isEnabled()) { + continue; } - $this->logToDrydock( - pht('%d Blueprints Enabled', count($blueprints))); + // Don't use blueprint types which can't allocate the correct kind of + // resource. + if ($impl->getType() != $lease->getResourceType()) { + continue; + } - foreach ($blueprints as $key => $candidate_blueprint) { - if (!$candidate_blueprint->canAllocateMoreResources($pool)) { - unset($blueprints[$key]); - continue; - } + if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { + continue; } - $this->logToDrydock( - pht('%d Blueprints Can Allocate', count($blueprints))); + $keep[$key] = $impl; + } - if (!$blueprints) { - $lease->setStatus(DrydockLeaseStatus::STATUS_BROKEN); - $lease->save(); + return $keep; + } - $this->logToDrydock( - pht( - "There are no resources of type '%s' available, and no ". - "blueprints which can allocate new ones.", - $type)); - return; + /** + * Remove blueprints which are too heavily allocated to build a resource for + * a lease from a list of blueprints. + * + * @param list List of blueprints. + * @param list List with fully allocated blueprints + * removed. + */ + private function removeOverallocatedBlueprints( + array $blueprints, + DrydockLease $lease) { + assert_instances_of($blueprints, 'DrydockBlueprint'); + + $keep = array(); + foreach ($blueprints as $key => $blueprint) { + if (!$blueprint->canAllocateResourceForLease($lease)) { + continue; } - // TODO: Rank intelligently. - shuffle($blueprints); - - $blueprint = head($blueprints); - $resource = $blueprint->allocateResource($lease); - - if (!$blueprint->allocateLease($resource, $lease)) { - // TODO: This "should" happen only if we lost a race with another lease, - // which happened to acquire this resource immediately after we - // allocated it. In this case, the right behavior is to retry - // immediately. However, other things like a blueprint allocating a - // resource it can't actually allocate the lease on might be happening - // too, in which case we'd just allocate infinite resources. Probably - // what we should do is test for an active or allocated lease and retry - // if we find one (although it might have already been released by now) - // and fail really hard ("your configuration is a huge broken mess") - // otherwise. But just throw for now since this stuff is all edge-casey. - // Alternatively we could bring resources up in a "BESPOKE" status - // and then switch them to "OPEN" only after the allocating lease gets - // its grubby mitts on the resource. This might make more sense but - // is a bit messy. - throw new Exception(pht('Lost an allocation race?')); - } + $keep[$key] = $blueprint; } - $blueprint = $resource->getBlueprint(); - $blueprint->acquireLease($resource, $lease); + return $keep; } }