diff --git a/resources/sql/autopatches/20150928.drydock.rexpire.1.sql b/resources/sql/autopatches/20150928.drydock.rexpire.1.sql new file mode 100644 index 0000000000..9321b9c0e1 --- /dev/null +++ b/resources/sql/autopatches/20150928.drydock.rexpire.1.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_drydock.drydock_resource + ADD until INT UNSIGNED; diff --git a/src/applications/drydock/capability/DrydockDefaultViewCapability.php b/src/applications/drydock/capability/DrydockDefaultViewCapability.php index af51edc38d..f43471766c 100644 --- a/src/applications/drydock/capability/DrydockDefaultViewCapability.php +++ b/src/applications/drydock/capability/DrydockDefaultViewCapability.php @@ -1,11 +1,15 @@ setBaseURI(new PhutilURI($this->getApplicationURI())); // These are only used on mobile. $nav->addFilter('blueprint', pht('Blueprints')); $nav->addFilter('resource', pht('Resources')); $nav->addFilter('lease', pht('Leases')); - $nav->addFilter('log', pht('Logs')); $nav->selectFilter(null); return $nav; } public function handleRequest(AphrontRequest $request) { $viewer = $request->getViewer(); $menu = id(new PHUIObjectItemListView()) ->setUser($viewer); $menu->addItem( id(new PHUIObjectItemView()) ->setHeader(pht('Blueprints')) + ->setFontIcon('fa-map-o') ->setHref($this->getApplicationURI('blueprint/')) ->addAttribute( pht( 'Configure blueprints so Drydock can build resources, like '. 'hosts and working copies.'))); $menu->addItem( id(new PHUIObjectItemView()) ->setHeader(pht('Resources')) + ->setFontIcon('fa-map') ->setHref($this->getApplicationURI('resource/')) ->addAttribute( pht('View and manage resources Drydock has built, like hosts.'))); $menu->addItem( id(new PHUIObjectItemView()) ->setHeader(pht('Leases')) + ->setFontIcon('fa-link') ->setHref($this->getApplicationURI('lease/')) ->addAttribute(pht('Manage leases on resources.'))); - $menu->addItem( - id(new PHUIObjectItemView()) - ->setHeader(pht('Logs')) - ->setHref($this->getApplicationURI('log/')) - ->addAttribute(pht('View logs.'))); - - $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Console')); $box = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Drydock Console')) ->setObjectList($menu); return $this->buildApplicationPage( array( $crumbs, $box, ), array( 'title' => pht('Drydock Console'), )); } } diff --git a/src/applications/drydock/controller/DrydockResourceViewController.php b/src/applications/drydock/controller/DrydockResourceViewController.php index 0641ced96d..d241b00808 100644 --- a/src/applications/drydock/controller/DrydockResourceViewController.php +++ b/src/applications/drydock/controller/DrydockResourceViewController.php @@ -1,178 +1,186 @@ getViewer(); $id = $request->getURIData('id'); $resource = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withIDs(array($id)) ->executeOne(); if (!$resource) { return new Aphront404Response(); } $title = pht('Resource %s %s', $resource->getID(), $resource->getName()); $header = id(new PHUIHeaderView()) ->setUser($viewer) ->setPolicyObject($resource) ->setHeader($title); $actions = $this->buildActionListView($resource); $properties = $this->buildPropertyListView($resource, $actions); $resource_uri = 'resource/'.$resource->getID().'/'; $resource_uri = $this->getApplicationURI($resource_uri); $pager = new PHUIPagerView(); $pager->setURI(new PhutilURI($resource_uri), 'offset'); $pager->setOffset($request->getInt('offset')); $logs = id(new DrydockLogQuery()) ->setViewer($viewer) ->withResourceIDs(array($resource->getID())) ->executeWithOffsetPager($pager); $log_table = id(new DrydockLogListView()) ->setUser($viewer) ->setLogs($logs) ->render(); $log_table->appendChild($pager); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Resource %d', $resource->getID())); $locks = $this->buildLocksTab($resource->getPHID()); $commands = $this->buildCommandsTab($resource->getPHID()); $object_box = id(new PHUIObjectBoxView()) ->setHeader($header) ->addPropertyList($properties, pht('Properties')) ->addPropertyList($locks, pht('Slot Locks')) ->addPropertyList($commands, pht('Commands')); $lease_box = $this->buildLeaseBox($resource); $log_box = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Resource Logs')) ->setTable($log_table); return $this->buildApplicationPage( array( $crumbs, $object_box, $lease_box, $log_box, ), array( 'title' => $title, )); } private function buildActionListView(DrydockResource $resource) { $viewer = $this->getViewer(); $view = id(new PhabricatorActionListView()) ->setUser($viewer) ->setObjectURI($this->getRequest()->getRequestURI()) ->setObject($resource); $can_release = $resource->canRelease(); $can_edit = PhabricatorPolicyFilter::hasCapability( $viewer, $resource, PhabricatorPolicyCapability::CAN_EDIT); $uri = '/resource/'.$resource->getID().'/release/'; $uri = $this->getApplicationURI($uri); $view->addAction( id(new PhabricatorActionView()) ->setHref($uri) ->setName(pht('Release Resource')) ->setIcon('fa-times') ->setWorkflow(true) ->setDisabled(!$can_release || !$can_edit)); return $view; } private function buildPropertyListView( DrydockResource $resource, PhabricatorActionListView $actions) { $viewer = $this->getViewer(); $view = id(new PHUIPropertyListView()) ->setActionList($actions); $status = $resource->getStatus(); $status = DrydockResourceStatus::getNameForStatus($status); $view->addProperty( pht('Status'), $status); + $until = $resource->getUntil(); + if ($until) { + $until_display = phabricator_datetime($until, $viewer); + } else { + $until_display = phutil_tag('em', array(), pht('Never')); + } + $view->addProperty(pht('Expires'), $until_display); + $view->addProperty( pht('Resource Type'), $resource->getType()); $view->addProperty( pht('Blueprint'), $viewer->renderHandle($resource->getBlueprintPHID())); $attributes = $resource->getAttributes(); if ($attributes) { $view->addSectionHeader( pht('Attributes'), 'fa-list-ul'); foreach ($attributes as $key => $value) { $view->addProperty($key, $value); } } return $view; } private function buildLeaseBox(DrydockResource $resource) { $viewer = $this->getViewer(); $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses( array( DrydockLeaseStatus::STATUS_PENDING, DrydockLeaseStatus::STATUS_ACQUIRED, DrydockLeaseStatus::STATUS_ACTIVE, )) ->setLimit(100) ->execute(); $id = $resource->getID(); $leases_uri = "resource/{$id}/leases/query/all/"; $leases_uri = $this->getApplicationURI($leases_uri); $lease_header = id(new PHUIHeaderView()) ->setHeader(pht('Active Leases')) ->addActionLink( id(new PHUIButtonView()) ->setTag('a') ->setHref($leases_uri) ->setIconFont('fa-search') ->setText(pht('View All Leases'))); $lease_list = id(new DrydockLeaseListView()) ->setUser($viewer) ->setLeases($leases) ->render() ->setNoDataString(pht('This resource has no active leases.')); return id(new PHUIObjectBoxView()) ->setHeader($lease_header) ->setObjectList($lease_list); } } diff --git a/src/applications/drydock/storage/DrydockLease.php b/src/applications/drydock/storage/DrydockLease.php index fe38f54fe0..bb65b982b6 100644 --- a/src/applications/drydock/storage/DrydockLease.php +++ b/src/applications/drydock/storage/DrydockLease.php @@ -1,366 +1,375 @@ releaseOnDestruction = true; return $this; } public function __destruct() { if (!$this->releaseOnDestruction) { return; } if (!$this->canRelease()) { return; } $actor = PhabricatorUser::getOmnipotentUser(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $command = DrydockCommand::initializeNewCommand($actor) ->setTargetPHID($this->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); $this->scheduleUpdate(); } public function getLeaseName() { return pht('Lease %d', $this->getID()); } protected function getConfiguration() { return array( self::CONFIG_AUX_PHID => true, self::CONFIG_SERIALIZATION => array( 'attributes' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'status' => 'text32', 'until' => 'epoch?', 'resourceType' => 'text128', 'ownerPHID' => 'phid?', 'resourcePHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_resource' => array( 'columns' => array('resourcePHID', 'status'), ), ), ) + parent::getConfiguration(); } public function setAttribute($key, $value) { $this->attributes[$key] = $value; return $this; } public function getAttribute($key, $default = null) { return idx($this->attributes, $key, $default); } public function generatePHID() { return PhabricatorPHID::generateNewPHID(DrydockLeasePHIDType::TYPECONST); } public function getInterface($type) { return $this->getResource()->getInterface($this, $type); } public function getResource() { return $this->assertAttached($this->resource); } public function attachResource(DrydockResource $resource = null) { $this->resource = $resource; return $this; } public function hasAttachedResource() { return ($this->resource !== null); } public function queueForActivation() { if ($this->getID()) { throw new Exception( pht('Only new leases may be queued for activation!')); } $this ->setStatus(DrydockLeaseStatus::STATUS_PENDING) ->save(); $task = PhabricatorWorker::scheduleTask( 'DrydockAllocatorWorker', array( 'leasePHID' => $this->getPHID(), ), array( 'objectPHID' => $this->getPHID(), )); return $this; } public function isActivating() { switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: return true; } return false; } public function isActive() { switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_ACTIVE: return true; } return false; } public function waitUntilActive() { while (true) { $lease = $this->reload(); if (!$lease) { throw new Exception(pht('Failed to reload lease.')); } $status = $lease->getStatus(); switch ($status) { case DrydockLeaseStatus::STATUS_ACTIVE: return; case DrydockLeaseStatus::STATUS_RELEASED: throw new Exception(pht('Lease has already been released!')); case DrydockLeaseStatus::STATUS_DESTROYED: throw new Exception(pht('Lease has already been destroyed!')); case DrydockLeaseStatus::STATUS_BROKEN: throw new Exception(pht('Lease has been broken!')); case DrydockLeaseStatus::STATUS_PENDING: case DrydockLeaseStatus::STATUS_ACQUIRED: break; default: throw new Exception( pht( 'Lease has unknown status "%s".', $status)); } sleep(1); } } public function setActivateWhenAcquired($activate) { $this->activateWhenAcquired = true; return $this; } public function needSlotLock($key) { $this->slotLocks[] = $key; return $this; } public function acquireOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to acquire a lease on a resource which is in the wrong '. 'state: status must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($this->activateWhenAcquired) { $new_status = DrydockLeaseStatus::STATUS_ACTIVE; } else { $new_status = DrydockLeaseStatus::STATUS_ACQUIRED; } if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { throw new Exception( pht( 'Trying to acquire an active lease on a pending resource. '. 'You can not immediately activate leases on resources which '. 'need time to start up.')); } } $this->openTransaction(); $this ->setResourcePHID($resource->getPHID()) ->setStatus($new_status) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isAcquired = true; if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { $this->didActivate(); } return $this; } public function isAcquiredLease() { return $this->isAcquired; } public function activateOnResource(DrydockResource $resource) { $expect_status = DrydockLeaseStatus::STATUS_ACQUIRED; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to activate a lease which has the wrong status: status '. 'must be "%s", actually "%s".', $expect_status, $actual_status)); } if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { // TODO: Be stricter about this? throw new Exception( pht( 'Trying to activate a lease on a pending resource.')); } $this->openTransaction(); $this ->setStatus(DrydockLeaseStatus::STATUS_ACTIVE) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isActivated = true; $this->didActivate(); return $this; } public function isActivatedLease() { return $this->isActivated; } public function canRelease() { if (!$this->getID()) { return false; } switch ($this->getStatus()) { case DrydockLeaseStatus::STATUS_RELEASED: case DrydockLeaseStatus::STATUS_DESTROYED: return false; default: return true; } } + public function canUpdate() { + switch ($this->getStatus()) { + case DrydockLeaseStatus::STATUS_ACTIVE: + return true; + default: + return false; + } + } + public function scheduleUpdate($epoch = null) { PhabricatorWorker::scheduleTask( 'DrydockLeaseUpdateWorker', array( 'leasePHID' => $this->getPHID(), 'isExpireTask' => ($epoch !== null), ), array( 'objectPHID' => $this->getPHID(), 'delayUntil' => $epoch, )); } private function didActivate() { $viewer = PhabricatorUser::getOmnipotentUser(); $need_update = false; $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($this->getPHID())) ->withConsumed(false) ->execute(); if ($commands) { $need_update = true; } if ($need_update) { $this->scheduleUpdate(); } $expires = $this->getUntil(); if ($expires) { $this->scheduleUpdate($expires); } } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { if ($this->getResource()) { return $this->getResource()->getPolicy($capability); } // TODO: Implement reasonable policies. return PhabricatorPolicies::getMostOpenPolicy(); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { if ($this->getResource()) { return $this->getResource()->hasAutomaticCapability($capability, $viewer); } return false; } public function describeAutomaticCapability($capability) { return pht('Leases inherit policies from the resources they lease.'); } } diff --git a/src/applications/drydock/storage/DrydockResource.php b/src/applications/drydock/storage/DrydockResource.php index f2be89a6f2..f46383a84c 100644 --- a/src/applications/drydock/storage/DrydockResource.php +++ b/src/applications/drydock/storage/DrydockResource.php @@ -1,238 +1,262 @@ true, self::CONFIG_SERIALIZATION => array( 'attributes' => self::SERIALIZATION_JSON, 'capabilities' => self::SERIALIZATION_JSON, ), self::CONFIG_COLUMN_SCHEMA => array( 'name' => 'text255', 'ownerPHID' => 'phid?', 'status' => 'text32', 'type' => 'text64', + 'until' => 'epoch?', ), self::CONFIG_KEY_SCHEMA => array( 'key_type' => array( 'columns' => array('type', 'status'), ), 'key_blueprint' => array( 'columns' => array('blueprintPHID', 'status'), ), ), ) + parent::getConfiguration(); } public function generatePHID() { return PhabricatorPHID::generateNewPHID(DrydockResourcePHIDType::TYPECONST); } public function getAttribute($key, $default = null) { return idx($this->attributes, $key, $default); } public function getAttributesForTypeSpec(array $attribute_names) { return array_select_keys($this->attributes, $attribute_names); } public function setAttribute($key, $value) { $this->attributes[$key] = $value; return $this; } public function getCapability($key, $default = null) { return idx($this->capbilities, $key, $default); } public function getInterface(DrydockLease $lease, $type) { return $this->getBlueprint()->getInterface($this, $lease, $type); } public function getBlueprint() { return $this->assertAttached($this->blueprint); } public function attachBlueprint(DrydockBlueprint $blueprint) { $this->blueprint = $blueprint; return $this; } public function setActivateWhenAllocated($activate) { $this->activateWhenAllocated = $activate; return $this; } public function needSlotLock($key) { $this->slotLocks[] = $key; return $this; } public function allocateResource() { if ($this->getID()) { throw new Exception( pht( 'Trying to allocate a resource which has already been persisted. '. 'Only new resources may be allocated.')); } $expect_status = DrydockResourceStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to allocate a resource from the wrong status. Status must '. 'be "%s", actually "%s".', $expect_status, $actual_status)); } if ($this->activateWhenAllocated) { $new_status = DrydockResourceStatus::STATUS_ACTIVE; } else { $new_status = DrydockResourceStatus::STATUS_PENDING; } $this->openTransaction(); $this ->setStatus($new_status) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isAllocated = true; + if ($new_status == DrydockResourceStatus::STATUS_ACTIVE) { + $this->didActivate(); + } + return $this; } public function isAllocatedResource() { return $this->isAllocated; } public function activateResource() { if (!$this->getID()) { throw new Exception( pht( 'Trying to activate a resource which has not yet been persisted.')); } $expect_status = DrydockResourceStatus::STATUS_PENDING; $actual_status = $this->getStatus(); if ($actual_status != $expect_status) { throw new Exception( pht( 'Trying to activate a resource from the wrong status. Status must '. 'be "%s", actually "%s".', $expect_status, $actual_status)); } $this->openTransaction(); $this ->setStatus(DrydockResourceStatus::STATUS_ACTIVE) ->save(); DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); $this->slotLocks = array(); $this->saveTransaction(); $this->isActivated = true; + $this->didActivate(); + return $this; } public function isActivatedResource() { return $this->isActivated; } public function canRelease() { switch ($this->getStatus()) { case DrydockResourceStatus::STATUS_RELEASED: case DrydockResourceStatus::STATUS_DESTROYED: return false; default: return true; } } - public function scheduleUpdate() { + public function scheduleUpdate($epoch = null) { PhabricatorWorker::scheduleTask( 'DrydockResourceUpdateWorker', array( 'resourcePHID' => $this->getPHID(), + 'isExpireTask' => ($epoch !== null), ), array( 'objectPHID' => $this->getPHID(), + 'delayUntil' => $epoch, )); } private function didActivate() { $viewer = PhabricatorUser::getOmnipotentUser(); $need_update = false; $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($this->getPHID())) ->withConsumed(false) ->execute(); if ($commands) { $need_update = true; } if ($need_update) { $this->scheduleUpdate(); } + + $expires = $this->getUntil(); + if ($expires) { + $this->scheduleUpdate($expires); + } + } + + public function canUpdate() { + switch ($this->getStatus()) { + case DrydockResourceStatus::STATUS_ACTIVE: + return true; + default: + return false; + } } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, ); } public function getPolicy($capability) { return $this->getBlueprint()->getPolicy($capability); } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return $this->getBlueprint()->hasAutomaticCapability( $capability, $viewer); } public function describeAutomaticCapability($capability) { return pht('Resources inherit the policies of their blueprints.'); } } diff --git a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php index 98ecf2b498..8f15201a2c 100644 --- a/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockLeaseUpdateWorker.php @@ -1,107 +1,83 @@ getTaskDataValue('leasePHID'); $hash = PhabricatorHash::digestForIndex($lease_phid); $lock_key = 'drydock.lease:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); try { $lease = $this->loadLease($lease_phid); $this->updateLease($lease); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); } private function updateLease(DrydockLease $lease) { - if ($lease->getStatus() != DrydockLeaseStatus::STATUS_ACTIVE) { + if (!$lease->canUpdate()) { return; } - $viewer = $this->getViewer(); - $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); - - // Check if the lease has expired. If it is, we're going to send it a - // release command. This command will be handled immediately below, it - // just generates a command log and improves consistency. - $now = PhabricatorTime::getNow(); - $expires = $lease->getUntil(); - if ($expires && ($expires <= $now)) { - $command = DrydockCommand::initializeNewCommand($viewer) - ->setTargetPHID($lease->getPHID()) - ->setAuthorPHID($drydock_phid) - ->setCommand(DrydockCommand::COMMAND_RELEASE) - ->save(); - } + $this->checkLeaseExpiration($lease); $commands = $this->loadCommands($lease->getPHID()); foreach ($commands as $command) { - if ($lease->getStatus() != DrydockLeaseStatus::STATUS_ACTIVE) { - // Leases can't receive commands before they activate or after they - // release. + if (!$lease->canUpdate()) { break; } $this->processCommand($lease, $command); $command ->setIsConsumed(true) ->save(); } - // If this is the task which will eventually release the lease after it - // expires but it is still active, reschedule the task to run after the - // lease expires. This can happen if the lease's expiration was pushed - // forward. - if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACTIVE) { - if ($this->getTaskDataValue('isExpireTask') && $expires) { - throw new PhabricatorWorkerYieldException($expires - $now); - } - } + $this->yieldIfExpiringLease($lease); } private function processCommand( DrydockLease $lease, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseLease($lease); break; } } private function releaseLease(DrydockLease $lease) { $lease->openTransaction(); $lease ->setStatus(DrydockLeaseStatus::STATUS_RELEASED) ->save(); // TODO: Hold slot locks until destruction? DrydockSlotLock::releaseLocks($lease->getPHID()); $lease->saveTransaction(); PhabricatorWorker::scheduleTask( 'DrydockLeaseDestroyWorker', array( 'leasePHID' => $lease->getPHID(), ), array( 'objectPHID' => $lease->getPHID(), )); $resource = $lease->getResource(); $blueprint = $resource->getBlueprint(); $blueprint->didReleaseLease($resource, $lease); } } diff --git a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php index 4afc51bc23..681741b6f1 100644 --- a/src/applications/drydock/worker/DrydockResourceUpdateWorker.php +++ b/src/applications/drydock/worker/DrydockResourceUpdateWorker.php @@ -1,99 +1,110 @@ getTaskDataValue('resourcePHID'); $hash = PhabricatorHash::digestForIndex($resource_phid); $lock_key = 'drydock.resource:'.$hash; $lock = PhabricatorGlobalLock::newLock($lock_key) ->lock(1); - $resource = $this->loadResource($resource_phid); - $this->updateResource($resource); + try { + $resource = $this->loadResource($resource_phid); + $this->updateResource($resource); + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } $lock->unlock(); } private function updateResource(DrydockResource $resource) { + if (!$resource->canUpdate()) { + return; + } + + $this->checkResourceExpiration($resource); + $commands = $this->loadCommands($resource->getPHID()); foreach ($commands as $command) { - if ($resource->getStatus() != DrydockResourceStatus::STATUS_ACTIVE) { - // Resources can't receive commands before they activate or after they - // release. + if (!$resource->canUpdate()) { break; } $this->processCommand($resource, $command); $command ->setIsConsumed(true) ->save(); } + + $this->yieldIfExpiringResource($resource); } private function processCommand( DrydockResource $resource, DrydockCommand $command) { switch ($command->getCommand()) { case DrydockCommand::COMMAND_RELEASE: $this->releaseResource($resource); break; } } private function releaseResource(DrydockResource $resource) { if ($resource->getStatus() != DrydockResourceStatus::STATUS_ACTIVE) { // If we had multiple release commands // This command is only meaningful to resources in the "Open" state. return; } $viewer = $this->getViewer(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $resource->openTransaction(); $resource ->setStatus(DrydockResourceStatus::STATUS_RELEASED) ->save(); // TODO: Hold slot locks until destruction? DrydockSlotLock::releaseLocks($resource->getPHID()); $resource->saveTransaction(); $statuses = array( DrydockLeaseStatus::STATUS_PENDING, DrydockLeaseStatus::STATUS_ACQUIRED, DrydockLeaseStatus::STATUS_ACTIVE, ); $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses($statuses) ->execute(); foreach ($leases as $lease) { $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($lease->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); $lease->scheduleUpdate(); } PhabricatorWorker::scheduleTask( 'DrydockResourceDestroyWorker', array( 'resourcePHID' => $resource->getPHID(), ), array( 'objectPHID' => $resource->getPHID(), )); } } diff --git a/src/applications/drydock/worker/DrydockWorker.php b/src/applications/drydock/worker/DrydockWorker.php index d41643de47..e64cfdafd7 100644 --- a/src/applications/drydock/worker/DrydockWorker.php +++ b/src/applications/drydock/worker/DrydockWorker.php @@ -1,53 +1,117 @@ getViewer(); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new PhabricatorWorkerPermanentFailureException( pht('No such lease "%s"!', $lease_phid)); } return $lease; } protected function loadResource($resource_phid) { $viewer = $this->getViewer(); $resource = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withPHIDs(array($resource_phid)) ->executeOne(); if (!$resource) { throw new PhabricatorWorkerPermanentFailureException( pht('No such resource "%s"!', $resource_phid)); } return $resource; } protected function loadCommands($target_phid) { $viewer = $this->getViewer(); $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($target_phid)) ->withConsumed(false) ->execute(); $commands = msort($commands, 'getID'); return $commands; } + protected function checkLeaseExpiration(DrydockLease $lease) { + $this->checkObjectExpiration($lease); + } + + protected function checkResourceExpiration(DrydockResource $resource) { + $this->checkObjectExpiration($resource); + } + + private function checkObjectExpiration($object) { + // Check if the resource or lease has expired. If it has, we're going to + // send it a release command. + + // This command is sent from within the update worker so it is handled + // immediately, but doing this generates a log and improves consistency. + + $expires = $object->getUntil(); + if (!$expires) { + return; + } + + $now = PhabricatorTime::getNow(); + if ($expires > $now) { + return; + } + + $viewer = $this->getViewer(); + $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); + + $command = DrydockCommand::initializeNewCommand($viewer) + ->setTargetPHID($object->getPHID()) + ->setAuthorPHID($drydock_phid) + ->setCommand(DrydockCommand::COMMAND_RELEASE) + ->save(); + } + + protected function yieldIfExpiringLease(DrydockLease $lease) { + if (!$lease->canUpdate()) { + return; + } + + $this->yieldIfExpiring($lease->getUntil()); + } + + protected function yieldIfExpiringResource(DrydockResource $resource) { + if (!$resource->canUpdate()) { + return; + } + + $this->yieldIfExpiring($resource->getUntil()); + } + + private function yieldIfExpiring($expires) { + if (!$expires) { + return; + } + + if (!$this->getTaskDataValue('isExpireTask')) { + return; + } + + $now = PhabricatorTime::getNow(); + throw new PhabricatorWorkerYieldException($expires - $now); + } + } diff --git a/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php b/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php index 9189489208..6357ebc5d3 100644 --- a/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php +++ b/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php @@ -1,49 +1,56 @@ 'id', 'param' => 'id', 'repeat' => true, 'help' => pht('Select one or more tasks by ID.'), ), ); } protected function loadTasks(PhutilArgumentParser $args) { $ids = $args->getArg('id'); if (!$ids) { throw new PhutilArgumentUsageException( pht('Use --id to select tasks by ID.')); } $active_tasks = id(new PhabricatorWorkerActiveTask())->loadAllWhere( 'id IN (%Ls)', $ids); $archive_tasks = id(new PhabricatorWorkerArchiveTaskQuery()) ->withIDs($ids) ->execute(); $tasks = mpull($active_tasks, null, 'getID') + mpull($archive_tasks, null, 'getID'); foreach ($ids as $id) { if (empty($tasks[$id])) { throw new PhutilArgumentUsageException( pht('No task exists with id "%s"!', $id)); } } + // When we lock tasks properly, this gets populated as a side effect. Just + // fake it when doing manual CLI stuff. This makes sure CLI yields have + // their expires times set properly. + foreach ($tasks as $task) { + $task->setServerTime(PhabricatorTime::getNow()); + } + return $tasks; } protected function describeTask(PhabricatorWorkerTask $task) { return pht('Task %d (%s)', $task->getID(), $task->getTaskClass()); } }