diff --git a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php index 6ffe90469d..2527a34cdc 100644 --- a/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php +++ b/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php @@ -1,269 +1,334 @@ getAttribute('repositoryPHID'); - $need_phid = $lease->getAttribute('repositoryPHID'); + $need_map = $lease->getAttribute('repositories.map'); + if (!is_array($need_map)) { + return false; + } - if ($need_phid !== $have_phid) { + $have_map = $resource->getAttribute('repositories.map'); + if (!is_array($have_map)) { + return false; + } + + $have_as = ipull($have_map, 'phid'); + $need_as = ipull($need_map, 'phid'); + + foreach ($need_as as $need_directory => $need_phid) { + if (empty($have_as[$need_directory])) { + // This resource is missing a required working copy. + return false; + } + + if ($have_as[$need_directory] != $need_phid) { + // This resource has a required working copy, but it contains + // the wrong repository. + return false; + } + + unset($have_as[$need_directory]); + } + + if ($have_as && $lease->getAttribute('repositories.strict')) { + // This resource has extra repositories, but the lease is strict about + // which repositories are allowed to exist. return false; } if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { return false; } return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->needSlotLock($this->getLeaseSlotLock($resource)) ->acquireOnResource($resource); } private function getLeaseSlotLock(DrydockResource $resource) { $resource_phid = $resource->getPHID(); return "workingcopy.lease({$resource_phid})"; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { - $repository_phid = $lease->getAttribute('repositoryPHID'); - $repository = $this->loadRepository($repository_phid); - $resource = $this->newResourceTemplate( $blueprint, - pht( - 'Working Copy (%s)', - $repository->getCallsign())); + pht('Working Copy')); $resource_phid = $resource->getPHID(); $host_lease = $this->newLease($blueprint) ->setResourceType('host') ->setOwnerPHID($resource_phid) ->setAttribute('workingcopy.resourcePHID', $resource_phid) ->queueForActivation(); // TODO: Add some limits to the number of working copies we can have at // once? + $map = $lease->getAttribute('repositories.map'); + foreach ($map as $key => $value) { + $map[$key] = array_select_keys( + $value, + array( + 'phid', + )); + } + return $resource - ->setAttribute('repositoryPHID', $repository->getPHID()) + ->setAttribute('repositories.map', $map) ->setAttribute('host.leasePHID', $host_lease->getPHID()) ->allocateResource(); } public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { $lease = $this->loadHostLease($resource); $this->requireActiveLease($lease); - $repository_phid = $resource->getAttribute('repositoryPHID'); - $repository = $this->loadRepository($repository_phid); - $repository_id = $repository->getID(); - $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); // TODO: Make this configurable. $resource_id = $resource->getID(); $root = "/var/drydock/workingcopy-{$resource_id}"; - $path = "{$root}/repo/{$repository_id}/"; - $interface->execx( - 'git clone -- %s %s', - (string)$repository->getCloneURIObject(), - $path); + $map = $resource->getAttribute('repositories.map'); + + $repositories = $this->loadRepositories(ipull($map, 'phid')); + foreach ($map as $directory => $spec) { + // TODO: Validate directory isn't goofy like "/etc" or "../../lol" + // somewhere? + + $repository = $repositories[$spec['phid']]; + $path = "{$root}/repo/{$directory}/"; + + // TODO: Run these in parallel? + $interface->execx( + 'git clone -- %s %s', + (string)$repository->getCloneURIObject(), + $path); + } $resource ->setAttribute('workingcopy.root', $root) - ->setAttribute('workingcopy.path', $path) ->activateResource(); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { $lease = $this->loadHostLease($resource); // Destroy the lease on the host. $lease->releaseOnDestruction(); // Destroy the working copy on disk. $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); $root_key = 'workingcopy.root'; $root = $resource->getAttribute($root_key); if (strlen($root)) { $interface->execx('rm -rf -- %s', $root); } } public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { + $host_lease = $this->loadHostLease($resource); $command_type = DrydockCommandInterface::INTERFACE_TYPE; - $interface = $lease->getInterface($command_type); + $interface = $host_lease->getInterface($command_type); + + $map = $lease->getAttribute('repositories.map'); + $root = $resource->getAttribute('workingcopy.root'); + + $default = null; + foreach ($map as $directory => $spec) { + $cmd = array(); + $arg = array(); + + $cmd[] = 'cd %s'; + $arg[] = "{$root}/repo/{$directory}/"; + + $cmd[] = 'git clean -d --force'; + $cmd[] = 'git fetch'; - $cmd = array(); - $arg = array(); + $commit = idx($spec, 'commit'); + $branch = idx($spec, 'branch'); - $cmd[] = 'git clean -d --force'; - $cmd[] = 'git reset --hard HEAD'; - $cmd[] = 'git fetch'; + if ($commit !== null) { + $cmd[] = 'git reset --hard %s'; + $arg[] = $commit; + } else if ($branch !== null) { + $cmd[] = 'git reset --hard %s'; + $arg[] = $branch; + } else { + $cmd[] = 'git reset --hard HEAD'; + } - $commit = $lease->getAttribute('commit'); - $branch = $lease->getAttribute('branch'); + $cmd = implode(' && ', $cmd); + $argv = array_merge(array($cmd), $arg); - if ($commit !== null) { - $cmd[] = 'git reset --hard %s'; - $arg[] = $commit; - } else if ($branch !== null) { - $cmd[] = 'git reset --hard %s'; - $arg[] = $branch; + $result = call_user_func_array( + array($interface, 'execx'), + $argv); + + if (idx($spec, 'default')) { + $default = $directory; + } } - $cmd = implode(' && ', $cmd); - $argv = array_merge(array($cmd), $arg); + if ($default === null) { + $default = head_key($map); + } - $result = call_user_func_array( - array($interface, 'execx'), - $argv); + // TODO: Use working storage? + $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/"); $lease->activateOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We leave working copies around even if there are no leases on them, // since the cost to maintain them is nearly zero but rebuilding them is // moderately expensive and it's likely that they'll be reused. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // When we activate a lease we just reset the working copy state and do // not create any new state, so we don't need to do anything special when // destroying a lease. return; } public function getType() { return 'working-copy'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $host_lease = $this->loadHostLease($resource); $command_interface = $host_lease->getInterface($type); - $path = $resource->getAttribute('workingcopy.path'); + $path = $lease->getAttribute('workingcopy.default'); $command_interface->setWorkingDirectory($path); return $command_interface; } } - private function loadRepository($repository_phid) { - $repository = id(new PhabricatorRepositoryQuery()) + private function loadRepositories(array $phids) { + $repositories = id(new PhabricatorRepositoryQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) - ->withPHIDs(array($repository_phid)) - ->executeOne(); - if (!$repository) { - // TODO: Permanent failure. - throw new Exception( - pht( - 'Repository PHID "%s" does not exist.', - $repository_phid)); - } + ->withPHIDs($phids) + ->execute(); + $repositories = mpull($repositories, null, 'getPHID'); - switch ($repository->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - break; - default: + foreach ($phids as $phid) { + if (empty($repositories[$phid])) { // TODO: Permanent failure. - throw new Exception(pht('Unsupported VCS!')); + throw new Exception( + pht( + 'Repository PHID "%s" does not exist.', + $phid)); + } + } + + foreach ($repositories as $repository) { + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + break; + default: + // TODO: Permanent failure. + throw new Exception(pht('Unsupported VCS!')); + } } - return $repository; + return $repositories; } private function loadHostLease(DrydockResource $resource) { $viewer = PhabricatorUser::getOmnipotentUser(); $lease_phid = $resource->getAttribute('host.leasePHID'); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { // TODO: Permanent failure. throw new Exception(pht('Unable to load lease "%s".', $lease_phid)); } return $lease; } } diff --git a/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php b/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php index b01ae26da6..69f9cab6c1 100644 --- a/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php +++ b/src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php @@ -1,111 +1,137 @@ getSettings(); // TODO: We should probably have a separate temporary storage area for // execution stuff that doesn't step on configuration state? $lease_phid = $build_target->getDetail('exec.leasePHID'); if ($lease_phid) { $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new PhabricatorWorkerPermanentFailureException( pht( 'Lease "%s" could not be loaded.', $lease_phid)); } } else { $working_copy_type = id(new DrydockWorkingCopyBlueprintImplementation()) ->getType(); $lease = id(new DrydockLease()) ->setResourceType($working_copy_type) ->setOwnerPHID($build_target->getPHID()); - $variables = $build_target->getVariables(); + $map = $this->buildRepositoryMap($build_target); - $repository_phid = idx($variables, 'repository.phid'); - $commit = idx($variables, 'repository.commit'); - - $lease - ->setAttribute('repositoryPHID', $repository_phid) - ->setAttribute('commit', $commit); + $lease->setAttribute('repositories.map', $map); $task_id = $this->getCurrentWorkerTaskID(); if ($task_id) { $lease->setAwakenTaskIDs(array($task_id)); } $lease->queueForActivation(); $build_target ->setDetail('exec.leasePHID', $lease->getPHID()) ->save(); } if ($lease->isActivating()) { // TODO: Smart backoff? throw new PhabricatorWorkerYieldException(15); } if (!$lease->isActive()) { // TODO: We could just forget about this lease and retry? throw new PhabricatorWorkerPermanentFailureException( pht( 'Lease "%s" never activated.', $lease->getPHID())); } $artifact = $build_target->createArtifact( $viewer, $settings['name'], HarbormasterWorkingCopyArtifact::ARTIFACTCONST, array( 'drydockLeasePHID' => $lease->getPHID(), )); } public function getArtifactOutputs() { return array( array( 'name' => pht('Working Copy'), 'key' => $this->getSetting('name'), 'type' => HarbormasterWorkingCopyArtifact::ARTIFACTCONST, ), ); } public function getFieldSpecifications() { return array( 'name' => array( 'name' => pht('Artifact Name'), 'type' => 'text', 'required' => true, ), ); } + private function buildRepositoryMap(HarbormasterBuildTarget $build_target) { + $viewer = PhabricatorUser::getOmnipotentUser(); + $variables = $build_target->getVariables(); + + $repository_phid = idx($variables, 'repository.phid'); + + $repository = id(new PhabricatorRepositoryQuery()) + ->setViewer($viewer) + ->withPHIDs(array($repository_phid)) + ->executeOne(); + if (!$repository) { + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Unable to load repository with PHID "%s".', + $repository_phid)); + } + + $commit = idx($variables, 'repository.commit'); + + $map = array(); + + $directory = $repository->getCloneName(); + $map[$directory] = array( + 'phid' => $repository->getPHID(), + 'commit' => $commit, + 'default' => true, + ); + + return $map; + } + }