diff --git a/scripts/daemon/phabricator_daemon_launcher.php b/scripts/daemon/phabricator_daemon_launcher.php index 5bcb312005..ef2806d5ba 100755 --- a/scripts/daemon/phabricator_daemon_launcher.php +++ b/scripts/daemon/phabricator_daemon_launcher.php @@ -1,260 +1,207 @@ #!/usr/bin/env php executeListCommand(); exit($err); case 'status': $err = $control->executeStatusCommand(); exit($err); case 'stop': $err = $control->executeStopCommand(); exit($err); case 'repository-launch-readonly': $need_launch = phd_load_tracked_repositories_of_type('git'); if (!$need_launch) { echo "There are no repositories with tracking enabled.\n"; } else { foreach ($need_launch as $repository) { $name = $repository->getName(); $callsign = $repository->getCallsign(); $desc = "'{$name}' ({$callsign})"; $phid = $repository->getPHID(); echo "Launching 'git fetch' daemon on the {$desc} repository...\n"; $control->launchDaemon( 'PhabricatorRepositoryGitFetchDaemon', array( $phid, )); } } break; case 'repository-launch-master': $need_launch = phd_load_tracked_repositories(); if (!$need_launch) { echo "There are no repositories with tracking enabled.\n"; } else { foreach ($need_launch as $repository) { $name = $repository->getName(); $callsign = $repository->getCallsign(); $desc = "'{$name}' ({$callsign})"; $phid = $repository->getPHID(); switch ($repository->getVersionControlSystem()) { case 'git': echo "Launching 'git fetch' daemon on the {$desc} repository...\n"; $control->launchDaemon( 'PhabricatorRepositoryGitFetchDaemon', array( $phid, )); echo "Launching discovery daemon on the {$desc} repository...\n"; $control->launchDaemon( 'PhabricatorRepositoryGitCommitDiscoveryDaemon', array( $phid, )); break; case 'svn': echo "Launching discovery daemon on the {$desc} repository...\n"; $control->launchDaemon( 'PhabricatorRepositorySvnCommitDiscoveryDaemon', array( $phid, )); break; } } echo "Launching CommitTask daemon...\n"; $control->launchDaemon( 'PhabricatorRepositoryCommitTaskDaemon', array()); echo "Done.\n"; } break; case 'launch': $daemon = idx($argv, 2); if (!$daemon) { throw new Exception("Daemon name required!"); } $pass_argv = array_slice($argv, 3); $n = 1; if (is_numeric($daemon)) { $n = $daemon; if ($n < 1) { throw new Exception("Count must be at least 1!"); } $daemon = idx($argv, 3); if (!$daemon) { throw new Exception("Daemon name required!"); } $pass_argv = array_slice($argv, 4); } $loader = new PhutilSymbolLoader(); $symbols = $loader ->setAncestorClass('PhutilDaemon') ->selectSymbolsWithoutLoading(); $symbols = ipull($symbols, 'name'); $match = array(); foreach ($symbols as $symbol) { if (stripos($symbol, $daemon) !== false) { if (strtolower($symbol) == strtolower($daemon)) { $match = array($symbol); break; } else { $match[] = $symbol; } } } if (count($match) == 0) { throw new Exception( "No daemons match! Use 'phd list' for a list of daemons."); } else if (count($match) > 1) { throw new Exception( "Which of these daemons did you mean?\n". " ".implode("\n ", $match)); } else { $daemon = reset($match); } echo "Launching {$n} x {$daemon}"; for ($ii = 0; $ii < $n; $ii++) { $control->launchDaemon($daemon, $pass_argv); echo "."; } echo "\n"; echo "Done.\n"; break; - case 'parse-commit': - $commit = isset($argv[2]) ? $argv[2] : null; - if (!$commit) { - throw new Exception("Provide a commit to parse!"); - } - $matches = null; - if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) { - throw new Exception("Can't parse commit identifier!"); - } - $repo = id(new PhabricatorRepository())->loadOneWhere( - 'callsign = %s', - $matches[1]); - if (!$repo) { - throw new Exception("Unknown repository!"); - } - $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( - 'repositoryID = %d AND commitIdentifier = %s', - $repo->getID(), - $matches[2]); - if (!$commit) { - throw new Exception('Unknown commit.'); - } - - $workers = array(); - - - switch ($repo->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker( - $commit->getID()); - $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker( - $commit->getID()); - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker( - $commit->getID()); - $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker( - $commit->getID()); - break; - default: - throw new Exception("Unknown repository type!"); - } - - ExecFuture::pushEchoMode(true); - - foreach ($workers as $worker) { - echo "Running ".get_class($worker)."...\n"; - $worker->doWork(); - } - - echo "Done.\n"; - - break; case '--help': case 'help': default: $err = $control->executeHelpCommand(); exit($err); } function phd_load_tracked_repositories_of_type($type) { $repositories = phd_load_tracked_repositories(); foreach ($repositories as $key => $repository) { if ($repository->getVersionControlSystem() != $type) { unset($repositories[$key]); } } return $repositories; } function phd_load_tracked_repositories() { phutil_require_module( 'phabricator', 'applications/repository/storage/repository'); $repositories = id(new PhabricatorRepository())->loadAll(); foreach ($repositories as $key => $repository) { if (!$repository->getDetail('tracking-enabled')) { unset($repositories[$key]); } } return $repositories; } diff --git a/scripts/repository/parse_one_commit.php b/scripts/repository/parse_one_commit.php new file mode 100755 index 0000000000..56f29156a6 --- /dev/null +++ b/scripts/repository/parse_one_commit.php @@ -0,0 +1,83 @@ +#!/usr/bin/env php +\n"; + die(1); +} + +$commit = isset($argv[1]) ? $argv[1] : null; +if (!$commit) { + throw new Exception("Provide a commit to parse!"); +} +$matches = null; +if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $commit, $matches)) { + throw new Exception("Can't parse commit identifier!"); +} +$repo = id(new PhabricatorRepository())->loadOneWhere( + 'callsign = %s', + $matches[1]); +if (!$repo) { + throw new Exception("Unknown repository!"); +} +$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %d AND commitIdentifier = %s', + $repo->getID(), + $matches[2]); +if (!$commit) { + throw new Exception('Unknown commit.'); +} + +$workers = array(); + +$spec = array( + 'commitID' => $commit->getID(), + 'only' => true, +); + +switch ($repo->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $workers[] = new PhabricatorRepositoryGitCommitMessageParserWorker( + $spec); + $workers[] = new PhabricatorRepositoryGitCommitChangeParserWorker( + $spec); + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $workers[] = new PhabricatorRepositorySvnCommitMessageParserWorker( + $spec); + $workers[] = new PhabricatorRepositorySvnCommitChangeParserWorker( + $spec); + break; + default: + throw new Exception("Unknown repository type!"); +} + +ExecFuture::pushEchoMode(true); + +foreach ($workers as $worker) { + echo "Running ".get_class($worker)."...\n"; + $worker->doWork(); +} + +echo "Done.\n"; + diff --git a/scripts/repository/reparse_all_commit_messages.php b/scripts/repository/reparse_all_commit_messages.php new file mode 100755 index 0000000000..b9fac56381 --- /dev/null +++ b/scripts/repository/reparse_all_commit_messages.php @@ -0,0 +1,95 @@ +#!/usr/bin/env php +\n"; + exit(1); +} + + +echo phutil_console_format( + 'This script will queue tasks to reparse every commit message known to '. + 'Diffusion. Once the tasks have been inserted, you need to start '. + 'Taskmaster daemons to execute them.'); + +$ok = phutil_console_confirm('Do you want to continue?'); +if (!$ok) { + die(1); +} + +if ($argv[1] == 'all') { + echo "Loading all repositories...\n"; + $repositories = id(new PhabricatorRepository())->loadAll(); + echo "Loading all commits...\n"; + $commits = id(new PhabricatorRepositoryCommit())->loadAll(); +} else { + $callsign = $argv[1]; + echo "Loading '{$callsign}' repository...\n"; + $repository = id(new PhabricatorRepository())->loadOneWhere( + 'callsign = %s', + $argv[1]); + if (!$repository) { + throw new Exception("No such repository exists!"); + } + $repositories = array( + $repository->getID() => $repository, + ); + echo "Loading commits in '{$callsign}' repository...\n"; + $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( + 'repositoryID = %d', + $repository->getID()); +} + +echo "Inserting tasks for ".count($commits)." commits"; +foreach ($commits as $commit) { + echo "."; + $id = $commit->getID(); + $repo = idx($repositories, $commit->getRepositoryID()); + if (!$repo) { + echo "\nWarning: Commit #{$id} has an invalid repository ID.\n"; + } + + switch ($repo->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + $task_class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + $task_class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; + break; + default: + throw new Exception("Unknown repository type!"); + } + + $task = new PhabricatorWorkerTask(); + $task->setTaskClass($task_class); + $task->setData( + array( + 'commitID' => $commit->getID(), + 'only' => true, + )); + $task->save(); +} +echo "\nDone.\n"; diff --git a/src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php b/src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php index 9e0aab76cc..79b977c197 100644 --- a/src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php +++ b/src/applications/repository/daemon/committask/PhabricatorRepositoryCommitTaskDaemon.php @@ -1,71 +1,77 @@ getData(); if (!$data) { // TODO: This event can't be processed, provide some way to // communicate that? continue; } $commit = id(new PhabricatorRepositoryCommit())->load($data['id']); if (!$commit) { // TODO: Same as above. continue; } // TODO: Cache these. $repository = id(new PhabricatorRepository())->load( $commit->getRepositoryID()); $vcs = $repository->getVersionControlSystem(); switch ($vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: $class = 'PhabricatorRepositoryGitCommitMessageParserWorker'; $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); - $task->setData($commit->getID()); + $task->setData( + array( + 'commitID' => $commit->getID(), + )); $task->save(); break; case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: $class = 'PhabricatorRepositorySvnCommitMessageParserWorker'; $task = new PhabricatorWorkerTask(); $task->setTaskClass($class); - $task->setData($commit->getID()); + $task->setData( + array( + 'commitID' => $commit->getID(), + )); $task->save(); break; default: throw new Exception("Unknown repository type."); } $this->stillWorking(); } sleep(1); $this->stillWorking(); } while (true); } } diff --git a/src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php b/src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php index fade2cda75..20994b2afb 100644 --- a/src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php +++ b/src/applications/repository/worker/base/PhabricatorRepositoryCommitParserWorker.php @@ -1,106 +1,110 @@ getTaskData(); + $commit_id = idx($this->getTaskData(), 'commitID'); if (!$commit_id) { return; } $commit = id(new PhabricatorRepositoryCommit())->load($commit_id); if (!$commit) { // TODO: Communicate permanent failure? return; } $this->commit = $commit; $repository = id(new PhabricatorRepository())->load( $commit->getRepositoryID()); if (!$repository) { return; } $this->repository = $repository; return $this->parseCommit($repository, $commit); } + final protected function shouldQueueFollowupTasks() { + return !!idx($this->getTaskData(), 'only'); + } + abstract protected function parseCommit( PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit); /** * This method is kind of awkward here but both the SVN message and * change parsers use it. */ protected function getSVNLogXMLObject($uri, $revision, $verbose = false) { if ($verbose) { $verbose = '--verbose'; } try { list($xml) = execx( "svn log --xml {$verbose} --limit 1 --non-interactive %s@%d", $uri, $revision); } catch (CommandException $ex) { // HTTPS is generally faster and more reliable than svn+ssh, but some // commit messages with non-UTF8 text can't be retrieved over HTTPS, see // Facebook rE197184 for one example. Make an attempt to fall back to // svn+ssh if we've failed outright to retrieve the message. $fallback_uri = new PhutilURI($uri); if ($fallback_uri->getProtocol() != 'https') { throw $ex; } $fallback_uri->setProtocol('svn+ssh'); list($xml) = execx( "svn log --xml {$verbose} --limit 1 --non-interactive %s@%d", $fallback_uri, $revision); } // Subversion may send us back commit messages which won't parse because // they have non UTF-8 garbage in them. Slam them into valid UTF-8. $xml = phutil_utf8ize($xml); return new SimpleXMLElement($xml); } protected function isBadCommit($full_commit_name) { $repository = new PhabricatorRepository(); $bad_commit = queryfx_one( $repository->establishConnection('w'), 'SELECT * FROM %T WHERE fullCommitName = %s', PhabricatorRepository::TABLE_BADCOMMIT, $full_commit_name); return (bool)$bad_commit; } } diff --git a/src/applications/repository/worker/commitmessageparser/git/PhabricatorRepositoryGitCommitMessageParserWorker.php b/src/applications/repository/worker/commitmessageparser/git/PhabricatorRepositoryGitCommitMessageParserWorker.php index 0ad674752b..2989452786 100644 --- a/src/applications/repository/worker/commitmessageparser/git/PhabricatorRepositoryGitCommitMessageParserWorker.php +++ b/src/applications/repository/worker/commitmessageparser/git/PhabricatorRepositoryGitCommitMessageParserWorker.php @@ -1,45 +1,53 @@ getDetail('local-path'); list($info) = execx( '(cd %s && git log -n 1 --pretty=format:%%an%%x00%%B %s)', $local_path, $commit->getCommitIdentifier()); - // TODO: Need to slam UTF8? list($author, $message) = explode("\0", $info); + // Make sure these are valid UTF-8. + $author = phutil_utf8ize($author); + $message = phutil_utf8ize($message); + $this->updateCommitData($author, $message); - $task = new PhabricatorWorkerTask(); - $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker'); - $task->setData($commit->getID()); - $task->save(); + if ($this->shouldQueueFollowupTasks()) { + $task = new PhabricatorWorkerTask(); + $task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker'); + $task->setData( + array( + 'commitID' => $commit->getID(), + )); + $task->save(); + } } } diff --git a/src/applications/repository/worker/commitmessageparser/svn/PhabricatorRepositorySvnCommitMessageParserWorker.php b/src/applications/repository/worker/commitmessageparser/svn/PhabricatorRepositorySvnCommitMessageParserWorker.php index 75da4e66ed..c8c8d95478 100644 --- a/src/applications/repository/worker/commitmessageparser/svn/PhabricatorRepositorySvnCommitMessageParserWorker.php +++ b/src/applications/repository/worker/commitmessageparser/svn/PhabricatorRepositorySvnCommitMessageParserWorker.php @@ -1,46 +1,51 @@ getDetail('remote-uri'); $log = $this->getSVNLogXMLObject( $uri, $commit->getCommitIdentifier(), $verbose = false); $entry = $log->logentry[0]; $author = (string)$entry->author; $message = (string)$entry->msg; $this->updateCommitData($author, $message); - $task = new PhabricatorWorkerTask(); - $task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker'); - $task->setData($commit->getID()); - $task->save(); + if ($this->shouldQueueFollowupTasks()) { + $task = new PhabricatorWorkerTask(); + $task->setTaskClass('PhabricatorRepositorySvnCommitChangeParserWorker'); + $task->setData( + array( + 'commitID' => $commit->getID(), + )); + $task->save(); + } } } diff --git a/src/infrastructure/daemon/control/PhabricatorDaemonControl.php b/src/infrastructure/daemon/control/PhabricatorDaemonControl.php index ce72b27b26..03170ba872 100644 --- a/src/infrastructure/daemon/control/PhabricatorDaemonControl.php +++ b/src/infrastructure/daemon/control/PhabricatorDaemonControl.php @@ -1,247 +1,244 @@ loadAvailableDaemonClasses(); $symbols = igroup($symbols, 'library'); echo "\n"; foreach ($symbols as $library => $symbol_list) { echo phutil_console_format("Daemons in library __%s__:\n", $library); foreach ($symbol_list as $symbol) { echo " ".$symbol['name']."\n"; } echo "\n"; } return 0; } public function executeStatusCommand() { $daemons = $this->loadRunningDaemons(); if (!$daemons) { echo "There are no running Phabricator daemons.\n"; return 0; } printf( "%-5s\t%-24s\t%s\n", "PID", "Started", "Daemon"); foreach ($daemons as $daemon) { $name = $daemon->getName(); if (!$daemon->isRunning()) { $name = ' '.$name; if ($daemon->getPIDFile()) { Filesystem::remove($daemon->getPIDFile()); } } printf( "%5s\t%-24s\t%s\n", $daemon->getPID(), $daemon->getEpochStarted() ? date('M j Y, g:i:s A', $daemon->getEpochStarted()) : null, $name); } return 0; } public function executeStopCommand() { $daemons = $this->loadRunningDaemons(); if (!$daemons) { echo "There are no running Phabricator daemons.\n"; return 0; } $running = $daemons; foreach ($running as $key => $daemon) { $pid = $daemon->getPID(); $name = $daemon->getName(); echo "Stopping daemon '{$name}' ({$pid})...\n"; if (!$daemon->isRunning()) { echo "Daemon is not running.\n"; unset($running[$key]); } else { posix_kill($pid, SIGINT); } } $start = time(); do { foreach ($running as $key => $daemon) { $pid = $daemon->getPID(); if (!$daemon->isRunning()) { echo "Daemon {$pid} exited normally.\n"; unset($running[$key]); } } if (empty($running)) { break; } usleep(100000); } while (time() < $start + 15); foreach ($running as $key => $daemon) { $pid = $daemon->getPID(); echo "KILLing daemon {$pid}.\n"; posix_kill($pid, SIGKILL); } foreach ($daemons as $daemon) { if ($daemon->getPIDFile()) { Filesystem::remove($daemon->getPIDFile()); } } } public function executeHelpCommand() { echo phutil_console_format(<<loadAvailableDaemonClasses(); $symbols = ipull($symbols, 'name', 'name'); if (empty($symbols[$daemon])) { throw new Exception("Daemon '{$daemon}' is not known."); } $pid_dir = $this->getControlDirectory('pid'); $libphutil_root = dirname(phutil_get_library_root('phutil')); $launch_daemon = $libphutil_root.'/scripts/daemon/'; // TODO: This should be a much better user experience. Filesystem::assertExists($pid_dir); Filesystem::assertIsDirectory($pid_dir); Filesystem::assertWritable($pid_dir); foreach ($argv as $key => $arg) { $argv[$key] = escapeshellarg($arg); } $future = new ExecFuture( "./launch_daemon.php ". "%s ". "--load-phutil-library=%s ". "--conduit-uri=%s ". "--daemonize ". "--phd=%s ". implode(' ', $argv), $daemon, phutil_get_library_root('phabricator'), PhabricatorEnv::getURI('/api/'), $pid_dir); // Play games to keep 'ps' looking reasonable. $future->setCWD($launch_daemon); $future->resolvex(); } protected function getControlDirectory($dir) { return PhabricatorEnv::getEnvConfig('phd.pid-directory').'/'.$dir; } protected function loadAvailableDaemonClasses() { $loader = new PhutilSymbolLoader(); return $loader ->setAncestorClass('PhutilDaemon') ->selectSymbolsWithoutLoading(); } protected function loadRunningDaemons() { $results = array(); $pid_dir = $this->getControlDirectory('pid'); $pid_files = Filesystem::listDirectory($pid_dir); if (!$pid_files) { return $results; } foreach ($pid_files as $pid_file) { $pid_data = Filesystem::readFile($pid_dir.'/'.$pid_file); $dict = json_decode($pid_data, true); if (!is_array($dict)) { // Just return a hanging reference, since control code needs to be // robust against unusual system states. $dict = array(); } $ref = PhabricatorDaemonReference::newFromDictionary($dict); $ref->setPIDFile($pid_dir.'/'.$pid_file); $results[] = $ref; } return $results; } protected function killDaemon(PhabricatorDaemonReference $ref) { } }