diff --git a/src/difference/ArcanistDiffUtils.php b/src/difference/ArcanistDiffUtils.php index 4006235e..bff4f63c 100644 --- a/src/difference/ArcanistDiffUtils.php +++ b/src/difference/ArcanistDiffUtils.php @@ -1,332 +1,334 @@ '; $highlight_c = ''; $n = strlen($str); for ($i = 0; $i < $n; $i++) { if ($p == $e) { do { if (empty($intra_stack)) { $buf .= substr($str, $i); break 2; } $stack = array_shift($intra_stack); $s = $e; $e += $stack[1]; } while ($stack[0] == 0); } if (!$highlight && !$tag && !$ent && $p == $s) { $buf .= $highlight_o; $highlight = true; } if ($str[$i] == '<') { $tag = true; if ($highlight) { $buf .= $highlight_c; } } if (!$tag) { if ($str[$i] == '&') { $ent = true; } if ($ent && $str[$i] == ';') { $ent = false; } if (!$ent) { $p++; } } $buf .= $str[$i]; if ($tag && $str[$i] == '>') { $tag = false; if ($highlight) { $buf .= $highlight_o; } } if ($highlight && ($p == $e || $i == $n - 1)) { $buf .= $highlight_c; $highlight = false; } } return $buf; } private static function collapseIntralineRuns($runs) { $count = count($runs); for ($ii = 0; $ii < $count - 1; $ii++) { if ($runs[$ii][0] == $runs[$ii + 1][0]) { $runs[$ii + 1][1] += $runs[$ii][1]; unset($runs[$ii]); } } return array_values($runs); } public static function buildLevenshteinDifferenceString($o, $n) { $olt = strlen($o); $nlt = strlen($n); if (!$olt) { return str_repeat('i', $nlt); } if (!$nlt) { return str_repeat('d', $olt); } $min = min($olt, $nlt); $t_start = microtime(true); $pre = 0; while ($pre < $min && $o[$pre] == $n[$pre]) { $pre++; } $end = 0; while ($end < $min && $o[($olt - 1) - $end] == $n[($nlt - 1) - $end]) { $end++; } if ($end + $pre >= $min) { $end = min($end, $min - $pre); $prefix = str_repeat('s', $pre); $suffix = str_repeat('s', $end); $infix = null; if ($olt > $nlt) { $infix = str_repeat('d', $olt - ($end + $pre)); } else if ($nlt > $olt) { $infix = str_repeat('i', $nlt - ($end + $pre)); } return $prefix.$infix.$suffix; } if ($min - ($end + $pre) > 80) { $max = max($olt, $nlt); return str_repeat('x', $min) . str_repeat($olt < $nlt ? 'i' : 'd', $max - $min); } $prefix = str_repeat('s', $pre); $suffix = str_repeat('s', $end); $o = substr($o, $pre, $olt - $end - $pre); $n = substr($n, $pre, $nlt - $end - $pre); $ol = strlen($o); $nl = strlen($n); $m = array_fill(0, $ol + 1, array_fill(0, $nl + 1, array())); $t_d = 'd'; $t_i = 'i'; $t_s = 's'; $t_x = 'x'; $m[0][0] = array( 0, null); for ($ii = 1; $ii <= $ol; $ii++) { $m[$ii][0] = array( $ii * 1000, $t_d); } for ($jj = 1; $jj <= $nl; $jj++) { $m[0][$jj] = array( $jj * 1000, $t_i); } $ii = 1; do { $jj = 1; do { if ($o[$ii - 1] == $n[$jj - 1]) { $sub_t_cost = $m[$ii - 1][$jj - 1][0] + 0; $sub_t = $t_s; } else { $sub_t_cost = $m[$ii - 1][$jj - 1][0] + 2000; $sub_t = $t_x; } if ($m[$ii - 1][$jj - 1][1] != $sub_t) { $sub_t_cost += 1; } $del_t_cost = $m[$ii - 1][$jj][0] + 1000; if ($m[$ii - 1][$jj][1] != $t_d) { $del_t_cost += 1; } $ins_t_cost = $m[$ii][$jj - 1][0] + 1000; if ($m[$ii][$jj - 1][1] != $t_i) { $ins_t_cost += 1; } if ($sub_t_cost <= $del_t_cost && $sub_t_cost <= $ins_t_cost) { $m[$ii][$jj] = array( $sub_t_cost, $sub_t); } else if ($ins_t_cost <= $del_t_cost) { $m[$ii][$jj] = array( $ins_t_cost, $t_i); } else { $m[$ii][$jj] = array( $del_t_cost, $t_d); } } while ($jj++ < $nl); } while ($ii++ < $ol); $result = ''; $ii = $ol; $jj = $nl; do { $r = $m[$ii][$jj][1]; $result .= $r; switch ($r) { case $t_s: case $t_x: $ii--; $jj--; break; case $t_i: $jj--; break; case $t_d: $ii--; break; } } while ($ii || $jj); return $prefix.strrev($result).$suffix; } } diff --git a/src/hgdaemon/ArcanistHgProxyServer.php b/src/hgdaemon/ArcanistHgProxyServer.php index 5e41f441..04097cdb 100644 --- a/src/hgdaemon/ArcanistHgProxyServer.php +++ b/src/hgdaemon/ArcanistHgProxyServer.php @@ -1,488 +1,487 @@ workingCopy = Filesystem::resolvePath($working_copy); } /* -( Configuration )------------------------------------------------------ */ /** * Disable status messages to stdout. Controlled with `--quiet`. * * @param bool True to disable status messages. * @return this * * @task config */ public function setQuiet($quiet) { $this->quiet = $quiet; return $this; } /** * Configure a client limit. After serving this many clients, the server * will exit. Controlled with `--client-limit`. * * You can use `--client-limit 1` with `--xprofile` and `--do-not-daemonize` * to profile the server. * * @param int Client limit, or 0 to disable limit. * @return this * * @task config */ public function setClientLimit($limit) { $this->clientLimit = $limit; return $this; } /** * Configure an idle time limit. After this many seconds idle, the server * will exit. Controlled with `--idle-limit`. * * @param int Idle limit, or 0 to disable limit. * @return this * * @task config */ public function setIdleLimit($limit) { $this->idleLimit = $limit; return $this; } /** * When clients connect, do not send the "capabilities" message expected by * the Mercurial protocol. This deviates from the protocol and will only work * if the clients are also configured not to expect the message, but slightly * improves performance. Controlled with --skip-hello. * * @param bool True to skip the "capabilities" message. * @return this * * @task config */ public function setSkipHello($skip) { $this->skipHello = $skip; return $this; } /** * Configure whether the server runs in the foreground or daemonizes. * Controlled by --do-not-daemonize. Primarily useful for debugging. * * @param bool True to run in the foreground. * @return this * * @task config */ public function setDoNotDaemonize($do_not_daemonize) { $this->doNotDaemonize = $do_not_daemonize; return $this; } /* -( Serving Requests )--------------------------------------------------- */ /** * Start the server. This method returns after the client limit or idle * limit are exceeded. If neither limit is configured, this method does not * exit. * * @return null * * @task server */ public function start() { // Create the unix domain socket in the working copy to listen for clients. $socket = $this->startWorkingCopySocket(); $this->socket = $socket; if (!$this->doNotDaemonize) { $this->daemonize(); } // Start the Mercurial process which we'll forward client requests to. $hg = $this->startMercurialProcess(); $clients = array(); $this->log(null, 'Listening'); $this->idleSince = time(); while (true) { // Wait for activity on any active clients, the Mercurial process, or // the listening socket where new clients connect. PhutilChannel::waitForAny( array_merge($clients, array($hg)), array( 'read' => $socket ? array($socket) : array(), 'except' => $socket ? array($socket) : array() )); if (!$hg->update()) { throw new Exception("Server exited unexpectedly!"); } // Accept any new clients. while ($socket && ($client = $this->acceptNewClient($socket))) { $clients[] = $client; $key = last_key($clients); $client->setName($key); $this->log($client, 'Connected'); $this->idleSince = time(); // Check if we've hit the client limit. If there's a configured // client limit and we've hit it, stop accepting new connections // and close the socket. $this->lifetimeClientCount++; if ($this->clientLimit) { if ($this->lifetimeClientCount >= $this->clientLimit) { $this->closeSocket(); $socket = null; } } } // Update all the active clients. foreach ($clients as $key => $client) { if ($this->updateClient($client, $hg)) { // In this case, the client is still connected so just move on to // the next one. Otherwise we continue below and handle the disconect. continue; } $this->log($client, 'Disconnected'); unset($clients[$key]); // If we have a client limit and we've served that many clients, exit. if ($this->clientLimit) { if ($this->lifetimeClientCount >= $this->clientLimit) { if (!$clients) { $this->log(null, 'Exiting (Client Limit)'); return; } } } } // If we have an idle limit and haven't had any activity in at least // that long, exit. if ($this->idleLimit) { $remaining = $this->idleLimit - (time() - $this->idleSince); if ($remaining <= 0) { $this->log(null, 'Exiting (Idle Limit)'); return; } if ($remaining <= 5) { $this->log(null, 'Exiting in '.$remaining.' seconds'); } } } } /** * Update one client, processing any commands it has sent us. We fully * process all commands we've received here before returning to the main * server loop. * * @param ArcanistHgClientChannel The client to update. * @param ArcanistHgServerChannel The Mercurial server. * * @task server */ private function updateClient( ArcanistHgClientChannel $client, ArcanistHgServerChannel $hg) { if (!$client->update()) { // Client has disconnected, don't bother proceeding. return false; } // Read a command from the client if one is available. Note that we stop // updating other clients or accepting new connections while processing a // command, since there isn't much we can do with them until the server // finishes executing this command. $message = $client->read(); if (!$message) { return true; } $this->log($client, '$ '.$message[0].' '.$message[1]); $t_start = microtime(true); // Forward the command to the server. $hg->write($message); while (true) { PhutilChannel::waitForAny(array($client, $hg)); if (!$client->update() || !$hg->update()) { // If either the client or server has exited, bail. return false; } $response = $hg->read(); if (!$response) { continue; } // Forward the response back to the client. $client->write($response); // If the response was on the 'r'esult channel, it indicates the end // of the command output. We can process the next command (if any // remain) or go back to accepting new connections and servicing // other clients. if ($response[0] == 'r') { // Update the client immediately to try to get the bytes on the wire // as quickly as possible. This gives us slightly more throughput. $client->update(); break; } } // Log the elapsed time. $t_end = microtime(true); $t = 1000000 * ($t_end - $t_start); $this->log($client, '< '.number_format($t, 0).'us'); $this->idleSince = time(); return true; } /* -( Managing Clients )--------------------------------------------------- */ /** * @task client */ public static function getPathToSocket($working_copy) { return $working_copy.'/.hg/hgdaemon-socket'; } /** * @task client */ private function startWorkingCopySocket() { $errno = null; $errstr = null; $socket_path = self::getPathToSocket($this->workingCopy); $socket_uri = 'unix://'.$socket_path; $socket = @stream_socket_server($socket_uri, $errno, $errstr); if ($errno || !$socket) { Filesystem::remove($socket_path); $socket = @stream_socket_server($socket_uri, $errno, $errstr); } if ($errno || !$socket) { throw new Exception( "Unable to start socket! Error #{$errno}: {$errstr}"); } $ok = stream_set_blocking($socket, 0); if ($ok === false) { throw new Exception("Unable to set socket nonblocking!"); } return $socket; } /** * @task client */ private function acceptNewClient($socket) { // NOTE: stream_socket_accept() always blocks, even when the socket has // been set nonblocking. $new_client = @stream_socket_accept($socket, $timeout = 0); if (!$new_client) { return null; } $channel = new PhutilSocketChannel($new_client); $client = new ArcanistHgClientChannel($channel); if (!$this->skipHello) { $client->write($this->hello); } return $client; } /* -( Managing Mercurial )------------------------------------------------- */ /** * Starts a Mercurial process which can actually handle requests. * * @return ArcanistHgServerChannel Channel to the Mercurial server. * @task hg */ private function startMercurialProcess() { // NOTE: "cmdserver.log=-" makes Mercurial use the 'd'ebug channel for // log messages. - $command = 'HGPLAIN=1 hg --config cmdserver.log=- serve --cmdserver pipe'; - - $future = new ExecFuture($command); + $future = new ExecFuture( + 'HGPLAIN=1 hg --config cmdserver.log=- serve --cmdserver pipe'); $future->setCWD($this->workingCopy); $channel = new PhutilExecChannel($future); $hg = new ArcanistHgServerChannel($channel); // The server sends a "hello" message with capability and encoding // information. Save it and forward it to clients when they connect. $this->hello = $hg->waitForMessage(); return $hg; } /* -( Internals )---------------------------------------------------------- */ /** * Close and remove the unix domain socket in the working copy. * * @task internal */ public function __destruct() { $this->closeSocket(); } private function closeSocket() { if ($this->socket) { @stream_socket_shutdown($this->socket, STREAM_SHUT_RDWR); @fclose($this->socket); Filesystem::remove(self::getPathToSocket($this->workingCopy)); $this->socket = null; } } private function log($client, $message) { if ($this->quiet) { return; } if ($client) { $message = '[Client '.$client->getName().'] '.$message; } else { $message = '[Server] '.$message; } echo $message."\n"; } private function daemonize() { // Keep stdout if it's been redirected somewhere, otherwise shut it down. $keep_stdout = false; $keep_stderr = false; if (function_exists('posix_isatty')) { if (!posix_isatty(STDOUT)) { $keep_stdout = true; } if (!posix_isatty(STDERR)) { $keep_stderr = true; } } $pid = pcntl_fork(); if ($pid === -1) { throw new Exception("Unable to fork!"); } else if ($pid) { // We're the parent; exit. First, drop our reference to the socket so // our __destruct() doesn't tear it down; the child will tear it down // later. $this->socket = null; exit(0); } // We're the child; continue. fclose(STDIN); if (!$keep_stdout) { fclose(STDOUT); $this->quiet = true; } if (!$keep_stderr) { fclose(STDERR); } } } diff --git a/src/lint/linter/ArcanistJSHintLinter.php b/src/lint/linter/ArcanistJSHintLinter.php index 7aa9c561..ac6ecc31 100644 --- a/src/lint/linter/ArcanistJSHintLinter.php +++ b/src/lint/linter/ArcanistJSHintLinter.php @@ -1,161 +1,163 @@ ArcanistLintSeverity::SEVERITY_ERROR ); } public function getLintNameMap() { return array( self::JSHINT_ERROR => "JSHint Error" ); } public function getJSHintOptions() { $working_copy = $this->getEngine()->getWorkingCopy(); $options = '--reporter '.dirname(realpath(__FILE__)).'/reporter.js'; $config = $working_copy->getConfig('lint.jshint.config'); if ($config !== null) { $config = Filesystem::resolvePath( $config, $working_copy->getProjectRoot()); if (!Filesystem::pathExists($config)) { throw new ArcanistUsageException( "Unable to find custom options file defined by ". "'lint.jshint.config'. Make sure that the path is correct."); } $options .= ' --config '.$config; } return $options; } private function getJSHintPath() { $working_copy = $this->getEngine()->getWorkingCopy(); $prefix = $working_copy->getConfig('lint.jshint.prefix'); $bin = $working_copy->getConfig('lint.jshint.bin'); if ($bin === null) { $bin = "jshint"; } if ($prefix !== null) { $bin = $prefix."/".$bin; if (!Filesystem::pathExists($bin)) { throw new ArcanistUsageException( "Unable to find JSHint binary in a specified directory. Make sure ". "that 'lint.jshint.prefix' and 'lint.jshint.bin' keys are set ". "correctly. If you'd rather use a copy of JSHint installed ". "globally, you can just remove these keys from your .arcconfig"); } return $bin; } // Look for globally installed JSHint - $cmd = (phutil_is_windows()) ? 'where %s' : 'which %s'; - list($err) = exec_manual($cmd, $bin); + list($err) = (phutil_is_windows() + ? exec_manual('where %s', $bin) + : exec_manual('which %s', $bin)); + if ($err) { throw new ArcanistUsageException( "JSHint does not appear to be installed on this system. Install it ". "(e.g., with 'npm install jshint -g') or configure ". "'lint.jshint.prefix' in your .arcconfig to point to the directory ". "where it resides."); } return $bin; } public function willLintPaths(array $paths) { $jshint_bin = $this->getJSHintPath(); $jshint_options = $this->getJSHintOptions(); $futures = array(); foreach ($paths as $path) { $filepath = $this->getEngine()->getFilePathOnDisk($path); $futures[$path] = new ExecFuture( "%s %s %C", $jshint_bin, $filepath, $jshint_options); } foreach (Futures($futures)->limit(8) as $path => $future) { $this->results[$path] = $future->resolve(); } } public function lintPath($path) { list($rc, $stdout, $stderr) = $this->results[$path]; if ($rc === 0) { return; } $errors = json_decode($stdout); if (!is_array($errors)) { // Something went wrong and we can't decode the output. Exit abnormally. throw new ArcanistUsageException( "JSHint returned unparseable output.\n". "stdout:\n\n{$stdout}". "stderr:\n\n{$stderr}"); } foreach ($errors as $err) { $this->raiseLintAtLine( $err->line, $err->col, self::JSHINT_ERROR, $err->reason); } } } diff --git a/src/lint/linter/ArcanistPyFlakesLinter.php b/src/lint/linter/ArcanistPyFlakesLinter.php index 3ae6cdc3..99c867a1 100644 --- a/src/lint/linter/ArcanistPyFlakesLinter.php +++ b/src/lint/linter/ArcanistPyFlakesLinter.php @@ -1,110 +1,112 @@ getEngine()->getWorkingCopy(); $pyflakes_path = $working_copy->getConfig('lint.pyflakes.path'); $pyflakes_prefix = $working_copy->getConfig('lint.pyflakes.prefix'); // Default to just finding pyflakes in the users path $pyflakes_bin = 'pyflakes'; $python_path = array(); // If a pyflakes path was specified, then just use that as the // pyflakes binary and assume that the libraries will be imported // correctly. // // If no pyflakes path was specified and a pyflakes prefix was // specified, then use the binary from this prefix and add it to // the PYTHONPATH environment variable so that the libs are imported // correctly. This is useful when pyflakes is installed into a // non-default location. if ($pyflakes_path !== null) { $pyflakes_bin = $pyflakes_path; } else if ($pyflakes_prefix !== null) { $pyflakes_bin = $pyflakes_prefix.'/bin/pyflakes'; $python_path[] = $pyflakes_prefix.'/lib/python2.7/site-packages'; $python_path[] = $pyflakes_prefix.'/lib/python2.7/dist-packages'; $python_path[] = $pyflakes_prefix.'/lib/python2.6/site-packages'; $python_path[] = $pyflakes_prefix.'/lib/python2.6/dist-packages'; } $python_path[] = ''; $python_path = implode(':', $python_path); $options = $this->getPyFlakesOptions(); $f = new ExecFuture( - "/usr/bin/env PYTHONPATH=%s\$PYTHONPATH ". - "{$pyflakes_bin} {$options}", $python_path); + '/usr/bin/env PYTHONPATH=%s$PYTHONPATH %s %C', + $python_path, + $pyflakes_bin, + $options); $f->write($this->getData($path)); try { list($stdout, $_) = $f->resolvex(); } catch (CommandException $e) { // PyFlakes will return an exit code of 1 if warnings/errors // are found but print nothing to stderr in this case. Therefore, // if we see any output on stderr or a return code other than 1 or 0, // pyflakes failed. if ($e->getError() !== 1 || $e->getStderr() !== '') { throw $e; } else { $stdout = $e->getStdout(); } } $lines = explode("\n", $stdout); $messages = array(); foreach ($lines as $line) { $matches = null; if (!preg_match('/^(.*?):(\d+): (.*)$/', $line, $matches)) { continue; } foreach ($matches as $key => $match) { $matches[$key] = trim($match); } $severity = ArcanistLintSeverity::SEVERITY_WARNING; $description = $matches[3]; $error_regexp = '/(^undefined|^duplicate|before assignment$)/'; if (preg_match($error_regexp, $description)) { $severity = ArcanistLintSeverity::SEVERITY_ERROR; } $message = new ArcanistLintMessage(); $message->setPath($path); $message->setLine($matches[2]); $message->setCode($this->getLinterName()); $message->setDescription($description); $message->setSeverity($severity); $this->addLintMessage($message); } } } diff --git a/src/lint/linter/ArcanistPyLintLinter.php b/src/lint/linter/ArcanistPyLintLinter.php index 88b45c18..ad885fb0 100644 --- a/src/lint/linter/ArcanistPyLintLinter.php +++ b/src/lint/linter/ArcanistPyLintLinter.php @@ -1,248 +1,250 @@ getEngine()->getWorkingCopy(); $error_regexp = $working_copy->getConfig('lint.pylint.codes.error'); $warning_regexp = $working_copy->getConfig('lint.pylint.codes.warning'); $advice_regexp = $working_copy->getConfig('lint.pylint.codes.advice'); if (!$error_regexp && !$warning_regexp && !$advice_regexp) { throw new ArcanistUsageException( "You are invoking the PyLint linter but have not configured any of ". "'lint.pylint.codes.error', 'lint.pylint.codes.warning', or ". "'lint.pylint.codes.advice'. Consult the documentation for ". "ArcanistPyLintLinter."); } $code_map = array( ArcanistLintSeverity::SEVERITY_ERROR => $error_regexp, ArcanistLintSeverity::SEVERITY_WARNING => $warning_regexp, ArcanistLintSeverity::SEVERITY_ADVICE => $advice_regexp, ); foreach ($code_map as $sev => $codes) { if ($codes === null) { continue; } if (!is_array($codes)) { $codes = array($codes); } foreach ($codes as $code_re) { if (preg_match("/{$code_re}/", $code)) { return $sev; } } } // If the message code doesn't match any of the provided regex's, // then just disable it. return ArcanistLintSeverity::SEVERITY_DISABLED; } private function getPyLintPath() { $pylint_bin = "pylint"; // Use the PyLint prefix specified in the config file $working_copy = $this->getEngine()->getWorkingCopy(); $prefix = $working_copy->getConfig('lint.pylint.prefix'); if ($prefix !== null) { $pylint_bin = $prefix."/bin/".$pylint_bin; } if (!Filesystem::pathExists($pylint_bin)) { list($err) = exec_manual('which %s', $pylint_bin); if ($err) { throw new ArcanistUsageException( "PyLint does not appear to be installed on this system. Install it ". "(e.g., with 'sudo easy_install pylint') or configure ". "'lint.pylint.prefix' in your .arcconfig to point to the directory ". "where it resides."); } } return $pylint_bin; } private function getPyLintPythonPath() { // Get non-default install locations for pylint and its dependencies // libraries. $working_copy = $this->getEngine()->getWorkingCopy(); $prefixes = array( $working_copy->getConfig('lint.pylint.prefix'), $working_copy->getConfig('lint.pylint.logilab_astng.prefix'), $working_copy->getConfig('lint.pylint.logilab_common.prefix'), ); // Add the libraries to the python search path $python_path = array(); foreach ($prefixes as $prefix) { if ($prefix !== null) { $python_path[] = $prefix.'/lib/python2.7/site-packages'; $python_path[] = $prefix.'/lib/python2.7/dist-packages'; $python_path[] = $prefix.'/lib/python2.6/site-packages'; $python_path[] = $prefix.'/lib/python2.6/dist-packages'; } } $config_paths = $working_copy->getConfig('lint.pylint.pythonpath'); if ($config_paths !== null) { foreach ($config_paths as $config_path) { if ($config_path !== null) { $python_path[] = Filesystem::resolvePath($config_path, $working_copy->getProjectRoot()); } } } $python_path[] = ''; return implode(":", $python_path); } private function getPyLintOptions() { // '-rn': don't print lint report/summary at end // '-iy': show message codes for lint warnings/errors $options = array('-rn', '-iy'); $working_copy = $this->getEngine()->getWorkingCopy(); // Specify an --rcfile, either absolute or relative to the project root. // Stupidly, the command line args above are overridden by rcfile, so be // careful. $rcfile = $working_copy->getConfig('lint.pylint.rcfile'); if ($rcfile !== null) { $rcfile = Filesystem::resolvePath( $rcfile, $working_copy->getProjectRoot()); $options[] = csprintf('--rcfile=%s', $rcfile); } // Add any options defined in the config file for PyLint $config_options = $working_copy->getConfig('lint.pylint.options'); if ($config_options !== null) { $options = array_merge($options, $config_options); } return implode(" ", $options); } public function willLintPaths(array $paths) { return; } public function getLinterName() { return 'PyLint'; } public function getLintSeverityMap() { return array(); } public function getLintNameMap() { return array(); } public function lintPath($path) { $pylint_bin = $this->getPyLintPath(); $python_path = $this->getPyLintPythonPath(); $options = $this->getPyLintOptions(); $path_on_disk = $this->getEngine()->getFilePathOnDisk($path); try { list($stdout, $_) = execx( - "/usr/bin/env PYTHONPATH=%s\$PYTHONPATH ". - "{$pylint_bin} {$options} {$path_on_disk}", - $python_path); + '/usr/bin/env PYTHONPATH=%s$PYTHONPATH %s %C %s', + $python_path, + $pylint_bin, + $options, + $path_on_disk); } catch (CommandException $e) { if ($e->getError() == 32) { // According to ##man pylint## the exit status of 32 means there was a // usage error. That's bad, so actually exit abnormally. throw $e; } else { // The other non-zero exit codes mean there were messages issued, // which is expected, so don't exit. $stdout = $e->getStdout(); } } $lines = explode("\n", $stdout); $messages = array(); foreach ($lines as $line) { $matches = null; if (!preg_match( '/([A-Z]\d+): *(\d+)(?:|,\d*): *(.*)$/', $line, $matches)) { continue; } foreach ($matches as $key => $match) { $matches[$key] = trim($match); } $message = new ArcanistLintMessage(); $message->setPath($path); $message->setLine($matches[2]); $message->setCode($matches[1]); $message->setName($this->getLinterName()." ".$matches[1]); $message->setDescription($matches[3]); $message->setSeverity($this->getMessageCodeSeverity($matches[1])); $this->addLintMessage($message); } } } diff --git a/src/parser/ArcanistBaseCommitParser.php b/src/parser/ArcanistBaseCommitParser.php index 215e9ee1..24cf6c73 100644 --- a/src/parser/ArcanistBaseCommitParser.php +++ b/src/parser/ArcanistBaseCommitParser.php @@ -1,176 +1,176 @@ api = $api; return $this; } private function tokenizeBaseCommitSpecification($raw_spec) { if (!$raw_spec) { return array(); } $spec = preg_split('/\s*,\s*/', $raw_spec); $spec = array_filter($spec); foreach ($spec as $rule) { if (strpos($rule, ':') === false) { throw new ArcanistUsageException( "Rule '{$rule}' is invalid, it must have a type and name like ". "'arc:upstream'."); } } return $spec; } private function log($message) { if ($this->verbose) { fwrite(STDERR, $message."\n"); } } public function resolveBaseCommit(array $specs) { $specs += array( 'args' => '', 'local' => '', 'project' => '', 'global' => '', 'system' => '', ); foreach ($specs as $source => $spec) { $specs[$source] = self::tokenizeBaseCommitSpecification($spec); } $this->try = array( 'args', 'local', 'project', 'global', 'system', ); while ($this->try) { $source = head($this->try); if (!idx($specs, $source)) { $this->log("No rules left from source '{$source}'."); array_shift($this->try); continue; } $this->log("Trying rules from source '{$source}'."); $rules = &$specs[$source]; while ($rule = array_shift($rules)) { $this->log("Trying rule '{$rule}'."); $commit = $this->resolveRule($rule, $source); if ($commit === false) { // If a rule returns false, it means to go to the next ruleset. break; } else if ($commit !== null) { $this->log("Resolved commit '{$commit}' from rule '{$rule}'."); return $commit; } } } return null; } /** * Handle resolving individual rules. */ private function resolveRule($rule, $source) { // NOTE: Returning `null` from this method means "no match". // Returning `false` from this method means "stop current ruleset". list($type, $name) = explode(':', $rule, 2); switch ($type) { case 'literal': return $name; case 'git': case 'hg': return $this->api->resolveBaseCommitRule($rule, $source); case 'arc': return $this->resolveArcRule($rule, $name, $source); default: throw new ArcanistUsageException( "Base commit rule '{$rule}' (from source '{$source}') ". "is not a recognized rule."); } } /** * Handle resolving "arc:*" rules. */ private function resolveArcRule($rule, $name, $source) { switch ($name) { case 'verbose': $this->verbose = true; $this->log("Enabled verbose mode."); break; case 'prompt': $reason = "it is what you typed when prompted."; $this->api->setBaseCommitExplanation($reason); return phutil_console_prompt('Against which commit?'); case 'local': case 'global': case 'project': case 'args': case 'system': // Push the other source on top of the list. array_unshift($this->try, $name); $this->log("Switching to source '{$name}'."); return false; case 'yield': // Cycle this source to the end of the list. $this->try[] = array_shift($this->try); $this->log("Yielding processing of rules from '{$source}'."); return false; case 'halt': // Dump the whole stack. $this->try = array(); $this->log("Halting all rule processing."); return false; case 'skip': return null; case 'empty': case 'upstream': case 'outgoing': case 'bookmark': case 'amended': case 'this': return $this->api->resolveBaseCommitRule($rule, $source); default: $matches = null; if (preg_match('/^exec\((.*)\)$/', $name, $matches)) { $root = $this->api->getWorkingCopyIdentity()->getProjectRoot(); - $future = new ExecFuture($matches[1]); + $future = new ExecFuture('%C', $matches[1]); $future->setCWD($root); list($err, $stdout) = $future->resolve(); if (!$err) { return trim($stdout); } else { return null; } } throw new ArcanistUsageException( "Base commit rule '{$rule}' (from source '{$source}') ". "is not a recognized rule."); } } } diff --git a/src/parser/ArcanistBundle.php b/src/parser/ArcanistBundle.php index 9218675d..c0000c9f 100644 --- a/src/parser/ArcanistBundle.php +++ b/src/parser/ArcanistBundle.php @@ -1,842 +1,839 @@ author = $author; return $this; } public function getAuthor() { return $this->author; } public function setConduit(ConduitClient $conduit) { $this->conduit = $conduit; return $this; } public function setProjectID($project_id) { $this->projectID = $project_id; return $this; } public function getProjectID() { return $this->projectID; } public function setBaseRevision($base_revision) { $this->baseRevision = $base_revision; return $this; } public function setEncoding($encoding) { $this->encoding = $encoding; return $this; } public function getEncoding() { return $this->encoding; } public function getBaseRevision() { return $this->baseRevision; } public function setRevisionID($revision_id) { $this->revisionID = $revision_id; return $this; } public function getRevisionID() { return $this->revisionID; } public static function newFromChanges(array $changes) { $obj = new ArcanistBundle(); $obj->changes = $changes; return $obj; } private function getEOL($patch_type) { // NOTE: Git always generates "\n" line endings, even under Windows, and // can not parse certain patches with "\r\n" line endings. SVN generates // patches with "\n" line endings on Mac or Linux and "\r\n" line endings // on Windows. (This EOL style is used only for patch metadata lines, not // for the actual patch content.) // (On Windows, Mercurial generates \n newlines for `--git` diffs, as it // must, but also \n newlines for unified diffs. We never need to deal with // these as we use Git format for Mercurial, so this case is currently // ignored.) switch ($patch_type) { case 'git': return "\n"; case 'unified': return phutil_is_windows() ? "\r\n" : "\n"; default: throw new Exception( "Unknown patch type '{$patch_type}'!"); } } public static function newFromArcBundle($path) { $path = Filesystem::resolvePath($path); $future = new ExecFuture( - csprintf( - 'tar tfO %s', - $path)); + 'tar tfO %s', + $path); list($stdout, $file_list) = $future->resolvex(); $file_list = explode("\n", trim($file_list)); if (in_array('meta.json', $file_list)) { $future = new ExecFuture( - csprintf( - 'tar xfO %s meta.json', - $path)); + 'tar xfO %s meta.json', + $path); $meta_info = $future->resolveJSON(); $version = idx($meta_info, 'version', 0); $project_name = idx($meta_info, 'projectName'); $base_revision = idx($meta_info, 'baseRevision'); $revision_id = idx($meta_info, 'revisionID'); $encoding = idx($meta_info, 'encoding'); $author = idx($meta_info, 'author'); // this arc bundle was probably made before we started storing meta info } else { $version = 0; $project_name = null; $base_revision = null; $revision_id = null; $encoding = null; $author = null; } $future = new ExecFuture( - csprintf( - 'tar xfO %s changes.json', - $path)); + 'tar xfO %s changes.json', + $path); $changes = $future->resolveJSON(); foreach ($changes as $change_key => $change) { foreach ($change['hunks'] as $key => $hunk) { list($hunk_data) = execx('tar xfO %s hunks/%s', $path, $hunk['corpus']); $changes[$change_key]['hunks'][$key]['corpus'] = $hunk_data; } } foreach ($changes as $change_key => $change) { $changes[$change_key] = ArcanistDiffChange::newFromDictionary($change); } $obj = new ArcanistBundle(); $obj->changes = $changes; $obj->diskPath = $path; $obj->setProjectID($project_name); $obj->setBaseRevision($base_revision); $obj->setRevisionID($revision_id); $obj->setEncoding($encoding); return $obj; } public static function newFromDiff($data) { $obj = new ArcanistBundle(); $parser = new ArcanistDiffParser(); $obj->changes = $parser->parseDiff($data); return $obj; } private function __construct() { } public function writeToDisk($path) { $changes = $this->getChanges(); $change_list = array(); foreach ($changes as $change) { $change_list[] = $change->toDictionary(); } $hunks = array(); foreach ($change_list as $change_key => $change) { foreach ($change['hunks'] as $key => $hunk) { $hunks[] = $hunk['corpus']; $change_list[$change_key]['hunks'][$key]['corpus'] = count($hunks) - 1; } } $blobs = array(); foreach ($change_list as $change) { if (!empty($change['metadata']['old:binary-phid'])) { $blobs[$change['metadata']['old:binary-phid']] = null; } if (!empty($change['metadata']['new:binary-phid'])) { $blobs[$change['metadata']['new:binary-phid']] = null; } } foreach ($blobs as $phid => $null) { $blobs[$phid] = $this->getBlob($phid); } $meta_info = array( 'version' => 4, 'projectName' => $this->getProjectID(), 'baseRevision' => $this->getBaseRevision(), 'revisionID' => $this->getRevisionID(), 'encoding' => $this->getEncoding(), 'author' => $this->getAuthor(), ); $dir = Filesystem::createTemporaryDirectory(); Filesystem::createDirectory($dir.'/hunks'); Filesystem::createDirectory($dir.'/blobs'); Filesystem::writeFile($dir.'/changes.json', json_encode($change_list)); Filesystem::writeFile($dir.'/meta.json', json_encode($meta_info)); foreach ($hunks as $key => $hunk) { Filesystem::writeFile($dir.'/hunks/'.$key, $hunk); } foreach ($blobs as $key => $blob) { Filesystem::writeFile($dir.'/blobs/'.$key, $blob); } execx( '(cd %s; tar -czf %s *)', $dir, Filesystem::resolvePath($path)); Filesystem::remove($dir); } public function toUnifiedDiff() { $eol = $this->getEOL('unified'); $result = array(); $changes = $this->getChanges(); foreach ($changes as $change) { $hunk_changes = $this->buildHunkChanges($change->getHunks(), $eol); if (!$hunk_changes) { continue; } $old_path = $this->getOldPath($change); $cur_path = $this->getCurrentPath($change); $index_path = $cur_path; if ($index_path === null) { $index_path = $old_path; } $result[] = 'Index: '.$index_path; $result[] = $eol; $result[] = str_repeat('=', 67); $result[] = $eol; if ($old_path === null) { $old_path = '/dev/null'; } if ($cur_path === null) { $cur_path = '/dev/null'; } // When the diff is used by `patch`, `patch` ignores what is listed as the // current path and just makes changes to the file at the old path (unless // the current path is '/dev/null'. // If the old path and the current path aren't the same (and neither is // /dev/null), this indicates the file was moved or copied. By listing // both paths as the new file, `patch` will apply the diff to the new // file. if ($cur_path !== '/dev/null' && $old_path !== '/dev/null') { $old_path = $cur_path; } $result[] = '--- '.$old_path.$eol; $result[] = '+++ '.$cur_path.$eol; $result[] = $hunk_changes; } if (!$result) { return ''; } $diff = implode('', $result); return $this->convertNonUTF8Diff($diff); } public function toGitPatch() { $eol = $this->getEOL('git'); $result = array(); $changes = $this->getChanges(); $binary_sources = array(); foreach ($changes as $change) { if (!$this->isGitBinaryChange($change)) { continue; } $type = $change->getType(); if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY || $type == ArcanistDiffChangeType::TYPE_COPY_AWAY || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { foreach ($change->getAwayPaths() as $path) { $binary_sources[$path] = $change; } } } foreach (array_keys($changes) as $multicopy_key) { $multicopy_change = $changes[$multicopy_key]; $type = $multicopy_change->getType(); if ($type != ArcanistDiffChangeType::TYPE_MULTICOPY) { continue; } // Decompose MULTICOPY into one MOVE_HERE and several COPY_HERE because // we need more information than we have in order to build a delete patch // and represent it as a bunch of COPY_HERE plus a delete. For details, // see T419. // Basically, MULTICOPY means there are 2 or more corresponding COPY_HERE // changes, so find one of them arbitrarily and turn it into a MOVE_HERE. // TODO: We might be able to do this more cleanly after T230 is resolved. $decompose_okay = false; foreach ($changes as $change_key => $change) { if ($change->getType() != ArcanistDiffChangeType::TYPE_COPY_HERE) { continue; } if ($change->getOldPath() != $multicopy_change->getCurrentPath()) { continue; } $decompose_okay = true; $change = clone $change; $change->setType(ArcanistDiffChangeType::TYPE_MOVE_HERE); $changes[$change_key] = $change; // The multicopy is now fully represented by MOVE_HERE plus one or more // COPY_HERE, so throw it away. unset($changes[$multicopy_key]); break; } if (!$decompose_okay) { throw new Exception( "Failed to decompose multicopy changeset in order to generate diff."); } } foreach ($changes as $change) { $type = $change->getType(); $file_type = $change->getFileType(); if ($file_type == ArcanistDiffChangeType::FILE_DIRECTORY) { // TODO: We should raise a FYI about this, so the user is aware // that we omitted it, if the directory is empty or has permissions // which git can't represent. // Git doesn't support empty directories, so we simply ignore them. If // the directory is nonempty, 'git apply' will create it when processing // the changesets for files inside it. continue; } if ($type == ArcanistDiffChangeType::TYPE_MOVE_AWAY) { // Git will apply this in the corresponding MOVE_HERE. continue; } $old_mode = idx($change->getOldProperties(), 'unix:filemode', '100644'); $new_mode = idx($change->getNewProperties(), 'unix:filemode', '100644'); $is_binary = $this->isGitBinaryChange($change); if ($is_binary) { $old_binary = idx($binary_sources, $this->getCurrentPath($change)); $change_body = $this->buildBinaryChange($change, $old_binary); } else { $change_body = $this->buildHunkChanges($change->getHunks(), $eol); } if ($type == ArcanistDiffChangeType::TYPE_COPY_AWAY) { // TODO: This is only relevant when patching old Differential diffs // which were created prior to arc pruning TYPE_COPY_AWAY for files // with no modifications. if (!strlen($change_body) && ($old_mode == $new_mode)) { continue; } } $old_path = $this->getOldPath($change); $cur_path = $this->getCurrentPath($change); if ($old_path === null) { $old_index = 'a/'.$cur_path; $old_target = '/dev/null'; } else { $old_index = 'a/'.$old_path; $old_target = 'a/'.$old_path; } if ($cur_path === null) { $cur_index = 'b/'.$old_path; $cur_target = '/dev/null'; } else { $cur_index = 'b/'.$cur_path; $cur_target = 'b/'.$cur_path; } $result[] = "diff --git {$old_index} {$cur_index}".$eol; if ($type == ArcanistDiffChangeType::TYPE_ADD) { $result[] = "new file mode {$new_mode}".$eol; } if ($type == ArcanistDiffChangeType::TYPE_COPY_HERE || $type == ArcanistDiffChangeType::TYPE_MOVE_HERE || $type == ArcanistDiffChangeType::TYPE_COPY_AWAY || $type == ArcanistDiffChangeType::TYPE_CHANGE) { if ($old_mode !== $new_mode) { $result[] = "old mode {$old_mode}".$eol; $result[] = "new mode {$new_mode}".$eol; } } if ($type == ArcanistDiffChangeType::TYPE_COPY_HERE) { $result[] = "copy from {$old_path}".$eol; $result[] = "copy to {$cur_path}".$eol; } else if ($type == ArcanistDiffChangeType::TYPE_MOVE_HERE) { $result[] = "rename from {$old_path}".$eol; $result[] = "rename to {$cur_path}".$eol; } else if ($type == ArcanistDiffChangeType::TYPE_DELETE || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { $old_mode = idx($change->getOldProperties(), 'unix:filemode'); if ($old_mode) { $result[] = "deleted file mode {$old_mode}".$eol; } } if ($change_body) { if (!$is_binary) { $result[] = "--- {$old_target}".$eol; $result[] = "+++ {$cur_target}".$eol; } $result[] = $change_body; } } $diff = implode('', $result).$eol; return $this->convertNonUTF8Diff($diff); } private function isGitBinaryChange(ArcanistDiffChange $change) { $file_type = $change->getFileType(); return ($file_type == ArcanistDiffChangeType::FILE_BINARY || $file_type == ArcanistDiffChangeType::FILE_IMAGE); } private function convertNonUTF8Diff($diff) { if ($this->encoding) { $diff = phutil_utf8_convert($diff, $this->encoding, 'UTF-8'); } return $diff; } public function getChanges() { return $this->changes; } private function breakHunkIntoSmallHunks(ArcanistDiffHunk $base_hunk) { $context = 3; $results = array(); $lines = phutil_split_lines($base_hunk->getCorpus()); $n = count($lines); $old_offset = $base_hunk->getOldOffset(); $new_offset = $base_hunk->getNewOffset(); $ii = 0; $jj = 0; while ($ii < $n) { // Skip lines until we find the next line with changes. Note: this skips // both ' ' (no changes) and '\' (no newline at end of file) lines. If we // don't skip the latter, we may incorrectly generate a terminal hunk // that has no actual change information when a file doesn't have a // terminal newline and not changed near the end of the file. 'patch' will // fail to apply the diff if we generate a hunk that does not actually // contain changes. for ($jj = $ii; $jj < $n; ++$jj) { $char = $lines[$jj][0]; if ($char == '-' || $char == '+') { break; } } if ($jj >= $n) { break; } $hunk_start = max($jj - $context, 0); // NOTE: There are two tricky considerations here. // We can not generate a patch with overlapping hunks, or 'git apply' // rejects it after 1.7.3.4. // We can not generate a patch with too much trailing context, or // 'patch' rejects it. // So we need to ensure that we generate disjoint hunks, but don't // generate any hunks with too much context. $old_lines = 0; $new_lines = 0; $hunk_adjust = 0; $last_change = $jj; $break_here = null; for (; $jj < $n; ++$jj) { if ($lines[$jj][0] == ' ') { if ($jj - $last_change > $context) { if ($break_here === null) { // We haven't seen a change in $context lines, so this is a // potential place to break the hunk. However, we need to keep // looking in case there is another change fewer than $context // lines away, in which case we have to merge the hunks. $break_here = $jj; } } if ($jj - $last_change > (($context + 1) * 2)) { // We definitely aren't going to merge this with the next hunk, so // break out of the loop. We'll end the hunk at $break_here. break; } } else { $break_here = null; $last_change = $jj; if ($lines[$jj][0] == '\\') { // When we have a "\ No newline at end of file" line, it does not // contribute to either hunk length. ++$hunk_adjust; } else if ($lines[$jj][0] == '-') { ++$old_lines; } else if ($lines[$jj][0] == '+') { ++$new_lines; } } } if ($break_here !== null) { $jj = $break_here; } $hunk_length = min($jj, $n) - $hunk_start; $count_length = ($hunk_length - $hunk_adjust); $hunk = new ArcanistDiffHunk(); $hunk->setOldOffset($old_offset + $hunk_start - $ii); $hunk->setNewOffset($new_offset + $hunk_start - $ii); $hunk->setOldLength($count_length - $new_lines); $hunk->setNewLength($count_length - $old_lines); $corpus = array_slice($lines, $hunk_start, $hunk_length); $corpus = implode('', $corpus); $hunk->setCorpus($corpus); $results[] = $hunk; $old_offset += ($jj - $ii) - $new_lines; $new_offset += ($jj - $ii) - $old_lines; $ii = $jj; } return $results; } private function getOldPath(ArcanistDiffChange $change) { $old_path = $change->getOldPath(); $type = $change->getType(); if (!strlen($old_path) || $type == ArcanistDiffChangeType::TYPE_ADD) { $old_path = null; } return $old_path; } private function getCurrentPath(ArcanistDiffChange $change) { $cur_path = $change->getCurrentPath(); $type = $change->getType(); if (!strlen($cur_path) || $type == ArcanistDiffChangeType::TYPE_DELETE || $type == ArcanistDiffChangeType::TYPE_MULTICOPY) { $cur_path = null; } return $cur_path; } private function buildHunkChanges(array $hunks, $eol) { assert_instances_of($hunks, 'ArcanistDiffHunk'); $result = array(); foreach ($hunks as $hunk) { $small_hunks = $this->breakHunkIntoSmallHunks($hunk); foreach ($small_hunks as $small_hunk) { $o_off = $small_hunk->getOldOffset(); $o_len = $small_hunk->getOldLength(); $n_off = $small_hunk->getNewOffset(); $n_len = $small_hunk->getNewLength(); $corpus = $small_hunk->getCorpus(); // NOTE: If the length is 1 it can be omitted. Since git does this, // we also do it so that "arc export --git" diffs are as similar to // real git diffs as possible, which helps debug issues. if ($o_len == 1) { $o_head = "{$o_off}"; } else { $o_head = "{$o_off},{$o_len}"; } if ($n_len == 1) { $n_head = "{$n_off}"; } else { $n_head = "{$n_off},{$n_len}"; } $result[] = "@@ -{$o_head} +{$n_head} @@".$eol; $result[] = $corpus; $last = substr($corpus, -1); if ($last !== false && $last != "\r" && $last != "\n") { $result[] = $eol; } } } return implode('', $result); } public function setLoadFileDataCallback($callback) { $this->loadFileDataCallback = $callback; return $this; } private function getBlob($phid, $name = null) { if ($this->loadFileDataCallback) { return call_user_func($this->loadFileDataCallback, $phid); } if ($this->diskPath) { list($blob_data) = execx('tar xfO %s blobs/%s', $this->diskPath, $phid); return $blob_data; } $console = PhutilConsole::getConsole(); if ($this->conduit) { if ($name) { $console->writeErr("Downloading binary data for '%s'...\n", $name); } else { $console->writeErr("Downloading binary data...\n"); } $data_base64 = $this->conduit->callMethodSynchronous( 'file.download', array( 'phid' => $phid, )); return base64_decode($data_base64); } throw new Exception("Nowhere to load blob '{$phid}' from!"); } private function buildBinaryChange(ArcanistDiffChange $change, $old_binary) { $eol = $this->getEOL('git'); // In Git, when we write out a binary file move or copy, we need the // original binary for the source and the current binary for the // destination. if ($old_binary) { if ($old_binary->getOriginalFileData() !== null) { $old_data = $old_binary->getOriginalFileData(); $old_phid = null; } else { $old_data = null; $old_phid = $old_binary->getMetadata('old:binary-phid'); } } else { $old_data = $change->getOriginalFileData(); $old_phid = $change->getMetadata('old:binary-phid'); } if ($old_data === null && $old_phid) { $name = basename($change->getOldPath()); $old_data = $this->getBlob($old_phid, $name); } $old_length = strlen($old_data); if ($old_data === null) { $old_data = ''; $old_sha1 = str_repeat('0', 40); } else { $old_sha1 = sha1("blob {$old_length}\0{$old_data}"); } $new_phid = $change->getMetadata('new:binary-phid'); $new_data = null; if ($change->getCurrentFileData() !== null) { $new_data = $change->getCurrentFileData(); } else if ($new_phid) { $name = basename($change->getCurrentPath()); $new_data = $this->getBlob($new_phid, $name); } $new_length = strlen($new_data); if ($new_data === null) { $new_data = ''; $new_sha1 = str_repeat('0', 40); } else { $new_sha1 = sha1("blob {$new_length}\0{$new_data}"); } $content = array(); $content[] = "index {$old_sha1}..{$new_sha1}".$eol; $content[] = "GIT binary patch".$eol; $content[] = "literal {$new_length}".$eol; $content[] = $this->emitBinaryDiffBody($new_data).$eol; $content[] = "literal {$old_length}".$eol; $content[] = $this->emitBinaryDiffBody($old_data).$eol; return implode('', $content); } private function emitBinaryDiffBody($data) { $eol = $this->getEOL('git'); if (!function_exists('gzcompress')) { throw new Exception( "This patch has binary data. The PHP zlib extension is required to ". "apply patches with binary data to git. Install the PHP zlib ". "extension to continue."); } // See emit_binary_diff_body() in diff.c for git's implementation. $buf = ''; $deflated = gzcompress($data); $lines = str_split($deflated, 52); foreach ($lines as $line) { $len = strlen($line); // The first character encodes the line length. if ($len <= 26) { $buf .= chr($len + ord('A') - 1); } else { $buf .= chr($len - 26 + ord('a') - 1); } $buf .= self::encodeBase85($line); $buf .= $eol; } return $buf; } public static function encodeBase85($data) { // This is implemented awkwardly in order to closely mirror git's // implementation in base85.c // It is also implemeted awkwardly to work correctly on 32-bit machines. // Broadly, this algorithm converts the binary input to printable output // by transforming each 4 binary bytes of input to 5 printable bytes of // output, one piece at a time. // // To do this, we convert the 4 bytes into a 32-bit integer, then use // modulus and division by 85 to pick out printable bytes (85^5 is slightly // larger than 2^32). In C, this algorithm is fairly easy to implement // because the accumulator can be made unsigned. // // In PHP, there are no unsigned integers, so values larger than 2^31 break // on 32-bit systems under modulus: // // $ php -r 'print (1 << 31) % 13;' # On a 32-bit machine. // -11 // // However, PHP's float type is an IEEE 754 64-bit double precision float, // so we can safely store integers up to around 2^53 without loss of // precision. To work around the lack of an unsigned type, we just use a // double and perform the modulus with fmod(). // // (Since PHP overflows integer operations into floats, we don't need much // additional casting.) static $map = array( '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '!', '#', '$', '%', '&', '(', ')', '*', '+', '-', ';', '<', '=', '>', '?', '@', '^', '_', '`', '{', '|', '}', '~', ); $buf = ''; $pos = 0; $bytes = strlen($data); while ($bytes) { $accum = 0; for ($count = 24; $count >= 0; $count -= 8) { $val = ord($data[$pos++]); $val = $val * (1 << $count); $accum = $accum + $val; if (--$bytes == 0) { break; } } $slice = ''; for ($count = 4; $count >= 0; $count--) { $val = (int)fmod($accum, 85.0); $accum = floor($accum / 85.0); $slice .= $map[$val]; } $buf .= strrev($slice); } return $buf; } } diff --git a/src/workflow/ArcanistAnoidWorkflow.php b/src/workflow/ArcanistAnoidWorkflow.php index 4a114ea0..91793891 100644 --- a/src/workflow/ArcanistAnoidWorkflow.php +++ b/src/workflow/ArcanistAnoidWorkflow.php @@ -1,32 +1,32 @@ null, 'unit' => null); private $testResults; private $diffID; private $revisionID; private $postponedLinters; private $haveUncommittedChanges = false; private $diffPropertyFutures = array(); private $commitMessageFromRevision; public function getWorkflowName() { return 'diff'; } public function getCommandSynopses() { return phutil_console_format(<<isRawDiffSource(); } public function requiresConduit() { return true; } public function requiresAuthentication() { return true; } public function requiresRepositoryAPI() { if (!$this->isRawDiffSource()) { return true; } if ($this->getArgument('use-commit-message')) { return true; } return false; } public function getDiffID() { return $this->diffID; } public function getArguments() { $arguments = array( 'message' => array( 'short' => 'm', 'param' => 'message', 'help' => "When updating a revision, use the specified message instead of ". "prompting.", ), 'message-file' => array( 'short' => 'F', 'param' => 'file', 'paramtype' => 'file', 'help' => 'When creating a revision, read revision information '. 'from this file.', ), 'use-commit-message' => array( 'supports' => array( 'git', // TODO: Support mercurial. ), 'short' => 'C', 'param' => 'commit', 'help' => 'Read revision information from a specific commit.', 'conflicts' => array( 'only' => null, 'preview' => null, 'update' => null, ), ), 'edit' => array( 'supports' => array( 'git', ), 'nosupport' => array( 'svn' => 'Edit revisions via the web interface when using SVN.', ), 'help' => "When updating a revision under git, edit revision information ". "before updating.", ), 'raw' => array( 'help' => "Read diff from stdin, not from the working copy. This disables ". "many Arcanist/Phabricator features which depend on having access ". "to the working copy.", 'conflicts' => array( 'less-context' => null, 'apply-patches' => '--raw disables lint.', 'never-apply-patches' => '--raw disables lint.', 'advice' => '--raw disables lint.', 'lintall' => '--raw disables lint.', 'create' => '--raw and --create both need stdin. '. 'Use --raw-command.', 'edit' => '--raw and --edit both need stdin. '. 'Use --raw-command.', 'raw-command' => null, ), ), 'raw-command' => array( 'param' => 'command', 'help' => "Generate diff by executing a specified command, not from the ". "working copy. This disables many Arcanist/Phabricator features ". "which depend on having access to the working copy.", 'conflicts' => array( 'less-context' => null, 'apply-patches' => '--raw-command disables lint.', 'never-apply-patches' => '--raw-command disables lint.', 'advice' => '--raw-command disables lint.', 'lintall' => '--raw-command disables lint.', ), ), 'create' => array( 'help' => "Always create a new revision.", 'conflicts' => array( 'edit' => '--create can not be used with --edit.', 'only' => '--create can not be used with --only.', 'preview' => '--create can not be used with --preview.', 'update' => '--create can not be used with --update.', ), ), 'update' => array( 'param' => 'revision_id', 'help' => "Always update a specific revision.", ), 'nounit' => array( 'help' => "Do not run unit tests.", ), 'nolint' => array( 'help' => "Do not run lint.", 'conflicts' => array( 'lintall' => '--nolint suppresses lint.', 'advice' => '--nolint suppresses lint.', 'apply-patches' => '--nolint suppresses lint.', 'never-apply-patches' => '--nolint suppresses lint.', ), ), 'only' => array( 'help' => "Only generate a diff, without running lint, unit tests, or other ". "auxiliary steps. See also --preview.", 'conflicts' => array( 'preview' => null, 'message' => '--only does not affect revisions.', 'edit' => '--only does not affect revisions.', 'lintall' => '--only suppresses lint.', 'advice' => '--only suppresses lint.', 'apply-patches' => '--only suppresses lint.', 'never-apply-patches' => '--only suppresses lint.', ), ), 'preview' => array( 'help' => "Instead of creating or updating a revision, only create a diff, ". "which you may later attach to a revision. This still runs lint ". "unit tests. See also --only.", 'conflicts' => array( 'only' => null, 'edit' => '--preview does affect revisions.', 'message' => '--preview does not update any revision.', ), ), 'plan-changes' => array( 'help' => "Create or update a revision without requesting a code review.", 'conflicts' => array( 'only' => '--only does not affect revisions.', 'preview' => '--preview does not affect revisions.', ), ), 'encoding' => array( 'param' => 'encoding', 'help' => "Attempt to convert non UTF-8 hunks into specified encoding.", ), 'allow-untracked' => array( 'help' => "Skip checks for untracked files in the working copy.", ), 'excuse' => array( 'param' => 'excuse', 'help' => 'Provide a prepared in advance excuse for any lints/tests'. ' shall they fail.', ), 'less-context' => array( 'help' => "Normally, files are diffed with full context: the entire file is ". "sent to Differential so reviewers can 'show more' and see it. If ". "you are making changes to very large files with tens of thousands ". "of lines, this may not work well. With this flag, a diff will ". "be created that has only a few lines of context.", ), 'lintall' => array( 'help' => "Raise all lint warnings, not just those on lines you changed.", 'passthru' => array( 'lint' => true, ), ), 'advice' => array( 'help' => "Require excuse for lint advice in addition to lint warnings and ". "errors.", ), 'only-new' => array( 'param' => 'bool', 'help' => 'Display only lint messages not present in the original code.', 'passthru' => array( 'lint' => true, ), ), 'apply-patches' => array( 'help' => 'Apply patches suggested by lint to the working copy without '. 'prompting.', 'conflicts' => array( 'never-apply-patches' => true, ), 'passthru' => array( 'lint' => true, ), ), 'never-apply-patches' => array( 'help' => 'Never apply patches suggested by lint.', 'conflicts' => array( 'apply-patches' => true, ), 'passthru' => array( 'lint' => true, ), ), 'amend-all' => array( 'help' => 'When linting git repositories, amend HEAD with all patches '. 'suggested by lint without prompting.', 'passthru' => array( 'lint' => true, ), ), 'amend-autofixes' => array( 'help' => 'When linting git repositories, amend HEAD with autofix '. 'patches suggested by lint without prompting.', 'passthru' => array( 'lint' => true, ), ), 'add-all' => array( 'help' => 'Automatically add all untracked, unstaged and uncommitted files to '. 'the commit.', ), 'json' => array( 'help' => 'Emit machine-readable JSON. EXPERIMENTAL! Probably does not work!', ), 'no-amend' => array( 'help' => 'Never amend commits in the working copy with lint patches.', ), 'uncommitted' => array( 'help' => 'Suppress warning about uncommitted changes.', 'supports' => array( 'hg', ), ), 'verbatim' => array( 'help' => 'When creating a revision, try to use the working copy '. 'commit message verbatim, without prompting to edit it. '. 'When updating a revision, update some fields from the '. 'local commit message.', 'supports' => array( 'hg', 'git', ), 'conflicts' => array( 'use-commit-message' => true, 'update' => true, 'only' => true, 'preview' => true, 'raw' => true, 'raw-command' => true, 'message-file' => true, ), ), 'reviewers' => array( 'param' => 'usernames', 'help' => 'When creating a revision, add reviewers.', 'conflicts' => array( 'only' => true, 'preview' => true, 'update' => true, ), ), 'cc' => array( 'param' => 'usernames', 'help' => 'When creating a revision, add CCs.', 'conflicts' => array( 'only' => true, 'preview' => true, 'update' => true, ), ), 'skip-binaries' => array( 'help' => 'Do not upload binaries (like images).', ), 'ignore-unsound-tests' => array( 'help' => 'Ignore unsound test failures without prompting.', ), 'base' => array( 'param' => 'rules', 'help' => 'Additional rules for determining base revision.', 'nosupport' => array( 'svn' => 'Subversion does not use base commits.', ), 'supports' => array('git', 'hg'), ), 'no-diff' => array( 'help' => 'Only run lint and unit tests. Intended for internal use.', ), 'background' => array( 'param' => 'bool', 'help' => 'Run lint and unit tests on background. '. '"0" to disable, "1" to enable (default).', ), 'cache' => array( 'param' => 'bool', 'help' => "0 to disable lint cache, 1 to enable (default).", 'passthru' => array( 'lint' => true, ), ), 'coverage' => array( 'help' => 'Always enable coverage information.', 'conflicts' => array( 'no-coverage' => null, ), 'passthru' => array( 'unit' => true, ), ), 'no-coverage' => array( 'help' => 'Always disable coverage information.', 'passthru' => array( 'unit' => true, ), ), '*' => 'paths', ); if (phutil_is_windows()) { unset($arguments['background']); } return $arguments; } public function isRawDiffSource() { return $this->getArgument('raw') || $this->getArgument('raw-command'); } public function run() { $this->console = PhutilConsole::getConsole(); $this->runRepositoryAPISetup(); if ($this->getArgument('no-diff')) { $this->removeScratchFile('diff-result.json'); $data = $this->runLintUnit(); $this->writeScratchJSONFile('diff-result.json', $data); return 0; } $this->runDiffSetupBasics(); $background = $this->getArgument('background', true); if ($this->isRawDiffSource() || phutil_is_windows()) { $background = false; } if ($background) { $argv = $this->getPassedArguments(); if (!PhutilConsoleFormatter::getDisableANSI()) { array_unshift($argv, '--ansi'); } $script = phutil_get_library_root('arcanist').'/../scripts/arcanist.php'; if ($argv) { $lint_unit = new ExecFuture( 'php %s --recon diff --no-diff %Ls', $script, $argv); } else { $lint_unit = new ExecFuture( 'php %s --recon diff --no-diff', $script); } $lint_unit->write('', true); $lint_unit->start(); } $commit_message = $this->buildCommitMessage(); $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_DIDBUILDMESSAGE, array( 'message' => $commit_message, )); if (!$this->shouldOnlyCreateDiff()) { $revision = $this->buildRevisionFromCommitMessage($commit_message); } if ($background) { $server = new PhutilConsoleServer(); $server->addExecFutureClient($lint_unit); $server->setHandler(array($this, 'handleServerMessage')); $server->run(); list($err) = $lint_unit->resolve(); $data = $this->readScratchJSONFile('diff-result.json'); if ($err || !$data) { throw new Exception( 'Unable to read results from background linting and unit testing. '. 'You can try running arc diff again with --background 0'); } } else { $server = $this->console->getServer(); $server->setHandler(array($this, 'handleServerMessage')); $data = $this->runLintUnit(); } $lint_result = $data['lintResult']; $this->unresolvedLint = $data['unresolvedLint']; $this->postponedLinters = $data['postponedLinters']; $unit_result = $data['unitResult']; $this->testResults = $data['testResults']; if ($this->getArgument('nolint')) { $this->excuses['lint'] = $this->getSkipExcuse( 'Provide explanation for skipping lint or press Enter to abort:', 'lint-excuses'); } if ($this->getArgument('nounit')) { $this->excuses['unit'] = $this->getSkipExcuse( 'Provide explanation for skipping unit tests or press Enter to abort:', 'unit-excuses'); } $changes = $this->generateChanges(); if (!$changes) { throw new ArcanistUsageException( "There are no changes to generate a diff from!"); } $diff_spec = array( 'changes' => mpull($changes, 'toDictionary'), 'lintStatus' => $this->getLintStatus($lint_result), 'unitStatus' => $this->getUnitStatus($unit_result), ) + $this->buildDiffSpecification(); $conduit = $this->getConduit(); $diff_info = $conduit->callMethodSynchronous( 'differential.creatediff', $diff_spec); $this->diffID = $diff_info['diffid']; $event = $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_WASCREATED, array( 'diffID' => $diff_info['diffid'], 'lintResult' => $lint_result, 'unitResult' => $unit_result, )); $this->updateLintDiffProperty(); $this->updateUnitDiffProperty(); $this->updateLocalDiffProperty(); $this->resolveDiffPropertyUpdates(); $output_json = $this->getArgument('json'); if ($this->shouldOnlyCreateDiff()) { if (!$output_json) { echo phutil_console_format( "Created a new Differential diff:\n". " **Diff URI:** __%s__\n\n", $diff_info['uri']); } else { $human = ob_get_clean(); echo json_encode(array( 'diffURI' => $diff_info['uri'], 'diffID' => $this->getDiffID(), 'human' => $human, ))."\n"; ob_start(); } } else { $revision['diffid'] = $this->getDiffID(); if ($commit_message->getRevisionID()) { $result = $conduit->callMethodSynchronous( 'differential.updaterevision', $revision); foreach (array('edit-messages.json', 'update-messages.json') as $file) { $messages = $this->readScratchJSONFile($file); unset($messages[$revision['id']]); $this->writeScratchJSONFile($file, $messages); } echo "Updated an existing Differential revision:\n"; } else { $revision = $this->dispatchWillCreateRevisionEvent($revision); $result = $conduit->callMethodSynchronous( 'differential.createrevision', $revision); $revised_message = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $result['revisionid'], )); if ($this->shouldAmend()) { $repository_api = $this->getRepositoryAPI(); if ($repository_api->supportsAmend()) { echo "Updating commit message...\n"; $repository_api->amendCommit($revised_message); } else { echo "Commit message was not amended. Amending commit message is ". "only supported in git and hg (version 2.2 or newer)"; } } echo "Created a new Differential revision:\n"; } $uri = $result['uri']; echo phutil_console_format( " **Revision URI:** __%s__\n\n", $uri); if ($this->getArgument('plan-changes')) { $conduit->callMethodSynchronous( 'differential.createcomment', array( 'revision_id' => $result['revisionid'], 'action' => 'rethink', )); echo "Planned changes to the revision.\n"; } } echo "Included changes:\n"; foreach ($changes as $change) { echo ' '.$change->renderTextSummary()."\n"; } if ($output_json) { ob_get_clean(); } $this->removeScratchFile('create-message'); return 0; } private function runRepositoryAPISetup() { if (!$this->requiresRepositoryAPI()) { return; } $repository_api = $this->getRepositoryAPI(); if ($this->getArgument('less-context')) { $repository_api->setDiffLinesOfContext(3); } $repository_api->setBaseCommitArgumentRules( $this->getArgument('base', '')); if ($repository_api->supportsCommitRanges()) { $this->parseBaseCommitArgument($this->getArgument('paths')); } } private function runDiffSetupBasics() { $output_json = $this->getArgument('json'); if ($output_json) { // TODO: We should move this to a higher-level and put an indirection // layer between echoing stuff and stdout. ob_start(); } if ($this->requiresWorkingCopy()) { $repository_api = $this->getRepositoryAPI(); try { if ($this->getArgument('add-all')) { $this->setCommitMode(self::COMMIT_ENABLE); } else if ($this->getArgument('uncommitted')) { $this->setCommitMode(self::COMMIT_DISABLE); } else { $this->setCommitMode(self::COMMIT_ALLOW); } if ($repository_api instanceof ArcanistSubversionAPI) { $repository_api->limitStatusToPaths($this->getArgument('paths')); } $this->requireCleanWorkingCopy(); } catch (ArcanistUncommittedChangesException $ex) { if ($repository_api instanceof ArcanistMercurialAPI) { // Some Mercurial users prefer to use it like SVN, where they don't // commit changes before sending them for review. This would be a // pretty bad workflow in Git, but Mercurial users are significantly // more expert at change management. $use_dirty_changes = false; if ($this->getArgument('uncommitted')) { // OK. } else { $ok = phutil_console_confirm( "You have uncommitted changes in your working copy. You can ". "include them in the diff, or abort and deal with them. (Use ". "'--uncommitted' to include them and skip this prompt.) ". "Do you want to include uncommitted changes in the diff?"); if (!$ok) { throw $ex; } } $repository_api->setIncludeDirectoryStateInDiffs(true); $this->haveUncommittedChanges = true; } else { throw $ex; } } } $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_DIDCOLLECTCHANGES, array()); } private function buildRevisionFromCommitMessage( ArcanistDifferentialCommitMessage $message) { $conduit = $this->getConduit(); $revision_id = $message->getRevisionID(); $revision = array( 'fields' => $message->getFields(), ); if ($revision_id) { // With '--verbatim', pass the (possibly modified) local fields. This // allows the user to edit some fields (like "title" and "summary") // locally without '--edit' and have changes automatically synchronized. // Without '--verbatim', we do not update the revision to reflect local // commit message changes. if ($this->getArgument('verbatim')) { $use_fields = $message->getFields(); } else { $use_fields = array(); } $should_edit = $this->getArgument('edit'); $edit_messages = $this->readScratchJSONFile('edit-messages.json'); $remote_corpus = idx($edit_messages, $revision_id); if (!$should_edit || !$remote_corpus || $use_fields) { if ($this->commitMessageFromRevision) { $remote_corpus = $this->commitMessageFromRevision; } else { $remote_corpus = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $revision_id, 'edit' => 'edit', 'fields' => $use_fields, )); } } if ($should_edit) { $edited = $this->newInteractiveEditor($remote_corpus) ->setName('differential-edit-revision-info') ->editInteractively(); if ($edited != $remote_corpus) { $remote_corpus = $edited; $edit_messages[$revision_id] = $remote_corpus; $this->writeScratchJSONFile('edit-messages.json', $edit_messages); } } if ($this->commitMessageFromRevision == $remote_corpus) { $new_message = $message; } else { $new_message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $remote_corpus); $new_message->pullDataFromConduit($conduit); } $revision['fields'] = $new_message->getFields(); $revision['id'] = $revision_id; $this->revisionID = $revision_id; $revision['message'] = $this->getArgument('message'); if (!strlen($revision['message'])) { $update_messages = $this->readScratchJSONFile('update-messages.json'); $update_messages[$revision_id] = $this->getUpdateMessage( $revision['fields'], idx($update_messages, $revision_id)); $revision['message'] = ArcanistCommentRemover::removeComments( $update_messages[$revision_id]); if (!strlen(trim($revision['message']))) { throw new ArcanistUserAbortException(); } $this->writeScratchJSONFile('update-messages.json', $update_messages); } } return $revision; } protected function shouldOnlyCreateDiff() { if ($this->getArgument('create')) { return false; } if ($this->getArgument('update')) { return false; } if ($this->getArgument('use-commit-message')) { return false; } if ($this->isRawDiffSource()) { return true; } return $this->getArgument('preview') || $this->getArgument('only'); } private function generateAffectedPaths() { if ($this->isRawDiffSource()) { return array(); } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistSubversionAPI) { $file_list = new FileList($this->getArgument('paths', array())); $paths = $repository_api->getSVNStatus($externals = true); foreach ($paths as $path => $mask) { if (!$file_list->contains($repository_api->getPath($path), true)) { unset($paths[$path]); } } $warn_externals = array(); foreach ($paths as $path => $mask) { $any_mod = ($mask & ArcanistRepositoryAPI::FLAG_ADDED) || ($mask & ArcanistRepositoryAPI::FLAG_MODIFIED) || ($mask & ArcanistRepositoryAPI::FLAG_DELETED); if ($mask & ArcanistRepositoryAPI::FLAG_EXTERNALS) { unset($paths[$path]); if ($any_mod) { $warn_externals[] = $path; } } } if ($warn_externals && !$this->hasWarnedExternals) { echo phutil_console_format( "The working copy includes changes to 'svn:externals' paths. These ". "changes will not be included in the diff because SVN can not ". "commit 'svn:externals' changes alongside normal changes.". "\n\n". "Modified 'svn:externals' files:". "\n\n". phutil_console_wrap(implode("\n", $warn_externals), 8)); $prompt = "Generate a diff (with just local changes) anyway?"; if (!phutil_console_confirm($prompt)) { throw new ArcanistUserAbortException(); } else { $this->hasWarnedExternals = true; } } } else { $paths = $repository_api->getWorkingCopyStatus(); } foreach ($paths as $path => $mask) { if ($mask & ArcanistRepositoryAPI::FLAG_UNTRACKED) { unset($paths[$path]); } } return $paths; } protected function generateChanges() { $parser = $this->newDiffParser(); $is_raw = $this->isRawDiffSource(); if ($is_raw) { if ($this->getArgument('raw')) { fwrite(STDERR, "Reading diff from stdin...\n"); $raw_diff = file_get_contents('php://stdin'); } else if ($this->getArgument('raw-command')) { - list($raw_diff) = execx($this->getArgument('raw-command')); + list($raw_diff) = execx('%C', $this->getArgument('raw-command')); } else { throw new Exception("Unknown raw diff source."); } $changes = $parser->parseDiff($raw_diff); foreach ($changes as $key => $change) { // Remove "message" changes, e.g. from "git show". if ($change->getType() == ArcanistDiffChangeType::TYPE_MESSAGE) { unset($changes[$key]); } } return $changes; } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistSubversionAPI) { $paths = $this->generateAffectedPaths(); $this->primeSubversionWorkingCopyData($paths); // Check to make sure the user is diffing from a consistent base revision. // This is mostly just an abuse sanity check because it's silly to do this // and makes the code more difficult to effectively review, but it also // affects patches and makes them nonportable. $bases = $repository_api->getSVNBaseRevisions(); // Remove all files with baserev "0"; these files are new. foreach ($bases as $path => $baserev) { if ($bases[$path] <= 0) { unset($bases[$path]); } } if ($bases) { $rev = reset($bases); $revlist = array(); foreach ($bases as $path => $baserev) { $revlist[] = " Revision {$baserev}, {$path}"; } $revlist = implode("\n", $revlist); foreach ($bases as $path => $baserev) { if ($baserev !== $rev) { throw new ArcanistUsageException( "Base revisions of changed paths are mismatched. Update all ". "paths to the same base revision before creating a diff: ". "\n\n". $revlist); } } // If you have a change which affects several files, all of which are // at a consistent base revision, treat that revision as the effective // base revision. The use case here is that you made a change to some // file, which updates it to HEAD, but want to be able to change it // again without updating the entire working copy. This is a little // sketchy but it arises in Facebook Ops workflows with config files and // doesn't have any real material tradeoffs (e.g., these patches are // perfectly applyable). $repository_api->overrideSVNBaseRevisionNumber($rev); } $changes = $parser->parseSubversionDiff( $repository_api, $paths); } else if ($repository_api instanceof ArcanistGitAPI) { $diff = $repository_api->getFullGitDiff(); if (!strlen($diff)) { throw new ArcanistUsageException( "No changes found. (Did you specify the wrong commit range?)"); } $changes = $parser->parseDiff($diff); } else if ($repository_api instanceof ArcanistMercurialAPI) { $diff = $repository_api->getFullMercurialDiff(); if (!strlen($diff)) { throw new ArcanistUsageException( "No changes found. (Did you specify the wrong commit range?)"); } $changes = $parser->parseDiff($diff); } else { throw new Exception("Repository API is not supported."); } if (count($changes) > 250) { $count = number_format(count($changes)); $link = "http://www.phabricator.com/docs/phabricator/article/". "Differential_User_Guide_Large_Changes.html"; $message = "This diff has a very large number of changes ({$count}). ". "Differential works best for changes which will receive detailed ". "human review, and not as well for large automated changes or ". "bulk checkins. See {$link} for information about reviewing big ". "checkins. Continue anyway?"; if (!phutil_console_confirm($message)) { throw new ArcanistUsageException( "Aborted generation of gigantic diff."); } } $limit = 1024 * 1024 * 4; foreach ($changes as $change) { $size = 0; foreach ($change->getHunks() as $hunk) { $size += strlen($hunk->getCorpus()); } if ($size > $limit) { $file_name = $change->getCurrentPath(); $change_size = number_format($size); $byte_warning = "Diff for '{$file_name}' with context is {$change_size} bytes in ". "length. Generally, source changes should not be this large."; if (!$this->getArgument('less-context')) { $byte_warning .= " If this file is a huge text file, try using the ". "'--less-context' flag."; } if ($repository_api instanceof ArcanistSubversionAPI) { throw new ArcanistUsageException( "{$byte_warning} If the file is not a text file, mark it as ". "binary with:". "\n\n". " $ svn propset svn:mime-type application/octet-stream ". "\n"); } else { $confirm = "{$byte_warning} If the file is not a text file, you can ". "mark it 'binary'. Mark this file as 'binary' and continue?"; if (phutil_console_confirm($confirm)) { $change->convertToBinaryChange(); } else { throw new ArcanistUsageException( "Aborted generation of gigantic diff."); } } } } $try_encoding = nonempty($this->getArgument('encoding'), null); $utf8_problems = array(); foreach ($changes as $change) { foreach ($change->getHunks() as $hunk) { $corpus = $hunk->getCorpus(); if (!phutil_is_utf8($corpus)) { // If this corpus is heuristically binary, don't try to convert it. // mb_check_encoding() and mb_convert_encoding() are both very very // liberal about what they're willing to process. $is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus); if (!$is_binary) { if (!$try_encoding) { try { $try_encoding = $this->getRepositoryEncoding(); } catch (ConduitClientException $e) { if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') { echo phutil_console_wrap( "Lookup of encoding in arcanist project failed\n". $e->getMessage()); } else { throw $e; } } } if ($try_encoding) { $corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding); $name = $change->getCurrentPath(); if (phutil_is_utf8($corpus)) { $this->writeStatusMessage( "Converted a '{$name}' hunk from '{$try_encoding}' ". "to UTF-8.\n"); $hunk->setCorpus($corpus); continue; } } } $utf8_problems[] = $change; break; } } } // If there are non-binary files which aren't valid UTF-8, warn the user // and treat them as binary changes. See D327 for discussion of why Arcanist // has this behavior. if ($utf8_problems) { $utf8_warning = pht( "This diff includes file(s) which are not valid UTF-8 (they contain ". "invalid byte sequences). You can either stop this workflow and ". "fix these files, or continue. If you continue, these files will ". "be marked as binary.", count($utf8_problems))."\n\n". "You can learn more about how Phabricator handles character encodings ". "(and how to configure encoding settings and detect and correct ". "encoding problems) by reading 'User Guide: UTF-8 and Character ". "Encoding' in the Phabricator documentation.\n\n"; " ".pht('AFFECTED FILE(S)', count($utf8_problems))."\n"; $confirm = pht( 'Do you want to mark these files as binary and continue?', count($utf8_problems)); echo phutil_console_format("**Invalid Content Encoding (Non-UTF8)**\n"); echo phutil_console_wrap($utf8_warning); $file_list = mpull($utf8_problems, 'getCurrentPath'); $file_list = ' '.implode("\n ", $file_list); echo $file_list; if (!phutil_console_confirm($confirm, $default_no = false)) { throw new ArcanistUsageException("Aborted workflow to fix UTF-8."); } else { foreach ($utf8_problems as $change) { $change->convertToBinaryChange(); } } } foreach ($changes as $change) { if ($change->getFileType() != ArcanistDiffChangeType::FILE_BINARY) { continue; } $path = $change->getCurrentPath(); $name = basename($path); $old_file = $change->getOriginalFileData(); $old_dict = $this->uploadFile($old_file, $name, 'old binary'); if ($old_dict['guid']) { $change->setMetadata('old:binary-phid', $old_dict['guid']); } $change->setMetadata('old:file:size', $old_dict['size']); $change->setMetadata('old:file:mime-type', $old_dict['mime']); $new_file = $change->getCurrentFileData(); $new_dict = $this->uploadFile($new_file, $name, 'new binary'); if ($new_dict['guid']) { $change->setMetadata('new:binary-phid', $new_dict['guid']); } $change->setMetadata('new:file:size', $new_dict['size']); $change->setMetadata('new:file:mime-type', $new_dict['mime']); $mime_type = coalesce($new_dict['mime'], $old_dict['mime']); if (preg_match('@^image/@', $mime_type)) { $change->setFileType(ArcanistDiffChangeType::FILE_IMAGE); } } return $changes; } private function uploadFile($data, $name, $desc) { $result = array( 'guid' => null, 'mime' => null, 'size' => null ); if ($this->getArgument('skip-binaries')) { return $result; } $result['size'] = $size = strlen($data); if (!$size) { return $result; } $tmp = new TempFile(); Filesystem::writeFile($tmp, $data); $mime_type = Filesystem::getMimeType($tmp); $result['mime'] = $mime_type; echo "Uploading {$desc} '{$name}' ({$mime_type}, {$size} bytes)...\n"; try { $guid = $this->getConduit()->callMethodSynchronous( 'file.upload', array( 'data_base64' => base64_encode($data), 'name' => $name, )); $result['guid'] = $guid; } catch (Exception $e) { echo "Failed to upload {$desc} '{$name}'.\n"; if (!phutil_console_confirm('Continue?', $default_no = false)) { throw new ArcanistUsageException( 'Aborted due to file upload failure. You can use --skip-binaries '. 'to skip binary uploads.'); } } return $result; } private function getGitParentLogInfo() { $info = array( 'parent' => null, 'base_revision' => null, 'base_path' => null, 'uuid' => null, ); $conduit = $this->getConduit(); $repository_api = $this->getRepositoryAPI(); $parser = $this->newDiffParser(); $history_messages = $repository_api->getGitHistoryLog(); if (!$history_messages) { // This can occur on the initial commit. return $info; } $history_messages = $parser->parseDiff($history_messages); foreach ($history_messages as $key => $change) { try { $message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $change->getMetadata('message')); if ($message->getRevisionID() && $info['parent'] === null) { $info['parent'] = $message->getRevisionID(); } if ($message->getGitSVNBaseRevision() && $info['base_revision'] === null) { $info['base_revision'] = $message->getGitSVNBaseRevision(); $info['base_path'] = $message->getGitSVNBasePath(); } if ($message->getGitSVNUUID()) { $info['uuid'] = $message->getGitSVNUUID(); } if ($info['parent'] && $info['base_revision']) { break; } } catch (ArcanistDifferentialCommitMessageParserException $ex) { // Ignore. } catch (ArcanistUsageException $ex) { // Ignore an invalid Differential Revision field in the parent commit } } return $info; } protected function primeSubversionWorkingCopyData($paths) { $repository_api = $this->getRepositoryAPI(); $futures = array(); $targets = array(); foreach ($paths as $path => $mask) { $futures[] = $repository_api->buildDiffFuture($path); $targets[] = array('command' => 'diff', 'path' => $path); $futures[] = $repository_api->buildInfoFuture($path); $targets[] = array('command' => 'info', 'path' => $path); } foreach (Futures($futures)->limit(8) as $key => $future) { $target = $targets[$key]; if ($target['command'] == 'diff') { $repository_api->primeSVNDiffResult( $target['path'], $future->resolve()); } else { $repository_api->primeSVNInfoResult( $target['path'], $future->resolve()); } } } private function shouldAmend() { if ($this->haveUncommittedChanges) { return false; } if ($this->isHistoryImmutable()) { return false; } if ($this->getArgument('no-amend')) { return false; } if ($this->isRawDiffSource()) { return false; } return true; } /* -( Lint and Unit Tests )------------------------------------------------ */ /** * @task lintunit */ private function runLintUnit() { $lint_result = $this->runLint(); $unit_result = $this->runUnit(); return array( 'lintResult' => $lint_result, 'unresolvedLint' => $this->unresolvedLint, 'postponedLinters' => $this->postponedLinters, 'unitResult' => $unit_result, 'testResults' => $this->testResults, ); } /** * @task lintunit */ private function runLint() { if ($this->getArgument('nolint') || $this->getArgument('only') || $this->isRawDiffSource()) { return ArcanistLintWorkflow::RESULT_SKIP; } $repository_api = $this->getRepositoryAPI(); $this->console->writeOut("Linting...\n"); try { $argv = $this->getPassthruArgumentsAsArgv('lint'); if ($repository_api->supportsCommitRanges()) { $argv[] = '--rev'; $argv[] = $repository_api->getBaseCommit(); } $lint_workflow = $this->buildChildWorkflow('lint', $argv); if ($this->shouldAmend()) { // TODO: We should offer to create a checkpoint commit. $lint_workflow->setShouldAmendChanges(true); } $lint_result = $lint_workflow->run(); switch ($lint_result) { case ArcanistLintWorkflow::RESULT_OKAY: if ($this->getArgument('advice') && $lint_workflow->getUnresolvedMessages()) { $this->getErrorExcuse( 'lint', "Lint issued unresolved advice.", 'lint-excuses'); } else { $this->console->writeOut( "** LINT OKAY ** No lint problems.\n"); } break; case ArcanistLintWorkflow::RESULT_WARNINGS: $this->getErrorExcuse( 'lint', "Lint issued unresolved warnings.", 'lint-excuses'); break; case ArcanistLintWorkflow::RESULT_ERRORS: $this->console->writeOut( "** LINT ERRORS ** Lint raised errors!\n"); $this->getErrorExcuse( 'lint', "Lint issued unresolved errors!", 'lint-excuses'); break; case ArcanistLintWorkflow::RESULT_POSTPONED: $this->console->writeOut( "** LINT POSTPONED ** ". "Lint results are postponed.\n"); break; } $this->unresolvedLint = array(); foreach ($lint_workflow->getUnresolvedMessages() as $message) { $this->unresolvedLint[] = $message->toDictionary(); } $this->postponedLinters = $lint_workflow->getPostponedLinters(); return $lint_result; } catch (ArcanistNoEngineException $ex) { $this->console->writeOut("No lint engine configured for this project.\n"); } catch (ArcanistNoEffectException $ex) { $this->console->writeOut($ex->getMessage()."\n"); } return null; } /** * @task lintunit */ private function runUnit() { if ($this->getArgument('nounit') || $this->getArgument('only') || $this->isRawDiffSource()) { return ArcanistUnitWorkflow::RESULT_SKIP; } $repository_api = $this->getRepositoryAPI(); $this->console->writeOut("Running unit tests...\n"); try { $argv = $this->getPassthruArgumentsAsArgv('unit'); if ($repository_api->supportsCommitRanges()) { $argv[] = '--rev'; $argv[] = $repository_api->getBaseCommit(); } $unit_workflow = $this->buildChildWorkflow('unit', $argv); $unit_result = $unit_workflow->run(); switch ($unit_result) { case ArcanistUnitWorkflow::RESULT_OKAY: $this->console->writeOut( "** UNIT OKAY ** No unit test failures.\n"); break; case ArcanistUnitWorkflow::RESULT_UNSOUND: if ($this->getArgument('ignore-unsound-tests')) { echo phutil_console_format( "** UNIT UNSOUND ** Unit testing raised errors, ". "but all failing tests are unsound.\n"); } else { $continue = $this->console->confirm( "Unit test results included failures, but all failing tests ". "are known to be unsound. Ignore unsound test failures?"); if (!$continue) { throw new ArcanistUserAbortException(); } } break; case ArcanistUnitWorkflow::RESULT_FAIL: $this->console->writeOut( "** UNIT ERRORS ** Unit testing raised errors!\n"); $this->getErrorExcuse( 'unit', "Unit test results include failures!", 'unit-excuses'); break; } $this->testResults = array(); foreach ($unit_workflow->getTestResults() as $test) { $this->testResults[] = array( 'name' => $test->getName(), 'link' => $test->getLink(), 'result' => $test->getResult(), 'userdata' => $test->getUserData(), 'coverage' => $test->getCoverage(), 'extra' => $test->getExtraData(), ); } return $unit_result; } catch (ArcanistNoEngineException $ex) { $this->console->writeOut( "No unit test engine is configured for this project.\n"); } catch (ArcanistNoEffectException $ex) { $this->console->writeOut($ex->getMessage()."\n"); } return null; } public function getTestResults() { return $this->testResults; } private function getSkipExcuse($prompt, $history) { $excuse = $this->getArgument('excuse'); if ($excuse === null) { $history = $this->getRepositoryAPI()->getScratchFilePath($history); $excuse = phutil_console_prompt($prompt, $history); if ($excuse == '') { throw new ArcanistUserAbortException(); } } return $excuse; } private function getErrorExcuse($type, $prompt, $history) { if ($this->getArgument('excuse')) { $this->console->sendMessage(array( 'type' => $type, 'confirm' => $prompt." Ignore them?", )); return; } $history = $this->getRepositoryAPI()->getScratchFilePath($history); $prompt .= " Provide explanation to continue or press Enter to abort."; $this->console->writeOut("\n\n%s", phutil_console_wrap($prompt)); $this->console->sendMessage(array( 'type' => $type, 'prompt' => "Explanation:", 'history' => $history, )); } public function handleServerMessage(PhutilConsoleMessage $message) { $data = $message->getData(); $response = ''; if (isset($data['prompt'])) { $response = phutil_console_prompt($data['prompt'], idx($data, 'history')); } else if (phutil_console_confirm($data['confirm'])) { $response = $this->getArgument('excuse'); } if ($response == '') { throw new ArcanistUserAbortException(); } $this->excuses[$data['type']] = $response; return null; } /* -( Commit and Update Messages )----------------------------------------- */ /** * @task message */ private function buildCommitMessage() { if ($this->getArgument('preview') || $this->getArgument('only')) { return null; } $is_create = $this->getArgument('create'); $is_update = $this->getArgument('update'); $is_raw = $this->isRawDiffSource(); $is_message = $this->getArgument('use-commit-message'); $is_verbatim = $this->getArgument('verbatim'); if ($is_message) { return $this->getCommitMessageFromCommit($is_message); } if ($is_verbatim) { return $this->getCommitMessageFromUser(); } if (!$is_raw && !$is_create && !$is_update) { $repository_api = $this->getRepositoryAPI(); $revisions = $repository_api->loadWorkingCopyDifferentialRevisions( $this->getConduit(), array( 'authors' => array($this->getUserPHID()), 'status' => 'status-open', )); if (!$revisions) { $is_create = true; } else if (count($revisions) == 1) { $revision = head($revisions); $is_update = $revision['id']; } else { throw new ArcanistUsageException( "There are several revisions which match the working copy:\n\n". $this->renderRevisionList($revisions)."\n". "Use '--update' to choose one, or '--create' to create a new ". "revision."); } } $message = null; if ($is_create) { $message_file = $this->getArgument('message-file'); if ($message_file) { return $this->getCommitMessageFromFile($message_file); } else { return $this->getCommitMessageFromUser(); } } else if ($is_update) { $revision_id = $this->normalizeRevisionID($is_update); if (!is_numeric($revision_id)) { throw new ArcanistUsageException( 'Parameter to --update must be a Differential Revision number'); } return $this->getCommitMessageFromRevision($revision_id); } else { // This is --raw without enough info to create a revision, so force just // a diff. return null; } } /** * @task message */ private function getCommitMessageFromCommit($commit) { $text = $this->getRepositoryAPI()->getCommitMessage($commit); $message = ArcanistDifferentialCommitMessage::newFromRawCorpus($text); $message->pullDataFromConduit($this->getConduit()); $this->validateCommitMessage($message); return $message; } /** * @task message */ private function getCommitMessageFromUser() { $conduit = $this->getConduit(); $template = null; if (!$this->getArgument('verbatim')) { $saved = $this->readScratchFile('create-message'); if ($saved) { $where = $this->getReadableScratchFilePath('create-message'); $preview = explode("\n", $saved); $preview = array_shift($preview); $preview = trim($preview); $preview = phutil_utf8_shorten($preview, 64); if ($preview) { $preview = "Message begins:\n\n {$preview}\n\n"; } else { $preview = null; } echo "You have a saved revision message in '{$where}'.\n". "{$preview}". "You can use this message, or discard it."; $use = phutil_console_confirm( "Do you want to use this message?", $default_no = false); if ($use) { $template = $saved; } else { $this->removeScratchFile('create-message'); } } } $template_is_default = false; $notes = array(); $included = array(); list($fields, $notes, $included_commits) = $this->getDefaultCreateFields(); if ($template) { $fields = array(); $notes = array(); } else { if (!$fields) { $template_is_default = true; } if ($notes) { $commit = head($this->getRepositoryAPI()->getLocalCommitInformation()); $template = $commit['message']; } else { $template = $conduit->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => null, 'edit' => 'create', 'fields' => $fields, )); } } $old_message = $template; $included = array(); if ($included_commits) { foreach ($included_commits as $commit) { $included[] = ' '.$commit; } $in_branch = ''; if (!$this->isRawDiffSource()) { $in_branch = ' in branch '.$this->getRepositoryAPI()->getBranchName(); } $included = array_merge( array( "", "Included commits{$in_branch}:", "", ), $included); } $issues = array_merge( array( 'NEW DIFFERENTIAL REVISION', 'Describe the changes in this new revision.', ), $included, array( '', 'arc could not identify any existing revision in your working copy.', 'If you intended to update an existing revision, use:', '', ' $ arc diff --update ', )); if ($notes) { $issues = array_merge($issues, array(''), $notes); } $done = false; $first = true; while (!$done) { $template = rtrim($template, "\r\n")."\n\n"; foreach ($issues as $issue) { $template .= '# '.$issue."\n"; } $template .= "\n"; if ($first && $this->getArgument('verbatim') && !$template_is_default) { $new_template = $template; } else { $new_template = $this->newInteractiveEditor($template) ->setName('new-commit') ->editInteractively(); } $first = false; if ($template_is_default && ($new_template == $template)) { throw new ArcanistUsageException("Template not edited."); } $template = ArcanistCommentRemover::removeComments($new_template); $repository_api = $this->getRepositoryAPI(); // special check for whether to amend here. optimizes a common git // workflow. we can't do this for mercurial because the mq extension // is popular and incompatible with hg commit --amend ; see T2011. $should_amend = (count($included_commits) == 1 && $repository_api instanceof ArcanistGitAPI && $this->shouldAmend()); if ($should_amend) { $wrote = (rtrim($old_message) != rtrim($template)); if ($wrote) { $repository_api->amendCommit($template); $where = 'commit message'; } } else { $wrote = $this->writeScratchFile('create-message', $template); $where = "'".$this->getReadableScratchFilePath('create-message')."'"; } try { $message = ArcanistDifferentialCommitMessage::newFromRawCorpus( $template); $message->pullDataFromConduit($conduit); $this->validateCommitMessage($message); $done = true; } catch (ArcanistDifferentialCommitMessageParserException $ex) { echo "Commit message has errors:\n\n"; $issues = array('Resolve these errors:'); foreach ($ex->getParserErrors() as $error) { echo phutil_console_wrap("- ".$error."\n", 6); $issues[] = ' - '.$error; } echo "\n"; echo "You must resolve these errors to continue."; $again = phutil_console_confirm( "Do you want to edit the message?", $default_no = false); if ($again) { // Keep going. } else { $saved = null; if ($wrote) { $saved = "A copy was saved to {$where}."; } throw new ArcanistUsageException( "Message has unresolved errrors. {$saved}"); } } catch (Exception $ex) { if ($wrote) { echo phutil_console_wrap("(Message saved to {$where}.)\n"); } throw $ex; } } return $message; } /** * @task message */ private function getCommitMessageFromFile($file) { $conduit = $this->getConduit(); $data = Filesystem::readFile($file); $message = ArcanistDifferentialCommitMessage::newFromRawCorpus($data); $message->pullDataFromConduit($conduit); $this->validateCommitMessage($message); return $message; } /** * @task message */ private function getCommitMessageFromRevision($revision_id) { $id = $revision_id; $revision = $this->getConduit()->callMethodSynchronous( 'differential.query', array( 'ids' => array($id), )); $revision = head($revision); if (!$revision) { throw new ArcanistUsageException( "Revision '{$revision_id}' does not exist!"); } $this->checkRevisionOwnership($revision); $message = $this->getConduit()->callMethodSynchronous( 'differential.getcommitmessage', array( 'revision_id' => $id, 'edit' => false, )); $this->commitMessageFromRevision = $message; $obj = ArcanistDifferentialCommitMessage::newFromRawCorpus($message); $obj->pullDataFromConduit($this->getConduit()); return $obj; } /** * @task message */ private function validateCommitMessage( ArcanistDifferentialCommitMessage $message) { $futures = array(); $revision_id = $message->getRevisionID(); if ($revision_id) { $futures['revision'] = $this->getConduit()->callMethod( 'differential.query', array( 'ids' => array($revision_id), )); } $reviewers = $message->getFieldValue('reviewerPHIDs'); if (!$reviewers) { $confirm = "You have not specified any reviewers. Continue anyway?"; if (!phutil_console_confirm($confirm)) { throw new ArcanistUsageException('Specify reviewers and retry.'); } } else { $futures['reviewers'] = $this->getConduit()->callMethod( 'user.query', array( 'phids' => $reviewers, )); } foreach (Futures($futures) as $key => $future) { $result = $future->resolve(); switch ($key) { case 'revision': if (empty($result)) { throw new ArcanistUsageException( "There is no revision D{$revision_id}."); } $this->checkRevisionOwnership(head($result)); break; case 'reviewers': $untils = array(); foreach ($result as $user) { if (idx($user, 'currentStatus') == 'away') { $untils[] = $user['currentStatusUntil']; } } if (count($untils) == count($reviewers)) { $until = date('l, M j Y', min($untils)); $confirm = "All reviewers are away until {$until}. ". "Continue anyway?"; if (!phutil_console_confirm($confirm)) { throw new ArcanistUsageException( 'Specify available reviewers and retry.'); } } break; } } } /** * @task message */ private function getUpdateMessage(array $fields, $template = '') { if ($this->getArgument('raw')) { throw new ArcanistUsageException( "When using '--raw' to update a revision, specify an update message ". "with '--message'. (Normally, we'd launch an editor to ask you for a ". "message, but can not do that because stdin is the diff source.)"); } // When updating a revision using git without specifying '--message', try // to prefill with the message in HEAD if it isn't a template message. The // idea is that if you do: // // $ git commit -a -m 'fix some junk' // $ arc diff // // ...you shouldn't have to retype the update message. Similar things apply // to Mercurial. if ($template == '') { $comments = $this->getDefaultUpdateMessage(); $template = rtrim($comments). "\n\n". "# Updating D{$fields['revisionID']}: {$fields['title']}\n". "#\n". "# Enter a brief description of the changes included in this update.\n". "# The first line is used as subject, next lines as comment.\n". "#\n". "# If you intended to create a new revision, use:\n". "# $ arc diff --create\n". "\n"; } $comments = $this->newInteractiveEditor($template) ->setName('differential-update-comments') ->editInteractively(); return $comments; } private function getDefaultCreateFields() { $result = array(array(), array(), array()); if ($this->isRawDiffSource()) { return $result; } $repository_api = $this->getRepositoryAPI(); $local = $repository_api->getLocalCommitInformation(); if ($local) { $result = $this->parseCommitMessagesIntoFields($local); } $result[0] = $this->dispatchWillBuildEvent($result[0]); return $result; } /** * Convert a list of commits from `getLocalCommitInformation()` into * a format usable by arc to create a new diff. Specifically, we emit: * * - A dictionary of commit message fields. * - A list of errors encountered while parsing the messages. * - A human-readable list of the commits themselves. * * For example, if the user runs "arc diff HEAD^^^" and selects a diff range * which includes several diffs, we attempt to merge them somewhat * intelligently into a single message, because we can only send one * "Summary:", "Reviewers:", etc., field to Differential. We also return * errors (e.g., if the user typed a reviewer name incorrectly) and a * summary of the commits themselves. * * @param dict Local commit information. * @return list Complex output, see summary. * @task message */ private function parseCommitMessagesIntoFields(array $local) { $conduit = $this->getConduit(); $local = ipull($local, null, 'commit'); // If the user provided "--reviewers" or "--ccs", add a faux message to // the list with the implied fields. $faux_message = array(); if ($this->getArgument('reviewers')) { $faux_message[] = 'Reviewers: '.$this->getArgument('reviewers'); } if ($this->getArgument('cc')) { $faux_message[] = 'CC: '.$this->getArgument('cc'); } if ($faux_message) { $faux_message = implode("\n\n", $faux_message); $local = array( '(Flags) ' => array( 'message' => $faux_message, 'summary' => 'Command-Line Flags', ), ) + $local; } // Build a human-readable list of the commits, so we can show the user which // commits are included in the diff. $included = array(); foreach ($local as $hash => $info) { $included[] = substr($hash, 0, 12).' '.$info['summary']; } // Parse all of the messages into fields. $messages = array(); foreach ($local as $hash => $info) { $text = $info['message']; if (trim($text) == self::AUTO_COMMIT_TITLE) { continue; } $obj = ArcanistDifferentialCommitMessage::newFromRawCorpus($text); $messages[$hash] = $obj; } $notes = array(); $fields = array(); foreach ($messages as $hash => $message) { try { $message->pullDataFromConduit($conduit, $partial = true); $fields[$hash] = $message->getFields(); } catch (ArcanistDifferentialCommitMessageParserException $ex) { if ($this->getArgument('verbatim')) { // In verbatim mode, just bail when we hit an error. The user can // rerun without --verbatim if they want to fix it manually. Most // users will probably `git commit --amend` instead. throw $ex; } $fields[$hash] = $message->getFields(); $frev = substr($hash, 0, 12); $notes[] = "NOTE: commit {$frev} could not be completely parsed:"; foreach ($ex->getParserErrors() as $error) { $notes[] = " - {$error}"; } } } // Merge commit message fields. We do this somewhat-intelligently so that // multiple "Reviewers" or "CC" fields will merge into the concatenation // of all values. // We have special parsing rules for 'title' because we can't merge // multiple titles, and one-line commit messages like "fix stuff" will // parse as titles. Instead, pick the first title we encounter. When we // encounter subsequent titles, treat them as part of the summary. Then // we merge all the summaries together below. $result = array(); // Process fields in oldest-first order, so earlier commits get to set the // title of record and reviewers/ccs are listed in chronological order. $fields = array_reverse($fields); foreach ($fields as $hash => $dict) { $title = idx($dict, 'title'); if (!strlen($title)) { continue; } if (!isset($result['title'])) { // We don't have a title yet, so use this one. $result['title'] = $title; } else { // We already have a title, so merge this new title into the summary. $summary = idx($dict, 'summary'); if ($summary) { $summary = $title."\n\n".$summary; } else { $summary = $title; } $fields[$hash]['summary'] = $summary; } } // Now, merge all the other fields in a general sort of way. foreach ($fields as $hash => $dict) { foreach ($dict as $key => $value) { if ($key == 'title') { // This has been handled above, and either assigned directly or // merged into the summary. continue; } if (is_array($value)) { // For array values, merge the arrays, appending the new values. // Examples are "Reviewers" and "Cc", where this produces a list of // all users specified as reviewers. $cur = idx($result, $key, array()); $new = array_merge($cur, $value); $result[$key] = $new; continue; } else { if (!strlen(trim($value))) { // Ignore empty fields. continue; } // For string values, append the new field to the old field with // a blank line separating them. Examples are "Test Plan" and // "Summary". $cur = idx($result, $key, ''); if (strlen($cur)) { $new = $cur."\n\n".$value; } else { $new = $value; } $result[$key] = $new; } } } return array($result, $notes, $included); } private function getDefaultUpdateMessage() { if ($this->isRawDiffSource()) { return null; } $repository_api = $this->getRepositoryAPI(); if ($repository_api instanceof ArcanistGitAPI) { return $this->getGitUpdateMessage(); } if ($repository_api instanceof ArcanistMercurialAPI) { return $this->getMercurialUpdateMessage(); } return null; } /** * Retrieve the git messages between HEAD and the last update. * * @task message */ private function getGitUpdateMessage() { $repository_api = $this->getRepositoryAPI(); $parser = $this->newDiffParser(); $commit_messages = $repository_api->getGitCommitLog(); $commit_messages = $parser->parseDiff($commit_messages); if (count($commit_messages) == 1) { // If there's only one message, assume this is an amend-based workflow and // that using it to prefill doesn't make sense. return null; } // We have more than one message, so figure out which ones are new. We // do this by pulling the current diff and comparing commit hashes in the // working copy with attached commit hashes. It's not super important that // we always get this 100% right, we're just trying to do something // reasonable. $local = $this->loadActiveLocalCommitInfo(); $hashes = ipull($local, null, 'commit'); $usable = array(); foreach ($commit_messages as $message) { $text = $message->getMetadata('message'); $parsed = ArcanistDifferentialCommitMessage::newFromRawCorpus($text); if ($parsed->getRevisionID()) { // If this is an amended commit message with a revision ID, it's // certainly not new. Stop marking commits as usable and break out. break; } if (isset($hashes[$message->getCommitHash()])) { // If this commit is currently part of the diff, stop using commit // messages, since anything older than this isn't new. break; } // Otherwise, this looks new, so it's a usable commit message. $usable[] = $text; } if (!$usable) { // No new commit messages, so we don't have anywhere to start from. return null; } return $this->formatUsableLogs($usable); } /** * Retrieve the hg messages between tip and the last update. * * @task message */ private function getMercurialUpdateMessage() { $repository_api = $this->getRepositoryAPI(); $messages = $repository_api->getCommitMessageLog(); $local = $this->loadActiveLocalCommitInfo(); $hashes = ipull($local, null, 'commit'); $usable = array(); foreach ($messages as $rev => $message) { if (isset($hashes[$rev])) { // If this commit is currently part of the active diff on the revision, // stop using commit messages, since anything older than this isn't new. break; } // Otherwise, this looks new, so it's a usable commit message. $usable[] = $message; } if (!$usable) { // No new commit messages, so we don't have anywhere to start from. return null; } return $this->formatUsableLogs($usable); } /** * Format log messages to prefill a diff update. * * @task message */ private function formatUsableLogs(array $usable) { // Flip messages so they'll read chronologically (oldest-first) in the // template, e.g.: // // - Added foobar. // - Fixed foobar bug. // - Documented foobar. $usable = array_reverse($usable); $default = array(); foreach ($usable as $message) { // Pick the first line out of each message. $text = trim($message); if ($text == self::AUTO_COMMIT_TITLE) { continue; } $text = head(explode("\n", $text)); $default[] = ' - '.$text."\n"; } return implode('', $default); } private function loadActiveLocalCommitInfo() { $current_diff = $this->getConduit()->callMethodSynchronous( 'differential.getdiff', array( 'revision_id' => $this->revisionID, )); $properties = idx($current_diff, 'properties', array()); return idx($properties, 'local:commits', array()); } /* -( Diff Specification )------------------------------------------------- */ /** * @task diffspec */ private function getLintStatus($lint_result) { $map = array( ArcanistLintWorkflow::RESULT_OKAY => 'okay', ArcanistLintWorkflow::RESULT_ERRORS => 'fail', ArcanistLintWorkflow::RESULT_WARNINGS => 'warn', ArcanistLintWorkflow::RESULT_SKIP => 'skip', ArcanistLintWorkflow::RESULT_POSTPONED => 'postponed', ); return idx($map, $lint_result, 'none'); } /** * @task diffspec */ private function getUnitStatus($unit_result) { $map = array( ArcanistUnitWorkflow::RESULT_OKAY => 'okay', ArcanistUnitWorkflow::RESULT_FAIL => 'fail', ArcanistUnitWorkflow::RESULT_UNSOUND => 'warn', ArcanistUnitWorkflow::RESULT_SKIP => 'skip', ArcanistUnitWorkflow::RESULT_POSTPONED => 'postponed', ); return idx($map, $unit_result, 'none'); } /** * @task diffspec */ private function buildDiffSpecification() { $base_revision = null; $base_path = null; $vcs = null; $repo_uuid = null; $parent = null; $source_path = null; $branch = null; $bookmark = null; if (!$this->isRawDiffSource()) { $repository_api = $this->getRepositoryAPI(); $base_revision = $repository_api->getSourceControlBaseRevision(); $base_path = $repository_api->getSourceControlPath(); $vcs = $repository_api->getSourceControlSystemName(); $source_path = $repository_api->getPath(); $branch = $repository_api->getBranchName(); if ($repository_api instanceof ArcanistGitAPI) { $info = $this->getGitParentLogInfo(); if ($info['parent']) { $parent = $info['parent']; } if ($info['base_revision']) { $base_revision = $info['base_revision']; } if ($info['base_path']) { $base_path = $info['base_path']; } if ($info['uuid']) { $repo_uuid = $info['uuid']; } } else if ($repository_api instanceof ArcanistSubversionAPI) { $repo_uuid = $repository_api->getRepositorySVNUUID(); } else if ($repository_api instanceof ArcanistMercurialAPI) { $bookmark = $repository_api->getActiveBookmark(); $svn_info = $repository_api->getSubversionInfo(); $repo_uuid = idx($svn_info, 'uuid'); $base_path = idx($svn_info, 'base_path', $base_path); $base_revision = idx($svn_info, 'base_revision', $base_revision); // TODO: provide parent info } else { throw new Exception("Unsupported repository API!"); } } $project_id = null; if ($this->requiresWorkingCopy()) { $project_id = $this->getWorkingCopy()->getProjectID(); } return array( 'sourceMachine' => php_uname('n'), 'sourcePath' => $source_path, 'branch' => $branch, 'bookmark' => $bookmark, 'sourceControlSystem' => $vcs, 'sourceControlPath' => $base_path, 'sourceControlBaseRevision' => $base_revision, 'parentRevisionID' => $parent, 'repositoryUUID' => $repo_uuid, 'creationMethod' => 'arc', 'arcanistProject' => $project_id, 'authorPHID' => $this->getUserPHID(), ); } /* -( Diff Properties )---------------------------------------------------- */ /** * Update lint information for the diff. * * @return void * * @task diffprop */ private function updateLintDiffProperty() { if (strlen($this->excuses['lint'])) { $this->updateDiffProperty('arc:lint-excuse', json_encode($this->excuses['lint'])); } if ($this->unresolvedLint) { $this->updateDiffProperty('arc:lint', json_encode($this->unresolvedLint)); } $postponed = $this->postponedLinters; if ($postponed) { $this->updateDiffProperty('arc:lint-postponed', json_encode($postponed)); } } /** * Update unit test information for the diff. * * @return void * * @task diffprop */ private function updateUnitDiffProperty() { if (strlen($this->excuses['unit'])) { $this->updateDiffProperty('arc:unit-excuse', json_encode($this->excuses['unit'])); } if ($this->testResults) { $this->updateDiffProperty('arc:unit', json_encode($this->testResults)); } } /** * Update local commit information for the diff. * * @task diffprop */ private function updateLocalDiffProperty() { if ($this->isRawDiffSource()) { return; } $local_info = $this->getRepositoryAPI()->getLocalCommitInformation(); if (!$local_info) { return; } $this->updateDiffProperty('local:commits', json_encode($local_info)); } /** * Update an arbitrary diff property. * * @param string Diff property name. * @param string Diff property value. * @return void * * @task diffprop */ private function updateDiffProperty($name, $data) { $this->diffPropertyFutures[] = $this->getConduit()->callMethod( 'differential.setdiffproperty', array( 'diff_id' => $this->getDiffID(), 'name' => $name, 'data' => $data, )); } /** * Wait for finishing all diff property updates. * * @return void * * @task diffprop */ private function resolveDiffPropertyUpdates() { Futures($this->diffPropertyFutures)->resolveAll(); $this->diffPropertyFutures = array(); } private function dispatchWillCreateRevisionEvent(array $fields) { $event = $this->dispatchEvent( ArcanistEventType::TYPE_REVISION_WILLCREATEREVISION, array( 'specification' => $fields, )); return $event->getValue('specification'); } private function dispatchWillBuildEvent(array $fields) { $event = $this->dispatchEvent( ArcanistEventType::TYPE_DIFF_WILLBUILDMESSAGE, array( 'fields' => $fields, )); return $event->getValue('fields'); } private function checkRevisionOwnership(array $revision) { if ($revision['authorPHID'] == $this->getUserPHID()) { return; } $id = $revision['id']; $title = $revision['title']; throw new ArcanistUsageException( "You don't own revision D{$id} '{$title}'. You can only update ". "revisions you own. You can 'Commandeer' this revision from the web ". "interface if you want to become the owner."); } }