diff --git a/src/lint/ArcanistLintResult.php b/src/lint/ArcanistLintResult.php index ab3a5d1e..dcb0968a 100644 --- a/src/lint/ArcanistLintResult.php +++ b/src/lint/ArcanistLintResult.php @@ -1,97 +1,107 @@ path = $path; return $this; } public function getPath() { return $this->path; } public function addMessage(ArcanistLintMessage $message) { $this->messages[] = $message; $this->needsSort = true; return $this; } public function getMessages() { if ($this->needsSort) { $this->sortAndFilterMessages(); } return $this->effectiveMessages; } public function setData($data) { $this->data = $data; return $this; } public function getData() { return $this->data; } public function setFilePathOnDisk($file_path_on_disk) { $this->filePathOnDisk = $file_path_on_disk; return $this; } public function getFilePathOnDisk() { return $this->filePathOnDisk; } + public function setCacheVersion($version) { + $this->cacheVersion = $version; + return $this; + } + + public function getCacheVersion() { + return $this->cacheVersion; + } + public function isPatchable() { foreach ($this->messages as $message) { if ($message->isPatchable()) { return true; } } return false; } public function isAllAutofix() { foreach ($this->messages as $message) { if (!$message->isAutofix()) { return false; } } return true; } private function sortAndFilterMessages() { $messages = $this->messages; foreach ($messages as $key => $message) { if ($message->getObsolete()) { unset($messages[$key]); continue; } } $map = array(); foreach ($messages as $key => $message) { $map[$key] = ($message->getLine() * (2 << 12)) + $message->getChar(); } asort($map); $messages = array_select_keys($messages, array_keys($map)); $this->effectiveMessages = $messages; $this->needsSort = false; } } diff --git a/src/lint/engine/ArcanistLintEngine.php b/src/lint/engine/ArcanistLintEngine.php index 255b0493..cbf7670a 100644 --- a/src/lint/engine/ArcanistLintEngine.php +++ b/src/lint/engine/ArcanistLintEngine.php @@ -1,352 +1,383 @@ workingCopy = $working_copy; return $this; } public function getWorkingCopy() { return $this->workingCopy; } public function setPaths($paths) { $this->paths = $paths; return $this; } public function getPaths() { return $this->paths; } public function setPathChangedLines($path, $changed) { if ($changed === null) { $this->changedLines[$path] = null; } else { $this->changedLines[$path] = array_fill_keys($changed, true); } return $this; } public function getPathChangedLines($path) { return idx($this->changedLines, $path); } public function setFileData($data) { $this->fileData = $data + $this->fileData; return $this; } public function setCommitHookMode($mode) { $this->commitHookMode = $mode; return $this; } public function setHookAPI(ArcanistHookAPI $hook_api) { $this->hookAPI = $hook_api; return $this; } public function getHookAPI() { return $this->hookAPI; } public function setEnableAsyncLint($enable_async_lint) { $this->enableAsyncLint = $enable_async_lint; return $this; } public function getEnableAsyncLint() { return $this->enableAsyncLint; } public function loadData($path) { if (!isset($this->fileData[$path])) { if ($this->getCommitHookMode()) { $this->fileData[$path] = $this->getHookAPI() ->getCurrentFileData($path); } else { $disk_path = $this->getFilePathOnDisk($path); $this->fileData[$path] = Filesystem::readFile($disk_path); } } return $this->fileData[$path]; } public function pathExists($path) { if ($this->getCommitHookMode()) { $file_data = $this->loadData($path); return ($file_data !== null); } else { $disk_path = $this->getFilePathOnDisk($path); return Filesystem::pathExists($disk_path); } } public function getFilePathOnDisk($path) { return Filesystem::resolvePath( $path, $this->getWorkingCopy()->getProjectRoot()); } public function setMinimumSeverity($severity) { $this->minimumSeverity = $severity; return $this; } public function getCommitHookMode() { return $this->commitHookMode; } public function run() { $stopped = array(); $linters = $this->buildLinters(); + if (!$linters) { + throw new ArcanistNoEffectException("No linters to run."); + } + + $have_paths = false; + foreach ($linters as $linter) { + if ($linter->getPaths()) { + $have_paths = true; + break; + } + } + + if (!$have_paths) { + throw new ArcanistNoEffectException("No paths are lintable."); + } + + $versions = array($this->getCacheVersion()); + foreach ($linters as $linter) { + $versions[] = get_class($linter).':'.$linter->getCacheVersion(); + } + $this->cacheVersion = crc32(implode("\n", $versions)); + $exceptions = array(); foreach ($linters as $linter_name => $linter) { try { $linter->setEngine($this); if (!$linter->canRun()) { continue; } $paths = $linter->getPaths(); foreach ($paths as $key => $path) { // Make sure each path has a result generated, even if it is empty // (i.e., the file has no lint messages). $result = $this->getResultForPath($path); if (isset($stopped[$path])) { unset($paths[$key]); } + // TODO: Some linters work with the whole directory. + if (isset($this->cachedResults[$path][$this->cacheVersion])) { + unset($paths[$key]); + } } $paths = array_values($paths); if ($paths) { $linter->willLintPaths($paths); foreach ($paths as $path) { $linter->willLintPath($path); $linter->lintPath($path); if ($linter->didStopAllLinters()) { $stopped[$path] = true; } } } $minimum = $this->minimumSeverity; foreach ($linter->getLintMessages() as $message) { if (!ArcanistLintSeverity::isAtLeastAsSevere($message, $minimum)) { continue; } if (!$this->isRelevantMessage($message)) { continue; } $result = $this->getResultForPath($message->getPath()); $result->addMessage($message); } } catch (Exception $ex) { if (!is_string($linter_name)) { $linter_name = get_class($linter); } $exceptions[$linter_name] = $ex; } } + if ($this->cachedResults) { + foreach ($this->cachedResults as $path => $messages) { + foreach (idx($messages, $this->cacheVersion, array()) as $message) { + $this->getResultForPath($path)->addMessage( + ArcanistLintMessage::newFromDictionary($message)); + } + } + } + foreach ($this->results as $path => $result) { $disk_path = $this->getFilePathOnDisk($path); $result->setFilePathOnDisk($disk_path); if (isset($this->fileData[$path])) { $result->setData($this->fileData[$path]); } else if ($disk_path && Filesystem::pathExists($disk_path)) { // TODO: this may cause us to, e.g., load a large binary when we only // raised an error about its filename. We could refine this by looking // through the lint messages and doing this load only if any of them // have original/replacement text or something like that. try { $this->fileData[$path] = Filesystem::readFile($disk_path); $result->setData($this->fileData[$path]); } catch (FilesystemException $ex) { // Ignore this, it's noncritical that we access this data and it // might be unreadable or a directory or whatever else for plenty // of legitimate reasons. } } } - if (!$linters) { - throw new ArcanistNoEffectException("No linters to run."); - } - - $have_paths = false; - foreach ($linters as $linter) { - if ($linter->getPaths()) { - $have_paths = true; - break; - } - } - - if (!$have_paths) { - throw new ArcanistNoEffectException("No paths are lintable."); - } - if ($exceptions) { throw new PhutilAggregateException('Some linters failed:', $exceptions); } return $this->results; } + /** + * @param dict>> + * @return this + */ + public function setCachedResults(array $results) { + $this->cachedResults = $results; + return $this; + } + public function getResults() { return $this->results; } abstract protected function buildLinters(); private function isRelevantMessage($message) { // When a user runs "arc lint", we default to raising only warnings on // lines they have changed (errors are still raised anywhere in the // file). The list of $changed lines may be null, to indicate that the // path is a directory or a binary file so we should not exclude // warnings. $changed = $this->getPathChangedLines($message->getPath()); if ($changed === null || $message->isError() || !$message->getLine()) { return true; } $last_line = $message->getLine(); if ($message->getOriginalText()) { $last_line += substr_count($message->getOriginalText(), "\n"); } for ($l = $message->getLine(); $l <= $last_line; $l++) { if (!empty($changed[$l])) { return true; } } return false; } - public function getResultForPath($path) { + protected function getResultForPath($path) { if (empty($this->results[$path])) { $result = new ArcanistLintResult(); $result->setPath($path); + $result->setCacheVersion($this->cacheVersion); $this->results[$path] = $result; } return $this->results[$path]; } public function getLineAndCharFromOffset($path, $offset) { if (!isset($this->charToLine[$path])) { $char_to_line = array(); $line_to_first_char = array(); $lines = explode("\n", $this->loadData($path)); $line_number = 0; $line_start = 0; foreach ($lines as $line) { $len = strlen($line) + 1; // Account for "\n". $line_to_first_char[] = $line_start; $line_start += $len; for ($ii = 0; $ii < $len; $ii++) { $char_to_line[] = $line_number; } $line_number++; } $this->charToLine[$path] = $char_to_line; $this->lineToFirstChar[$path] = $line_to_first_char; } $line = $this->charToLine[$path][$offset]; $char = $offset - $this->lineToFirstChar[$path][$line]; return array($line, $char); } public function getPostponedLinters() { return $this->postponedLinters; } public function setPostponedLinters(array $linters) { $this->postponedLinters = $linters; return $this; } - public function getCacheVersion() { + protected function getCacheVersion() { return 0; } protected function getPEP8WithTextOptions() { // E101 is subset of TXT2 (Tab Literal). // E501 is same as TXT3 (Line Too Long). // W291 is same as TXT6 (Trailing Whitespace). // W292 is same as TXT4 (File Does Not End in Newline). // W293 is same as TXT6 (Trailing Whitespace). return '--ignore=E101,E501,W291,W292,W293'; } } diff --git a/src/lint/linter/ArcanistLinter.php b/src/lint/linter/ArcanistLinter.php index 2b5dd53b..0c20fab9 100644 --- a/src/lint/linter/ArcanistLinter.php +++ b/src/lint/linter/ArcanistLinter.php @@ -1,210 +1,214 @@ customSeverityMap = $map; return $this; } public function setConfig(array $config) { $this->config = $config; return $this; } protected function getConfig($key, $default = null) { return idx($this->config, $key, $default); } public function getActivePath() { return $this->activePath; } public function stopAllLinters() { $this->stopAllLinters = true; return $this; } public function didStopAllLinters() { return $this->stopAllLinters; } public function addPath($path) { $this->paths[$path] = $path; return $this; } public function setPaths(array $paths) { $this->paths = $paths; return $this; } public function getPaths() { return array_values($this->paths); } public function addData($path, $data) { $this->data[$path] = $data; return $this; } protected function getData($path) { if (!array_key_exists($path, $this->data)) { $this->data[$path] = $this->getEngine()->loadData($path); } return $this->data[$path]; } public function setEngine(ArcanistLintEngine $engine) { $this->engine = $engine; return $this; } protected function getEngine() { return $this->engine; } + public function getCacheVersion() { + return 0; + } + public function getLintMessageFullCode($short_code) { return $this->getLinterName().$short_code; } public function getLintMessageSeverity($code) { $map = $this->customSeverityMap; if (isset($map[$code])) { return $map[$code]; } $map = $this->getLintSeverityMap(); if (isset($map[$code])) { return $map[$code]; } return ArcanistLintSeverity::SEVERITY_ERROR; } public function isMessageEnabled($code) { return ($this->getLintMessageSeverity($code) !== ArcanistLintSeverity::SEVERITY_DISABLED); } public function getLintMessageName($code) { $map = $this->getLintNameMap(); if (isset($map[$code])) { return $map[$code]; } return "Unknown lint message!"; } protected function addLintMessage(ArcanistLintMessage $message) { if (!$this->getEngine()->getCommitHookMode()) { $root = $this->getEngine()->getWorkingCopy()->getProjectRoot(); $path = Filesystem::resolvePath($message->getPath(), $root); $message->setPath(Filesystem::readablePath($path, $root)); } $this->messages[] = $message; return $message; } public function getLintMessages() { return $this->messages; } protected function raiseLintAtLine( $line, $char, $code, $desc, $original = null, $replacement = null) { $dict = array( 'path' => $this->getActivePath(), 'line' => $line, 'char' => $char, 'code' => $this->getLintMessageFullCode($code), 'severity' => $this->getLintMessageSeverity($code), 'name' => $this->getLintMessageName($code), 'description' => $desc, ); if ($original !== null) { $dict['original'] = $original; } if ($replacement !== null) { $dict['replacement'] = $replacement; } return $this->addLintMessage(ArcanistLintMessage::newFromDictionary($dict)); } protected function raiseLintAtPath( $code, $desc) { return $this->raiseLintAtLine(null, null, $code, $desc, null, null); } protected function raiseLintAtOffset( $offset, $code, $desc, $original = null, $replacement = null) { $path = $this->getActivePath(); $engine = $this->getEngine(); if ($offset === null) { $line = null; $char = null; } else { list($line, $char) = $engine->getLineAndCharFromOffset($path, $offset); } return $this->raiseLintAtLine( $line + 1, $char + 1, $code, $desc, $original, $replacement); } public function willLintPath($path) { $this->stopAllLinters = false; $this->activePath = $path; } public function canRun() { return true; } abstract public function willLintPaths(array $paths); abstract public function lintPath($path); abstract public function getLinterName(); public function getLintSeverityMap() { return array(); } public function getLintNameMap() { return array(); } } diff --git a/src/workflow/ArcanistLintWorkflow.php b/src/workflow/ArcanistLintWorkflow.php index 1bb1934b..10e8a37b 100644 --- a/src/workflow/ArcanistLintWorkflow.php +++ b/src/workflow/ArcanistLintWorkflow.php @@ -1,451 +1,446 @@ shouldAmendChanges = $should_amend; return $this; } public function setShouldAmendWithoutPrompt($should_amend) { $this->shouldAmendWithoutPrompt = $should_amend; return $this; } public function setShouldAmendAutofixesWithoutPrompt($should_amend) { $this->shouldAmendAutofixesWithoutPrompt = $should_amend; return $this; } public function getCommandSynopses() { return phutil_console_format(<< array( 'help' => "Show all lint warnings, not just those on changed lines." ), 'rev' => array( 'param' => 'revision', 'help' => "Lint changes since a specific revision.", 'supports' => array( 'git', 'hg', ), 'nosupport' => array( 'svn' => "Lint does not currently support --rev in SVN.", ), ), 'output' => array( 'param' => 'format', 'help' => "With 'summary', show lint warnings in a more compact format. ". "With 'json', show lint warnings in machine-readable JSON format. ". "With 'compiler', show lint warnings in suitable for your editor." ), 'engine' => array( 'param' => 'classname', 'help' => "Override configured lint engine for this project." ), 'apply-patches' => array( 'help' => 'Apply patches suggested by lint to the working copy without '. 'prompting.', 'conflicts' => array( 'never-apply-patches' => true, ), ), 'never-apply-patches' => array( 'help' => 'Never apply patches suggested by lint.', 'conflicts' => array( 'apply-patches' => true, ), ), 'amend-all' => array( 'help' => 'When linting git repositories, amend HEAD with all patches '. 'suggested by lint without prompting.', ), 'amend-autofixes' => array( 'help' => 'When linting git repositories, amend HEAD with autofix '. 'patches suggested by lint without prompting.', ), 'severity' => array( 'param' => 'string', 'help' => "Set minimum message severity. One of: '". implode( "', '", array_keys(ArcanistLintSeverity::getLintSeverities())). "'. Defaults to '".self::DEFAULT_SEVERITY."'.", ), 'cache' => array( 'param' => 'bool', 'help' => "0 to disable cache (default), 1 to enable.", ), '*' => 'paths', ); } public function requiresWorkingCopy() { return true; } public function requiresRepositoryAPI() { return true; } private function getCacheKey() { return implode("\n", array( get_class($this->engine), $this->getArgument('severity', self::DEFAULT_SEVERITY), - $this->getArgument('lintall'), - $this->engine->getCacheVersion(), + $this->shouldLintAll, )); } public function run() { $working_copy = $this->getWorkingCopy(); $engine = $this->getArgument('engine'); if (!$engine) { $engine = $working_copy->getConfigFromAnySource('lint.engine'); if (!$engine) { throw new ArcanistNoEngineException( "No lint engine configured for this project. Edit .arcconfig to ". "specify a lint engine."); } } $rev = $this->getArgument('rev'); $paths = $this->getArgument('paths'); if ($rev && $paths) { throw new ArcanistUsageException("Specify either --rev or paths."); } - $should_lint_all = $this->getArgument('lintall'); + $this->shouldLintAll = $this->getArgument('lintall'); if ($paths) { // NOTE: When the user specifies paths, we imply --lintall and show all // warnings for the paths in question. This is easier to deal with for // us and less confusing for users. - $should_lint_all = true; + $this->shouldLintAll = true; } $paths = $this->selectPathsForWorkflow($paths, $rev); if (!class_exists($engine) || !is_subclass_of($engine, 'ArcanistLintEngine')) { throw new ArcanistUsageException( "Configured lint engine '{$engine}' is not a subclass of ". "'ArcanistLintEngine'."); } $engine = newv($engine, array()); $this->engine = $engine; $engine->setWorkingCopy($working_copy); $engine->setMinimumSeverity( $this->getArgument('severity', self::DEFAULT_SEVERITY)); - $cached = false; if ($this->getArgument('cache')) { - $paths = array_combine($paths, $paths); $cache = $this->readScratchJSONFile('lint-cache.json'); $cache = idx($cache, $this->getCacheKey(), array()); + $cached = array(); foreach ($cache as $path => $messages) { $messages = idx($messages, md5_file($engine->getFilePathOnDisk($path))); - // TODO: Some linters work with the whole directory. if ($messages !== null) { - foreach ($messages as $message) { - $engine->getResultForPath($path)->addMessage( - ArcanistLintMessage::newFromDictionary($message)); - } - $cached = true; - unset($paths[$path]); + $cached[$path] = $messages; } } + $engine->setCachedResults($cached); } // Propagate information about which lines changed to the lint engine. // This is used so that the lint engine can drop warning messages // concerning lines that weren't in the change. $engine->setPaths($paths); - if (!$should_lint_all) { + if (!$this->shouldLintAll) { foreach ($paths as $path) { // Note that getChangedLines() returns null to indicate that a file // is binary or a directory (i.e., changed lines are not relevant). $engine->setPathChangedLines( $path, $this->getChangedLines($path, 'new')); } } // Enable possible async linting only for 'arc diff' not 'arc lint' if ($this->getParentWorkflow()) { $engine->setEnableAsyncLint(true); } else { $engine->setEnableAsyncLint(false); } $failed = null; try { $engine->run(); } catch (Exception $ex) { - if ($ex instanceof ArcanistNoEffectException && $cached) { - // Swallow. - } else { - $failed = $ex; - } + $failed = $ex; } $results = $engine->getResults(); // It'd be nice to just return a single result from the run method above // which contains both the lint messages and the postponed linters. // However, to maintain compatibility with existing lint subclasses, use // a separate method call to grab the postponed linters. $this->postponedLinters = $engine->getPostponedLinters(); if ($this->getArgument('never-apply-patches')) { $apply_patches = false; } else { $apply_patches = true; } if ($this->getArgument('apply-patches')) { $prompt_patches = false; } else { $prompt_patches = true; } if ($this->getArgument('amend-all')) { $this->shouldAmendChanges = true; $this->shouldAmendWithoutPrompt = true; } if ($this->getArgument('amend-autofixes')) { $prompt_autofix_patches = false; $this->shouldAmendChanges = true; $this->shouldAmendAutofixesWithoutPrompt = true; } else { $prompt_autofix_patches = true; } $wrote_to_disk = false; switch ($this->getArgument('output')) { case 'json': $renderer = new ArcanistLintJSONRenderer(); $prompt_patches = false; $apply_patches = $this->getArgument('apply-patches'); break; case 'summary': $renderer = new ArcanistLintSummaryRenderer(); break; case 'compiler': $renderer = new ArcanistLintLikeCompilerRenderer(); $prompt_patches = false; $apply_patches = $this->getArgument('apply-patches'); break; default: $renderer = new ArcanistLintConsoleRenderer(); $renderer->setShowAutofixPatches($prompt_autofix_patches); break; } $all_autofix = true; $console = PhutilConsole::getConsole(); foreach ($results as $result) { $result_all_autofix = $result->isAllAutofix(); if (!$result->getMessages() && !$result_all_autofix) { continue; } if (!$result_all_autofix) { $all_autofix = false; } $lint_result = $renderer->renderLintResult($result); if ($lint_result) { $console->writeOut('%s', $lint_result); } if ($apply_patches && $result->isPatchable()) { $patcher = ArcanistLintPatcher::newFromArcanistLintResult($result); if ($prompt_patches && !($result_all_autofix && !$prompt_autofix_patches)) { $old_file = $result->getFilePathOnDisk(); if (!Filesystem::pathExists($old_file)) { $old_file = '/dev/null'; } $new_file = new TempFile(); $new = $patcher->getModifiedFileContent(); Filesystem::writeFile($new_file, $new); // TODO: Improve the behavior here, make it more like // difference_render(). list(, $stdout, $stderr) = exec_manual("diff -u %s %s", $old_file, $new_file); $console->writeOut('%s', $stdout); $console->writeErr('%s', $stderr); $prompt = phutil_console_format( "Apply this patch to __%s__?", $result->getPath()); if (!$console->confirm($prompt, $default_no = false)) { continue; } } $patcher->writePatchToDisk(); $wrote_to_disk = true; } } $repository_api = $this->getRepositoryAPI(); if ($wrote_to_disk && ($repository_api instanceof ArcanistGitAPI) && $this->shouldAmendChanges) { if ($this->shouldAmendWithoutPrompt || ($this->shouldAmendAutofixesWithoutPrompt && $all_autofix)) { $console->writeOut( "** LINT NOTICE ** Automatically amending HEAD ". "with lint patches.\n"); $amend = true; } else { $amend = $console->confirm("Amend HEAD with lint patches?"); } if ($amend) { execx( '(cd %s; git commit -a --amend -C HEAD)', $repository_api->getPath()); } else { throw new ArcanistUsageException( "Sort out the lint changes that were applied to the working ". "copy and relint."); } } if ($this->getArgument('output') == 'json') { // NOTE: Required by save_lint.php in Phabricator. return 0; } if ($failed) { throw $failed; } $unresolved = array(); $has_warnings = false; $has_errors = false; foreach ($results as $result) { foreach ($result->getMessages() as $message) { if (!$message->isPatchApplied()) { if ($message->isError()) { $has_errors = true; } else if ($message->isWarning()) { $has_warnings = true; } $unresolved[] = $message; } } } $this->unresolvedMessages = $unresolved; - if ($this->getArgument('cache')) { - $cached = array(); + $cache = $this->readScratchJSONFile('lint-cache.json'); + $cached = idx($cache, $this->getCacheKey(), array()); + if ($cached || $this->getArgument('cache')) { foreach ($results as $result) { $path = $result->getPath(); + if (!$this->getArgument('cache')) { + unset($cached[$path]); + continue; + } $hash = md5_file($engine->getFilePathOnDisk($path)); - $cached[$path] = array($hash => array()); + $version = $result->getCacheVersion(); + $cached[$path] = array($hash => array($version => array())); foreach ($result->getMessages() as $message) { if (!$message->isPatchApplied()) { - $cached[$path][$hash][] = $message->toDictionary(); + $cached[$path][$hash][$version][] = $message->toDictionary(); } } } - $cache = $this->readScratchJSONFile('lint-cache.json'); $cache[$this->getCacheKey()] = $cached; // TODO: Garbage collection. $this->writeScratchJSONFile('lint-cache.json', $cache); } // Take the most severe lint message severity and use that // as the result code. if ($has_errors) { $result_code = self::RESULT_ERRORS; } else if ($has_warnings) { $result_code = self::RESULT_WARNINGS; } else if (!empty($this->postponedLinters)) { $result_code = self::RESULT_POSTPONED; } else { $result_code = self::RESULT_OKAY; } if (!$this->getParentWorkflow()) { if ($result_code == self::RESULT_OKAY) { $console->writeOut('%s', $renderer->renderOkayResult()); } } return $result_code; } public function getUnresolvedMessages() { return $this->unresolvedMessages; } public function getPostponedLinters() { return $this->postponedLinters; } }