diff --git a/src/applications/diviner/publisher/DivinerLivePublisher.php b/src/applications/diviner/publisher/DivinerLivePublisher.php index 71002ccd2a..25d1464b4e 100644 --- a/src/applications/diviner/publisher/DivinerLivePublisher.php +++ b/src/applications/diviner/publisher/DivinerLivePublisher.php @@ -1,152 +1,150 @@ book) { $book_name = $this->getConfig('name'); $book = id(new DivinerLiveBook())->loadOneWhere('name = %s', $book_name); if (!$book) { $book = id(new DivinerLiveBook()) ->setName($book_name) ->setViewPolicy(PhabricatorPolicies::POLICY_USER) ->save(); } $book->setConfigurationData($this->getConfigurationData())->save(); $this->book = $book; } return $this->book; } private function loadSymbolForAtom(DivinerAtom $atom) { $symbol = id(new DivinerAtomQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withBookPHIDs(array($this->loadBook()->getPHID())) ->withTypes(array($atom->getType())) ->withNames(array($atom->getName())) ->withContexts(array($atom->getContext())) ->withIndexes(array($this->getAtomSimilarIndex($atom))) - ->withIncludeUndocumentable(true) - ->withIncludeGhosts(true) ->executeOne(); if ($symbol) { return $symbol; } return id(new DivinerLiveSymbol()) ->setBookPHID($this->loadBook()->getPHID()) ->setType($atom->getType()) ->setName($atom->getName()) ->setContext($atom->getContext()) ->setAtomIndex($this->getAtomSimilarIndex($atom)); } private function loadAtomStorageForSymbol(DivinerLiveSymbol $symbol) { $storage = id(new DivinerLiveAtom())->loadOneWhere( 'symbolPHID = %s', $symbol->getPHID()); if ($storage) { return $storage; } return id(new DivinerLiveAtom()) ->setSymbolPHID($symbol->getPHID()); } protected function loadAllPublishedHashes() { $symbols = id(new DivinerAtomQuery()) ->setViewer(PhabricatorUser::getOmnipotentUser()) ->withBookPHIDs(array($this->loadBook()->getPHID())) ->withGhosts(false) ->execute(); return mpull($symbols, 'getGraphHash'); } protected function deleteDocumentsByHash(array $hashes) { $atom_table = new DivinerLiveAtom(); $symbol_table = new DivinerLiveSymbol(); $conn_w = $symbol_table->establishConnection('w'); $strings = array(); foreach ($hashes as $hash) { $strings[] = qsprintf($conn_w, '%s', $hash); } foreach (PhabricatorLiskDAO::chunkSQL($strings, ', ') as $chunk) { queryfx( $conn_w, 'UPDATE %T SET graphHash = NULL, nodeHash = NULL WHERE graphHash IN (%Q)', $symbol_table->getTableName(), $chunk); } queryfx( $conn_w, 'DELETE a FROM %T a LEFT JOIN %T s ON a.symbolPHID = s.phid WHERE s.graphHash IS NULL', $atom_table->getTableName(), $symbol_table->getTableName()); } protected function createDocumentsByHash(array $hashes) { foreach ($hashes as $hash) { $atom = $this->getAtomFromGraphHash($hash); $ref = $atom->getRef(); $symbol = $this->loadSymbolForAtom($atom); $is_documentable = $this->shouldGenerateDocumentForAtom($atom); $symbol ->setGraphHash($hash) ->setIsDocumentable((int)$is_documentable) ->setTitle($ref->getTitle()) ->setGroupName($ref->getGroup()) ->setNodeHash($atom->getHash()); if ($atom->getType() !== DivinerAtom::TYPE_FILE) { $renderer = $this->getRenderer(); $summary = $renderer->getAtomSummary($atom); $symbol->setSummary($summary); } else { $symbol->setSummary(''); } $symbol->save(); // TODO: We probably need a finer-grained sense of what "documentable" // atoms are. Neither files nor methods are currently considered // documentable, but for different reasons: files appear nowhere, while // methods just don't appear at the top level. These are probably // separate concepts. Since we need atoms in order to build method // documentation, we insert them here. This also means we insert files, // which are unnecessary and unused. Make sure this makes sense, but then // probably introduce separate "isTopLevel" and "isDocumentable" flags? // TODO: Yeah do that soon ^^^ if ($atom->getType() !== DivinerAtom::TYPE_FILE) { $storage = $this->loadAtomStorageForSymbol($symbol) ->setAtomData($atom->toDictionary()) ->setContent(null) ->save(); } } } public function findAtomByRef(DivinerAtomRef $ref) { // TODO: Actually implement this. return null; } } diff --git a/src/applications/diviner/query/DivinerAtomQuery.php b/src/applications/diviner/query/DivinerAtomQuery.php index 12600927ff..4af0f1b1fa 100644 --- a/src/applications/diviner/query/DivinerAtomQuery.php +++ b/src/applications/diviner/query/DivinerAtomQuery.php @@ -1,447 +1,446 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withBookPHIDs(array $phids) { $this->bookPHIDs = $phids; return $this; } public function withTypes(array $types) { $this->types = $types; return $this; } public function withNames(array $names) { $this->names = $names; return $this; } public function withContexts(array $contexts) { $this->contexts = $contexts; return $this; } public function withIndexes(array $indexes) { $this->indexes = $indexes; return $this; } public function withNodeHashes(array $hashes) { $this->nodeHashes = $hashes; return $this; } public function withTitles($titles) { $this->titles = $titles; return $this; } public function withNameContains($text) { $this->nameContains = $text; return $this; } public function needAtoms($need) { $this->needAtoms = $need; return $this; } public function needChildren($need) { $this->needChildren = $need; return $this; } /** * Include or exclude "ghosts", which are symbols which used to exist but do * not exist currently (for example, a function which existed in an older * version of the codebase but was deleted). * * These symbols had PHIDs assigned to them, and may have other sorts of * metadata that we don't want to lose (like comments or flags), so we don't * delete them outright. They might also come back in the future: the change * which deleted the symbol might be reverted, or the documentation might * have been generated incorrectly by accident. In these cases, we can * restore the original data. * * @param bool * @return this */ public function withGhosts($ghosts) { $this->isGhost = $ghosts; return $this; } public function needExtends($need) { $this->needExtends = $need; return $this; } public function withIsDocumentable($documentable) { $this->isDocumentable = $documentable; return $this; } protected function loadPage() { $table = new DivinerLiveSymbol(); $conn_r = $table->establishConnection('r'); $data = queryfx_all( $conn_r, 'SELECT * FROM %T %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r)); return $table->loadAllFromArray($data); } protected function willFilterPage(array $atoms) { $books = array_unique(mpull($atoms, 'getBookPHID')); $books = id(new DivinerBookQuery()) ->setViewer($this->getViewer()) ->withPHIDs($books) ->execute(); $books = mpull($books, null, 'getPHID'); foreach ($atoms as $key => $atom) { $book = idx($books, $atom->getBookPHID()); if (!$book) { unset($atoms[$key]); continue; } $atom->attachBook($book); } if ($this->needAtoms) { $atom_data = id(new DivinerLiveAtom())->loadAllWhere( 'symbolPHID IN (%Ls)', mpull($atoms, 'getPHID')); $atom_data = mpull($atom_data, null, 'getSymbolPHID'); foreach ($atoms as $key => $atom) { $data = idx($atom_data, $atom->getPHID()); if (!$data) { unset($atoms[$key]); continue; } $atom->attachAtom($data); } } // Load all of the symbols this symbol extends, recursively. Commonly, // this means all the ancestor classes and interfaces it extends and // implements. if ($this->needExtends) { // First, load all the matching symbols by name. This does 99% of the // work in most cases, assuming things are named at all reasonably. $names = array(); foreach ($atoms as $atom) { foreach ($atom->getAtom()->getExtends() as $xref) { $names[] = $xref->getName(); } } if ($names) { $xatoms = id(new DivinerAtomQuery()) ->setViewer($this->getViewer()) ->withNames($names) ->needExtends(true) ->needAtoms(true) ->needChildren($this->needChildren) ->execute(); $xatoms = mgroup($xatoms, 'getName', 'getType', 'getBookPHID'); } else { $xatoms = array(); } foreach ($atoms as $atom) { $alang = $atom->getAtom()->getLanguage(); $extends = array(); foreach ($atom->getAtom()->getExtends() as $xref) { // If there are no symbols of the matching name and type, we can't // resolve this. if (empty($xatoms[$xref->getName()][$xref->getType()])) { continue; } // If we found matches in the same documentation book, prefer them // over other matches. Otherwise, look at all the the matches. $matches = $xatoms[$xref->getName()][$xref->getType()]; if (isset($matches[$atom->getBookPHID()])) { $maybe = $matches[$atom->getBookPHID()]; } else { $maybe = array_mergev($matches); } if (!$maybe) { continue; } // Filter out matches in a different language, since, e.g., PHP // classes can not implement JS classes. $same_lang = array(); foreach ($maybe as $xatom) { if ($xatom->getAtom()->getLanguage() == $alang) { $same_lang[] = $xatom; } } if (!$same_lang) { continue; } // If we have duplicates remaining, just pick the first one. There's // nothing more we can do to figure out which is the real one. $extends[] = head($same_lang); } $atom->attachExtends($extends); } } if ($this->needChildren) { $child_hashes = $this->getAllChildHashes($atoms, $this->needExtends); if ($child_hashes) { $children = id(new DivinerAtomQuery()) ->setViewer($this->getViewer()) - ->withIncludeUndocumentable(true) ->withNodeHashes($child_hashes) ->needAtoms($this->needAtoms) ->execute(); $children = mpull($children, null, 'getNodeHash'); } else { $children = array(); } $this->attachAllChildren($atoms, $children, $this->needExtends); } return $atoms; } protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->ids) { $where[] = qsprintf( $conn_r, 'id IN (%Ld)', $this->ids); } if ($this->phids) { $where[] = qsprintf( $conn_r, 'phid IN (%Ls)', $this->phids); } if ($this->bookPHIDs) { $where[] = qsprintf( $conn_r, 'bookPHID IN (%Ls)', $this->bookPHIDs); } if ($this->types) { $where[] = qsprintf( $conn_r, 'type IN (%Ls)', $this->types); } if ($this->names) { $where[] = qsprintf( $conn_r, 'name IN (%Ls)', $this->names); } if ($this->titles) { $hashes = array(); foreach ($this->titles as $title) { $slug = DivinerAtomRef::normalizeTitleString($title); $hash = PhabricatorHash::digestForIndex($slug); $hashes[] = $hash; } $where[] = qsprintf( $conn_r, 'titleSlugHash in (%Ls)', $hashes); } if ($this->contexts) { $with_null = false; $contexts = $this->contexts; foreach ($contexts as $key => $value) { if ($value === null) { unset($contexts[$key]); $with_null = true; continue; } } if ($contexts && $with_null) { $where[] = qsprintf( $conn_r, 'context IN (%Ls) OR context IS NULL', $contexts); } else if ($contexts) { $where[] = qsprintf( $conn_r, 'context IN (%Ls)', $contexts); } else if ($with_null) { $where[] = qsprintf( $conn_r, 'context IS NULL'); } } if ($this->indexes) { $where[] = qsprintf( $conn_r, 'atomIndex IN (%Ld)', $this->indexes); } if ($this->isDocumentable !== null) { $where[] = qsprintf( $conn_r, 'isDocumentable = %d', (int)$this->isDocumentable); } if ($this->isGhost !== null) { if ($this->isGhost) { $where[] = qsprintf($conn_r, 'graphHash IS NULL'); } else { $where[] = qsprintf($conn_r, 'graphHash IS NOT NULL'); } } if ($this->nodeHashes) { $where[] = qsprintf( $conn_r, 'nodeHash IN (%Ls)', $this->nodeHashes); } if ($this->nameContains) { // NOTE: This CONVERT() call makes queries case-insensitive, since the // column has binary collation. Eventually, this should move into // fulltext. $where[] = qsprintf( $conn_r, 'CONVERT(name USING utf8) LIKE %~', $this->nameContains); } $where[] = $this->buildPagingClause($conn_r); return $this->formatWhereClause($where); } /** * Walk a list of atoms and collect all the node hashes of the atoms' * children. When recursing, also walk up the tree and collect children of * atoms they extend. * * @param list List of symbols to collect child hashes of. * @param bool True to collect children of extended atoms, * as well. * @return map Hashes of atoms' children. */ private function getAllChildHashes(array $symbols, $recurse_up) { assert_instances_of($symbols, 'DivinerLiveSymbol'); $hashes = array(); foreach ($symbols as $symbol) { foreach ($symbol->getAtom()->getChildHashes() as $hash) { $hashes[$hash] = $hash; } if ($recurse_up) { $hashes += $this->getAllChildHashes($symbol->getExtends(), true); } } return $hashes; } /** * Attach child atoms to existing atoms. In recursive mode, also attach child * atoms to atoms that these atoms extend. * * @param list List of symbols to attach children to. * @param map Map of symbols, keyed by node hash. * @param bool True to attach children to extended atoms, as well. * @return void */ private function attachAllChildren( array $symbols, array $children, $recurse_up) { assert_instances_of($symbols, 'DivinerLiveSymbol'); assert_instances_of($children, 'DivinerLiveSymbol'); foreach ($symbols as $symbol) { $symbol_children = array(); foreach ($symbol->getAtom()->getChildHashes() as $hash) { if (isset($children[$hash])) { $symbol_children[] = $children[$hash]; } } $symbol->attachChildren($symbol_children); if ($recurse_up) { $this->attachAllChildren($symbol->getExtends(), $children, true); } } } public function getQueryApplicationClass() { return 'PhabricatorDivinerApplication'; } }