diff --git a/resources/sql/autopatches/20151023.patchduration.sql b/resources/sql/autopatches/20151023.patchduration.sql new file mode 100644 index 0000000000..3e0c363931 --- /dev/null +++ b/resources/sql/autopatches/20151023.patchduration.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_meta_data.patch_status + ADD duration BIGINT UNSIGNED; diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php index 0d8fc79bb0..76e8e0a057 100644 --- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php +++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php @@ -1,304 +1,336 @@ disableUTF8MB4 = $disable_utf8_mb4; return $this; } public function getDisableUTF8MB4() { return $this->disableUTF8MB4; } public function setNamespace($namespace) { $this->namespace = $namespace; PhabricatorLiskDAO::pushStorageNamespace($namespace); return $this; } public function getNamespace() { return $this->namespace; } public function setUser($user) { $this->user = $user; return $this; } public function getUser() { return $this->user; } public function setPassword($password) { $this->password = $password; return $this; } public function getPassword() { return $this->password; } public function setHost($host) { $this->host = $host; return $this; } public function getHost() { return $this->host; } public function setPort($port) { $this->port = $port; return $this; } public function getPort() { return $this->port; } public function getDatabaseName($fragment) { return $this->namespace.'_'.$fragment; } public function getDatabaseList(array $patches, $only_living = false) { assert_instances_of($patches, 'PhabricatorStoragePatch'); $list = array(); foreach ($patches as $patch) { if ($patch->getType() == 'db') { if ($only_living && $patch->isDead()) { continue; } $list[] = $this->getDatabaseName($patch->getName()); } } return $list; } public function getConn($fragment) { $database = $this->getDatabaseName($fragment); $return = &$this->conns[$this->host][$this->user][$database]; if (!$return) { $return = PhabricatorEnv::newObjectFromConfig( 'mysql.implementation', array( array( 'user' => $this->user, 'pass' => $this->password, 'host' => $this->host, 'port' => $this->port, 'database' => $fragment ? $database : null, ), )); } return $return; } public function getAppliedPatches() { try { $applied = queryfx_all( $this->getConn('meta_data'), - 'SELECT patch FROM patch_status'); + 'SELECT patch FROM %T', + self::TABLE_STATUS); return ipull($applied, 'patch'); } catch (AphrontQueryException $ex) { return null; } } + public function getPatchDurations() { + try { + $rows = queryfx_all( + $this->getConn('meta_data'), + 'SELECT patch, duration FROM %T WHERE duration IS NOT NULL', + self::TABLE_STATUS); + return ipull($rows, 'duration', 'patch'); + } catch (AphrontQueryException $ex) { + return array(); + } + } + public function createDatabase($fragment) { $info = $this->getCharsetInfo(); queryfx( $this->getConn(null), 'CREATE DATABASE IF NOT EXISTS %T COLLATE %T', $this->getDatabaseName($fragment), $info[self::COLLATE_TEXT]); } public function createTable($fragment, $table, array $cols) { queryfx( $this->getConn($fragment), 'CREATE TABLE IF NOT EXISTS %T.%T (%Q) '. 'ENGINE=InnoDB, COLLATE utf8_general_ci', $this->getDatabaseName($fragment), $table, implode(', ', $cols)); } public function getLegacyPatches(array $patches) { assert_instances_of($patches, 'PhabricatorStoragePatch'); try { $row = queryfx_one( $this->getConn('meta_data'), 'SELECT version FROM %T', 'schema_version'); $version = $row['version']; } catch (AphrontQueryException $ex) { return array(); } $legacy = array(); foreach ($patches as $key => $patch) { if ($patch->getLegacy() !== false && $patch->getLegacy() <= $version) { $legacy[] = $key; } } return $legacy; } - public function markPatchApplied($patch) { + public function markPatchApplied($patch, $duration = null) { + $conn = $this->getConn('meta_data'); + queryfx( - $this->getConn('meta_data'), + $conn, 'INSERT INTO %T (patch, applied) VALUES (%s, %d)', - 'patch_status', + self::TABLE_STATUS, $patch, time()); + + // We didn't add this column for a long time, so it may not exist yet. + if ($duration !== null) { + try { + queryfx( + $conn, + 'UPDATE %T SET duration = %d WHERE patch = %s', + self::TABLE_STATUS, + (int)floor($duration * 1000000), + $patch); + } catch (AphrontQueryException $ex) { + // Just ignore this, as it almost certainly indicates that we just + // don't have the column yet. + } + } } public function applyPatch(PhabricatorStoragePatch $patch) { $type = $patch->getType(); $name = $patch->getName(); switch ($type) { case 'db': $this->createDatabase($name); break; case 'sql': $this->applyPatchSQL($name); break; case 'php': $this->applyPatchPHP($name); break; default: throw new Exception(pht("Unable to apply patch of type '%s'.", $type)); } } public function applyPatchSQL($sql) { $sql = Filesystem::readFile($sql); $queries = preg_split('/;\s+/', $sql); $queries = array_filter($queries); $conn = $this->getConn(null); $charset_info = $this->getCharsetInfo(); foreach ($charset_info as $key => $value) { $charset_info[$key] = qsprintf($conn, '%T', $value); } foreach ($queries as $query) { $query = str_replace('{$NAMESPACE}', $this->namespace, $query); foreach ($charset_info as $key => $value) { $query = str_replace('{$'.$key.'}', $value, $query); } queryfx( $conn, '%Q', $query); } } public function applyPatchPHP($script) { $schema_conn = $this->getConn(null); require_once $script; } public function isCharacterSetAvailable($character_set) { if ($character_set == 'utf8mb4') { if ($this->getDisableUTF8MB4()) { return false; } } $conn = $this->getConn(null); return self::isCharacterSetAvailableOnConnection($character_set, $conn); } public static function isCharacterSetAvailableOnConnection( $character_set, AphrontDatabaseConnection $conn) { $result = queryfx_one( $conn, 'SELECT CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.CHARACTER_SETS WHERE CHARACTER_SET_NAME = %s', $character_set); return (bool)$result; } public function getCharsetInfo() { if ($this->isCharacterSetAvailable('utf8mb4')) { // If utf8mb4 is available, we use it with the utf8mb4_unicode_ci // collation. This is most correct, and will sort properly. $charset = 'utf8mb4'; $charset_sort = 'utf8mb4'; $charset_full = 'utf8mb4'; $collate_text = 'utf8mb4_bin'; $collate_sort = 'utf8mb4_unicode_ci'; $collate_full = 'utf8mb4_unicode_ci'; } else { // If utf8mb4 is not available, we use binary for most data. This allows // us to store 4-byte unicode characters. // // It's possible that strings will be truncated in the middle of a // character on insert. We encourage users to set STRICT_ALL_TABLES // to prevent this. // // For "fulltext" and "sort" columns, we don't use binary. // // With "fulltext", we can not use binary because MySQL won't let us. // We use 3-byte utf8 instead and accept being unable to index 4-byte // characters. // // With "sort", if we use binary we lose case insensitivity (for // example, "ALincoln@logcabin.com" and "alincoln@logcabin.com" would no // longer be identified as the same email address). This can be very // confusing and is far worse overall than not supporting 4-byte unicode // characters, so we use 3-byte utf8 and accept limited 4-byte support as // a tradeoff to get sensible collation behavior. Many columns where // collation is important rarely contain 4-byte characters anyway, so we // are not giving up too much. $charset = 'binary'; $charset_sort = 'utf8'; $charset_full = 'utf8'; $collate_text = 'binary'; $collate_sort = 'utf8_general_ci'; $collate_full = 'utf8_general_ci'; } return array( self::CHARSET_DEFAULT => $charset, self::CHARSET_SORT => $charset_sort, self::CHARSET_FULLTEXT => $charset_full, self::COLLATE_TEXT => $collate_text, self::COLLATE_SORT => $collate_sort, self::COLLATE_FULLTEXT => $collate_full, ); } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php index ebff080bd4..774ae8445c 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php @@ -1,50 +1,61 @@ setName('status') ->setExamples('**status** [__options__]') ->setSynopsis(pht('Show patch application status.')); } public function execute(PhutilArgumentParser $args) { $api = $this->getAPI(); $patches = $this->getPatches(); $applied = $api->getAppliedPatches(); if ($applied === null) { echo phutil_console_format( "**%s**: %s\n", pht('Database Not Initialized'), pht('Run **%s** to initialize.', 'storage upgrade')); return 1; } $table = id(new PhutilConsoleTable()) ->setShowHeader(false) ->addColumn('id', array('title' => pht('ID'))) ->addColumn('status', array('title' => pht('Status'))) + ->addColumn('duration', array('title' => pht('Duration'))) ->addColumn('type', array('title' => pht('Type'))) ->addColumn('name', array('title' => pht('Name'))); + $durations = $api->getPatchDurations(); + foreach ($patches as $patch) { + $duration = idx($durations, $patch->getFullKey()); + if ($duration === null) { + $duration = '-'; + } else { + $duration = pht('%s us', new PhutilNumber($duration)); + } + $table->addRow(array( 'id' => $patch->getFullKey(), 'status' => in_array($patch->getFullKey(), $applied) ? pht('Applied') : pht('Not Applied'), + 'duration' => $duration, 'type' => $patch->getType(), 'name' => $patch->getName(), )); } $table->draw(); return 0; } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php index 55cf3b223e..d5a4f41cad 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php @@ -1,226 +1,230 @@ setName('upgrade') ->setExamples('**upgrade** [__options__]') ->setSynopsis(pht('Upgrade database schemata.')) ->setArguments( array( array( 'name' => 'apply', 'param' => 'patch', 'help' => pht( 'Apply __patch__ explicitly. This is an advanced feature for '. 'development and debugging; you should not normally use this '. 'flag. This skips adjustment.'), ), array( 'name' => 'no-quickstart', 'help' => pht( 'Build storage patch-by-patch from scatch, even if it could '. 'be loaded from the quickstart template.'), ), array( 'name' => 'init-only', 'help' => pht( 'Initialize storage only; do not apply patches or adjustments.'), ), array( 'name' => 'no-adjust', 'help' => pht( 'Do not apply storage adjustments after storage upgrades.'), ), )); } public function execute(PhutilArgumentParser $args) { $is_dry = $args->getArg('dryrun'); $is_force = $args->getArg('force'); $api = $this->getAPI(); $patches = $this->getPatches(); if (!$is_dry && !$is_force) { echo phutil_console_wrap( pht( 'Before running storage upgrades, you should take down the '. 'Phabricator web interface and stop any running Phabricator '. 'daemons (you can disable this warning with %s).', '--force')); if (!phutil_console_confirm(pht('Are you ready to continue?'))) { echo pht('Cancelled.')."\n"; return 1; } } $apply_only = $args->getArg('apply'); if ($apply_only) { if (empty($patches[$apply_only])) { throw new PhutilArgumentUsageException( pht( "%s argument '%s' is not a valid patch. ". "Use '%s' to show patch status.", '--apply', $apply_only, 'storage status')); } } $no_quickstart = $args->getArg('no-quickstart'); $init_only = $args->getArg('init-only'); $no_adjust = $args->getArg('no-adjust'); $applied = $api->getAppliedPatches(); if ($applied === null) { if ($is_dry) { echo pht( "DRYRUN: Patch metadata storage doesn't exist yet, ". "it would be created.\n"); return 0; } if ($apply_only) { throw new PhutilArgumentUsageException( pht( 'Storage has not been initialized yet, you must initialize '. 'storage before selectively applying patches.')); return 1; } $legacy = $api->getLegacyPatches($patches); if ($legacy || $no_quickstart || $init_only) { // If we have legacy patches, we can't quickstart. $api->createDatabase('meta_data'); $api->createTable( 'meta_data', 'patch_status', array( 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci', 'applied INT UNSIGNED NOT NULL', )); foreach ($legacy as $patch) { $api->markPatchApplied($patch); } } else { echo pht('Loading quickstart template...')."\n"; $root = dirname(phutil_get_library_root('phabricator')); $sql = $root.'/resources/sql/quickstart.sql'; $api->applyPatchSQL($sql); } } if ($init_only) { echo pht('Storage initialized.')."\n"; return 0; } $applied = $api->getAppliedPatches(); $applied = array_fuse($applied); $skip_mark = false; if ($apply_only) { if (isset($applied[$apply_only])) { unset($applied[$apply_only]); $skip_mark = true; if (!$is_force && !$is_dry) { echo phutil_console_wrap( pht( "Patch '%s' has already been applied. Are you sure you want ". "to apply it again? This may put your storage in a state ". "that the upgrade scripts can not automatically manage.", $apply_only)); if (!phutil_console_confirm(pht('Apply patch again?'))) { echo pht('Cancelled.')."\n"; return 1; } } } } while (true) { $applied_something = false; foreach ($patches as $key => $patch) { if (isset($applied[$key])) { unset($patches[$key]); continue; } if ($apply_only && $apply_only != $key) { unset($patches[$key]); continue; } $can_apply = true; foreach ($patch->getAfter() as $after) { if (empty($applied[$after])) { if ($apply_only) { echo pht( "Unable to apply patch '%s' because it depends ". "on patch '%s', which has not been applied.\n", $apply_only, $after); return 1; } $can_apply = false; break; } } if (!$can_apply) { continue; } $applied_something = true; if ($is_dry) { echo pht("DRYRUN: Would apply patch '%s'.", $key)."\n"; } else { echo pht("Applying patch '%s'...", $key)."\n"; + + $t_begin = microtime(true); $api->applyPatch($patch); + $t_end = microtime(true); + if (!$skip_mark) { - $api->markPatchApplied($key); + $api->markPatchApplied($key, ($t_end - $t_begin)); } } unset($patches[$key]); $applied[$key] = true; } if (!$applied_something) { if (count($patches)) { throw new Exception( pht( 'Some patches could not be applied: %s', implode(', ', array_keys($patches)))); } else if (!$is_dry && !$apply_only) { echo pht( "Storage is up to date. Use '%s' for details.", 'storage status')."\n"; } break; } } $console = PhutilConsole::getConsole(); if ($no_adjust || $init_only || $apply_only) { $console->writeOut( "%s\n", pht('Declining to apply storage adjustments.')); return 0; } else { return $this->adjustSchemata($is_force, $unsafe = false, $is_dry); } } } diff --git a/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php b/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php index 886df14e2c..df48ce3812 100644 --- a/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php +++ b/src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php @@ -1,22 +1,23 @@ buildRawSchema( 'meta_data', 'patch_status', array( 'patch' => 'text128', 'applied' => 'uint32', + 'duration' => 'uint64?', ), array( 'PRIMARY' => array( 'columns' => array('patch'), 'unique' => true, ), )); } }