diff --git a/src/upload/ArcanistFileDataRef.php b/src/upload/ArcanistFileDataRef.php index 795fbd67..318418ec 100644 --- a/src/upload/ArcanistFileDataRef.php +++ b/src/upload/ArcanistFileDataRef.php @@ -1,313 +1,357 @@ name = $name; return $this; } /** * @task config */ public function getName() { return $this->name; } /** * Set the data to upload as a single raw blob. * * You can specify file data by calling this method with a single blob of * data, or by calling @{method:setPath} and providing a path to a file on * disk. * * @param bytes Blob of file data. * @task config */ public function setData($data) { $this->data = $data; return $this; } /** * @task config */ public function getData() { return $this->data; } /** * Set the data to upload by pointing to a file on disk. * * You can specify file data by calling this method with a path, or by * providing a blob of raw data to @{method:setData}. * * The path itself only provides data. If you want to name the file, you * should also call @{method:setName}. * * @param string Path on disk to a file containing data to upload. * @return this * @task config */ public function setPath($path) { $this->path = $path; return $this; } /** * @task config */ public function getPath() { return $this->path; } + /** + * @task config + */ + public function setViewPolicy($view_policy) { + $this->viewPolicy = $view_policy; + return $this; + } + + + /** + * @task config + */ + public function getViewPolicy() { + return $this->viewPolicy; + } + + + /** + * Configure a file to be temporary instead of permanent. + * + * By default, files are retained indefinitely until explicitly deleted. If + * you want to upload a temporary file instead, you can specify an epoch + * timestamp. The file will be deleted after this time. + * + * @param int Epoch timestamp to retain the file until. + * @return this + * @task config + */ + public function setDeleteAfterEpoch($epoch) { + $this->deleteAfterEpoch = $epoch; + return $this; + } + + + /** + * @task config + */ + public function getDeleteAfterEpoch() { + return $this->deleteAfterEpoch; + } + + /* -( Handling Upload Results )-------------------------------------------- */ /** * @task results */ public function getErrors() { return $this->errors; } /** * @task results */ public function getPHID() { return $this->phid; } /* -( Uploader API )------------------------------------------------------- */ /** * @task uploader */ public function willUpload() { $have_data = ($this->data !== null); $have_path = ($this->path !== null); if (!$have_data && !$have_path) { throw new Exception( pht( 'Specify setData() or setPath() when building a file data '. 'reference.')); } if ($have_data && $have_path) { throw new Exception( pht( 'Specify either setData() or setPath() when building a file data '. 'reference, but not both.')); } if ($have_path) { $path = $this->path; if (!Filesystem::pathExists($path)) { throw new Exception( pht( 'Unable to upload file: path "%s" does not exist.', $path)); } try { Filesystem::assertIsFile($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not a file.', $path)); } try { Filesystem::assertReadable($path); } catch (FilesystemException $ex) { throw new Exception( pht( 'Unable to upload file: path "%s" is not readable.', $path)); } $hash = @sha1_file($path); if ($hash === false) { throw new Exception( pht( 'Unable to upload file: failed to calculate file data hash for '. 'path "%s".', $path)); } $size = @filesize($path); if ($size === false) { throw new Exception( pht( 'Unable to upload file: failed to determine filesize of '. 'path "%s".', $path)); } $this->hash = $hash; $this->size = $size; } else { $data = $this->data; $this->hash = sha1($data); $this->size = strlen($data); } } /** * @task uploader */ public function didFail($error) { $this->errors[] = $error; return $this; } /** * @task uploader */ public function setPHID($phid) { $this->phid = $phid; return $this; } /** * @task uploader */ public function getByteSize() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->size; } /** * @task uploader */ public function getContentHash() { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } return $this->hash; } /** * @task uploader */ public function didUpload() { if ($this->fileHandle) { @fclose($this->fileHandle); $this->fileHandle = null; } } /** * @task uploader */ public function readBytes($start, $end) { if ($this->size === null) { throw new PhutilInvalidStateException('willUpload'); } $len = ($end - $start); if ($this->data !== null) { return substr($this->data, $start, $len); } $path = $this->path; if ($this->fileHandle === null) { $f = @fopen($path, 'rb'); if (!$f) { throw new Exception( pht( 'Unable to upload file: failed to open path "%s" for reading.', $path)); } $this->fileHandle = $f; } $f = $this->fileHandle; $ok = @fseek($f, $start); if ($ok !== 0) { throw new Exception( pht( 'Unable to upload file: failed to fseek() to offset %d in file '. 'at path "%s".', $start, $path)); } $data = @fread($f, $len); if ($data === false) { throw new Exception( pht( 'Unable to upload file: failed to read %d bytes after offset %d '. 'from file at path "%s".', $len, $start, $path)); } return $data; } } diff --git a/src/upload/ArcanistFileUploader.php b/src/upload/ArcanistFileUploader.php index ba960775..27ec8d34 100644 --- a/src/upload/ArcanistFileUploader.php +++ b/src/upload/ArcanistFileUploader.php @@ -1,338 +1,313 @@ setConduitClient($conduit); * * // Queue one or more files to be uploaded. * $file = id(new ArcanistFileDataRef()) * ->setName('example.jpg') * ->setPath('/path/to/example.jpg'); * $uploader->addFile($file); * * // Upload the files. * $files = $uploader->uploadFiles(); * * For details about building file references, see @{class:ArcanistFileDataRef}. * * @task config Configuring the Uploader * @task add Adding Files * @task upload Uploading Files * @task internal Internals */ final class ArcanistFileUploader extends Phobject { private $conduit; private $files; - private $config = array(); /* -( Configuring the Uploader )------------------------------------------- */ /** * Provide a Conduit client to choose which server to upload files to. * * @param ConduitClient Configured client. * @return this * @task config */ public function setConduitClient(ConduitClient $conduit) { $this->conduit = $conduit; return $this; } /* -( Adding Files )------------------------------------------------------- */ /** * Add a file to the list of files to be uploaded. * * You can optionally provide an explicit key which will be used to identify * the file. After adding files, upload them with @{method:uploadFiles}. * * @param ArcanistFileDataRef File data to upload. * @param null|string Optional key to use to identify this file. * @return this * @task add */ public function addFile(ArcanistFileDataRef $file, $key = null) { if ($key === null) { $this->files[] = $file; } else { if (isset($this->files[$key])) { throw new Exception( pht( 'Two files were added with identical explicit keys ("%s"); each '. 'explicit key must be unique.', $key)); } $this->files[$key] = $file; } return $this; } - /** - * Configure a file to be temporary instead of permanent. - * - * By default, files are retained indefinitely until explicitly deleted. If - * you want to upload a temporary file instead, you can specify an epoch - * timestamp. The file will be deleted after this time. - * - * @param string Key identifying the file you want to make temporary, as - * passed to @{method:addFile}. - * @param int Epoch timestamp to retain the file until. - * @return this - * @task add - */ - public function setDeleteFileAfterEpoch($file_key, $epoch) { - if (empty($this->files[$file_key])) { - throw new Exception( - pht( - 'No file with given key ("%s") has been added to this uploader.', - $file_key)); - } - - $this->config[$file_key]['deleteAfterEpoch'] = $epoch; - - return $this; - } - - /* -( Uploading Files )---------------------------------------------------- */ /** * Upload files to the server. * * This transfers all files which have been queued with @{method:addFiles} * over the Conduit link configured with @{method:setConduitClient}. * * This method returns a map of all file data references. If references were * added with an explicit key when @{method:addFile} was called, the key is * retained in the result map. * * On return, files are either populated with a PHID (indicating a successful * upload) or a list of errors. See @{class:ArcanistFileDataRef} for * details. * * @return map Files with results populated. * @task upload */ public function uploadFiles() { if (!$this->conduit) { throw new PhutilInvalidStateException('setConduitClient'); } $files = $this->files; foreach ($files as $key => $file) { try { $file->willUpload(); } catch (Exception $ex) { $file->didFail($ex->getMessage()); unset($files[$key]); } } $conduit = $this->conduit; $futures = array(); foreach ($files as $key => $file) { - $config = idx($this->config, $key, array()); - $params = array( 'name' => $file->getName(), 'contentLength' => $file->getByteSize(), 'contentHash' => $file->getContentHash(), ); - $delete_after = idx($config, 'deleteAfterEpoch'); + $delete_after = $file->getDeleteAfterEpoch(); if ($delete_after !== null) { $params['deleteAfterEpoch'] = $delete_after; } + $view_policy = $file->getViewPolicy(); + if ($view_policy !== null) { + $params['viewPolicy'] = $view_policy; + } + $futures[$key] = $conduit->callMethod('file.allocate', $params); } $iterator = id(new FutureIterator($futures))->limit(4); $chunks = array(); foreach ($iterator as $key => $future) { try { $result = $future->resolve(); } catch (Exception $ex) { // The most likely cause for a failure here is that the server does // not support `file.allocate`. In this case, we'll try the older // upload method below. continue; } $phid = $result['filePHID']; $file = $files[$key]; // We don't need to upload any data. Figure out why not: this can either // be because of an error (server can't accept the data) or because the // server already has the data. if (!$result['upload']) { if (!$phid) { $file->didFail( pht( 'Unable to upload file: the server refused to accept file '. '"%s". This usually means it is too large.', $file->getName())); } else { // These server completed the upload by creating a reference to known // file data. We don't need to transfer the actual data, and are all // set. $file->setPHID($phid); } unset($files[$key]); continue; } // The server wants us to do an upload. if ($phid) { $chunks[$key] = array( 'file' => $file, 'phid' => $phid, ); } } foreach ($chunks as $key => $chunk) { $file = $chunk['file']; $phid = $chunk['phid']; try { $this->uploadChunks($file, $phid); $file->setPHID($phid); } catch (Exception $ex) { $file->didFail( pht( 'Unable to upload file chunks: %s', $ex->getMessage())); } unset($files[$key]); } foreach ($files as $key => $file) { try { $phid = $this->uploadData($file); $file->setPHID($phid); } catch (Exception $ex) { $file->didFail( pht( 'Unable to upload file data: %s', $ex->getMessage())); } unset($files[$key]); } foreach ($this->files as $file) { $file->didUpload(); } return $this->files; } /* -( Internals )---------------------------------------------------------- */ /** * Upload missing chunks of a large file by calling `file.uploadchunk` over * Conduit. * * @task internal */ private function uploadChunks(ArcanistFileDataRef $file, $file_phid) { $conduit = $this->conduit; $chunks = $conduit->callMethodSynchronous( 'file.querychunks', array( 'filePHID' => $file_phid, )); $remaining = array(); foreach ($chunks as $chunk) { if (!$chunk['complete']) { $remaining[] = $chunk; } } $done = (count($chunks) - count($remaining)); if ($done) { $this->writeStatus( pht( 'Resuming upload (%d of %d chunks remain).', new PhutilNumber(count($remaining)), new PhutilNumber(count($chunks)))); } else { $this->writeStatus( pht( 'Uploading chunks (%d chunks to upload).', new PhutilNumber(count($remaining)))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($chunks)); for ($ii = 0; $ii < $done; $ii++) { $progress->update(1); } $progress->draw(); // TODO: We could do these in parallel to improve upload performance. foreach ($remaining as $chunk) { $data = $file->readBytes($chunk['byteStart'], $chunk['byteEnd']); $conduit->callMethodSynchronous( 'file.uploadchunk', array( 'filePHID' => $file_phid, 'byteStart' => $chunk['byteStart'], 'dataEncoding' => 'base64', 'data' => base64_encode($data), )); $progress->update(1); } } /** * Upload an entire file by calling `file.upload` over Conduit. * * @task internal */ private function uploadData(ArcanistFileDataRef $file) { $conduit = $this->conduit; $data = $file->readBytes(0, $file->getByteSize()); return $conduit->callMethodSynchronous( 'file.upload', array( 'name' => $file->getName(), 'data_base64' => base64_encode($data), )); } /** * Write a status message. * * @task internal */ private function writeStatus($message) { fwrite(STDERR, $message."\n"); } } diff --git a/src/workflow/ArcanistUploadWorkflow.php b/src/workflow/ArcanistUploadWorkflow.php index b4414a39..a31ce0f0 100644 --- a/src/workflow/ArcanistUploadWorkflow.php +++ b/src/workflow/ArcanistUploadWorkflow.php @@ -1,199 +1,198 @@ array( 'help' => pht('Output upload information in JSON format.'), ), 'temporary' => array( 'help' => pht( 'Mark the file as temporary. Temporary files will be deleted '. 'automatically after 24 hours.'), ), '*' => 'paths', ); } protected function didParseArguments() { if (!$this->getArgument('paths')) { throw new ArcanistUsageException( pht('Specify one or more files to upload.')); } $this->paths = $this->getArgument('paths'); $this->json = $this->getArgument('json'); } public function requiresAuthentication() { return true; } public function run() { $is_temporary = $this->getArgument('temporary'); $conduit = $this->getConduit(); $results = array(); $uploader = id(new ArcanistFileUploader()) ->setConduitClient($conduit); foreach ($this->paths as $key => $path) { $file = id(new ArcanistFileDataRef()) ->setName(basename($path)) ->setPath($path); - $uploader->addFile($file, $key); - if ($is_temporary) { - $uploader->setDeleteFileAfterEpoch( - $key, - time() + phutil_units('24 hours in seconds')); + $expires_at = time() + phutil_units('24 hours in seconds'); + $file->setDeleteAfterEpoch($expires_at); } + + $uploader->addFile($file); } $files = $uploader->uploadFiles(); $results = array(); foreach ($files as $file) { // TODO: This could be handled more gracefully; just preserving behavior // until we introduce `file.query` and modernize this. if ($file->getErrors()) { throw new Exception(implode("\n", $file->getErrors())); } $phid = $file->getPHID(); $name = $file->getName(); $info = $conduit->callMethodSynchronous( 'file.info', array( 'phid' => $phid, )); $results[$path] = $info; if (!$this->json) { $id = $info['id']; echo " F{$id} {$name}: ".$info['uri']."\n\n"; } } if ($this->json) { echo json_encode($results)."\n"; } else { $this->writeStatus(pht('Done.')); } return 0; } private function writeStatus($line) { $this->writeStatusMessage($line."\n"); } private function uploadChunks($file_phid, $path) { $conduit = $this->getConduit(); $f = @fopen($path, 'rb'); if (!$f) { throw new Exception(pht('Unable to open file "%s"', $path)); } $this->writeStatus(pht('Beginning chunked upload of large file...')); $chunks = $conduit->callMethodSynchronous( 'file.querychunks', array( 'filePHID' => $file_phid, )); $remaining = array(); foreach ($chunks as $chunk) { if (!$chunk['complete']) { $remaining[] = $chunk; } } $done = (count($chunks) - count($remaining)); if ($done) { $this->writeStatus( pht( 'Resuming upload (%d of %d chunks remain).', new PhutilNumber(count($remaining)), new PhutilNumber(count($chunks)))); } else { $this->writeStatus( pht( 'Uploading chunks (%d chunks to upload).', new PhutilNumber(count($remaining)))); } $progress = new PhutilConsoleProgressBar(); $progress->setTotal(count($chunks)); for ($ii = 0; $ii < $done; $ii++) { $progress->update(1); } $progress->draw(); // TODO: We could do these in parallel to improve upload performance. foreach ($remaining as $chunk) { $offset = $chunk['byteStart']; $ok = fseek($f, $offset); if ($ok !== 0) { throw new Exception( pht( 'Failed to %s!', 'fseek()')); } $data = fread($f, $chunk['byteEnd'] - $chunk['byteStart']); if ($data === false) { throw new Exception( pht( 'Failed to %s!', 'fread()')); } $conduit->callMethodSynchronous( 'file.uploadchunk', array( 'filePHID' => $file_phid, 'byteStart' => $offset, 'dataEncoding' => 'base64', 'data' => base64_encode($data), )); $progress->update(1); } } }