diff --git a/resources/sql/autopatches/20160921.fileexternalrequest.sql b/resources/sql/autopatches/20160921.fileexternalrequest.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160921.fileexternalrequest.sql @@ -0,0 +1,11 @@ +CREATE TABLE {$NAMESPACE}_file.file_externalrequest ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + filePHID VARBINARY(64), + ttl INT UNSIGNED NOT NULL, + uri LONGTEXT NOT NULL, + uriIndex BINARY(12) NOT NULL, + isSuccessful BOOL NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_uriindex` (uriIndex) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2553,10 +2553,12 @@ 'PhabricatorFileDropUploadController' => 'applications/files/controller/PhabricatorFileDropUploadController.php', 'PhabricatorFileEditController' => 'applications/files/controller/PhabricatorFileEditController.php', 'PhabricatorFileEditor' => 'applications/files/editor/PhabricatorFileEditor.php', + 'PhabricatorFileExternalRequest' => 'applications/files/storage/PhabricatorFileExternalRequest.php', 'PhabricatorFileFilePHIDType' => 'applications/files/phid/PhabricatorFileFilePHIDType.php', 'PhabricatorFileHasObjectEdgeType' => 'applications/files/edge/PhabricatorFileHasObjectEdgeType.php', 'PhabricatorFileIconSetSelectController' => 'applications/files/controller/PhabricatorFileIconSetSelectController.php', 'PhabricatorFileImageMacro' => 'applications/macro/storage/PhabricatorFileImageMacro.php', + 'PhabricatorFileImageProxyController' => 'applications/files/controller/PhabricatorFileImageProxyController.php', 'PhabricatorFileImageTransform' => 'applications/files/transform/PhabricatorFileImageTransform.php', 'PhabricatorFileInfoController' => 'applications/files/controller/PhabricatorFileInfoController.php', 'PhabricatorFileLinkView' => 'view/layout/PhabricatorFileLinkView.php', @@ -2572,6 +2574,7 @@ 'PhabricatorFileStorageEngineTestCase' => 'applications/files/engine/__tests__/PhabricatorFileStorageEngineTestCase.php', 'PhabricatorFileStorageFormat' => 'applications/files/format/PhabricatorFileStorageFormat.php', 'PhabricatorFileStorageFormatTestCase' => 'applications/files/format/__tests__/PhabricatorFileStorageFormatTestCase.php', + 'PhabricatorFileTempGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTempGarbageCollector.php', 'PhabricatorFileTemporaryGarbageCollector' => 'applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php', 'PhabricatorFileTestCase' => 'applications/files/storage/__tests__/PhabricatorFileTestCase.php', 'PhabricatorFileTestDataGenerator' => 'applications/files/lipsum/PhabricatorFileTestDataGenerator.php', @@ -7368,6 +7371,10 @@ 'PhabricatorFileDropUploadController' => 'PhabricatorFileController', 'PhabricatorFileEditController' => 'PhabricatorFileController', 'PhabricatorFileEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhabricatorFileExternalRequest' => array( + 'PhabricatorFileDAO', + 'PhabricatorDestructibleInterface', + ), 'PhabricatorFileFilePHIDType' => 'PhabricatorPHIDType', 'PhabricatorFileHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorFileIconSetSelectController' => 'PhabricatorFileController', @@ -7379,6 +7386,7 @@ 'PhabricatorTokenReceiverInterface', 'PhabricatorPolicyInterface', ), + 'PhabricatorFileImageProxyController' => 'PhabricatorFileController', 'PhabricatorFileImageTransform' => 'PhabricatorFileTransform', 'PhabricatorFileInfoController' => 'PhabricatorFileController', 'PhabricatorFileLinkView' => 'AphrontView', @@ -7394,6 +7402,7 @@ 'PhabricatorFileStorageEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorFileStorageFormat' => 'Phobject', 'PhabricatorFileStorageFormatTestCase' => 'PhabricatorTestCase', + 'PhabricatorFileTempGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorFileTemporaryGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorFileTestCase' => 'PhabricatorTestCase', 'PhabricatorFileTestDataGenerator' => 'PhabricatorTestDataGenerator', diff --git a/src/applications/files/application/PhabricatorFilesApplication.php b/src/applications/files/application/PhabricatorFilesApplication.php --- a/src/applications/files/application/PhabricatorFilesApplication.php +++ b/src/applications/files/application/PhabricatorFilesApplication.php @@ -78,7 +78,7 @@ 'delete/(?P[1-9]\d*)/' => 'PhabricatorFileDeleteController', 'edit/(?P[1-9]\d*)/' => 'PhabricatorFileEditController', 'info/(?P[^/]+)/' => 'PhabricatorFileInfoController', - 'proxy/' => 'PhabricatorFileProxyController', + 'imageproxy/' => 'PhabricatorFileImageProxyController', 'transforms/(?P[1-9]\d*)/' => 'PhabricatorFileTransformListController', 'uploaddialog/(?Psingle/)?' diff --git a/src/applications/files/controller/PhabricatorFileImageProxyController.php b/src/applications/files/controller/PhabricatorFileImageProxyController.php new file mode 100644 --- /dev/null +++ b/src/applications/files/controller/PhabricatorFileImageProxyController.php @@ -0,0 +1,116 @@ +getViewer(); + $img_uri = $request->getStr('uri'); + + // Check if we already have the specified image URI downloaded + $cached_request = id(new PhabricatorFileExternalRequest())->loadOneWhere( + 'uri = %s LIMIT 1', + $img_uri); + if ($cached_request) { + if ($cached_request->getIsSuccessful()) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($viewer) + ->withPHIDs(array($cached_request->getFilePHID())) + ->execute(); + if ($file) { + $file = head($file); + return id(new AphrontRedirectResponse()) + ->setIsExternal(true) + ->setURI($file->getViewURI()); + } + } else { + // Handle the case somehow where the request wasn't successful + } + } + + // Cache missed so we'll need to validate and download the image + PhabricatorEnv::requireValidRemoteURIForLink($img_uri); + $uri = new PhutilURI($img_uri); + $proto = $uri->getProtocol(); + if (!in_array($proto, array('http', 'https'))) { + throw new Exception( + pht('The provided image URI must be either http or https')); + } + + try { + // Rate limit outbound fetches to make this mechanism less useful for + // scanning networks and ports. + PhabricatorSystemActionEngine::willTakeAction( + array($viewer->getPHID()), + new PhabricatorFilesOutboundRequestAction(), + 1); + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + $file = PhabricatorFile::newFromFileDownload( + $uri, + array( + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + 'canCDN' => true, + )); + if (!$file->isViewableInBrowser()) { + $mime_type = $file->getMimeType(); + $engine = new PhabricatorDestructionEngine(); + $engine->destroyObject($file); + $file = null; + throw new Exception( + pht( + 'The URI "%s" does not correspond to a valid image file, got '. + 'a file with MIME type "%s". You must specify the URI of a '. + 'valid image file.', + $uri, + $mime_type)); + } else { + $file + ->setAuthorPHID($viewer->getPHID()) + ->save(); + } + + // Not sure what a reasonable ttl would be here. + $ttl = PhabricatorTime::getNow() + phutil_units('7 days in seconds'); + id(new PhabricatorFileExternalRequest()) + ->setURI($img_uri) + ->setTTL($ttl) + ->setIsSuccessful(true) + ->setFilePHID($file->getPHID()) + ->save(); + unset($unguarded); + return id(new AphrontRedirectResponse()) + ->setIsExternal(true) + ->setURI($file->getViewURI()); + } catch (HTTPFutureHTTPResponseStatus $status) { + $ttl = PhabricatorTime::getNow() + phutil_units('7 days in seconds'); + id(new PhabricatorFileExternalRequest()) + ->setURI($img_uri) + ->setTTL($ttl) + ->setIsSuccessful(false) + ->save(); + throw new Exception(pht( + 'The URI "%s" could not be loaded, got %s error.', + $uri, + $status->getStatusCode())); + } catch (Exception $ex) { + throw new Exception(pht( + "The URI '%s' could not be loaded due to the following error:\n %s", + $uri, + $ex->getMessage())); + } + } +} diff --git a/src/applications/files/garbagecollector/PhabricatorFileTempGarbageCollector.php b/src/applications/files/garbagecollector/PhabricatorFileTempGarbageCollector.php new file mode 100644 --- /dev/null +++ b/src/applications/files/garbagecollector/PhabricatorFileTempGarbageCollector.php @@ -0,0 +1,28 @@ +loadAllWhere( + 'ttl < %d LIMIT 100', + PhabricatorTime::getNow()); + $engine = new PhabricatorDestructionEngine(); + foreach ($file_requests as $request) { + $engine->destroyObject($request); + } + + return (count($file_requests) == 100); + } + +} diff --git a/src/applications/files/storage/PhabricatorFileExternalRequest.php b/src/applications/files/storage/PhabricatorFileExternalRequest.php new file mode 100644 --- /dev/null +++ b/src/applications/files/storage/PhabricatorFileExternalRequest.php @@ -0,0 +1,55 @@ + array( + 'uri' => 'text', + 'uriIndex' => 'bytes12', + 'ttl' => 'epoch', + 'filePHID' => 'phid?', + 'isSuccessful' => 'bool', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_uriindex' => array( + 'columns' => array('uriIndex'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + public function save() { + $hash = PhabricatorHash::digestForIndex($this->getURI()); + $this->setURIIndex($hash); + return parent::save(); + } + +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + + $file_phid = $this->getFilePHID(); + if ($file_phid) { + $file = id(new PhabricatorFileQuery()) + ->setViewer($engine->getViewer()) + ->withPHIDs(array($file_phid)) + ->executeOne(); + if ($file) { + $engine->destroyObject($file); + } + } + $this->delete(); + } + +}