/**
* Class for an OpenStack Swift based file backend.
*
- * This requires that the php-cloudfiles library is present,
- * which is available at https://github.com/rackspace/php-cloudfiles.
- * All of the library classes must be registed in $wgAutoloadClasses.
+ * This requires the SwiftCloudFiles MediaWiki extension, which includes
+ * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
+ * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
*
* Status messages should avoid mentioning the Swift account name
* Likewise, error suppression should be used to avoid path disclosure.
class SwiftFileBackend extends FileBackend {
/** @var CF_Authentication */
protected $auth; // Swift authentication handler
+ protected $authTTL; // integer seconds
+ protected $swiftAnonUser; // string; username to handle unauthenticated requests
+ protected $maxContCacheSize = 20; // integer; max containers with entries
/** @var CF_Connection */
protected $conn; // Swift connection handle
protected $connStarted = 0; // integer UNIX timestamp
protected $connContainers = array(); // container object cache
- protected $connTTL = 120; // integer seconds
-
- protected $swiftProxyUser; // string
/**
* @see FileBackend::__construct()
* Additional $config params include:
* swiftAuthUrl : Swift authentication server URL
- * swiftUser : Swift user used by MediaWiki
+ * swiftUser : Swift user used by MediaWiki (account:username)
* swiftKey : Swift authentication key for the above user
- * swiftProxyUser : Swift user used for end-user hits to proxy server
+ * swiftAuthTTL : Swift authentication TTL (seconds)
+ * swiftAnonUser : Swift user used for end-user requests (account:username)
* shardViaHashLevels : Map of container names to the number of hash levels
*/
public function __construct( array $config ) {
parent::__construct( $config );
// Required settings
$this->auth = new CF_Authentication(
- $config['swiftUser'], $config['swiftKey'], null, $config['swiftAuthUrl'] );
+ $config['swiftUser'],
+ $config['swiftKey'],
+ null, // account; unused
+ $config['swiftAuthUrl']
+ );
// Optional settings
- $this->connTTL = isset( $config['connTTL'] )
- ? $config['connTTL']
- : 60; // some sane number
- $this->swiftProxyUser = isset( $config['swiftProxyUser'] )
- ? $config['swiftProxyUser']
+ $this->authTTL = isset( $config['swiftAuthTTL'] )
+ ? $config['swiftAuthTTL']
+ : 120; // some sane number
+ $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
+ ? $config['swiftAnonUser']
: '';
$this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
? $config['shardViaHashLevels']
return $relStoragePath;
}
+ /**
+ * @see FileBackend::isPathUsableInternal()
+ */
+ public function isPathUsableInternal( $storagePath ) {
+ list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
+ if ( $rel === null ) {
+ return false; // invalid
+ }
+
+ try {
+ $this->getContainer( $container );
+ return true; // container exists
+ } catch ( NoSuchContainerException $e ) {
+ } catch ( InvalidResponseException $e ) {
+ } catch ( Exception $e ) { // some other exception?
+ $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
+ }
+
+ return false;
+ }
+
/**
* @see FileBackend::doCopyInternal()
*/
protected function doCreateInternal( array $params ) {
$status = Status::newGood();
- list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] );
- if ( $destRel === null ) {
+ list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
+ if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
return $status;
}
- // (a) Check the destination container
+ // (a) Check the destination container and object
try {
$dContObj = $this->getContainer( $dstCont );
+ if ( empty( $params['overwrite'] ) ) {
+ $destObj = $dContObj->create_object( $dstRel );
+ // Check if the object already exists (fields populated)
+ if ( $destObj->last_modified ) {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ }
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
return $status;
return $status;
}
- // (b) Check if the destination object already exists
- try {
- $dContObj->get_object( $destRel ); // throws NoSuchObjectException
- // NoSuchObjectException not thrown: file must exist
- if ( empty( $params['overwriteDest'] ) ) {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
- } catch ( NoSuchObjectException $e ) {
- // NoSuchObjectException thrown: file does not exist
- } catch ( InvalidResponseException $e ) {
- $status->fatal( 'backend-fail-connect', $this->name );
- return $status;
- } catch ( Exception $e ) { // some other exception?
- $status->fatal( 'backend-fail-internal', $this->name );
- $this->logException( $e, __METHOD__, $params );
- return $status;
- }
-
- // (c) Get a SHA-1 hash of the object
+ // (b) Get a SHA-1 hash of the object
$sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
- // (d) Actually create the object
+ // (c) Actually create the object
try {
- $obj = $dContObj->create_object( $destRel );
+ // Create a fresh CF_Object with no fields preloaded.
+ // We don't want to preserve headers, metadata, and such.
+ $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
// Note: metadata keys stored as [Upper case char][[Lower case char]...]
$obj->metadata = array( 'Sha1base36' => $sha1Hash );
$obj->write( $params['content'] );
protected function doStoreInternal( array $params ) {
$status = Status::newGood();
- list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] );
- if ( $destRel === null ) {
+ list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
+ if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
return $status;
}
- // (a) Check the destination container
+ // (a) Check the destination container and object
try {
$dContObj = $this->getContainer( $dstCont );
+ if ( empty( $params['overwrite'] ) ) {
+ $destObj = $dContObj->create_object( $dstRel );
+ // Check if the object already exists (fields populated)
+ if ( $destObj->last_modified ) {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ }
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
return $status;
return $status;
}
- // (b) Check if the destination object already exists
- try {
- $dContObj->get_object( $destRel ); // throws NoSuchObjectException
- // NoSuchObjectException not thrown: file must exist
- if ( empty( $params['overwriteDest'] ) ) {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
- } catch ( NoSuchObjectException $e ) {
- // NoSuchObjectException thrown: file does not exist
- } catch ( InvalidResponseException $e ) {
- $status->fatal( 'backend-fail-connect', $this->name );
- return $status;
- } catch ( Exception $e ) { // some other exception?
- $status->fatal( 'backend-fail-internal', $this->name );
- $this->logException( $e, __METHOD__, $params );
- return $status;
- }
-
- // (c) Get a SHA-1 hash of the object
+ // (b) Get a SHA-1 hash of the object
$sha1Hash = sha1_file( $params['src'] );
if ( $sha1Hash === false ) { // source doesn't exist?
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
}
$sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
- // (d) Actually store the object
+ // (c) Actually store the object
try {
- $obj = $dContObj->create_object( $destRel );
+ // Create a fresh CF_Object with no fields preloaded.
+ // We don't want to preserve headers, metadata, and such.
+ $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
// Note: metadata keys stored as [Upper case char][[Lower case char]...]
$obj->metadata = array( 'Sha1base36' => $sha1Hash );
$obj->load_from_filename( $params['src'], True ); // calls $obj->write()
return $status;
}
- list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] );
- if ( $destRel === null ) {
+ list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
+ if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
return $status;
}
- // (a) Check the source and destination containers
+ // (a) Check the source/destination containers and destination object
try {
$sContObj = $this->getContainer( $srcCont );
$dContObj = $this->getContainer( $dstCont );
+ if ( empty( $params['overwrite'] ) ) {
+ $destObj = $dContObj->create_object( $dstRel );
+ // Check if the object already exists (fields populated)
+ if ( $destObj->last_modified ) {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ }
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
return $status;
return $status;
}
- // (b) Check if the destination object already exists
- try {
- $dContObj->get_object( $destRel ); // throws NoSuchObjectException
- // NoSuchObjectException not thrown: file must exist
- if ( empty( $params['overwriteDest'] ) ) {
- $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
- return $status;
- }
- } catch ( NoSuchObjectException $e ) {
- // NoSuchObjectException thrown: file does not exist
- } catch ( InvalidResponseException $e ) {
- $status->fatal( 'backend-fail-connect', $this->name );
- return $status;
- } catch ( Exception $e ) { // some other exception?
- $status->fatal( 'backend-fail-internal', $this->name );
- $this->logException( $e, __METHOD__, $params );
- return $status;
- }
-
- // (c) Actually copy the file to the destination
+ // (b) Actually copy the file to the destination
try {
- $sContObj->copy_object_to( $srcRel, $dContObj, $destRel );
+ $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
} catch ( NoSuchObjectException $e ) { // source object does not exist
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
} catch ( InvalidResponseException $e ) {
return $status;
}
- // (a) Check the source container
try {
$sContObj = $this->getContainer( $srcCont );
+ $sContObj->delete_object( $srcRel );
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
- return $status;
- } catch ( InvalidResponseException $e ) {
- $status->fatal( 'backend-fail-connect', $this->name );
- return $status;
- } catch ( Exception $e ) { // some other exception?
- $status->fatal( 'backend-fail-internal', $this->name );
- $this->logException( $e, __METHOD__, $params );
- return $status;
- }
-
- // (b) Actually delete the object
- try {
- $sContObj->delete_object( $srcRel );
} catch ( NoSuchObjectException $e ) {
if ( empty( $params['ignoreMissingSource'] ) ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
protected function doPrepareInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
+ // (a) Check if container already exists
try {
- $this->createContainer( $fullCont );
+ $contObj = $this->getContainer( $fullCont );
+ // NoSuchContainerException not thrown: container must exist
+ return $status; // already exists
+ } catch ( NoSuchContainerException $e ) {
+ // NoSuchContainerException thrown: container does not exist
} catch ( InvalidResponseException $e ) {
$status->fatal( 'backend-fail-connect', $this->name );
+ return $status;
} catch ( Exception $e ) { // some other exception?
$status->fatal( 'backend-fail-internal', $this->name );
$this->logException( $e, __METHOD__, $params );
+ return $status;
+ }
+
+ // (b) Create container as needed
+ try {
+ $contObj = $this->createContainer( $fullCont );
+ if ( $this->swiftAnonUser != '' ) {
+ // Make container public to end-users...
+ $status->merge( $this->setContainerAccess(
+ $contObj,
+ array( $this->auth->username, $this->swiftAnonUser ), // read
+ array( $this->auth->username ) // write
+ ) );
+ }
+ } catch ( InvalidResponseException $e ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
+ return $status;
+ } catch ( Exception $e ) { // some other exception?
+ $status->fatal( 'backend-fail-internal', $this->name );
+ $this->logException( $e, __METHOD__, $params );
+ return $status;
}
return $status;
*/
protected function doSecureInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
- // @TODO: restrict container from $this->swiftProxyUser
+
+ if ( $this->swiftAnonUser != '' ) {
+ // Restrict container from end-users...
+ try {
+ // doPrepareInternal() should have been called,
+ // so the Swift container should already exist...
+ $contObj = $this->getContainer( $fullCont ); // normally a cache hit
+ // NoSuchContainerException not thrown: container must exist
+ if ( !isset( $contObj->mw_wasSecured ) ) {
+ $status->merge( $this->setContainerAccess(
+ $contObj,
+ array( $this->auth->username ), // read
+ array( $this->auth->username ) // write
+ ) );
+ // @TODO: when php-cloudfiles supports container
+ // metadata, we can make use of that to avoid RTTs
+ $contObj->mw_wasSecured = true; // avoid useless RTTs
+ }
+ } catch ( InvalidResponseException $e ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
+ } catch ( Exception $e ) { // some other exception?
+ $status->fatal( 'backend-fail-internal', $this->name );
+ $this->logException( $e, __METHOD__, $params );
+ }
+ }
+
return $status;
}
protected function doCleanInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
+ // Only containers themselves can be removed, all else is virtual
+ if ( $dir != '' ) {
+ return $status; // nothing to do
+ }
+
// (a) Check the container
try {
$contObj = $this->getContainer( $fullCont, true );
return $status;
}
- // (c) Delete the container if empty
+ // (b) Delete the container if empty
if ( $contObj->object_count == 0 ) {
try {
$this->deleteContainer( $fullCont );
$stat = false;
try {
- $container = $this->getContainer( $srcCont );
- // @TODO: handle 'latest' param as "X-Newest: true"
- $obj = $container->get_object( $srcRel );
- // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
- $date = DateTime::createFromFormat( 'D, d F Y G:i:s e', $obj->last_modified );
- if ( $date ) {
- $stat = array(
- 'mtime' => $date->format( 'YmdHis' ),
- 'size' => $obj->content_length,
- 'sha1' => $obj->metadata['Sha1base36']
- );
- } else { // exception will be caught below
- throw new Exception( "Could not parse date for object {$srcRel}" );
- }
+ $contObj = $this->getContainer( $srcCont );
+ $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
+ $stat = array(
+ // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
+ 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
+ 'size' => $srcObj->content_length,
+ 'sha1' => $srcObj->metadata['Sha1base36']
+ );
} catch ( NoSuchContainerException $e ) {
} catch ( NoSuchObjectException $e ) {
} catch ( InvalidResponseException $e ) {
*/
public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) {
$files = array();
+
try {
$container = $this->getContainer( $fullCont );
- $files = $container->list_objects( $limit, $after, "{$dir}/" );
+ $prefix = ( $dir == '' ) ? null : "{$dir}/";
+ $files = $container->list_objects( $limit, $after, $prefix );
} catch ( NoSuchContainerException $e ) {
} catch ( NoSuchObjectException $e ) {
} catch ( InvalidResponseException $e ) {
} catch ( Exception $e ) { // some other exception?
- $this->logException( $e, __METHOD__, $params );
+ $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
}
return $files;
try {
$cont = $this->getContainer( $srcCont );
- $obj = $cont->get_object( $srcRel );
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-stream', $params['src'] );
return $status;
- } catch ( NoSuchObjectException $e ) {
- $status->fatal( 'backend-fail-stream', $params['src'] );
- return $status;
- } catch ( IOException $e ) {
- $status->fatal( 'backend-fail-stream', $params['src'] );
+ } catch ( InvalidResponseException $e ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
return $status;
} catch ( Exception $e ) { // some other exception?
$status->fatal( 'backend-fail-stream', $params['src'] );
try {
$output = fopen( 'php://output', 'w' );
+ // FileBackend::streamFile() already checks existence
+ $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
$obj->stream( $output, $this->headersFromParams( $params ) );
- } catch ( InvalidResponseException $e ) {
- $status->fatal( 'backend-fail-connect', $this->name );
+ } catch ( InvalidResponseException $e ) { // 404? connection problem?
+ $status->fatal( 'backend-fail-stream', $params['src'] );
} catch ( Exception $e ) { // some other exception?
$status->fatal( 'backend-fail-stream', $params['src'] );
$this->logException( $e, __METHOD__, $params );
return null;
}
- // Get source file extension
- $ext = FileBackend::extensionFromPath( $srcRel );
- // Create a new temporary file...
- $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
- if ( !$tmpFile ) {
- return null;
- }
-
+ $tmpFile = null;
try {
$cont = $this->getContainer( $srcCont );
$obj = $cont->get_object( $srcRel );
- $handle = fopen( $tmpFile->getPath(), 'w' );
- if ( $handle ) {
- $obj->stream( $handle, $this->headersFromParams( $params ) );
- fclose( $handle );
- } else {
- $tmpFile = null; // couldn't open temp file
+ // Get source file extension
+ $ext = FileBackend::extensionFromPath( $srcRel );
+ // Create a new temporary file...
+ $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
+ if ( $tmpFile ) {
+ $handle = fopen( $tmpFile->getPath(), 'w' );
+ if ( $handle ) {
+ $obj->stream( $handle, $this->headersFromParams( $params ) );
+ fclose( $handle );
+ } else {
+ $tmpFile = null; // couldn't open temp file
+ }
}
} catch ( NoSuchContainerException $e ) {
$tmpFile = null;
return $hdrs;
}
+ /**
+ * Set read/write permissions for a Swift container
+ *
+ * @param $contObj CF_Container Swift container
+ * @param $readGrps Array Swift users who can read (account:user)
+ * @param $writeGrps Array Swift users who can write (account:user)
+ * @return Status
+ */
+ protected function setContainerAccess(
+ CF_Container $contObj, array $readGrps, array $writeGrps
+ ) {
+ $creds = $contObj->cfs_auth->export_credentials();
+
+ $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
+
+ // Note: 10 second timeout consistent with php-cloudfiles
+ $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
+ $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
+ $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
+ $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
+
+ return $req->execute(); // should return 204
+ }
+
/**
* Get a connection to the Swift proxy
*
*/
protected function getConnection() {
if ( $this->conn === false ) {
- return false; // failed last attempt
+ throw new InvalidResponseException; // failed last attempt
+ }
+ // Session keys expire after a while, so we renew them periodically
+ if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
+ $this->conn->close(); // close active cURL connections
+ $this->conn = null;
}
- // Authenticate with proxy and get a session key.
- // Session keys expire after a while, so we renew them periodically.
- if ( $this->conn === null || ( time() - $this->connStarted ) > $this->connTTL ) {
+ // Authenticate with proxy and get a session key...
+ if ( $this->conn === null ) {
$this->connContainers = array();
try {
$this->auth->authenticate();
return $this->conn;
}
+ /**
+ * @see FileBackend::doClearCache()
+ */
+ protected function doClearCache( array $paths = null ) {
+ $this->connContainers = array(); // clear container object cache
+ }
+
/**
* Get a Swift container object, possibly from process cache.
* Use $reCache if the file count or byte count is needed.
}
if ( !isset( $this->connContainers[$container] ) ) {
$contObj = $conn->get_container( $container );
- // Exception not thrown: container must exist
+ // NoSuchContainerException not thrown: container must exist
+ if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
+ reset( $this->connContainers );
+ $key = key( $this->connContainers );
+ unset( $this->connContainers[$key] );
+ }
$this->connContainers[$container] = $contObj; // cache it
}
return $this->connContainers[$container];
if ( substr( $this->dir, -1 ) === '/' ) {
$this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
}
- $this->suffixStart = strlen( $dir ) + 1; // size of "path/to/dir/"
+ if ( $this->dir == '' ) { // whole container
+ $this->suffixStart = 0;
+ } else { // dir within container
+ $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
+ }
}
public function current() {
dépôts / lhc / web / wiklou.git / blobdiff