includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftTempUrlKey; // string; shared secret value for making temp urls
  44         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  45         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  46         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  47         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  48
  49         // Rados Gateway specific options
  50         protected $rgwS3AccessKey; // string; S3 access key
  51         protected $rgwS3SecretKey; // string; S3 authentication key
  52
  53         /** @var CF_Connection */
  54         protected $conn; // Swift connection handle
  55         protected $sessionStarted = 0; // integer UNIX timestamp
  56
  57         /** @var CloudFilesException */
  58         protected $connException;
  59         protected $connErrorTime = 0; // UNIX timestamp
  60
  61         /** @var BagOStuff */
  62         protected $srvCache;
  63
  64         /** @var ProcessCacheLRU */
  65         protected $connContainerCache; // container object cache
  66
  67         /**
  68          * @see FileBackendStore::__construct()
  69          * Additional $config params include:
  70          *   - swiftAuthUrl       : Swift authentication server URL
  71          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  72          *   - swiftKey           : Swift authentication key for the above user
  73          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  74          *   - swiftTempUrlKey    : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
  75          *                          Do not set this until it has been set in the backend.
  76          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  77          *                          If set, then views of public containers are assumed to go
  78          *                          through this user. If not set, then public containers are
  79          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  80          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  81          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  82          *                          If files may likely change, this should probably not exceed
  83          *                          a few days. For example, deletions may take this long to apply.
  84          *                          If object purging is enabled, however, this is not an issue.
  85          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  86          *   - shardViaHashLevels : Map of container names to sharding config with:
  87          *                             - base   : base of hash characters, 16 or 36
  88          *                             - levels : the number of hash levels (and digits)
  89          *                             - repeat : hash subdirectories are prefixed with all the
  90          *                                        parent hash directory names (e.g. "a/ab/abc")
  91          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC, XCache, ect.
  92          *                          If those are not available, then the main cache will be used.
  93          *                          This is probably insecure in shared hosting environments.
  94          *   - rgwS3AccessKey     : Ragos Gateway S3 "access key" value on the account.
  95          *                          Do not set this until it has been set in the backend.
  96          *                          This is used for generating expiring pre-authenticated URLs.
  97          *                          Only use this when using rgw and to work around
  98          *                          http://tracker.newdream.net/issues/3454.
  99          *   - rgwS3SecretKey     : Ragos Gateway S3 "secret key" value on the account.
 100          *                          Do not set this until it has been set in the backend.
 101          *                          This is used for generating expiring pre-authenticated URLs.
 102          *                          Only use this when using rgw and to work around
 103          *                          http://tracker.newdream.net/issues/3454.
 104          */
 105         public function __construct( array $config ) {
 106                 parent::__construct( $config );
 107                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
 108                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
 109                 }
 110                 // Required settings
 111                 $this->auth = new CF_Authentication(
 112                         $config['swiftUser'],
 113                         $config['swiftKey'],
 114                         null, // account; unused
 115                         $config['swiftAuthUrl']
 116                 );
 117                 // Optional settings
 118                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 119                         ? $config['swiftAuthTTL']
 120                         : 5 * 60; // some sane number
 121                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 122                         ? $config['swiftAnonUser']
 123                         : '';
 124                 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
 125                         ? $config['swiftTempUrlKey']
 126                         : '';
 127                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 128                         ? $config['shardViaHashLevels']
 129                         : '';
 130                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 131                         ? $config['swiftUseCDN']
 132                         : false;
 133                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 134                         ? $config['swiftCDNExpiry']
 135                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 136                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 137                         ? $config['swiftCDNPurgable']
 138                         : true;
 139                 $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
 140                         ? $config['rgwS3AccessKey']
 141                         : '';
 142                 $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
 143                         ? $config['rgwS3SecretKey']
 144                         : '';
 145                 // Cache container information to mask latency
 146                 $this->memCache = wfGetMainCache();
 147                 // Process cache for container info
 148                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 149                 // Cache auth token information to avoid RTTs
 150                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 151                         if ( php_sapi_name() === 'cli' ) {
 152                                 $this->srvCache = wfGetMainCache(); // preferrably memcached
 153                         } else {
 154                                 try { // look for APC, XCache, WinCache, ect...
 155                                         $this->srvCache = ObjectCache::newAccelerator( array() );
 156                                 } catch ( Exception $e ) {}
 157                         }
 158                 }
 159                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 160         }
 161
 162         /**
 163          * @see FileBackendStore::resolveContainerPath()
 164          * @return null
 165          */
 166         protected function resolveContainerPath( $container, $relStoragePath ) {
 167                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 168                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 169                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 170                         return null; // too long for Swift
 171                 }
 172                 return $relStoragePath;
 173         }
 174
 175         /**
 176          * @see FileBackendStore::isPathUsableInternal()
 177          * @return bool
 178          */
 179         public function isPathUsableInternal( $storagePath ) {
 180                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 181                 if ( $rel === null ) {
 182                         return false; // invalid
 183                 }
 184
 185                 try {
 186                         $this->getContainer( $container );
 187                         return true; // container exists
 188                 } catch ( NoSuchContainerException $e ) {
 189                 } catch ( CloudFilesException $e ) { // some other exception?
 190                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 191                 }
 192
 193                 return false;
 194         }
 195
 196         /**
 197          * @param $disposition string Content-Disposition header value
 198          * @return string Truncated Content-Disposition header value to meet Swift limits
 199          */
 200         protected function truncDisp( $disposition ) {
 201                 $res = '';
 202                 foreach ( explode( ';', $disposition ) as $part ) {
 203                         $part = trim( $part );
 204                         $new  = ( $res === '' ) ? $part : "{$res};{$part}";
 205                         if ( strlen( $new ) <= 255 ) {
 206                                 $res = $new;
 207                         } else {
 208                                 break; // too long; sigh
 209                         }
 210                 }
 211                 return $res;
 212         }
 213
 214         /**
 215          * @see FileBackendStore::doCreateInternal()
 216          * @return Status
 217          */
 218         protected function doCreateInternal( array $params ) {
 219                 $status = Status::newGood();
 220
 221                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 222                 if ( $dstRel === null ) {
 223                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 224                         return $status;
 225                 }
 226
 227                 // (a) Check the destination container and object
 228                 try {
 229                         $dContObj = $this->getContainer( $dstCont );
 230                 } catch ( NoSuchContainerException $e ) {
 231                         $status->fatal( 'backend-fail-create', $params['dst'] );
 232                         return $status;
 233                 } catch ( CloudFilesException $e ) { // some other exception?
 234                         $this->handleException( $e, $status, __METHOD__, $params );
 235                         return $status;
 236                 }
 237
 238                 // (b) Get a SHA-1 hash of the object
 239                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 240
 241                 // (c) Actually create the object
 242                 try {
 243                         // Create a fresh CF_Object with no fields preloaded.
 244                         // We don't want to preserve headers, metadata, and such.
 245                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 246                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 247                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 248                         // The MD5 here will be checked within Swift against its own MD5.
 249                         $obj->set_etag( md5( $params['content'] ) );
 250                         // Use the same content type as StreamFile for security
 251                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 252                         if ( !strlen( $obj->content_type ) ) { // special case
 253                                 $obj->content_type = 'unknown/unknown';
 254                         }
 255                         // Set the Content-Disposition header if requested
 256                         if ( isset( $params['disposition'] ) ) {
 257                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 258                         }
 259                         // Set any other custom headers if requested
 260                         if ( isset( $params['headers'] ) ) {
 261                                 $obj->headers += $params['headers'];
 262                         }
 263                         if ( !empty( $params['async'] ) ) { // deferred
 264                                 $op = $obj->write_async( $params['content'] );
 265                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
 266                                 $status->value->affectedObjects[] = $obj;
 267                         } else { // actually write the object in Swift
 268                                 $obj->write( $params['content'] );
 269                                 $this->purgeCDNCache( array( $obj ) );
 270                         }
 271                 } catch ( CDNNotEnabledException $e ) {
 272                         // CDN not enabled; nothing to see here
 273                 } catch ( BadContentTypeException $e ) {
 274                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 275                 } catch ( CloudFilesException $e ) { // some other exception?
 276                         $this->handleException( $e, $status, __METHOD__, $params );
 277                 }
 278
 279                 return $status;
 280         }
 281
 282         /**
 283          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 284          */
 285         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 286                 try {
 287                         $cfOp->getLastResponse();
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 }
 291         }
 292
 293         /**
 294          * @see FileBackendStore::doStoreInternal()
 295          * @return Status
 296          */
 297         protected function doStoreInternal( array $params ) {
 298                 $status = Status::newGood();
 299
 300                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 301                 if ( $dstRel === null ) {
 302                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 303                         return $status;
 304                 }
 305
 306                 // (a) Check the destination container and object
 307                 try {
 308                         $dContObj = $this->getContainer( $dstCont );
 309                 } catch ( NoSuchContainerException $e ) {
 310                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 311                         return $status;
 312                 } catch ( CloudFilesException $e ) { // some other exception?
 313                         $this->handleException( $e, $status, __METHOD__, $params );
 314                         return $status;
 315                 }
 316
 317                 // (b) Get a SHA-1 hash of the object
 318                 $sha1Hash = sha1_file( $params['src'] );
 319                 if ( $sha1Hash === false ) { // source doesn't exist?
 320                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 321                         return $status;
 322                 }
 323                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 324
 325                 // (c) Actually store the object
 326                 try {
 327                         // Create a fresh CF_Object with no fields preloaded.
 328                         // We don't want to preserve headers, metadata, and such.
 329                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 330                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 331                         // The MD5 here will be checked within Swift against its own MD5.
 332                         $obj->set_etag( md5_file( $params['src'] ) );
 333                         // Use the same content type as StreamFile for security
 334                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 335                         if ( !strlen( $obj->content_type ) ) { // special case
 336                                 $obj->content_type = 'unknown/unknown';
 337                         }
 338                         // Set the Content-Disposition header if requested
 339                         if ( isset( $params['disposition'] ) ) {
 340                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 341                         }
 342                         // Set any other custom headers if requested
 343                         if ( isset( $params['headers'] ) ) {
 344                                 $obj->headers += $params['headers'];
 345                         }
 346                         if ( !empty( $params['async'] ) ) { // deferred
 347                                 wfSuppressWarnings();
 348                                 $fp = fopen( $params['src'], 'rb' );
 349                                 wfRestoreWarnings();
 350                                 if ( !$fp ) {
 351                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 352                                 } else {
 353                                         $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
 354                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
 355                                         $status->value->resourcesToClose[] = $fp;
 356                                         $status->value->affectedObjects[] = $obj;
 357                                 }
 358                         } else { // actually write the object in Swift
 359                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 360                                 $this->purgeCDNCache( array( $obj ) );
 361                         }
 362                 } catch ( CDNNotEnabledException $e ) {
 363                         // CDN not enabled; nothing to see here
 364                 } catch ( BadContentTypeException $e ) {
 365                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 366                 } catch ( IOException $e ) {
 367                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 368                 } catch ( CloudFilesException $e ) { // some other exception?
 369                         $this->handleException( $e, $status, __METHOD__, $params );
 370                 }
 371
 372                 return $status;
 373         }
 374
 375         /**
 376          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 377          */
 378         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 379                 try {
 380                         $cfOp->getLastResponse();
 381                 } catch ( BadContentTypeException $e ) {
 382                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 383                 } catch ( IOException $e ) {
 384                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 385                 }
 386         }
 387
 388         /**
 389          * @see FileBackendStore::doCopyInternal()
 390          * @return Status
 391          */
 392         protected function doCopyInternal( array $params ) {
 393                 $status = Status::newGood();
 394
 395                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 396                 if ( $srcRel === null ) {
 397                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 398                         return $status;
 399                 }
 400
 401                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 402                 if ( $dstRel === null ) {
 403                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 404                         return $status;
 405                 }
 406
 407                 // (a) Check the source/destination containers and destination object
 408                 try {
 409                         $sContObj = $this->getContainer( $srcCont );
 410                         $dContObj = $this->getContainer( $dstCont );
 411                 } catch ( NoSuchContainerException $e ) {
 412                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 413                         return $status;
 414                 } catch ( CloudFilesException $e ) { // some other exception?
 415                         $this->handleException( $e, $status, __METHOD__, $params );
 416                         return $status;
 417                 }
 418
 419                 // (b) Actually copy the file to the destination
 420                 try {
 421                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 422                         $hdrs = array(); // source file headers to override with new values
 423                         if ( isset( $params['disposition'] ) ) {
 424                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 425                         }
 426                         if ( !empty( $params['async'] ) ) { // deferred
 427                                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 428                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
 429                                 $status->value->affectedObjects[] = $dstObj;
 430                         } else { // actually write the object in Swift
 431                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 432                                 $this->purgeCDNCache( array( $dstObj ) );
 433                         }
 434                 } catch ( CDNNotEnabledException $e ) {
 435                         // CDN not enabled; nothing to see here
 436                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 437                         if ( empty( $params['ignoreMissingSource'] ) ) {
 438                                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 439                         }
 440                 } catch ( CloudFilesException $e ) { // some other exception?
 441                         $this->handleException( $e, $status, __METHOD__, $params );
 442                 }
 443
 444                 return $status;
 445         }
 446
 447         /**
 448          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 449          */
 450         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 451                 try {
 452                         $cfOp->getLastResponse();
 453                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 454                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 455                 }
 456         }
 457
 458         /**
 459          * @see FileBackendStore::doMoveInternal()
 460          * @return Status
 461          */
 462         protected function doMoveInternal( array $params ) {
 463                 $status = Status::newGood();
 464
 465                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 466                 if ( $srcRel === null ) {
 467                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 468                         return $status;
 469                 }
 470
 471                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 472                 if ( $dstRel === null ) {
 473                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 474                         return $status;
 475                 }
 476
 477                 // (a) Check the source/destination containers and destination object
 478                 try {
 479                         $sContObj = $this->getContainer( $srcCont );
 480                         $dContObj = $this->getContainer( $dstCont );
 481                 } catch ( NoSuchContainerException $e ) {
 482                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 483                         return $status;
 484                 } catch ( CloudFilesException $e ) { // some other exception?
 485                         $this->handleException( $e, $status, __METHOD__, $params );
 486                         return $status;
 487                 }
 488
 489                 // (b) Actually move the file to the destination
 490                 try {
 491                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 492                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 493                         $hdrs = array(); // source file headers to override with new values
 494                         if ( isset( $params['disposition'] ) ) {
 495                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 496                         }
 497                         if ( !empty( $params['async'] ) ) { // deferred
 498                                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 499                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
 500                                 $status->value->affectedObjects[] = $srcObj;
 501                                 $status->value->affectedObjects[] = $dstObj;
 502                         } else { // actually write the object in Swift
 503                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 504                                 $this->purgeCDNCache( array( $srcObj ) );
 505                                 $this->purgeCDNCache( array( $dstObj ) );
 506                         }
 507                 } catch ( CDNNotEnabledException $e ) {
 508                         // CDN not enabled; nothing to see here
 509                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 510                         if ( empty( $params['ignoreMissingSource'] ) ) {
 511                                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 512                         }
 513                 } catch ( CloudFilesException $e ) { // some other exception?
 514                         $this->handleException( $e, $status, __METHOD__, $params );
 515                 }
 516
 517                 return $status;
 518         }
 519
 520         /**
 521          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 522          */
 523         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 524                 try {
 525                         $cfOp->getLastResponse();
 526                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 527                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 528                 }
 529         }
 530
 531         /**
 532          * @see FileBackendStore::doDeleteInternal()
 533          * @return Status
 534          */
 535         protected function doDeleteInternal( array $params ) {
 536                 $status = Status::newGood();
 537
 538                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 539                 if ( $srcRel === null ) {
 540                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 541                         return $status;
 542                 }
 543
 544                 try {
 545                         $sContObj = $this->getContainer( $srcCont );
 546                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 547                         if ( !empty( $params['async'] ) ) { // deferred
 548                                 $op = $sContObj->delete_object_async( $srcRel );
 549                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
 550                                 $status->value->affectedObjects[] = $srcObj;
 551                         } else { // actually write the object in Swift
 552                                 $sContObj->delete_object( $srcRel );
 553                                 $this->purgeCDNCache( array( $srcObj ) );
 554                         }
 555                 } catch ( CDNNotEnabledException $e ) {
 556                         // CDN not enabled; nothing to see here
 557                 } catch ( NoSuchContainerException $e ) {
 558                         $status->fatal( 'backend-fail-delete', $params['src'] );
 559                 } catch ( NoSuchObjectException $e ) {
 560                         if ( empty( $params['ignoreMissingSource'] ) ) {
 561                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 562                         }
 563                 } catch ( CloudFilesException $e ) { // some other exception?
 564                         $this->handleException( $e, $status, __METHOD__, $params );
 565                 }
 566
 567                 return $status;
 568         }
 569
 570         /**
 571          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 572          */
 573         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 574                 try {
 575                         $cfOp->getLastResponse();
 576                 } catch ( NoSuchContainerException $e ) {
 577                         $status->fatal( 'backend-fail-delete', $params['src'] );
 578                 } catch ( NoSuchObjectException $e ) {
 579                         if ( empty( $params['ignoreMissingSource'] ) ) {
 580                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 581                         }
 582                 }
 583         }
 584
 585         /**
 586          * @see FileBackendStore::doDescribeInternal()
 587          * @return Status
 588          */
 589         protected function doDescribeInternal( array $params ) {
 590                 $status = Status::newGood();
 591
 592                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 593                 if ( $srcRel === null ) {
 594                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 595                         return $status;
 596                 }
 597
 598                 $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
 599                 // Set the Content-Disposition header if requested
 600                 if ( isset( $params['disposition'] ) ) {
 601                         $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 602                 }
 603
 604                 try {
 605                         $sContObj = $this->getContainer( $srcCont );
 606                         // Get the latest version of the current metadata
 607                         $srcObj = $sContObj->get_object( $srcRel,
 608                                 $this->headersFromParams( array( 'latest' => true ) ) );
 609                         // Merge in the metadata updates...
 610                         $srcObj->headers = $hdrs + $srcObj->headers;
 611                         $srcObj->sync_metadata(); // save to Swift
 612                         $this->purgeCDNCache( array( $srcObj ) );
 613                 } catch ( CDNNotEnabledException $e ) {
 614                         // CDN not enabled; nothing to see here
 615                 } catch ( NoSuchContainerException $e ) {
 616                         $status->fatal( 'backend-fail-describe', $params['src'] );
 617                 } catch ( NoSuchObjectException $e ) {
 618                         $status->fatal( 'backend-fail-describe', $params['src'] );
 619                 } catch ( CloudFilesException $e ) { // some other exception?
 620                         $this->handleException( $e, $status, __METHOD__, $params );
 621                 }
 622
 623                 return $status;
 624         }
 625
 626         /**
 627          * @see FileBackendStore::doPrepareInternal()
 628          * @return Status
 629          */
 630         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 631                 $status = Status::newGood();
 632
 633                 // (a) Check if container already exists
 634                 try {
 635                         $contObj = $this->getContainer( $fullCont );
 636                         // NoSuchContainerException not thrown: container must exist
 637                         return $status; // already exists
 638                 } catch ( NoSuchContainerException $e ) {
 639                         // NoSuchContainerException thrown: container does not exist
 640                 } catch ( CloudFilesException $e ) { // some other exception?
 641                         $this->handleException( $e, $status, __METHOD__, $params );
 642                         return $status;
 643                 }
 644
 645                 // (b) Create container as needed
 646                 try {
 647                         $contObj = $this->createContainer( $fullCont );
 648                         if ( !empty( $params['noAccess'] ) ) {
 649                                 // Make container private to end-users...
 650                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 651                         } else {
 652                                 // Make container public to end-users...
 653                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 654                         }
 655                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 656                                 $contObj->make_public( $this->swiftCDNExpiry );
 657                         }
 658                 } catch ( CDNNotEnabledException $e ) {
 659                         // CDN not enabled; nothing to see here
 660                 } catch ( CloudFilesException $e ) { // some other exception?
 661                         $this->handleException( $e, $status, __METHOD__, $params );
 662                         return $status;
 663                 }
 664
 665                 return $status;
 666         }
 667
 668         /**
 669          * @see FileBackendStore::doSecureInternal()
 670          * @return Status
 671          */
 672         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 673                 $status = Status::newGood();
 674                 if ( empty( $params['noAccess'] ) ) {
 675                         return $status; // nothing to do
 676                 }
 677
 678                 // Restrict container from end-users...
 679                 try {
 680                         // doPrepareInternal() should have been called,
 681                         // so the Swift container should already exist...
 682                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 683                         // NoSuchContainerException not thrown: container must exist
 684
 685                         // Make container private to end-users...
 686                         $status->merge( $this->setContainerAccess(
 687                                 $contObj,
 688                                 array( $this->auth->username ), // read
 689                                 array( $this->auth->username ) // write
 690                         ) );
 691                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 692                                 $contObj->make_private();
 693                         }
 694                 } catch ( CDNNotEnabledException $e ) {
 695                         // CDN not enabled; nothing to see here
 696                 } catch ( CloudFilesException $e ) { // some other exception?
 697                         $this->handleException( $e, $status, __METHOD__, $params );
 698                 }
 699
 700                 return $status;
 701         }
 702
 703         /**
 704          * @see FileBackendStore::doPublishInternal()
 705          * @return Status
 706          */
 707         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 708                 $status = Status::newGood();
 709
 710                 // Unrestrict container from end-users...
 711                 try {
 712                         // doPrepareInternal() should have been called,
 713                         // so the Swift container should already exist...
 714                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 715                         // NoSuchContainerException not thrown: container must exist
 716
 717                         // Make container public to end-users...
 718                         if ( $this->swiftAnonUser != '' ) {
 719                                 $status->merge( $this->setContainerAccess(
 720                                         $contObj,
 721                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 722                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 723                                 ) );
 724                         } else {
 725                                 $status->merge( $this->setContainerAccess(
 726                                         $contObj,
 727                                         array( $this->auth->username, '.r:*' ), // read
 728                                         array( $this->auth->username ) // write
 729                                 ) );
 730                         }
 731                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 732                                 $contObj->make_public();
 733                         }
 734                 } catch ( CDNNotEnabledException $e ) {
 735                         // CDN not enabled; nothing to see here
 736                 } catch ( CloudFilesException $e ) { // some other exception?
 737                         $this->handleException( $e, $status, __METHOD__, $params );
 738                 }
 739
 740                 return $status;
 741         }
 742
 743         /**
 744          * @see FileBackendStore::doCleanInternal()
 745          * @return Status
 746          */
 747         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 748                 $status = Status::newGood();
 749
 750                 // Only containers themselves can be removed, all else is virtual
 751                 if ( $dir != '' ) {
 752                         return $status; // nothing to do
 753                 }
 754
 755                 // (a) Check the container
 756                 try {
 757                         $contObj = $this->getContainer( $fullCont, true );
 758                 } catch ( NoSuchContainerException $e ) {
 759                         return $status; // ok, nothing to do
 760                 } catch ( CloudFilesException $e ) { // some other exception?
 761                         $this->handleException( $e, $status, __METHOD__, $params );
 762                         return $status;
 763                 }
 764
 765                 // (b) Delete the container if empty
 766                 if ( $contObj->object_count == 0 ) {
 767                         try {
 768                                 $this->deleteContainer( $fullCont );
 769                         } catch ( NoSuchContainerException $e ) {
 770                                 return $status; // race?
 771                         } catch ( NonEmptyContainerException $e ) {
 772                                 return $status; // race? consistency delay?
 773                         } catch ( CloudFilesException $e ) { // some other exception?
 774                                 $this->handleException( $e, $status, __METHOD__, $params );
 775                                 return $status;
 776                         }
 777                 }
 778
 779                 return $status;
 780         }
 781
 782         /**
 783          * @see FileBackendStore::doFileExists()
 784          * @return array|bool|null
 785          */
 786         protected function doGetFileStat( array $params ) {
 787                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 788                 if ( $srcRel === null ) {
 789                         return false; // invalid storage path
 790                 }
 791
 792                 $stat = false;
 793                 try {
 794                         $contObj = $this->getContainer( $srcCont );
 795                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 796                         $this->addMissingMetadata( $srcObj, $params['src'] );
 797                         $stat = array(
 798                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 799                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 800                                 'size'  => (int)$srcObj->content_length,
 801                                 'sha1'  => $srcObj->getMetadataValue( 'Sha1base36' )
 802                         );
 803                 } catch ( NoSuchContainerException $e ) {
 804                 } catch ( NoSuchObjectException $e ) {
 805                 } catch ( CloudFilesException $e ) { // some other exception?
 806                         $stat = null;
 807                         $this->handleException( $e, null, __METHOD__, $params );
 808                 }
 809
 810                 return $stat;
 811         }
 812
 813         /**
 814          * Fill in any missing object metadata and save it to Swift
 815          *
 816          * @param $obj CF_Object
 817          * @param $path string Storage path to object
 818          * @return bool Success
 819          * @throws Exception cloudfiles exceptions
 820          */
 821         protected function addMissingMetadata( CF_Object $obj, $path ) {
 822                 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
 823                         return true; // nothing to do
 824                 }
 825                 wfProfileIn( __METHOD__ );
 826                 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
 827                 $status = Status::newGood();
 828                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 829                 if ( $status->isOK() ) {
 830                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 831                         if ( $tmpFile ) {
 832                                 $hash = $tmpFile->getSha1Base36();
 833                                 if ( $hash !== false ) {
 834                                         $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
 835                                         $obj->sync_metadata(); // save to Swift
 836                                         wfProfileOut( __METHOD__ );
 837                                         return true; // success
 838                                 }
 839                         }
 840                 }
 841                 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
 842                 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
 843                 wfProfileOut( __METHOD__ );
 844                 return false; // failed
 845         }
 846
 847         /**
 848          * @see FileBackendStore::doGetFileContentsMulti()
 849          * @return Array
 850          */
 851         protected function doGetFileContentsMulti( array $params ) {
 852                 $contents = array();
 853
 854                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
 855                 // Blindly create tmp files and stream to them, catching any exception if the file does
 856                 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
 857                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
 858                         $cfOps = array(); // (path => CF_Async_Op)
 859
 860                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
 861                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
 862                                 if ( $srcRel === null ) {
 863                                         $contents[$path] = false;
 864                                         continue;
 865                                 }
 866                                 $data = false;
 867                                 try {
 868                                         $sContObj = $this->getContainer( $srcCont );
 869                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 870                                         // Get source file extension
 871                                         $ext = FileBackend::extensionFromPath( $path );
 872                                         // Create a new temporary memory file...
 873                                         $handle = fopen( 'php://temp', 'wb' );
 874                                         if ( $handle ) {
 875                                                 $headers = $this->headersFromParams( $params );
 876                                                 if ( count( $pathBatch ) > 1 ) {
 877                                                         $cfOps[$path] = $obj->stream_async( $handle, $headers );
 878                                                         $cfOps[$path]->_file_handle = $handle; // close this later
 879                                                 } else {
 880                                                         $obj->stream( $handle, $headers );
 881                                                         rewind( $handle ); // start from the beginning
 882                                                         $data = stream_get_contents( $handle );
 883                                                         fclose( $handle );
 884                                                 }
 885                                         } else {
 886                                                 $data = false;
 887                                         }
 888                                 } catch ( NoSuchContainerException $e ) {
 889                                         $data = false;
 890                                 } catch ( NoSuchObjectException $e ) {
 891                                         $data = false;
 892                                 } catch ( CloudFilesException $e ) { // some other exception?
 893                                         $data = false;
 894                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 895                                 }
 896                                 $contents[$path] = $data;
 897                         }
 898
 899                         $batch = new CF_Async_Op_Batch( $cfOps );
 900                         $cfOps = $batch->execute();
 901                         foreach ( $cfOps as $path => $cfOp ) {
 902                                 try {
 903                                         $cfOp->getLastResponse();
 904                                         rewind( $cfOp->_file_handle ); // start from the beginning
 905                                         $contents[$path] = stream_get_contents( $cfOp->_file_handle );
 906                                 } catch ( NoSuchContainerException $e ) {
 907                                         $contents[$path] = false;
 908                                 } catch ( NoSuchObjectException $e ) {
 909                                         $contents[$path] = false;
 910                                 } catch ( CloudFilesException $e ) { // some other exception?
 911                                         $contents[$path] = false;
 912                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 913                                 }
 914                                 fclose( $cfOp->_file_handle ); // close open handle
 915                         }
 916                 }
 917
 918                 return $contents;
 919         }
 920
 921         /**
 922          * @see FileBackendStore::doDirectoryExists()
 923          * @return bool|null
 924          */
 925         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 926                 try {
 927                         $container = $this->getContainer( $fullCont );
 928                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 929                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 930                 } catch ( NoSuchContainerException $e ) {
 931                         return false;
 932                 } catch ( CloudFilesException $e ) { // some other exception?
 933                         $this->handleException( $e, null, __METHOD__,
 934                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 935                 }
 936
 937                 return null; // error
 938         }
 939
 940         /**
 941          * @see FileBackendStore::getDirectoryListInternal()
 942          * @return SwiftFileBackendDirList
 943          */
 944         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 945                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 946         }
 947
 948         /**
 949          * @see FileBackendStore::getFileListInternal()
 950          * @return SwiftFileBackendFileList
 951          */
 952         public function getFileListInternal( $fullCont, $dir, array $params ) {
 953                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 954         }
 955
 956         /**
 957          * Do not call this function outside of SwiftFileBackendFileList
 958          *
 959          * @param $fullCont string Resolved container name
 960          * @param $dir string Resolved storage directory with no trailing slash
 961          * @param $after string|null Storage path of file to list items after
 962          * @param $limit integer Max number of items to list
 963          * @param $params Array Includes flag for 'topOnly'
 964          * @return Array List of relative paths of dirs directly under $dir
 965          */
 966         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 967                 $dirs = array();
 968                 if ( $after === INF ) {
 969                         return $dirs; // nothing more
 970                 }
 971                 wfProfileIn( __METHOD__ . '-' . $this->name );
 972
 973                 try {
 974                         $container = $this->getContainer( $fullCont );
 975                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 976                         // Non-recursive: only list dirs right under $dir
 977                         if ( !empty( $params['topOnly'] ) ) {
 978                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 979                                 foreach ( $objects as $object ) { // files and dirs
 980                                         if ( substr( $object, -1 ) === '/' ) {
 981                                                 $dirs[] = $object; // directories end in '/'
 982                                         }
 983                                 }
 984                         // Recursive: list all dirs under $dir and its subdirs
 985                         } else {
 986                                 // Get directory from last item of prior page
 987                                 $lastDir = $this->getParentDir( $after ); // must be first page
 988                                 $objects = $container->list_objects( $limit, $after, $prefix );
 989                                 foreach ( $objects as $object ) { // files
 990                                         $objectDir = $this->getParentDir( $object ); // directory of object
 991                                         if ( $objectDir !== false && $objectDir !== $dir ) {
 992                                                 // Swift stores paths in UTF-8, using binary sorting.
 993                                                 // See function "create_container_table" in common/db.py.
 994                                                 // If a directory is not "greater" than the last one,
 995                                                 // then it was already listed by the calling iterator.
 996                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
 997                                                         $pDir = $objectDir;
 998                                                         do { // add dir and all its parent dirs
 999                                                                 $dirs[] = "{$pDir}/";
1000                                                                 $pDir = $this->getParentDir( $pDir );
1001                                                         } while ( $pDir !== false // sanity
1002                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
1003                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
1004                                                         );
1005                                                 }
1006                                                 $lastDir = $objectDir;
1007                                         }
1008                                 }
1009                         }
1010                         if ( count( $objects ) < $limit ) {
1011                                 $after = INF; // avoid a second RTT
1012                         } else {
1013                                 $after = end( $objects ); // update last item
1014                         }
1015                 } catch ( NoSuchContainerException $e ) {
1016                 } catch ( CloudFilesException $e ) { // some other exception?
1017                         $this->handleException( $e, null, __METHOD__,
1018                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1019                 }
1020
1021                 wfProfileOut( __METHOD__ . '-' . $this->name );
1022                 return $dirs;
1023         }
1024
1025         protected function getParentDir( $path ) {
1026                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
1027         }
1028
1029         /**
1030          * Do not call this function outside of SwiftFileBackendFileList
1031          *
1032          * @param $fullCont string Resolved container name
1033          * @param $dir string Resolved storage directory with no trailing slash
1034          * @param $after string|null Storage path of file to list items after
1035          * @param $limit integer Max number of items to list
1036          * @param $params Array Includes flag for 'topOnly'
1037          * @return Array List of relative paths of files under $dir
1038          */
1039         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1040                 $files = array();
1041                 if ( $after === INF ) {
1042                         return $files; // nothing more
1043                 }
1044                 wfProfileIn( __METHOD__ . '-' . $this->name );
1045
1046                 try {
1047                         $container = $this->getContainer( $fullCont );
1048                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
1049                         // Non-recursive: only list files right under $dir
1050                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
1051                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
1052                                 foreach ( $objects as $object ) {
1053                                         if ( substr( $object, -1 ) !== '/' ) {
1054                                                 $files[] = $object; // directories end in '/'
1055                                         }
1056                                 }
1057                         // Recursive: list all files under $dir and its subdirs
1058                         } else { // files
1059                                 $objects = $container->list_objects( $limit, $after, $prefix );
1060                                 $files = $objects;
1061                         }
1062                         if ( count( $objects ) < $limit ) {
1063                                 $after = INF; // avoid a second RTT
1064                         } else {
1065                                 $after = end( $objects ); // update last item
1066                         }
1067                 } catch ( NoSuchContainerException $e ) {
1068                 } catch ( CloudFilesException $e ) { // some other exception?
1069                         $this->handleException( $e, null, __METHOD__,
1070                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1071                 }
1072
1073                 wfProfileOut( __METHOD__ . '-' . $this->name );
1074                 return $files;
1075         }
1076
1077         /**
1078          * @see FileBackendStore::doGetFileSha1base36()
1079          * @return bool
1080          */
1081         protected function doGetFileSha1base36( array $params ) {
1082                 $stat = $this->getFileStat( $params );
1083                 if ( $stat ) {
1084                         return $stat['sha1'];
1085                 } else {
1086                         return false;
1087                 }
1088         }
1089
1090         /**
1091          * @see FileBackendStore::doStreamFile()
1092          * @return Status
1093          */
1094         protected function doStreamFile( array $params ) {
1095                 $status = Status::newGood();
1096
1097                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1098                 if ( $srcRel === null ) {
1099                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1100                 }
1101
1102                 try {
1103                         $cont = $this->getContainer( $srcCont );
1104                 } catch ( NoSuchContainerException $e ) {
1105                         $status->fatal( 'backend-fail-stream', $params['src'] );
1106                         return $status;
1107                 } catch ( CloudFilesException $e ) { // some other exception?
1108                         $this->handleException( $e, $status, __METHOD__, $params );
1109                         return $status;
1110                 }
1111
1112                 try {
1113                         $output = fopen( 'php://output', 'wb' );
1114                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1115                         $obj->stream( $output, $this->headersFromParams( $params ) );
1116                 } catch ( NoSuchObjectException $e ) {
1117                         $status->fatal( 'backend-fail-stream', $params['src'] );
1118                 } catch ( CloudFilesException $e ) { // some other exception?
1119                         $this->handleException( $e, $status, __METHOD__, $params );
1120                 }
1121
1122                 return $status;
1123         }
1124
1125         /**
1126          * @see FileBackendStore::doGetLocalCopyMulti()
1127          * @return null|TempFSFile
1128          */
1129         protected function doGetLocalCopyMulti( array $params ) {
1130                 $tmpFiles = array();
1131
1132                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1133                 // Blindly create tmp files and stream to them, catching any exception if the file does
1134                 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1135                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1136                         $cfOps = array(); // (path => CF_Async_Op)
1137
1138                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1139                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1140                                 if ( $srcRel === null ) {
1141                                         $tmpFiles[$path] = null;
1142                                         continue;
1143                                 }
1144                                 $tmpFile = null;
1145                                 try {
1146                                         $sContObj = $this->getContainer( $srcCont );
1147                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1148                                         // Get source file extension
1149                                         $ext = FileBackend::extensionFromPath( $path );
1150                                         // Create a new temporary file...
1151                                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1152                                         if ( $tmpFile ) {
1153                                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1154                                                 if ( $handle ) {
1155                                                         $headers = $this->headersFromParams( $params );
1156                                                         if ( count( $pathBatch ) > 1 ) {
1157                                                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1158                                                                 $cfOps[$path]->_file_handle = $handle; // close this later
1159                                                         } else {
1160                                                                 $obj->stream( $handle, $headers );
1161                                                                 fclose( $handle );
1162                                                         }
1163                                                 } else {
1164                                                         $tmpFile = null;
1165                                                 }
1166                                         }
1167                                 } catch ( NoSuchContainerException $e ) {
1168                                         $tmpFile = null;
1169                                 } catch ( NoSuchObjectException $e ) {
1170                                         $tmpFile = null;
1171                                 } catch ( CloudFilesException $e ) { // some other exception?
1172                                         $tmpFile = null;
1173                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1174                                 }
1175                                 $tmpFiles[$path] = $tmpFile;
1176                         }
1177
1178                         $batch = new CF_Async_Op_Batch( $cfOps );
1179                         $cfOps = $batch->execute();
1180                         foreach ( $cfOps as $path => $cfOp ) {
1181                                 try {
1182                                         $cfOp->getLastResponse();
1183                                 } catch ( NoSuchContainerException $e ) {
1184                                         $tmpFiles[$path] = null;
1185                                 } catch ( NoSuchObjectException $e ) {
1186                                         $tmpFiles[$path] = null;
1187                                 } catch ( CloudFilesException $e ) { // some other exception?
1188                                         $tmpFiles[$path] = null;
1189                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1190                                 }
1191                                 fclose( $cfOp->_file_handle ); // close open handle
1192                         }
1193                 }
1194
1195                 return $tmpFiles;
1196         }
1197
1198         /**
1199          * @see FileBackendStore::getFileHttpUrl()
1200          * @return string|null
1201          */
1202         public function getFileHttpUrl( array $params ) {
1203                 if ( $this->swiftTempUrlKey != '' ||
1204                         ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
1205                 {
1206                         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1207                         if ( $srcRel === null ) {
1208                                 return null; // invalid path
1209                         }
1210                         try {
1211                                 $sContObj = $this->getContainer( $srcCont );
1212                                 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1213                                 if ( $this->swiftTempUrlKey != '' ) {
1214                                         return $obj->get_temp_url( $this->swiftTempUrlKey, 86400, "GET" );
1215                                 } else { // give S3 API URL for rgw
1216                                         $expires = time() + 86400;
1217                                         // Path for signature starts with the bucket
1218                                         $spath = '/' . rawurlencode( $srcCont ) . '/' .
1219                                                 str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1220                                         // Calculate the hash
1221                                         $signature = base64_encode( hash_hmac(
1222                                                 'sha1',
1223                                                 "GET\n\n\n{$expires}\n{$spath}",
1224                                                 $this->rgwS3SecretKey,
1225                                                 true // raw
1226                                         ) );
1227                                         // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1228                                         // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1229                                         return wfAppendQuery(
1230                                                 str_replace( '/swift/v1', '', // S3 API is the rgw default
1231                                                         $sContObj->cfs_http->getStorageUrl() . $spath ),
1232                                                 array(
1233                                                         'Signature'      => $signature,
1234                                                         'Expires'        => $expires,
1235                                                         'AWSAccessKeyId' => $this->rgwS3AccessKey )
1236                                         );
1237                                 }
1238                         } catch ( NoSuchContainerException $e ) {
1239                         } catch ( CloudFilesException $e ) { // some other exception?
1240                                 $this->handleException( $e, null, __METHOD__, $params );
1241                         }
1242                 }
1243                 return null;
1244         }
1245
1246         /**
1247          * @see FileBackendStore::directoriesAreVirtual()
1248          * @return bool
1249          */
1250         protected function directoriesAreVirtual() {
1251                 return true;
1252         }
1253
1254         /**
1255          * Get headers to send to Swift when reading a file based
1256          * on a FileBackend params array, e.g. that of getLocalCopy().
1257          * $params is currently only checked for a 'latest' flag.
1258          *
1259          * @param $params Array
1260          * @return Array
1261          */
1262         protected function headersFromParams( array $params ) {
1263                 $hdrs = array();
1264                 if ( !empty( $params['latest'] ) ) {
1265                         $hdrs[] = 'X-Newest: true';
1266                 }
1267                 return $hdrs;
1268         }
1269
1270         /**
1271          * @see FileBackendStore::doExecuteOpHandlesInternal()
1272          * @return Array List of corresponding Status objects
1273          */
1274         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1275                 $statuses = array();
1276
1277                 $cfOps = array(); // list of CF_Async_Op objects
1278                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1279                         $cfOps[$index] = $fileOpHandle->cfOp;
1280                 }
1281                 $batch = new CF_Async_Op_Batch( $cfOps );
1282
1283                 $cfOps = $batch->execute();
1284                 foreach ( $cfOps as $index => $cfOp ) {
1285                         $status = Status::newGood();
1286                         $function = '_getResponse' . $fileOpHandles[$index]->call;
1287                         try { // catch exceptions; update status
1288                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1289                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1290                         } catch ( CloudFilesException $e ) { // some other exception?
1291                                 $this->handleException( $e, $status,
1292                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1293                         }
1294                         $statuses[$index] = $status;
1295                 }
1296
1297                 return $statuses;
1298         }
1299
1300         /**
1301          * Set read/write permissions for a Swift container.
1302          *
1303          * $readGrps is a list of the possible criteria for a request to have
1304          * access to read a container. Each item is one of the following formats:
1305          *   - account:user        : Grants access if the request is by the given user
1306          *   - ".r:<regex>"        : Grants access if the request is from a referrer host that
1307          *                           matches the expression and the request is not for a listing.
1308          *                           Setting this to '*' effectively makes a container public.
1309          *   -".rlistings:<regex>" : Grants access if the request is from a referrer host that
1310          *                           matches the expression and the request for a listing.
1311          *
1312          * $writeGrps is a list of the possible criteria for a request to have
1313          * access to write to a container. Each item is of the following format:
1314          *   - account:user       : Grants access if the request is by the given user
1315          *
1316          * @see http://swift.openstack.org/misc.html#acls
1317          *
1318          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1319          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1320          *
1321          * @param $contObj CF_Container Swift container
1322          * @param $readGrps Array List of read access routes
1323          * @param $writeGrps Array List of write access routes
1324          * @return Status
1325          */
1326         protected function setContainerAccess(
1327                 CF_Container $contObj, array $readGrps, array $writeGrps
1328         ) {
1329                 $creds = $contObj->cfs_auth->export_credentials();
1330
1331                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1332
1333                 // Note: 10 second timeout consistent with php-cloudfiles
1334                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1335                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1336                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1337                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1338
1339                 return $req->execute(); // should return 204
1340         }
1341
1342         /**
1343          * Purge the CDN cache of affected objects if CDN caching is enabled.
1344          * This is for Rackspace/Akamai CDNs.
1345          *
1346          * @param $objects Array List of CF_Object items
1347          * @return void
1348          */
1349         public function purgeCDNCache( array $objects ) {
1350                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1351                         foreach ( $objects as $object ) {
1352                                 try {
1353                                         $object->purge_from_cdn();
1354                                 } catch ( CDNNotEnabledException $e ) {
1355                                         // CDN not enabled; nothing to see here
1356                                 } catch ( CloudFilesException $e ) {
1357                                         $this->handleException( $e, null, __METHOD__,
1358                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1359                                 }
1360                         }
1361                 }
1362         }
1363
1364         /**
1365          * Get an authenticated connection handle to the Swift proxy
1366          *
1367          * @throws CloudFilesException
1368          * @throws CloudFilesException|Exception
1369          * @return CF_Connection|bool False on failure
1370          */
1371         protected function getConnection() {
1372                 if ( $this->connException instanceof CloudFilesException ) {
1373                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1374                                 throw $this->connException; // failed last attempt; don't bother
1375                         } else { // actually retry this time
1376                                 $this->connException = null;
1377                                 $this->connErrorTime = 0;
1378                         }
1379                 }
1380                 // Session keys expire after a while, so we renew them periodically
1381                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1382                 // Authenticate with proxy and get a session key...
1383                 if ( !$this->conn || $reAuth ) {
1384                         $this->sessionStarted = 0;
1385                         $this->connContainerCache->clear();
1386                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1387                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1388                         if ( is_array( $creds ) ) { // cache hit
1389                                 $this->auth->load_cached_credentials(
1390                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1391                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1392                         } else { // cache miss
1393                                 try {
1394                                         $this->auth->authenticate();
1395                                         $creds = $this->auth->export_credentials();
1396                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1397                                         $this->sessionStarted = time();
1398                                 } catch ( CloudFilesException $e ) {
1399                                         $this->connException = $e; // don't keep re-trying
1400                                         $this->connErrorTime = time();
1401                                         throw $e; // throw it back
1402                                 }
1403                         }
1404                         if ( $this->conn ) { // re-authorizing?
1405                                 $this->conn->close(); // close active cURL handles in CF_Http object
1406                         }
1407                         $this->conn = new CF_Connection( $this->auth );
1408                 }
1409                 return $this->conn;
1410         }
1411
1412         /**
1413          * Close the connection to the Swift proxy
1414          *
1415          * @return void
1416          */
1417         protected function closeConnection() {
1418                 if ( $this->conn ) {
1419                         $this->conn->close(); // close active cURL handles in CF_Http object
1420                         $this->conn = null;
1421                         $this->sessionStarted = 0;
1422                         $this->connContainerCache->clear();
1423                 }
1424         }
1425
1426         /**
1427          * Get the cache key for a container
1428          *
1429          * @param $username string
1430          * @return string
1431          */
1432         private function getCredsCacheKey( $username ) {
1433                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1434         }
1435
1436         /**
1437          * Get a Swift container object, possibly from process cache.
1438          * Use $reCache if the file count or byte count is needed.
1439          *
1440          * @param $container string Container name
1441          * @param $bypassCache bool Bypass all caches and load from Swift
1442          * @return CF_Container
1443          * @throws CloudFilesException
1444          */
1445         protected function getContainer( $container, $bypassCache = false ) {
1446                 $conn = $this->getConnection(); // Swift proxy connection
1447                 if ( $bypassCache ) { // purge cache
1448                         $this->connContainerCache->clear( $container );
1449                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1450                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1451                 }
1452                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1453                         $contObj = $conn->get_container( $container );
1454                         // NoSuchContainerException not thrown: container must exist
1455                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1456                         if ( !$bypassCache ) {
1457                                 $this->setContainerCache( $container, // update persistent cache
1458                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1459                                 );
1460                         }
1461                 }
1462                 return $this->connContainerCache->get( $container, 'obj' );
1463         }
1464
1465         /**
1466          * Create a Swift container
1467          *
1468          * @param $container string Container name
1469          * @return CF_Container
1470          * @throws CloudFilesException
1471          */
1472         protected function createContainer( $container ) {
1473                 $conn = $this->getConnection(); // Swift proxy connection
1474                 $contObj = $conn->create_container( $container );
1475                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1476                 return $contObj;
1477         }
1478
1479         /**
1480          * Delete a Swift container
1481          *
1482          * @param $container string Container name
1483          * @return void
1484          * @throws CloudFilesException
1485          */
1486         protected function deleteContainer( $container ) {
1487                 $conn = $this->getConnection(); // Swift proxy connection
1488                 $this->connContainerCache->clear( $container ); // purge
1489                 $conn->delete_container( $container );
1490         }
1491
1492         /**
1493          * @see FileBackendStore::doPrimeContainerCache()
1494          * @return void
1495          */
1496         protected function doPrimeContainerCache( array $containerInfo ) {
1497                 try {
1498                         $conn = $this->getConnection(); // Swift proxy connection
1499                         foreach ( $containerInfo as $container => $info ) {
1500                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1501                                         $container, $info['count'], $info['bytes'] );
1502                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1503                         }
1504                 } catch ( CloudFilesException $e ) { // some other exception?
1505                         $this->handleException( $e, null, __METHOD__, array() );
1506                 }
1507         }
1508
1509         /**
1510          * Log an unexpected exception for this backend.
1511          * This also sets the Status object to have a fatal error.
1512          *
1513          * @param $e Exception
1514          * @param $status Status|null
1515          * @param $func string
1516          * @param $params Array
1517          * @return void
1518          */
1519         protected function handleException( Exception $e, $status, $func, array $params ) {
1520                 if ( $status instanceof Status ) {
1521                         if ( $e instanceof AuthenticationException ) {
1522                                 $status->fatal( 'backend-fail-connect', $this->name );
1523                         } else {
1524                                 $status->fatal( 'backend-fail-internal', $this->name );
1525                         }
1526                 }
1527                 if ( $e->getMessage() ) {
1528                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1529                 }
1530                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1531                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1532                         $this->closeConnection(); // force a re-connect and re-auth next time
1533                 }
1534                 wfDebugLog( 'SwiftBackend',
1535                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1536                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1537                 );
1538         }
1539 }
1540
1541 /**
1542  * @see FileBackendStoreOpHandle
1543  */
1544 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1545         /** @var CF_Async_Op */
1546         public $cfOp;
1547         /** @var Array */
1548         public $affectedObjects = array();
1549
1550         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1551                 $this->backend = $backend;
1552                 $this->params = $params;
1553                 $this->call = $call;
1554                 $this->cfOp = $cfOp;
1555         }
1556 }
1557
1558 /**
1559  * SwiftFileBackend helper class to page through listings.
1560  * Swift also has a listing limit of 10,000 objects for sanity.
1561  * Do not use this class from places outside SwiftFileBackend.
1562  *
1563  * @ingroup FileBackend
1564  */
1565 abstract class SwiftFileBackendList implements Iterator {
1566         /** @var Array */
1567         protected $bufferIter = array();
1568         protected $bufferAfter = null; // string; list items *after* this path
1569         protected $pos = 0; // integer
1570         /** @var Array */
1571         protected $params = array();
1572
1573         /** @var SwiftFileBackend */
1574         protected $backend;
1575         protected $container; // string; container name
1576         protected $dir; // string; storage directory
1577         protected $suffixStart; // integer
1578
1579         const PAGE_SIZE = 9000; // file listing buffer size
1580
1581         /**
1582          * @param $backend SwiftFileBackend
1583          * @param $fullCont string Resolved container name
1584          * @param $dir string Resolved directory relative to container
1585          * @param $params Array
1586          */
1587         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1588                 $this->backend = $backend;
1589                 $this->container = $fullCont;
1590                 $this->dir = $dir;
1591                 if ( substr( $this->dir, -1 ) === '/' ) {
1592                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1593                 }
1594                 if ( $this->dir == '' ) { // whole container
1595                         $this->suffixStart = 0;
1596                 } else { // dir within container
1597                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1598                 }
1599                 $this->params = $params;
1600         }
1601
1602         /**
1603          * @see Iterator::key()
1604          * @return integer
1605          */
1606         public function key() {
1607                 return $this->pos;
1608         }
1609
1610         /**
1611          * @see Iterator::next()
1612          * @return void
1613          */
1614         public function next() {
1615                 // Advance to the next file in the page
1616                 next( $this->bufferIter );
1617                 ++$this->pos;
1618                 // Check if there are no files left in this page and
1619                 // advance to the next page if this page was not empty.
1620                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1621                         $this->bufferIter = $this->pageFromList(
1622                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1623                         ); // updates $this->bufferAfter
1624                 }
1625         }
1626
1627         /**
1628          * @see Iterator::rewind()
1629          * @return void
1630          */
1631         public function rewind() {
1632                 $this->pos = 0;
1633                 $this->bufferAfter = null;
1634                 $this->bufferIter = $this->pageFromList(
1635                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1636                 ); // updates $this->bufferAfter
1637         }
1638
1639         /**
1640          * @see Iterator::valid()
1641          * @return bool
1642          */
1643         public function valid() {
1644                 if ( $this->bufferIter === null ) {
1645                         return false; // some failure?
1646                 } else {
1647                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1648                 }
1649         }
1650
1651         /**
1652          * Get the given list portion (page)
1653          *
1654          * @param $container string Resolved container name
1655          * @param $dir string Resolved path relative to container
1656          * @param $after string|null
1657          * @param $limit integer
1658          * @param $params Array
1659          * @return Traversable|Array|null Returns null on failure
1660          */
1661         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1662 }
1663
1664 /**
1665  * Iterator for listing directories
1666  */
1667 class SwiftFileBackendDirList extends SwiftFileBackendList {
1668         /**
1669          * @see Iterator::current()
1670          * @return string|bool String (relative path) or false
1671          */
1672         public function current() {
1673                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1674         }
1675
1676         /**
1677          * @see SwiftFileBackendList::pageFromList()
1678          * @return Array|null
1679          */
1680         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1681                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1682         }
1683 }
1684
1685 /**
1686  * Iterator for listing regular files
1687  */
1688 class SwiftFileBackendFileList extends SwiftFileBackendList {
1689         /**
1690          * @see Iterator::current()
1691          * @return string|bool String (relative path) or false
1692          */
1693         public function current() {
1694                 return substr( current( $this->bufferIter ), $this->suffixStart );
1695         }
1696
1697         /**
1698          * @see SwiftFileBackendList::pageFromList()
1699          * @return Array|null
1700          */
1701         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1702                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1703         }
1704 }