includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftTempUrlKey; // string; shared secret value for making temp urls
  44         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  45         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  46         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  47         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  48
  49         /** @var CF_Connection */
  50         protected $conn; // Swift connection handle
  51         protected $sessionStarted = 0; // integer UNIX timestamp
  52
  53         /** @var CloudFilesException */
  54         protected $connException;
  55         protected $connErrorTime = 0; // UNIX timestamp
  56
  57         /** @var BagOStuff */
  58         protected $srvCache;
  59
  60         /** @var ProcessCacheLRU */
  61         protected $connContainerCache; // container object cache
  62
  63         /**
  64          * @see FileBackendStore::__construct()
  65          * Additional $config params include:
  66          *   - swiftAuthUrl       : Swift authentication server URL
  67          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  68          *   - swiftKey           : Swift authentication key for the above user
  69          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  70          *   - swiftTempUrlKey    : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
  71          *                          Do not set this until it has been set in the backend.
  72          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  73          *                          If set, then views of public containers are assumed to go
  74          *                          through this user. If not set, then public containers are
  75          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  76          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  77          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  78          *                          If files may likely change, this should probably not exceed
  79          *                          a few days. For example, deletions may take this long to apply.
  80          *                          If object purging is enabled, however, this is not an issue.
  81          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  82          *   - shardViaHashLevels : Map of container names to sharding config with:
  83          *                             - base   : base of hash characters, 16 or 36
  84          *                             - levels : the number of hash levels (and digits)
  85          *                             - repeat : hash subdirectories are prefixed with all the
  86          *                                        parent hash directory names (e.g. "a/ab/abc")
  87          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC, XCache, ect.
  88          *                          If those are not available, then the main cache will be used.
  89          *                          This is probably insecure in shared hosting environments.
  90          */
  91         public function __construct( array $config ) {
  92                 parent::__construct( $config );
  93                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  94                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  95                 }
  96                 // Required settings
  97                 $this->auth = new CF_Authentication(
  98                         $config['swiftUser'],
  99                         $config['swiftKey'],
 100                         null, // account; unused
 101                         $config['swiftAuthUrl']
 102                 );
 103                 // Optional settings
 104                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 105                         ? $config['swiftAuthTTL']
 106                         : 5 * 60; // some sane number
 107                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 108                         ? $config['swiftAnonUser']
 109                         : '';
 110                 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
 111                         ? $config['swiftTempUrlKey']
 112                         : '';
 113                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 114                         ? $config['shardViaHashLevels']
 115                         : '';
 116                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 117                         ? $config['swiftUseCDN']
 118                         : false;
 119                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 120                         ? $config['swiftCDNExpiry']
 121                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 122                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 123                         ? $config['swiftCDNPurgable']
 124                         : true;
 125                 // Cache container information to mask latency
 126                 $this->memCache = wfGetMainCache();
 127                 // Process cache for container info
 128                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 129                 // Cache auth token information to avoid RTTs
 130                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 131                         if ( php_sapi_name() === 'cli' ) {
 132                                 $this->srvCache = wfGetMainCache(); // preferrably memcached
 133                         } else {
 134                                 try { // look for APC, XCache, WinCache, ect...
 135                                         $this->srvCache = ObjectCache::newAccelerator( array() );
 136                                 } catch ( Exception $e ) {}
 137                         }
 138                 }
 139                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 140         }
 141
 142         /**
 143          * @see FileBackendStore::resolveContainerPath()
 144          * @return null
 145          */
 146         protected function resolveContainerPath( $container, $relStoragePath ) {
 147                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 148                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 149                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 150                         return null; // too long for Swift
 151                 }
 152                 return $relStoragePath;
 153         }
 154
 155         /**
 156          * @see FileBackendStore::isPathUsableInternal()
 157          * @return bool
 158          */
 159         public function isPathUsableInternal( $storagePath ) {
 160                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 161                 if ( $rel === null ) {
 162                         return false; // invalid
 163                 }
 164
 165                 try {
 166                         $this->getContainer( $container );
 167                         return true; // container exists
 168                 } catch ( NoSuchContainerException $e ) {
 169                 } catch ( CloudFilesException $e ) { // some other exception?
 170                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 171                 }
 172
 173                 return false;
 174         }
 175
 176         /**
 177          * @param $disposition string Content-Disposition header value
 178          * @return string Truncated Content-Disposition header value to meet Swift limits
 179          */
 180         protected function truncDisp( $disposition ) {
 181                 $res = '';
 182                 foreach ( explode( ';', $disposition ) as $part ) {
 183                         $part = trim( $part );
 184                         $new  = ( $res === '' ) ? $part : "{$res};{$part}";
 185                         if ( strlen( $new ) <= 255 ) {
 186                                 $res = $new;
 187                         } else {
 188                                 break; // too long; sigh
 189                         }
 190                 }
 191                 return $res;
 192         }
 193
 194         /**
 195          * @see FileBackendStore::doCreateInternal()
 196          * @return Status
 197          */
 198         protected function doCreateInternal( array $params ) {
 199                 $status = Status::newGood();
 200
 201                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 202                 if ( $dstRel === null ) {
 203                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 204                         return $status;
 205                 }
 206
 207                 // (a) Check the destination container and object
 208                 try {
 209                         $dContObj = $this->getContainer( $dstCont );
 210                 } catch ( NoSuchContainerException $e ) {
 211                         $status->fatal( 'backend-fail-create', $params['dst'] );
 212                         return $status;
 213                 } catch ( CloudFilesException $e ) { // some other exception?
 214                         $this->handleException( $e, $status, __METHOD__, $params );
 215                         return $status;
 216                 }
 217
 218                 // (b) Get a SHA-1 hash of the object
 219                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 220
 221                 // (c) Actually create the object
 222                 try {
 223                         // Create a fresh CF_Object with no fields preloaded.
 224                         // We don't want to preserve headers, metadata, and such.
 225                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 226                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 227                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 228                         // The MD5 here will be checked within Swift against its own MD5.
 229                         $obj->set_etag( md5( $params['content'] ) );
 230                         // Use the same content type as StreamFile for security
 231                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 232                         if ( !strlen( $obj->content_type ) ) { // special case
 233                                 $obj->content_type = 'unknown/unknown';
 234                         }
 235                         // Set the Content-Disposition header if requested
 236                         if ( isset( $params['disposition'] ) ) {
 237                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 238                         }
 239                         // Set any other custom headers if requested
 240                         if ( isset( $params['headers'] ) ) {
 241                                 $obj->headers += $params['headers'];
 242                         }
 243                         if ( !empty( $params['async'] ) ) { // deferred
 244                                 $op = $obj->write_async( $params['content'] );
 245                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
 246                                 $status->value->affectedObjects[] = $obj;
 247                         } else { // actually write the object in Swift
 248                                 $obj->write( $params['content'] );
 249                                 $this->purgeCDNCache( array( $obj ) );
 250                         }
 251                 } catch ( CDNNotEnabledException $e ) {
 252                         // CDN not enabled; nothing to see here
 253                 } catch ( BadContentTypeException $e ) {
 254                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 255                 } catch ( CloudFilesException $e ) { // some other exception?
 256                         $this->handleException( $e, $status, __METHOD__, $params );
 257                 }
 258
 259                 return $status;
 260         }
 261
 262         /**
 263          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 264          */
 265         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 266                 try {
 267                         $cfOp->getLastResponse();
 268                 } catch ( BadContentTypeException $e ) {
 269                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 270                 }
 271         }
 272
 273         /**
 274          * @see FileBackendStore::doStoreInternal()
 275          * @return Status
 276          */
 277         protected function doStoreInternal( array $params ) {
 278                 $status = Status::newGood();
 279
 280                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 281                 if ( $dstRel === null ) {
 282                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 283                         return $status;
 284                 }
 285
 286                 // (a) Check the destination container and object
 287                 try {
 288                         $dContObj = $this->getContainer( $dstCont );
 289                 } catch ( NoSuchContainerException $e ) {
 290                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 291                         return $status;
 292                 } catch ( CloudFilesException $e ) { // some other exception?
 293                         $this->handleException( $e, $status, __METHOD__, $params );
 294                         return $status;
 295                 }
 296
 297                 // (b) Get a SHA-1 hash of the object
 298                 $sha1Hash = sha1_file( $params['src'] );
 299                 if ( $sha1Hash === false ) { // source doesn't exist?
 300                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 301                         return $status;
 302                 }
 303                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 304
 305                 // (c) Actually store the object
 306                 try {
 307                         // Create a fresh CF_Object with no fields preloaded.
 308                         // We don't want to preserve headers, metadata, and such.
 309                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 310                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 311                         // The MD5 here will be checked within Swift against its own MD5.
 312                         $obj->set_etag( md5_file( $params['src'] ) );
 313                         // Use the same content type as StreamFile for security
 314                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 315                         if ( !strlen( $obj->content_type ) ) { // special case
 316                                 $obj->content_type = 'unknown/unknown';
 317                         }
 318                         // Set the Content-Disposition header if requested
 319                         if ( isset( $params['disposition'] ) ) {
 320                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 321                         }
 322                         // Set any other custom headers if requested
 323                         if ( isset( $params['headers'] ) ) {
 324                                 $obj->headers += $params['headers'];
 325                         }
 326                         if ( !empty( $params['async'] ) ) { // deferred
 327                                 wfSuppressWarnings();
 328                                 $fp = fopen( $params['src'], 'rb' );
 329                                 wfRestoreWarnings();
 330                                 if ( !$fp ) {
 331                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 332                                 } else {
 333                                         $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
 334                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
 335                                         $status->value->resourcesToClose[] = $fp;
 336                                         $status->value->affectedObjects[] = $obj;
 337                                 }
 338                         } else { // actually write the object in Swift
 339                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 340                                 $this->purgeCDNCache( array( $obj ) );
 341                         }
 342                 } catch ( CDNNotEnabledException $e ) {
 343                         // CDN not enabled; nothing to see here
 344                 } catch ( BadContentTypeException $e ) {
 345                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 346                 } catch ( IOException $e ) {
 347                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 348                 } catch ( CloudFilesException $e ) { // some other exception?
 349                         $this->handleException( $e, $status, __METHOD__, $params );
 350                 }
 351
 352                 return $status;
 353         }
 354
 355         /**
 356          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 357          */
 358         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 359                 try {
 360                         $cfOp->getLastResponse();
 361                 } catch ( BadContentTypeException $e ) {
 362                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 363                 } catch ( IOException $e ) {
 364                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 365                 }
 366         }
 367
 368         /**
 369          * @see FileBackendStore::doCopyInternal()
 370          * @return Status
 371          */
 372         protected function doCopyInternal( array $params ) {
 373                 $status = Status::newGood();
 374
 375                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 376                 if ( $srcRel === null ) {
 377                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 378                         return $status;
 379                 }
 380
 381                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 382                 if ( $dstRel === null ) {
 383                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 384                         return $status;
 385                 }
 386
 387                 // (a) Check the source/destination containers and destination object
 388                 try {
 389                         $sContObj = $this->getContainer( $srcCont );
 390                         $dContObj = $this->getContainer( $dstCont );
 391                 } catch ( NoSuchContainerException $e ) {
 392                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 393                         return $status;
 394                 } catch ( CloudFilesException $e ) { // some other exception?
 395                         $this->handleException( $e, $status, __METHOD__, $params );
 396                         return $status;
 397                 }
 398
 399                 // (b) Actually copy the file to the destination
 400                 try {
 401                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 402                         $hdrs = array(); // source file headers to override with new values
 403                         if ( isset( $params['disposition'] ) ) {
 404                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 405                         }
 406                         if ( !empty( $params['async'] ) ) { // deferred
 407                                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 408                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
 409                                 $status->value->affectedObjects[] = $dstObj;
 410                         } else { // actually write the object in Swift
 411                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 412                                 $this->purgeCDNCache( array( $dstObj ) );
 413                         }
 414                 } catch ( CDNNotEnabledException $e ) {
 415                         // CDN not enabled; nothing to see here
 416                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 417                         if ( empty( $params['ignoreMissingSource'] ) ) {
 418                                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 419                         }
 420                 } catch ( CloudFilesException $e ) { // some other exception?
 421                         $this->handleException( $e, $status, __METHOD__, $params );
 422                 }
 423
 424                 return $status;
 425         }
 426
 427         /**
 428          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 429          */
 430         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 431                 try {
 432                         $cfOp->getLastResponse();
 433                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 434                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 435                 }
 436         }
 437
 438         /**
 439          * @see FileBackendStore::doMoveInternal()
 440          * @return Status
 441          */
 442         protected function doMoveInternal( array $params ) {
 443                 $status = Status::newGood();
 444
 445                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 446                 if ( $srcRel === null ) {
 447                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 448                         return $status;
 449                 }
 450
 451                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 452                 if ( $dstRel === null ) {
 453                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 454                         return $status;
 455                 }
 456
 457                 // (a) Check the source/destination containers and destination object
 458                 try {
 459                         $sContObj = $this->getContainer( $srcCont );
 460                         $dContObj = $this->getContainer( $dstCont );
 461                 } catch ( NoSuchContainerException $e ) {
 462                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 463                         return $status;
 464                 } catch ( CloudFilesException $e ) { // some other exception?
 465                         $this->handleException( $e, $status, __METHOD__, $params );
 466                         return $status;
 467                 }
 468
 469                 // (b) Actually move the file to the destination
 470                 try {
 471                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 472                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 473                         $hdrs = array(); // source file headers to override with new values
 474                         if ( isset( $params['disposition'] ) ) {
 475                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 476                         }
 477                         if ( !empty( $params['async'] ) ) { // deferred
 478                                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 479                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
 480                                 $status->value->affectedObjects[] = $srcObj;
 481                                 $status->value->affectedObjects[] = $dstObj;
 482                         } else { // actually write the object in Swift
 483                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 484                                 $this->purgeCDNCache( array( $srcObj ) );
 485                                 $this->purgeCDNCache( array( $dstObj ) );
 486                         }
 487                 } catch ( CDNNotEnabledException $e ) {
 488                         // CDN not enabled; nothing to see here
 489                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 490                         if ( empty( $params['ignoreMissingSource'] ) ) {
 491                                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 492                         }
 493                 } catch ( CloudFilesException $e ) { // some other exception?
 494                         $this->handleException( $e, $status, __METHOD__, $params );
 495                 }
 496
 497                 return $status;
 498         }
 499
 500         /**
 501          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 502          */
 503         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 504                 try {
 505                         $cfOp->getLastResponse();
 506                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 507                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 508                 }
 509         }
 510
 511         /**
 512          * @see FileBackendStore::doDeleteInternal()
 513          * @return Status
 514          */
 515         protected function doDeleteInternal( array $params ) {
 516                 $status = Status::newGood();
 517
 518                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 519                 if ( $srcRel === null ) {
 520                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 521                         return $status;
 522                 }
 523
 524                 try {
 525                         $sContObj = $this->getContainer( $srcCont );
 526                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 527                         if ( !empty( $params['async'] ) ) { // deferred
 528                                 $op = $sContObj->delete_object_async( $srcRel );
 529                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
 530                                 $status->value->affectedObjects[] = $srcObj;
 531                         } else { // actually write the object in Swift
 532                                 $sContObj->delete_object( $srcRel );
 533                                 $this->purgeCDNCache( array( $srcObj ) );
 534                         }
 535                 } catch ( CDNNotEnabledException $e ) {
 536                         // CDN not enabled; nothing to see here
 537                 } catch ( NoSuchContainerException $e ) {
 538                         $status->fatal( 'backend-fail-delete', $params['src'] );
 539                 } catch ( NoSuchObjectException $e ) {
 540                         if ( empty( $params['ignoreMissingSource'] ) ) {
 541                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 542                         }
 543                 } catch ( CloudFilesException $e ) { // some other exception?
 544                         $this->handleException( $e, $status, __METHOD__, $params );
 545                 }
 546
 547                 return $status;
 548         }
 549
 550         /**
 551          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 552          */
 553         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 554                 try {
 555                         $cfOp->getLastResponse();
 556                 } catch ( NoSuchContainerException $e ) {
 557                         $status->fatal( 'backend-fail-delete', $params['src'] );
 558                 } catch ( NoSuchObjectException $e ) {
 559                         if ( empty( $params['ignoreMissingSource'] ) ) {
 560                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 561                         }
 562                 }
 563         }
 564
 565         /**
 566          * @see FileBackendStore::doDescribeInternal()
 567          * @return Status
 568          */
 569         protected function doDescribeInternal( array $params ) {
 570                 $status = Status::newGood();
 571
 572                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 573                 if ( $srcRel === null ) {
 574                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 575                         return $status;
 576                 }
 577
 578                 $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
 579                 // Set the Content-Disposition header if requested
 580                 if ( isset( $params['disposition'] ) ) {
 581                         $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 582                 }
 583
 584                 try {
 585                         $sContObj = $this->getContainer( $srcCont );
 586                         // Get the latest version of the current metadata
 587                         $srcObj = $sContObj->get_object( $srcRel,
 588                                 $this->headersFromParams( array( 'latest' => true ) ) );
 589                         // Merge in the metadata updates...
 590                         $srcObj->headers = $hdrs + $srcObj->headers;
 591                         $srcObj->sync_metadata(); // save to Swift
 592                         $this->purgeCDNCache( array( $srcObj ) );
 593                 } catch ( CDNNotEnabledException $e ) {
 594                         // CDN not enabled; nothing to see here
 595                 } catch ( NoSuchContainerException $e ) {
 596                         $status->fatal( 'backend-fail-describe', $params['src'] );
 597                 } catch ( NoSuchObjectException $e ) {
 598                         $status->fatal( 'backend-fail-describe', $params['src'] );
 599                 } catch ( CloudFilesException $e ) { // some other exception?
 600                         $this->handleException( $e, $status, __METHOD__, $params );
 601                 }
 602
 603                 return $status;
 604         }
 605
 606         /**
 607          * @see FileBackendStore::doPrepareInternal()
 608          * @return Status
 609          */
 610         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 611                 $status = Status::newGood();
 612
 613                 // (a) Check if container already exists
 614                 try {
 615                         $contObj = $this->getContainer( $fullCont );
 616                         // NoSuchContainerException not thrown: container must exist
 617                         return $status; // already exists
 618                 } catch ( NoSuchContainerException $e ) {
 619                         // NoSuchContainerException thrown: container does not exist
 620                 } catch ( CloudFilesException $e ) { // some other exception?
 621                         $this->handleException( $e, $status, __METHOD__, $params );
 622                         return $status;
 623                 }
 624
 625                 // (b) Create container as needed
 626                 try {
 627                         $contObj = $this->createContainer( $fullCont );
 628                         if ( !empty( $params['noAccess'] ) ) {
 629                                 // Make container private to end-users...
 630                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 631                         } else {
 632                                 // Make container public to end-users...
 633                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 634                         }
 635                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 636                                 $contObj->make_public( $this->swiftCDNExpiry );
 637                         }
 638                 } catch ( CDNNotEnabledException $e ) {
 639                         // CDN not enabled; nothing to see here
 640                 } catch ( CloudFilesException $e ) { // some other exception?
 641                         $this->handleException( $e, $status, __METHOD__, $params );
 642                         return $status;
 643                 }
 644
 645                 return $status;
 646         }
 647
 648         /**
 649          * @see FileBackendStore::doSecureInternal()
 650          * @return Status
 651          */
 652         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 653                 $status = Status::newGood();
 654                 if ( empty( $params['noAccess'] ) ) {
 655                         return $status; // nothing to do
 656                 }
 657
 658                 // Restrict container from end-users...
 659                 try {
 660                         // doPrepareInternal() should have been called,
 661                         // so the Swift container should already exist...
 662                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 663                         // NoSuchContainerException not thrown: container must exist
 664
 665                         // Make container private to end-users...
 666                         $status->merge( $this->setContainerAccess(
 667                                 $contObj,
 668                                 array( $this->auth->username ), // read
 669                                 array( $this->auth->username ) // write
 670                         ) );
 671                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 672                                 $contObj->make_private();
 673                         }
 674                 } catch ( CDNNotEnabledException $e ) {
 675                         // CDN not enabled; nothing to see here
 676                 } catch ( CloudFilesException $e ) { // some other exception?
 677                         $this->handleException( $e, $status, __METHOD__, $params );
 678                 }
 679
 680                 return $status;
 681         }
 682
 683         /**
 684          * @see FileBackendStore::doPublishInternal()
 685          * @return Status
 686          */
 687         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 688                 $status = Status::newGood();
 689
 690                 // Unrestrict container from end-users...
 691                 try {
 692                         // doPrepareInternal() should have been called,
 693                         // so the Swift container should already exist...
 694                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 695                         // NoSuchContainerException not thrown: container must exist
 696
 697                         // Make container public to end-users...
 698                         if ( $this->swiftAnonUser != '' ) {
 699                                 $status->merge( $this->setContainerAccess(
 700                                         $contObj,
 701                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 702                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 703                                 ) );
 704                         } else {
 705                                 $status->merge( $this->setContainerAccess(
 706                                         $contObj,
 707                                         array( $this->auth->username, '.r:*' ), // read
 708                                         array( $this->auth->username ) // write
 709                                 ) );
 710                         }
 711                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 712                                 $contObj->make_public();
 713                         }
 714                 } catch ( CDNNotEnabledException $e ) {
 715                         // CDN not enabled; nothing to see here
 716                 } catch ( CloudFilesException $e ) { // some other exception?
 717                         $this->handleException( $e, $status, __METHOD__, $params );
 718                 }
 719
 720                 return $status;
 721         }
 722
 723         /**
 724          * @see FileBackendStore::doCleanInternal()
 725          * @return Status
 726          */
 727         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 728                 $status = Status::newGood();
 729
 730                 // Only containers themselves can be removed, all else is virtual
 731                 if ( $dir != '' ) {
 732                         return $status; // nothing to do
 733                 }
 734
 735                 // (a) Check the container
 736                 try {
 737                         $contObj = $this->getContainer( $fullCont, true );
 738                 } catch ( NoSuchContainerException $e ) {
 739                         return $status; // ok, nothing to do
 740                 } catch ( CloudFilesException $e ) { // some other exception?
 741                         $this->handleException( $e, $status, __METHOD__, $params );
 742                         return $status;
 743                 }
 744
 745                 // (b) Delete the container if empty
 746                 if ( $contObj->object_count == 0 ) {
 747                         try {
 748                                 $this->deleteContainer( $fullCont );
 749                         } catch ( NoSuchContainerException $e ) {
 750                                 return $status; // race?
 751                         } catch ( NonEmptyContainerException $e ) {
 752                                 return $status; // race? consistency delay?
 753                         } catch ( CloudFilesException $e ) { // some other exception?
 754                                 $this->handleException( $e, $status, __METHOD__, $params );
 755                                 return $status;
 756                         }
 757                 }
 758
 759                 return $status;
 760         }
 761
 762         /**
 763          * @see FileBackendStore::doFileExists()
 764          * @return array|bool|null
 765          */
 766         protected function doGetFileStat( array $params ) {
 767                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 768                 if ( $srcRel === null ) {
 769                         return false; // invalid storage path
 770                 }
 771
 772                 $stat = false;
 773                 try {
 774                         $contObj = $this->getContainer( $srcCont );
 775                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 776                         $this->addMissingMetadata( $srcObj, $params['src'] );
 777                         $stat = array(
 778                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 779                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 780                                 'size'  => (int)$srcObj->content_length,
 781                                 'sha1'  => $srcObj->getMetadataValue( 'Sha1base36' )
 782                         );
 783                 } catch ( NoSuchContainerException $e ) {
 784                 } catch ( NoSuchObjectException $e ) {
 785                 } catch ( CloudFilesException $e ) { // some other exception?
 786                         $stat = null;
 787                         $this->handleException( $e, null, __METHOD__, $params );
 788                 }
 789
 790                 return $stat;
 791         }
 792
 793         /**
 794          * Fill in any missing object metadata and save it to Swift
 795          *
 796          * @param $obj CF_Object
 797          * @param $path string Storage path to object
 798          * @return bool Success
 799          * @throws Exception cloudfiles exceptions
 800          */
 801         protected function addMissingMetadata( CF_Object $obj, $path ) {
 802                 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
 803                         return true; // nothing to do
 804                 }
 805                 wfProfileIn( __METHOD__ );
 806                 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
 807                 $status = Status::newGood();
 808                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 809                 if ( $status->isOK() ) {
 810                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 811                         if ( $tmpFile ) {
 812                                 $hash = $tmpFile->getSha1Base36();
 813                                 if ( $hash !== false ) {
 814                                         $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
 815                                         $obj->sync_metadata(); // save to Swift
 816                                         wfProfileOut( __METHOD__ );
 817                                         return true; // success
 818                                 }
 819                         }
 820                 }
 821                 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
 822                 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
 823                 wfProfileOut( __METHOD__ );
 824                 return false; // failed
 825         }
 826
 827         /**
 828          * @see FileBackendStore::doGetFileContentsMulti()
 829          * @return Array
 830          */
 831         protected function doGetFileContentsMulti( array $params ) {
 832                 $contents = array();
 833
 834                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
 835                 // Blindly create tmp files and stream to them, catching any exception if the file does
 836                 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
 837                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
 838                         $cfOps = array(); // (path => CF_Async_Op)
 839
 840                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
 841                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
 842                                 if ( $srcRel === null ) {
 843                                         $contents[$path] = false;
 844                                         continue;
 845                                 }
 846                                 $data = false;
 847                                 try {
 848                                         $sContObj = $this->getContainer( $srcCont );
 849                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 850                                         // Get source file extension
 851                                         $ext = FileBackend::extensionFromPath( $path );
 852                                         // Create a new temporary memory file...
 853                                         $handle = fopen( 'php://temp', 'wb' );
 854                                         if ( $handle ) {
 855                                                 $headers = $this->headersFromParams( $params );
 856                                                 if ( count( $pathBatch ) > 1 ) {
 857                                                         $cfOps[$path] = $obj->stream_async( $handle, $headers );
 858                                                         $cfOps[$path]->_file_handle = $handle; // close this later
 859                                                 } else {
 860                                                         $obj->stream( $handle, $headers );
 861                                                         rewind( $handle ); // start from the beginning
 862                                                         $data = stream_get_contents( $handle );
 863                                                         fclose( $handle );
 864                                                 }
 865                                         } else {
 866                                                 $data = false;
 867                                         }
 868                                 } catch ( NoSuchContainerException $e ) {
 869                                         $data = false;
 870                                 } catch ( NoSuchObjectException $e ) {
 871                                         $data = false;
 872                                 } catch ( CloudFilesException $e ) { // some other exception?
 873                                         $data = false;
 874                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 875                                 }
 876                                 $contents[$path] = $data;
 877                         }
 878
 879                         $batch = new CF_Async_Op_Batch( $cfOps );
 880                         $cfOps = $batch->execute();
 881                         foreach ( $cfOps as $path => $cfOp ) {
 882                                 try {
 883                                         $cfOp->getLastResponse();
 884                                         rewind( $cfOp->_file_handle ); // start from the beginning
 885                                         $contents[$path] = stream_get_contents( $cfOp->_file_handle );
 886                                 } catch ( NoSuchContainerException $e ) {
 887                                         $contents[$path] = false;
 888                                 } catch ( NoSuchObjectException $e ) {
 889                                         $contents[$path] = false;
 890                                 } catch ( CloudFilesException $e ) { // some other exception?
 891                                         $contents[$path] = false;
 892                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 893                                 }
 894                                 fclose( $cfOp->_file_handle ); // close open handle
 895                         }
 896                 }
 897
 898                 return $contents;
 899         }
 900
 901         /**
 902          * @see FileBackendStore::doDirectoryExists()
 903          * @return bool|null
 904          */
 905         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 906                 try {
 907                         $container = $this->getContainer( $fullCont );
 908                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 909                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 910                 } catch ( NoSuchContainerException $e ) {
 911                         return false;
 912                 } catch ( CloudFilesException $e ) { // some other exception?
 913                         $this->handleException( $e, null, __METHOD__,
 914                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 915                 }
 916
 917                 return null; // error
 918         }
 919
 920         /**
 921          * @see FileBackendStore::getDirectoryListInternal()
 922          * @return SwiftFileBackendDirList
 923          */
 924         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 925                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 926         }
 927
 928         /**
 929          * @see FileBackendStore::getFileListInternal()
 930          * @return SwiftFileBackendFileList
 931          */
 932         public function getFileListInternal( $fullCont, $dir, array $params ) {
 933                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 934         }
 935
 936         /**
 937          * Do not call this function outside of SwiftFileBackendFileList
 938          *
 939          * @param $fullCont string Resolved container name
 940          * @param $dir string Resolved storage directory with no trailing slash
 941          * @param $after string|null Storage path of file to list items after
 942          * @param $limit integer Max number of items to list
 943          * @param $params Array Includes flag for 'topOnly'
 944          * @return Array List of relative paths of dirs directly under $dir
 945          */
 946         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 947                 $dirs = array();
 948                 if ( $after === INF ) {
 949                         return $dirs; // nothing more
 950                 }
 951                 wfProfileIn( __METHOD__ . '-' . $this->name );
 952
 953                 try {
 954                         $container = $this->getContainer( $fullCont );
 955                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 956                         // Non-recursive: only list dirs right under $dir
 957                         if ( !empty( $params['topOnly'] ) ) {
 958                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 959                                 foreach ( $objects as $object ) { // files and dirs
 960                                         if ( substr( $object, -1 ) === '/' ) {
 961                                                 $dirs[] = $object; // directories end in '/'
 962                                         }
 963                                 }
 964                         // Recursive: list all dirs under $dir and its subdirs
 965                         } else {
 966                                 // Get directory from last item of prior page
 967                                 $lastDir = $this->getParentDir( $after ); // must be first page
 968                                 $objects = $container->list_objects( $limit, $after, $prefix );
 969                                 foreach ( $objects as $object ) { // files
 970                                         $objectDir = $this->getParentDir( $object ); // directory of object
 971                                         if ( $objectDir !== false ) { // file has a parent dir
 972                                                 // Swift stores paths in UTF-8, using binary sorting.
 973                                                 // See function "create_container_table" in common/db.py.
 974                                                 // If a directory is not "greater" than the last one,
 975                                                 // then it was already listed by the calling iterator.
 976                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
 977                                                         $pDir = $objectDir;
 978                                                         do { // add dir and all its parent dirs
 979                                                                 $dirs[] = "{$pDir}/";
 980                                                                 $pDir = $this->getParentDir( $pDir );
 981                                                         } while ( $pDir !== false // sanity
 982                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
 983                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 984                                                         );
 985                                                 }
 986                                                 $lastDir = $objectDir;
 987                                         }
 988                                 }
 989                         }
 990                         if ( count( $objects ) < $limit ) {
 991                                 $after = INF; // avoid a second RTT
 992                         } else {
 993                                 $after = end( $objects ); // update last item
 994                         }
 995                 } catch ( NoSuchContainerException $e ) {
 996                 } catch ( CloudFilesException $e ) { // some other exception?
 997                         $this->handleException( $e, null, __METHOD__,
 998                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 999                 }
1000
1001                 wfProfileOut( __METHOD__ . '-' . $this->name );
1002                 return $dirs;
1003         }
1004
1005         protected function getParentDir( $path ) {
1006                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
1007         }
1008
1009         /**
1010          * Do not call this function outside of SwiftFileBackendFileList
1011          *
1012          * @param $fullCont string Resolved container name
1013          * @param $dir string Resolved storage directory with no trailing slash
1014          * @param $after string|null Storage path of file to list items after
1015          * @param $limit integer Max number of items to list
1016          * @param $params Array Includes flag for 'topOnly'
1017          * @return Array List of relative paths of files under $dir
1018          */
1019         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1020                 $files = array();
1021                 if ( $after === INF ) {
1022                         return $files; // nothing more
1023                 }
1024                 wfProfileIn( __METHOD__ . '-' . $this->name );
1025
1026                 try {
1027                         $container = $this->getContainer( $fullCont );
1028                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
1029                         // Non-recursive: only list files right under $dir
1030                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
1031                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
1032                                 foreach ( $objects as $object ) {
1033                                         if ( substr( $object, -1 ) !== '/' ) {
1034                                                 $files[] = $object; // directories end in '/'
1035                                         }
1036                                 }
1037                         // Recursive: list all files under $dir and its subdirs
1038                         } else { // files
1039                                 $objects = $container->list_objects( $limit, $after, $prefix );
1040                                 $files = $objects;
1041                         }
1042                         if ( count( $objects ) < $limit ) {
1043                                 $after = INF; // avoid a second RTT
1044                         } else {
1045                                 $after = end( $objects ); // update last item
1046                         }
1047                 } catch ( NoSuchContainerException $e ) {
1048                 } catch ( CloudFilesException $e ) { // some other exception?
1049                         $this->handleException( $e, null, __METHOD__,
1050                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1051                 }
1052
1053                 wfProfileOut( __METHOD__ . '-' . $this->name );
1054                 return $files;
1055         }
1056
1057         /**
1058          * @see FileBackendStore::doGetFileSha1base36()
1059          * @return bool
1060          */
1061         protected function doGetFileSha1base36( array $params ) {
1062                 $stat = $this->getFileStat( $params );
1063                 if ( $stat ) {
1064                         return $stat['sha1'];
1065                 } else {
1066                         return false;
1067                 }
1068         }
1069
1070         /**
1071          * @see FileBackendStore::doStreamFile()
1072          * @return Status
1073          */
1074         protected function doStreamFile( array $params ) {
1075                 $status = Status::newGood();
1076
1077                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1078                 if ( $srcRel === null ) {
1079                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1080                 }
1081
1082                 try {
1083                         $cont = $this->getContainer( $srcCont );
1084                 } catch ( NoSuchContainerException $e ) {
1085                         $status->fatal( 'backend-fail-stream', $params['src'] );
1086                         return $status;
1087                 } catch ( CloudFilesException $e ) { // some other exception?
1088                         $this->handleException( $e, $status, __METHOD__, $params );
1089                         return $status;
1090                 }
1091
1092                 try {
1093                         $output = fopen( 'php://output', 'wb' );
1094                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1095                         $obj->stream( $output, $this->headersFromParams( $params ) );
1096                 } catch ( NoSuchObjectException $e ) {
1097                         $status->fatal( 'backend-fail-stream', $params['src'] );
1098                 } catch ( CloudFilesException $e ) { // some other exception?
1099                         $this->handleException( $e, $status, __METHOD__, $params );
1100                 }
1101
1102                 return $status;
1103         }
1104
1105         /**
1106          * @see FileBackendStore::doGetLocalCopyMulti()
1107          * @return null|TempFSFile
1108          */
1109         protected function doGetLocalCopyMulti( array $params ) {
1110                 $tmpFiles = array();
1111
1112                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1113                 // Blindly create tmp files and stream to them, catching any exception if the file does
1114                 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1115                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1116                         $cfOps = array(); // (path => CF_Async_Op)
1117
1118                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1119                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1120                                 if ( $srcRel === null ) {
1121                                         $tmpFiles[$path] = null;
1122                                         continue;
1123                                 }
1124                                 $tmpFile = null;
1125                                 try {
1126                                         $sContObj = $this->getContainer( $srcCont );
1127                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1128                                         // Get source file extension
1129                                         $ext = FileBackend::extensionFromPath( $path );
1130                                         // Create a new temporary file...
1131                                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1132                                         if ( $tmpFile ) {
1133                                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1134                                                 if ( $handle ) {
1135                                                         $headers = $this->headersFromParams( $params );
1136                                                         if ( count( $pathBatch ) > 1 ) {
1137                                                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1138                                                                 $cfOps[$path]->_file_handle = $handle; // close this later
1139                                                         } else {
1140                                                                 $obj->stream( $handle, $headers );
1141                                                                 fclose( $handle );
1142                                                         }
1143                                                 } else {
1144                                                         $tmpFile = null;
1145                                                 }
1146                                         }
1147                                 } catch ( NoSuchContainerException $e ) {
1148                                         $tmpFile = null;
1149                                 } catch ( NoSuchObjectException $e ) {
1150                                         $tmpFile = null;
1151                                 } catch ( CloudFilesException $e ) { // some other exception?
1152                                         $tmpFile = null;
1153                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1154                                 }
1155                                 $tmpFiles[$path] = $tmpFile;
1156                         }
1157
1158                         $batch = new CF_Async_Op_Batch( $cfOps );
1159                         $cfOps = $batch->execute();
1160                         foreach ( $cfOps as $path => $cfOp ) {
1161                                 try {
1162                                         $cfOp->getLastResponse();
1163                                 } catch ( NoSuchContainerException $e ) {
1164                                         $tmpFiles[$path] = null;
1165                                 } catch ( NoSuchObjectException $e ) {
1166                                         $tmpFiles[$path] = null;
1167                                 } catch ( CloudFilesException $e ) { // some other exception?
1168                                         $tmpFiles[$path] = null;
1169                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1170                                 }
1171                                 fclose( $cfOp->_file_handle ); // close open handle
1172                         }
1173                 }
1174
1175                 return $tmpFiles;
1176         }
1177
1178         /**
1179          * @see FileBackendStore::getFileHttpUrl()
1180          * @return string|null
1181          */
1182         public function getFileHttpUrl( array $params ) {
1183                 if ( $this->swiftTempUrlKey != '' ) { // temp urls enabled
1184                         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1185                         if ( $srcRel === null ) {
1186                                 return null; // invalid path
1187                         }
1188                         try {
1189                                 $sContObj = $this->getContainer( $srcCont );
1190                                 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1191                                 return $obj->get_temp_url( $this->swiftTempUrlKey, 86400, "GET" );
1192                         } catch ( NoSuchContainerException $e ) {
1193                         } catch ( CloudFilesException $e ) { // some other exception?
1194                                 $this->handleException( $e, null, __METHOD__, $params );
1195                         }
1196                 }
1197                 return null;
1198         }
1199
1200         /**
1201          * @see FileBackendStore::directoriesAreVirtual()
1202          * @return bool
1203          */
1204         protected function directoriesAreVirtual() {
1205                 return true;
1206         }
1207
1208         /**
1209          * Get headers to send to Swift when reading a file based
1210          * on a FileBackend params array, e.g. that of getLocalCopy().
1211          * $params is currently only checked for a 'latest' flag.
1212          *
1213          * @param $params Array
1214          * @return Array
1215          */
1216         protected function headersFromParams( array $params ) {
1217                 $hdrs = array();
1218                 if ( !empty( $params['latest'] ) ) {
1219                         $hdrs[] = 'X-Newest: true';
1220                 }
1221                 return $hdrs;
1222         }
1223
1224         /**
1225          * @see FileBackendStore::doExecuteOpHandlesInternal()
1226          * @return Array List of corresponding Status objects
1227          */
1228         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1229                 $statuses = array();
1230
1231                 $cfOps = array(); // list of CF_Async_Op objects
1232                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1233                         $cfOps[$index] = $fileOpHandle->cfOp;
1234                 }
1235                 $batch = new CF_Async_Op_Batch( $cfOps );
1236
1237                 $cfOps = $batch->execute();
1238                 foreach ( $cfOps as $index => $cfOp ) {
1239                         $status = Status::newGood();
1240                         try { // catch exceptions; update status
1241                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1242                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1243                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1244                         } catch ( CloudFilesException $e ) { // some other exception?
1245                                 $this->handleException( $e, $status,
1246                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1247                         }
1248                         $statuses[$index] = $status;
1249                 }
1250
1251                 return $statuses;
1252         }
1253
1254         /**
1255          * Set read/write permissions for a Swift container.
1256          *
1257          * $readGrps is a list of the possible criteria for a request to have
1258          * access to read a container. Each item is one of the following formats:
1259          *   - account:user       : Grants access if the request is by the given user
1260          *   - ".r:<regex>"       : Grants access if the request is from a referrer host that
1261          *                          matches the expression and the request is not for a listing.
1262          *                          Setting this to '*' effectively makes a container public.
1263          *   -".rlistings:<regex>": Grants access if the request is from a referrer host that
1264          *                          matches the expression and the request for a listing.
1265          *
1266          * $writeGrps is a list of the possible criteria for a request to have
1267          * access to write to a container. Each item is of the following format:
1268          *   - account:user       : Grants access if the request is by the given user
1269          *
1270          * @see http://swift.openstack.org/misc.html#acls
1271          *
1272          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1273          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1274          *
1275          * @param $contObj CF_Container Swift container
1276          * @param $readGrps Array List of read access routes
1277          * @param $writeGrps Array List of write access routes
1278          * @return Status
1279          */
1280         protected function setContainerAccess(
1281                 CF_Container $contObj, array $readGrps, array $writeGrps
1282         ) {
1283                 $creds = $contObj->cfs_auth->export_credentials();
1284
1285                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1286
1287                 // Note: 10 second timeout consistent with php-cloudfiles
1288                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1289                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1290                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1291                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1292
1293                 return $req->execute(); // should return 204
1294         }
1295
1296         /**
1297          * Purge the CDN cache of affected objects if CDN caching is enabled.
1298          * This is for Rackspace/Akamai CDNs.
1299          *
1300          * @param $objects Array List of CF_Object items
1301          * @return void
1302          */
1303         public function purgeCDNCache( array $objects ) {
1304                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1305                         foreach ( $objects as $object ) {
1306                                 try {
1307                                         $object->purge_from_cdn();
1308                                 } catch ( CDNNotEnabledException $e ) {
1309                                         // CDN not enabled; nothing to see here
1310                                 } catch ( CloudFilesException $e ) {
1311                                         $this->handleException( $e, null, __METHOD__,
1312                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1313                                 }
1314                         }
1315                 }
1316         }
1317
1318         /**
1319          * Get an authenticated connection handle to the Swift proxy
1320          *
1321          * @throws CloudFilesException
1322          * @throws CloudFilesException|Exception
1323          * @return CF_Connection|bool False on failure
1324          */
1325         protected function getConnection() {
1326                 if ( $this->connException instanceof CloudFilesException ) {
1327                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1328                                 throw $this->connException; // failed last attempt; don't bother
1329                         } else { // actually retry this time
1330                                 $this->connException = null;
1331                                 $this->connErrorTime = 0;
1332                         }
1333                 }
1334                 // Session keys expire after a while, so we renew them periodically
1335                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1336                 // Authenticate with proxy and get a session key...
1337                 if ( !$this->conn || $reAuth ) {
1338                         $this->sessionStarted = 0;
1339                         $this->connContainerCache->clear();
1340                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1341                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1342                         if ( is_array( $creds ) ) { // cache hit
1343                                 $this->auth->load_cached_credentials(
1344                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1345                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1346                         } else { // cache miss
1347                                 try {
1348                                         $this->auth->authenticate();
1349                                         $creds = $this->auth->export_credentials();
1350                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1351                                         $this->sessionStarted = time();
1352                                 } catch ( CloudFilesException $e ) {
1353                                         $this->connException = $e; // don't keep re-trying
1354                                         $this->connErrorTime = time();
1355                                         throw $e; // throw it back
1356                                 }
1357                         }
1358                         if ( $this->conn ) { // re-authorizing?
1359                                 $this->conn->close(); // close active cURL handles in CF_Http object
1360                         }
1361                         $this->conn = new CF_Connection( $this->auth );
1362                 }
1363                 return $this->conn;
1364         }
1365
1366         /**
1367          * Close the connection to the Swift proxy
1368          *
1369          * @return void
1370          */
1371         protected function closeConnection() {
1372                 if ( $this->conn ) {
1373                         $this->conn->close(); // close active cURL handles in CF_Http object
1374                         $this->conn = null;
1375                         $this->sessionStarted = 0;
1376                         $this->connContainerCache->clear();
1377                 }
1378         }
1379
1380         /**
1381          * Get the cache key for a container
1382          *
1383          * @param $username string
1384          * @return string
1385          */
1386         private function getCredsCacheKey( $username ) {
1387                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1388         }
1389
1390         /**
1391          * Get a Swift container object, possibly from process cache.
1392          * Use $reCache if the file count or byte count is needed.
1393          *
1394          * @param $container string Container name
1395          * @param $bypassCache bool Bypass all caches and load from Swift
1396          * @return CF_Container
1397          * @throws CloudFilesException
1398          */
1399         protected function getContainer( $container, $bypassCache = false ) {
1400                 $conn = $this->getConnection(); // Swift proxy connection
1401                 if ( $bypassCache ) { // purge cache
1402                         $this->connContainerCache->clear( $container );
1403                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1404                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1405                 }
1406                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1407                         $contObj = $conn->get_container( $container );
1408                         // NoSuchContainerException not thrown: container must exist
1409                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1410                         if ( !$bypassCache ) {
1411                                 $this->setContainerCache( $container, // update persistent cache
1412                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1413                                 );
1414                         }
1415                 }
1416                 return $this->connContainerCache->get( $container, 'obj' );
1417         }
1418
1419         /**
1420          * Create a Swift container
1421          *
1422          * @param $container string Container name
1423          * @return CF_Container
1424          * @throws CloudFilesException
1425          */
1426         protected function createContainer( $container ) {
1427                 $conn = $this->getConnection(); // Swift proxy connection
1428                 $contObj = $conn->create_container( $container );
1429                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1430                 return $contObj;
1431         }
1432
1433         /**
1434          * Delete a Swift container
1435          *
1436          * @param $container string Container name
1437          * @return void
1438          * @throws CloudFilesException
1439          */
1440         protected function deleteContainer( $container ) {
1441                 $conn = $this->getConnection(); // Swift proxy connection
1442                 $this->connContainerCache->clear( $container ); // purge
1443                 $conn->delete_container( $container );
1444         }
1445
1446         /**
1447          * @see FileBackendStore::doPrimeContainerCache()
1448          * @return void
1449          */
1450         protected function doPrimeContainerCache( array $containerInfo ) {
1451                 try {
1452                         $conn = $this->getConnection(); // Swift proxy connection
1453                         foreach ( $containerInfo as $container => $info ) {
1454                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1455                                         $container, $info['count'], $info['bytes'] );
1456                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1457                         }
1458                 } catch ( CloudFilesException $e ) { // some other exception?
1459                         $this->handleException( $e, null, __METHOD__, array() );
1460                 }
1461         }
1462
1463         /**
1464          * Log an unexpected exception for this backend.
1465          * This also sets the Status object to have a fatal error.
1466          *
1467          * @param $e Exception
1468          * @param $status Status|null
1469          * @param $func string
1470          * @param $params Array
1471          * @return void
1472          */
1473         protected function handleException( Exception $e, $status, $func, array $params ) {
1474                 if ( $status instanceof Status ) {
1475                         if ( $e instanceof AuthenticationException ) {
1476                                 $status->fatal( 'backend-fail-connect', $this->name );
1477                         } else {
1478                                 $status->fatal( 'backend-fail-internal', $this->name );
1479                         }
1480                 }
1481                 if ( $e->getMessage() ) {
1482                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1483                 }
1484                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1485                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1486                         $this->closeConnection(); // force a re-connect and re-auth next time
1487                 }
1488                 wfDebugLog( 'SwiftBackend',
1489                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1490                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1491                 );
1492         }
1493 }
1494
1495 /**
1496  * @see FileBackendStoreOpHandle
1497  */
1498 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1499         /** @var CF_Async_Op */
1500         public $cfOp;
1501         /** @var Array */
1502         public $affectedObjects = array();
1503
1504         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1505                 $this->backend = $backend;
1506                 $this->params = $params;
1507                 $this->call = $call;
1508                 $this->cfOp = $cfOp;
1509         }
1510 }
1511
1512 /**
1513  * SwiftFileBackend helper class to page through listings.
1514  * Swift also has a listing limit of 10,000 objects for sanity.
1515  * Do not use this class from places outside SwiftFileBackend.
1516  *
1517  * @ingroup FileBackend
1518  */
1519 abstract class SwiftFileBackendList implements Iterator {
1520         /** @var Array */
1521         protected $bufferIter = array();
1522         protected $bufferAfter = null; // string; list items *after* this path
1523         protected $pos = 0; // integer
1524         /** @var Array */
1525         protected $params = array();
1526
1527         /** @var SwiftFileBackend */
1528         protected $backend;
1529         protected $container; // string; container name
1530         protected $dir; // string; storage directory
1531         protected $suffixStart; // integer
1532
1533         const PAGE_SIZE = 9000; // file listing buffer size
1534
1535         /**
1536          * @param $backend SwiftFileBackend
1537          * @param $fullCont string Resolved container name
1538          * @param $dir string Resolved directory relative to container
1539          * @param $params Array
1540          */
1541         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1542                 $this->backend = $backend;
1543                 $this->container = $fullCont;
1544                 $this->dir = $dir;
1545                 if ( substr( $this->dir, -1 ) === '/' ) {
1546                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1547                 }
1548                 if ( $this->dir == '' ) { // whole container
1549                         $this->suffixStart = 0;
1550                 } else { // dir within container
1551                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1552                 }
1553                 $this->params = $params;
1554         }
1555
1556         /**
1557          * @see Iterator::key()
1558          * @return integer
1559          */
1560         public function key() {
1561                 return $this->pos;
1562         }
1563
1564         /**
1565          * @see Iterator::next()
1566          * @return void
1567          */
1568         public function next() {
1569                 // Advance to the next file in the page
1570                 next( $this->bufferIter );
1571                 ++$this->pos;
1572                 // Check if there are no files left in this page and
1573                 // advance to the next page if this page was not empty.
1574                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1575                         $this->bufferIter = $this->pageFromList(
1576                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1577                         ); // updates $this->bufferAfter
1578                 }
1579         }
1580
1581         /**
1582          * @see Iterator::rewind()
1583          * @return void
1584          */
1585         public function rewind() {
1586                 $this->pos = 0;
1587                 $this->bufferAfter = null;
1588                 $this->bufferIter = $this->pageFromList(
1589                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1590                 ); // updates $this->bufferAfter
1591         }
1592
1593         /**
1594          * @see Iterator::valid()
1595          * @return bool
1596          */
1597         public function valid() {
1598                 if ( $this->bufferIter === null ) {
1599                         return false; // some failure?
1600                 } else {
1601                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1602                 }
1603         }
1604
1605         /**
1606          * Get the given list portion (page)
1607          *
1608          * @param $container string Resolved container name
1609          * @param $dir string Resolved path relative to container
1610          * @param $after string|null
1611          * @param $limit integer
1612          * @param $params Array
1613          * @return Traversable|Array|null Returns null on failure
1614          */
1615         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1616 }
1617
1618 /**
1619  * Iterator for listing directories
1620  */
1621 class SwiftFileBackendDirList extends SwiftFileBackendList {
1622         /**
1623          * @see Iterator::current()
1624          * @return string|bool String (relative path) or false
1625          */
1626         public function current() {
1627                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1628         }
1629
1630         /**
1631          * @see SwiftFileBackendList::pageFromList()
1632          * @return Array|null
1633          */
1634         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1635                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1636         }
1637 }
1638
1639 /**
1640  * Iterator for listing regular files
1641  */
1642 class SwiftFileBackendFileList extends SwiftFileBackendList {
1643         /**
1644          * @see Iterator::current()
1645          * @return string|bool String (relative path) or false
1646          */
1647         public function current() {
1648                 return substr( current( $this->bufferIter ), $this->suffixStart );
1649         }
1650
1651         /**
1652          * @see SwiftFileBackendList::pageFromList()
1653          * @return Array|null
1654          */
1655         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1656                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1657         }
1658 }