includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $maxContCacheSize = 300; // integer; max containers with entries
  46
  47         /** @var CF_Connection */
  48         protected $conn; // Swift connection handle
  49         protected $connStarted = 0; // integer UNIX timestamp
  50         protected $connContainers = array(); // container object cache
  51         protected $connException; // CloudFiles exception
  52
  53         /**
  54          * @see FileBackendStore::__construct()
  55          * Additional $config params include:
  56          *    swiftAuthUrl       : Swift authentication server URL
  57          *    swiftUser          : Swift user used by MediaWiki (account:username)
  58          *    swiftKey           : Swift authentication key for the above user
  59          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  60          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  61          *    swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  62          *    shardViaHashLevels : Map of container names to sharding config with:
  63          *                         'base'   : base of hash characters, 16 or 36
  64          *                         'levels' : the number of hash levels (and digits)
  65          *                         'repeat' : hash subdirectories are prefixed with all the
  66          *                                    parent hash directory names (e.g. "a/ab/abc")
  67          */
  68         public function __construct( array $config ) {
  69                 parent::__construct( $config );
  70                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  71                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  72                 }
  73                 // Required settings
  74                 $this->auth = new CF_Authentication(
  75                         $config['swiftUser'],
  76                         $config['swiftKey'],
  77                         null, // account; unused
  78                         $config['swiftAuthUrl']
  79                 );
  80                 // Optional settings
  81                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  82                         ? $config['swiftAuthTTL']
  83                         : 5 * 60; // some sane number
  84                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  85                         ? $config['swiftAnonUser']
  86                         : '';
  87                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  88                         ? $config['shardViaHashLevels']
  89                         : '';
  90                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
  91                         ? $config['swiftUseCDN']
  92                         : false;
  93                 // Cache container info to mask latency
  94                 $this->memCache = wfGetMainCache();
  95         }
  96
  97         /**
  98          * @see FileBackendStore::resolveContainerPath()
  99          * @return null
 100          */
 101         protected function resolveContainerPath( $container, $relStoragePath ) {
 102                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 103                         return null; // too long for Swift
 104                 }
 105                 return $relStoragePath;
 106         }
 107
 108         /**
 109          * @see FileBackendStore::isPathUsableInternal()
 110          * @return bool
 111          */
 112         public function isPathUsableInternal( $storagePath ) {
 113                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 114                 if ( $rel === null ) {
 115                         return false; // invalid
 116                 }
 117
 118                 try {
 119                         $this->getContainer( $container );
 120                         return true; // container exists
 121                 } catch ( NoSuchContainerException $e ) {
 122                 } catch ( CloudFilesException $e ) { // some other exception?
 123                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 124                 }
 125
 126                 return false;
 127         }
 128
 129         /**
 130          * @see FileBackendStore::doCreateInternal()
 131          * @return Status
 132          */
 133         protected function doCreateInternal( array $params ) {
 134                 $status = Status::newGood();
 135
 136                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 137                 if ( $dstRel === null ) {
 138                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 139                         return $status;
 140                 }
 141
 142                 // (a) Check the destination container and object
 143                 try {
 144                         $dContObj = $this->getContainer( $dstCont );
 145                         if ( empty( $params['overwrite'] ) &&
 146                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 147                         {
 148                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 149                                 return $status;
 150                         }
 151                 } catch ( NoSuchContainerException $e ) {
 152                         $status->fatal( 'backend-fail-create', $params['dst'] );
 153                         return $status;
 154                 } catch ( CloudFilesException $e ) { // some other exception?
 155                         $this->handleException( $e, $status, __METHOD__, $params );
 156                         return $status;
 157                 }
 158
 159                 // (b) Get a SHA-1 hash of the object
 160                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 161
 162                 // (c) Actually create the object
 163                 try {
 164                         // Create a fresh CF_Object with no fields preloaded.
 165                         // We don't want to preserve headers, metadata, and such.
 166                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 167                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 168                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 169                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 170                         // The MD5 here will be checked within Swift against its own MD5.
 171                         $obj->set_etag( md5( $params['content'] ) );
 172                         // Use the same content type as StreamFile for security
 173                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 174                         if ( !empty( $params['async'] ) ) { // deferred
 175                                 $handle = $obj->write_async( $params['content'] );
 176                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 177                                 $status->value->affectedObjects[] = $obj;
 178                         } else { // actually write the object in Swift
 179                                 $obj->write( $params['content'] );
 180                                 $this->purgeCDNCache( array( $obj ) );
 181                         }
 182                 } catch ( CDNNotEnabledException $e ) {
 183                         // CDN not enabled; nothing to see here
 184                 } catch ( BadContentTypeException $e ) {
 185                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 186                 } catch ( CloudFilesException $e ) { // some other exception?
 187                         $this->handleException( $e, $status, __METHOD__, $params );
 188                 }
 189
 190                 return $status;
 191         }
 192
 193         /**
 194          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 195          */
 196         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 197                 try {
 198                         $cfOp->getLastResponse();
 199                 } catch ( BadContentTypeException $e ) {
 200                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 201                 }
 202         }
 203
 204         /**
 205          * @see FileBackendStore::doStoreInternal()
 206          * @return Status
 207          */
 208         protected function doStoreInternal( array $params ) {
 209                 $status = Status::newGood();
 210
 211                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 212                 if ( $dstRel === null ) {
 213                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 214                         return $status;
 215                 }
 216
 217                 // (a) Check the destination container and object
 218                 try {
 219                         $dContObj = $this->getContainer( $dstCont );
 220                         if ( empty( $params['overwrite'] ) &&
 221                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 222                         {
 223                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 224                                 return $status;
 225                         }
 226                 } catch ( NoSuchContainerException $e ) {
 227                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 228                         return $status;
 229                 } catch ( CloudFilesException $e ) { // some other exception?
 230                         $this->handleException( $e, $status, __METHOD__, $params );
 231                         return $status;
 232                 }
 233
 234                 // (b) Get a SHA-1 hash of the object
 235                 $sha1Hash = sha1_file( $params['src'] );
 236                 if ( $sha1Hash === false ) { // source doesn't exist?
 237                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 238                         return $status;
 239                 }
 240                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 241
 242                 // (c) Actually store the object
 243                 try {
 244                         // Create a fresh CF_Object with no fields preloaded.
 245                         // We don't want to preserve headers, metadata, and such.
 246                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 247                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 248                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 249                         // The MD5 here will be checked within Swift against its own MD5.
 250                         $obj->set_etag( md5_file( $params['src'] ) );
 251                         // Use the same content type as StreamFile for security
 252                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 253                         if ( !empty( $params['async'] ) ) { // deferred
 254                                 wfSuppressWarnings();
 255                                 $fp = fopen( $params['src'], 'rb' );
 256                                 wfRestoreWarnings();
 257                                 if ( !$fp ) {
 258                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 259                                 } else {
 260                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 261                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 262                                         $status->value->resourcesToClose[] = $fp;
 263                                         $status->value->affectedObjects[] = $obj;
 264                                 }
 265                         } else { // actually write the object in Swift
 266                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 267                                 $this->purgeCDNCache( array( $obj ) );
 268                         }
 269                 } catch ( CDNNotEnabledException $e ) {
 270                         // CDN not enabled; nothing to see here
 271                 } catch ( BadContentTypeException $e ) {
 272                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 273                 } catch ( IOException $e ) {
 274                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 275                 } catch ( CloudFilesException $e ) { // some other exception?
 276                         $this->handleException( $e, $status, __METHOD__, $params );
 277                 }
 278
 279                 return $status;
 280         }
 281
 282         /**
 283          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 284          */
 285         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 286                 try {
 287                         $cfOp->getLastResponse();
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 } catch ( IOException $e ) {
 291                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 292                 }
 293         }
 294
 295         /**
 296          * @see FileBackendStore::doCopyInternal()
 297          * @return Status
 298          */
 299         protected function doCopyInternal( array $params ) {
 300                 $status = Status::newGood();
 301
 302                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 303                 if ( $srcRel === null ) {
 304                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 305                         return $status;
 306                 }
 307
 308                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 309                 if ( $dstRel === null ) {
 310                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 311                         return $status;
 312                 }
 313
 314                 // (a) Check the source/destination containers and destination object
 315                 try {
 316                         $sContObj = $this->getContainer( $srcCont );
 317                         $dContObj = $this->getContainer( $dstCont );
 318                         if ( empty( $params['overwrite'] ) &&
 319                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 320                         {
 321                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 322                                 return $status;
 323                         }
 324                 } catch ( NoSuchContainerException $e ) {
 325                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 326                         return $status;
 327                 } catch ( CloudFilesException $e ) { // some other exception?
 328                         $this->handleException( $e, $status, __METHOD__, $params );
 329                         return $status;
 330                 }
 331
 332                 // (b) Actually copy the file to the destination
 333                 try {
 334                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 335                         if ( !empty( $params['async'] ) ) { // deferred
 336                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 337                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 338                                 $status->value->affectedObjects[] = $dstObj;
 339                         } else { // actually write the object in Swift
 340                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 341                                 $this->purgeCDNCache( array( $dstObj ) );
 342                         }
 343                 } catch ( CDNNotEnabledException $e ) {
 344                         // CDN not enabled; nothing to see here
 345                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 346                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 347                 } catch ( CloudFilesException $e ) { // some other exception?
 348                         $this->handleException( $e, $status, __METHOD__, $params );
 349                 }
 350
 351                 return $status;
 352         }
 353
 354         /**
 355          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 356          */
 357         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 358                 try {
 359                         $cfOp->getLastResponse();
 360                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 361                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 362                 }
 363         }
 364
 365         /**
 366          * @see FileBackendStore::doMoveInternal()
 367          * @return Status
 368          */
 369         protected function doMoveInternal( array $params ) {
 370                 $status = Status::newGood();
 371
 372                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 373                 if ( $srcRel === null ) {
 374                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 375                         return $status;
 376                 }
 377
 378                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 379                 if ( $dstRel === null ) {
 380                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 381                         return $status;
 382                 }
 383
 384                 // (a) Check the source/destination containers and destination object
 385                 try {
 386                         $sContObj = $this->getContainer( $srcCont );
 387                         $dContObj = $this->getContainer( $dstCont );
 388                         if ( empty( $params['overwrite'] ) &&
 389                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 390                         {
 391                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 392                                 return $status;
 393                         }
 394                 } catch ( NoSuchContainerException $e ) {
 395                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 396                         return $status;
 397                 } catch ( CloudFilesException $e ) { // some other exception?
 398                         $this->handleException( $e, $status, __METHOD__, $params );
 399                         return $status;
 400                 }
 401
 402                 // (b) Actually move the file to the destination
 403                 try {
 404                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 405                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 406                         if ( !empty( $params['async'] ) ) { // deferred
 407                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 408                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 409                                 $status->value->affectedObjects[] = $srcObj;
 410                                 $status->value->affectedObjects[] = $dstObj;
 411                         } else { // actually write the object in Swift
 412                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 413                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 414                         }
 415                 } catch ( CDNNotEnabledException $e ) {
 416                         // CDN not enabled; nothing to see here
 417                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 418                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 419                 } catch ( CloudFilesException $e ) { // some other exception?
 420                         $this->handleException( $e, $status, __METHOD__, $params );
 421                 }
 422
 423                 return $status;
 424         }
 425
 426         /**
 427          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 428          */
 429         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 430                 try {
 431                         $cfOp->getLastResponse();
 432                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 433                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 434                 }
 435         }
 436
 437         /**
 438          * @see FileBackendStore::doDeleteInternal()
 439          * @return Status
 440          */
 441         protected function doDeleteInternal( array $params ) {
 442                 $status = Status::newGood();
 443
 444                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 445                 if ( $srcRel === null ) {
 446                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 447                         return $status;
 448                 }
 449
 450                 try {
 451                         $sContObj = $this->getContainer( $srcCont );
 452                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 453                         if ( !empty( $params['async'] ) ) { // deferred
 454                                 $handle = $sContObj->delete_object_async( $srcRel );
 455                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 456                                 $status->value->affectedObjects[] = $srcObj;
 457                         } else { // actually write the object in Swift
 458                                 $sContObj->delete_object( $srcRel );
 459                                 $this->purgeCDNCache( array( $srcObj ) );
 460                         }
 461                 } catch ( CDNNotEnabledException $e ) {
 462                         // CDN not enabled; nothing to see here
 463                 } catch ( NoSuchContainerException $e ) {
 464                         $status->fatal( 'backend-fail-delete', $params['src'] );
 465                 } catch ( NoSuchObjectException $e ) {
 466                         if ( empty( $params['ignoreMissingSource'] ) ) {
 467                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 468                         }
 469                 } catch ( CloudFilesException $e ) { // some other exception?
 470                         $this->handleException( $e, $status, __METHOD__, $params );
 471                 }
 472
 473                 return $status;
 474         }
 475
 476         /**
 477          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 478          */
 479         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 480                 try {
 481                         $cfOp->getLastResponse();
 482                 } catch ( NoSuchContainerException $e ) {
 483                         $status->fatal( 'backend-fail-delete', $params['src'] );
 484                 } catch ( NoSuchObjectException $e ) {
 485                         if ( empty( $params['ignoreMissingSource'] ) ) {
 486                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 487                         }
 488                 }
 489         }
 490
 491         /**
 492          * @see FileBackendStore::doPrepareInternal()
 493          * @return Status
 494          */
 495         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 496                 $status = Status::newGood();
 497
 498                 // (a) Check if container already exists
 499                 try {
 500                         $contObj = $this->getContainer( $fullCont );
 501                         // NoSuchContainerException not thrown: container must exist
 502                         return $status; // already exists
 503                 } catch ( NoSuchContainerException $e ) {
 504                         // NoSuchContainerException thrown: container does not exist
 505                 } catch ( CloudFilesException $e ) { // some other exception?
 506                         $this->handleException( $e, $status, __METHOD__, $params );
 507                         return $status;
 508                 }
 509
 510                 // (b) Create container as needed
 511                 try {
 512                         $contObj = $this->createContainer( $fullCont );
 513                         // Make container public to end-users...
 514                         if ( $this->swiftAnonUser != '' ) {
 515                                 $status->merge( $this->setContainerAccess(
 516                                         $contObj,
 517                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 518                                         array( $this->auth->username ) // write
 519                                 ) );
 520                         }
 521                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 522                                 $contObj->make_public();
 523                         }
 524                 } catch ( CDNNotEnabledException $e ) {
 525                         // CDN not enabled; nothing to see here
 526                 } catch ( CloudFilesException $e ) { // some other exception?
 527                         $this->handleException( $e, $status, __METHOD__, $params );
 528                         return $status;
 529                 }
 530
 531                 return $status;
 532         }
 533
 534         /**
 535          * @see FileBackendStore::doSecureInternal()
 536          * @return Status
 537          */
 538         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 539                 $status = Status::newGood();
 540
 541                 // Restrict container from end-users...
 542                 try {
 543                         // doPrepareInternal() should have been called,
 544                         // so the Swift container should already exist...
 545                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 546                         // NoSuchContainerException not thrown: container must exist
 547
 548                         // Make container private to end-users...
 549                         if ( $this->swiftAnonUser != '' && !isset( $contObj->mw_wasSecured ) ) {
 550                                 $status->merge( $this->setContainerAccess(
 551                                         $contObj,
 552                                         array( $this->auth->username ), // read
 553                                         array( $this->auth->username ) // write
 554                                 ) );
 555                                 // @TODO: when php-cloudfiles supports container
 556                                 // metadata, we can make use of that to avoid RTTs
 557                                 $contObj->mw_wasSecured = true; // avoid useless RTTs
 558                         }
 559                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 560                                 $contObj->make_private();
 561                         }
 562                 } catch ( CDNNotEnabledException $e ) {
 563                         // CDN not enabled; nothing to see here
 564                 } catch ( CloudFilesException $e ) { // some other exception?
 565                         $this->handleException( $e, $status, __METHOD__, $params );
 566                 }
 567
 568                 return $status;
 569         }
 570
 571         /**
 572          * @see FileBackendStore::doCleanInternal()
 573          * @return Status
 574          */
 575         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 576                 $status = Status::newGood();
 577
 578                 // Only containers themselves can be removed, all else is virtual
 579                 if ( $dir != '' ) {
 580                         return $status; // nothing to do
 581                 }
 582
 583                 // (a) Check the container
 584                 try {
 585                         $contObj = $this->getContainer( $fullCont, true );
 586                 } catch ( NoSuchContainerException $e ) {
 587                         return $status; // ok, nothing to do
 588                 } catch ( CloudFilesException $e ) { // some other exception?
 589                         $this->handleException( $e, $status, __METHOD__, $params );
 590                         return $status;
 591                 }
 592
 593                 // (b) Delete the container if empty
 594                 if ( $contObj->object_count == 0 ) {
 595                         try {
 596                                 $this->deleteContainer( $fullCont );
 597                         } catch ( NoSuchContainerException $e ) {
 598                                 return $status; // race?
 599                         } catch ( NonEmptyContainerException $e ) {
 600                                 return $status; // race? consistency delay?
 601                         } catch ( CloudFilesException $e ) { // some other exception?
 602                                 $this->handleException( $e, $status, __METHOD__, $params );
 603                                 return $status;
 604                         }
 605                 }
 606
 607                 return $status;
 608         }
 609
 610         /**
 611          * @see FileBackendStore::doFileExists()
 612          * @return array|bool|null
 613          */
 614         protected function doGetFileStat( array $params ) {
 615                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 616                 if ( $srcRel === null ) {
 617                         return false; // invalid storage path
 618                 }
 619
 620                 $stat = false;
 621                 try {
 622                         $contObj = $this->getContainer( $srcCont );
 623                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 624                         $this->addMissingMetadata( $srcObj, $params['src'] );
 625                         $stat = array(
 626                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 627                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 628                                 'size'  => (int)$srcObj->content_length,
 629                                 'sha1'  => $srcObj->metadata['Sha1base36']
 630                         );
 631                 } catch ( NoSuchContainerException $e ) {
 632                 } catch ( NoSuchObjectException $e ) {
 633                 } catch ( CloudFilesException $e ) { // some other exception?
 634                         $stat = null;
 635                         $this->handleException( $e, null, __METHOD__, $params );
 636                 }
 637
 638                 return $stat;
 639         }
 640
 641         /**
 642          * Fill in any missing object metadata and save it to Swift
 643          *
 644          * @param $obj CF_Object
 645          * @param $path string Storage path to object
 646          * @return bool Success
 647          * @throws Exception cloudfiles exceptions
 648          */
 649         protected function addMissingMetadata( CF_Object $obj, $path ) {
 650                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 651                         return true; // nothing to do
 652                 }
 653                 $status = Status::newGood();
 654                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 655                 if ( $status->isOK() ) {
 656                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 657                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 658                         if ( $tmpFile ) {
 659                                 $hash = $tmpFile->getSha1Base36();
 660                                 if ( $hash !== false ) {
 661                                         $obj->metadata['Sha1base36'] = $hash;
 662                                         $obj->sync_metadata(); // save to Swift
 663                                         return true; // success
 664                                 }
 665                         }
 666                 }
 667                 $obj->metadata['Sha1base36'] = false;
 668                 return false; // failed
 669         }
 670
 671         /**
 672          * @see FileBackend::getFileContents()
 673          * @return bool|null|string
 674          */
 675         public function getFileContents( array $params ) {
 676                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 677                 if ( $srcRel === null ) {
 678                         return false; // invalid storage path
 679                 }
 680
 681                 if ( !$this->fileExists( $params ) ) {
 682                         return null;
 683                 }
 684
 685                 $data = false;
 686                 try {
 687                         $sContObj = $this->getContainer( $srcCont );
 688                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 689                         $data = $obj->read( $this->headersFromParams( $params ) );
 690                 } catch ( NoSuchContainerException $e ) {
 691                 } catch ( CloudFilesException $e ) { // some other exception?
 692                         $this->handleException( $e, null, __METHOD__, $params );
 693                 }
 694
 695                 return $data;
 696         }
 697
 698         /**
 699          * @see FileBackendStore::doDirectoryExists()
 700          * @return bool|null
 701          */
 702         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 703                 try {
 704                         $container = $this->getContainer( $fullCont );
 705                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 706                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 707                 } catch ( NoSuchContainerException $e ) {
 708                         return false;
 709                 } catch ( CloudFilesException $e ) { // some other exception?
 710                         $this->handleException( $e, null, __METHOD__,
 711                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 712                 }
 713
 714                 return null; // error
 715         }
 716
 717         /**
 718          * @see FileBackendStore::getDirectoryListInternal()
 719          * @return SwiftFileBackendDirList
 720          */
 721         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 722                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 723         }
 724
 725         /**
 726          * @see FileBackendStore::getFileListInternal()
 727          * @return SwiftFileBackendFileList
 728          */
 729         public function getFileListInternal( $fullCont, $dir, array $params ) {
 730                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 731         }
 732
 733         /**
 734          * Do not call this function outside of SwiftFileBackendFileList
 735          *
 736          * @param $fullCont string Resolved container name
 737          * @param $dir string Resolved storage directory with no trailing slash
 738          * @param $after string|null Storage path of file to list items after
 739          * @param $limit integer Max number of items to list
 740          * @param $params Array Includes flag for 'topOnly'
 741          * @return Array List of relative paths of dirs directly under $dir
 742          */
 743         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 744                 $dirs = array();
 745                 if ( $after === INF ) {
 746                         return $dirs; // nothing more
 747                 }
 748                 wfProfileIn( __METHOD__ . '-' . $this->name );
 749
 750                 try {
 751                         $container = $this->getContainer( $fullCont );
 752                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 753                         // Non-recursive: only list dirs right under $dir
 754                         if ( !empty( $params['topOnly'] ) ) {
 755                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 756                                 foreach ( $objects as $object ) { // files and dirs
 757                                         if ( substr( $object, -1 ) === '/' ) {
 758                                                 $dirs[] = $object; // directories end in '/'
 759                                         }
 760                                 }
 761                         // Recursive: list all dirs under $dir and its subdirs
 762                         } else {
 763                                 // Get directory from last item of prior page
 764                                 $lastDir = $this->getParentDir( $after ); // must be first page
 765                                 $objects = $container->list_objects( $limit, $after, $prefix );
 766                                 foreach ( $objects as $object ) { // files
 767                                         $objectDir = $this->getParentDir( $object ); // directory of object
 768                                         if ( $objectDir !== false ) { // file has a parent dir
 769                                                 // Swift stores paths in UTF-8, using binary sorting.
 770                                                 // See function "create_container_table" in common/db.py.
 771                                                 // If a directory is not "greater" than the last one,
 772                                                 // then it was already listed by the calling iterator.
 773                                                 if ( $objectDir > $lastDir ) {
 774                                                         $pDir = $objectDir;
 775                                                         do { // add dir and all its parent dirs
 776                                                                 $dirs[] = "{$pDir}/";
 777                                                                 $pDir = $this->getParentDir( $pDir );
 778                                                         } while ( $pDir !== false // sanity
 779                                                                 && $pDir > $lastDir // not done already
 780                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 781                                                         );
 782                                                 }
 783                                                 $lastDir = $objectDir;
 784                                         }
 785                                 }
 786                         }
 787                         if ( count( $objects ) < $limit ) {
 788                                 $after = INF; // avoid a second RTT
 789                         } else {
 790                                 $after = end( $objects ); // update last item
 791                         }
 792                 } catch ( NoSuchContainerException $e ) {
 793                 } catch ( CloudFilesException $e ) { // some other exception?
 794                         $this->handleException( $e, null, __METHOD__,
 795                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 796                 }
 797
 798                 wfProfileOut( __METHOD__ . '-' . $this->name );
 799                 return $dirs;
 800         }
 801
 802         protected function getParentDir( $path ) {
 803                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 804         }
 805
 806         /**
 807          * Do not call this function outside of SwiftFileBackendFileList
 808          *
 809          * @param $fullCont string Resolved container name
 810          * @param $dir string Resolved storage directory with no trailing slash
 811          * @param $after string|null Storage path of file to list items after
 812          * @param $limit integer Max number of items to list
 813          * @param $params Array Includes flag for 'topOnly'
 814          * @return Array List of relative paths of files under $dir
 815          */
 816         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 817                 $files = array();
 818                 if ( $after === INF ) {
 819                         return $files; // nothing more
 820                 }
 821                 wfProfileIn( __METHOD__ . '-' . $this->name );
 822
 823                 try {
 824                         $container = $this->getContainer( $fullCont );
 825                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 826                         // Non-recursive: only list files right under $dir
 827                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 828                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 829                                 foreach ( $objects as $object ) {
 830                                         if ( substr( $object, -1 ) !== '/' ) {
 831                                                 $files[] = $object; // directories end in '/'
 832                                         }
 833                                 }
 834                         // Recursive: list all files under $dir and its subdirs
 835                         } else { // files
 836                                 $objects = $container->list_objects( $limit, $after, $prefix );
 837                                 $files = $objects;
 838                         }
 839                         if ( count( $objects ) < $limit ) {
 840                                 $after = INF; // avoid a second RTT
 841                         } else {
 842                                 $after = end( $objects ); // update last item
 843                         }
 844                 } catch ( NoSuchContainerException $e ) {
 845                 } catch ( CloudFilesException $e ) { // some other exception?
 846                         $this->handleException( $e, null, __METHOD__,
 847                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 848                 }
 849
 850                 wfProfileOut( __METHOD__ . '-' . $this->name );
 851                 return $files;
 852         }
 853
 854         /**
 855          * @see FileBackendStore::doGetFileSha1base36()
 856          * @return bool
 857          */
 858         protected function doGetFileSha1base36( array $params ) {
 859                 $stat = $this->getFileStat( $params );
 860                 if ( $stat ) {
 861                         return $stat['sha1'];
 862                 } else {
 863                         return false;
 864                 }
 865         }
 866
 867         /**
 868          * @see FileBackendStore::doStreamFile()
 869          * @return Status
 870          */
 871         protected function doStreamFile( array $params ) {
 872                 $status = Status::newGood();
 873
 874                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 875                 if ( $srcRel === null ) {
 876                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 877                 }
 878
 879                 try {
 880                         $cont = $this->getContainer( $srcCont );
 881                 } catch ( NoSuchContainerException $e ) {
 882                         $status->fatal( 'backend-fail-stream', $params['src'] );
 883                         return $status;
 884                 } catch ( CloudFilesException $e ) { // some other exception?
 885                         $this->handleException( $e, $status, __METHOD__, $params );
 886                         return $status;
 887                 }
 888
 889                 try {
 890                         $output = fopen( 'php://output', 'wb' );
 891                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 892                         $obj->stream( $output, $this->headersFromParams( $params ) );
 893                 } catch ( CloudFilesException $e ) { // some other exception?
 894                         $this->handleException( $e, $status, __METHOD__, $params );
 895                 }
 896
 897                 return $status;
 898         }
 899
 900         /**
 901          * @see FileBackendStore::getLocalCopy()
 902          * @return null|TempFSFile
 903          */
 904         public function getLocalCopy( array $params ) {
 905                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 906                 if ( $srcRel === null ) {
 907                         return null;
 908                 }
 909
 910                 # Check the recursion guard to avoid loops when filling metadata
 911                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 912                         return null;
 913                 }
 914
 915                 $tmpFile = null;
 916                 try {
 917                         $sContObj = $this->getContainer( $srcCont );
 918                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 919                         // Get source file extension
 920                         $ext = FileBackend::extensionFromPath( $srcRel );
 921                         // Create a new temporary file...
 922                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 923                         if ( $tmpFile ) {
 924                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 925                                 if ( $handle ) {
 926                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 927                                         fclose( $handle );
 928                                 } else {
 929                                         $tmpFile = null; // couldn't open temp file
 930                                 }
 931                         }
 932                 } catch ( NoSuchContainerException $e ) {
 933                         $tmpFile = null;
 934                 } catch ( CloudFilesException $e ) { // some other exception?
 935                         $tmpFile = null;
 936                         $this->handleException( $e, null, __METHOD__, $params );
 937                 }
 938
 939                 return $tmpFile;
 940         }
 941
 942         /**
 943          * @see FileBackendStore::directoriesAreVirtual()
 944          * @return bool
 945          */
 946         protected function directoriesAreVirtual() {
 947                 return true;
 948         }
 949
 950         /**
 951          * Get headers to send to Swift when reading a file based
 952          * on a FileBackend params array, e.g. that of getLocalCopy().
 953          * $params is currently only checked for a 'latest' flag.
 954          *
 955          * @param $params Array
 956          * @return Array
 957          */
 958         protected function headersFromParams( array $params ) {
 959                 $hdrs = array();
 960                 if ( !empty( $params['latest'] ) ) {
 961                         $hdrs[] = 'X-Newest: true';
 962                 }
 963                 return $hdrs;
 964         }
 965
 966         /**
 967          * @see FileBackendStore::doExecuteOpHandlesInternal()
 968          * @return Array List of corresponding Status objects
 969          */
 970         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 971                 $statuses = array();
 972
 973                 $cfOps = array(); // list of CF_Async_Op objects
 974                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 975                         $cfOps[$index] = $fileOpHandle->cfOp;
 976                 }
 977                 $batch = new CF_Async_Op_Batch( $cfOps );
 978
 979                 $cfOps = $batch->execute();
 980                 foreach ( $cfOps as $index => $cfOp ) {
 981                         $status = Status::newGood();
 982                         try { // catch exceptions; update status
 983                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 984                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 985                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
 986                         } catch ( CloudFilesException $e ) { // some other exception?
 987                                 $this->handleException( $e, $status,
 988                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
 989                         }
 990                         $statuses[$index] = $status;
 991                 }
 992
 993                 return $statuses;
 994         }
 995
 996         /**
 997          * Set read/write permissions for a Swift container
 998          *
 999          * @param $contObj CF_Container Swift container
1000          * @param $readGrps Array Swift users who can read (account:user)
1001          * @param $writeGrps Array Swift users who can write (account:user)
1002          * @return Status
1003          */
1004         protected function setContainerAccess(
1005                 CF_Container $contObj, array $readGrps, array $writeGrps
1006         ) {
1007                 $creds = $contObj->cfs_auth->export_credentials();
1008
1009                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1010
1011                 // Note: 10 second timeout consistent with php-cloudfiles
1012                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1013                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1014                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1015                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1016
1017                 return $req->execute(); // should return 204
1018         }
1019
1020         /**
1021          * Purge the CDN cache of affected objects if CDN caching is enabled
1022          *
1023          * @param $objects Array List of CF_Object items
1024          * @return void
1025          */
1026         public function purgeCDNCache( array $objects ) {
1027                 if ( $this->swiftUseCDN ) { // Rackspace style CDN
1028                         foreach ( $objects as $object ) {
1029                                 try {
1030                                         $object->purge_from_cdn();
1031                                 } catch ( CDNNotEnabledException $e ) {
1032                                         // CDN not enabled; nothing to see here
1033                                 } catch ( CloudFilesException $e ) {
1034                                         $this->handleException( $e, null, __METHOD__,
1035                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1036                                 }
1037                         }
1038                 }
1039         }
1040
1041         /**
1042          * Get a connection to the Swift proxy
1043          *
1044          * @return CF_Connection|bool False on failure
1045          * @throws CloudFilesException
1046          */
1047         protected function getConnection() {
1048                 if ( $this->connException instanceof Exception ) {
1049                         throw $this->connException; // failed last attempt
1050                 }
1051                 // Session keys expire after a while, so we renew them periodically
1052                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1053                         $this->conn->close(); // close active cURL connections
1054                         $this->conn = null;
1055                 }
1056                 // Authenticate with proxy and get a session key...
1057                 if ( !$this->conn ) {
1058                         $this->connStarted = 0;
1059                         $this->connContainers = array();
1060                         try {
1061                                 $this->auth->authenticate();
1062                                 $this->conn = new CF_Connection( $this->auth );
1063                                 $this->connStarted = time();
1064                         } catch ( CloudFilesException $e ) {
1065                                 $this->connException = $e; // don't keep re-trying
1066                                 throw $e; // throw it back
1067                         }
1068                 }
1069                 return $this->conn;
1070         }
1071
1072         /**
1073          * @see FileBackendStore::doClearCache()
1074          */
1075         protected function doClearCache( array $paths = null ) {
1076                 $this->connContainers = array(); // clear container object cache
1077         }
1078
1079         /**
1080          * Get a Swift container object, possibly from process cache.
1081          * Use $reCache if the file count or byte count is needed.
1082          *
1083          * @param $container string Container name
1084          * @param $bypassCache bool Bypass all caches and load from Swift
1085          * @return CF_Container
1086          * @throws CloudFilesException
1087          */
1088         protected function getContainer( $container, $bypassCache = false ) {
1089                 $conn = $this->getConnection(); // Swift proxy connection
1090                 if ( $bypassCache ) { // purge cache
1091                         unset( $this->connContainers[$container] );
1092                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1093                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1094                 }
1095                 if ( !isset( $this->connContainers[$container] ) ) {
1096                         $contObj = $conn->get_container( $container );
1097                         // NoSuchContainerException not thrown: container must exist
1098                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1099                                 reset( $this->connContainers );
1100                                 unset( $this->connContainers[key( $this->connContainers )] );
1101                         }
1102                         $this->connContainers[$container] = $contObj; // cache it
1103                         if ( !$bypassCache ) {
1104                                 $this->setContainerCache( $container, // update persistent cache
1105                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1106                                 );
1107                         }
1108                 }
1109                 return $this->connContainers[$container];
1110         }
1111
1112         /**
1113          * Create a Swift container
1114          *
1115          * @param $container string Container name
1116          * @return CF_Container
1117          * @throws InvalidResponseException
1118          */
1119         protected function createContainer( $container ) {
1120                 $conn = $this->getConnection(); // Swift proxy connection
1121                 $contObj = $conn->create_container( $container );
1122                 $this->connContainers[$container] = $contObj; // cache it
1123                 return $contObj;
1124         }
1125
1126         /**
1127          * Delete a Swift container
1128          *
1129          * @param $container string Container name
1130          * @return void
1131          * @throws InvalidResponseException
1132          */
1133         protected function deleteContainer( $container ) {
1134                 $conn = $this->getConnection(); // Swift proxy connection
1135                 $conn->delete_container( $container );
1136                 unset( $this->connContainers[$container] ); // purge cache
1137         }
1138
1139         /**
1140          * @see FileBackendStore::doPrimeContainerCache()
1141          * @return void
1142          */
1143         protected function doPrimeContainerCache( array $containerInfo ) {
1144                 try {
1145                         $conn = $this->getConnection(); // Swift proxy connection
1146                         foreach ( $containerInfo as $container => $info ) {
1147                                 $this->connContainers[$container] = new CF_Container(
1148                                         $conn->cfs_auth,
1149                                         $conn->cfs_http,
1150                                         $container,
1151                                         $info['count'],
1152                                         $info['bytes']
1153                                 );
1154                         }
1155                 } catch ( CloudFilesException $e ) { // some other exception?
1156                         $this->handleException( $e, null, __METHOD__, array() );
1157                 }
1158         }
1159
1160         /**
1161          * Log an unexpected exception for this backend.
1162          * This also sets the Status object to have a fatal error.
1163          *
1164          * @param $e Exception
1165          * @param $status Status|null
1166          * @param $func string
1167          * @param $params Array
1168          * @return void
1169          */
1170         protected function handleException( Exception $e, $status, $func, array $params ) {
1171                 if ( $status instanceof Status ) {
1172                         if ( $e instanceof AuthenticationException ) {
1173                                 $status->fatal( 'backend-fail-connect', $this->name );
1174                         } else {
1175                                 $status->fatal( 'backend-fail-internal', $this->name );
1176                         }
1177                 }
1178                 if ( $e->getMessage() ) {
1179                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1180                 }
1181                 wfDebugLog( 'SwiftBackend',
1182                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1183                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1184                 );
1185         }
1186 }
1187
1188 /**
1189  * @see FileBackendStoreOpHandle
1190  */
1191 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1192         /** @var CF_Async_Op */
1193         public $cfOp;
1194         /** @var Array */
1195         public $affectedObjects = array();
1196
1197         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1198                 $this->backend = $backend;
1199                 $this->params = $params;
1200                 $this->call = $call;
1201                 $this->cfOp = $cfOp;
1202         }
1203 }
1204
1205 /**
1206  * SwiftFileBackend helper class to page through listings.
1207  * Swift also has a listing limit of 10,000 objects for sanity.
1208  * Do not use this class from places outside SwiftFileBackend.
1209  *
1210  * @ingroup FileBackend
1211  */
1212 abstract class SwiftFileBackendList implements Iterator {
1213         /** @var Array */
1214         protected $bufferIter = array();
1215         protected $bufferAfter = null; // string; list items *after* this path
1216         protected $pos = 0; // integer
1217         /** @var Array */
1218         protected $params = array();
1219
1220         /** @var SwiftFileBackend */
1221         protected $backend;
1222         protected $container; // string; container name
1223         protected $dir; // string; storage directory
1224         protected $suffixStart; // integer
1225
1226         const PAGE_SIZE = 5000; // file listing buffer size
1227
1228         /**
1229          * @param $backend SwiftFileBackend
1230          * @param $fullCont string Resolved container name
1231          * @param $dir string Resolved directory relative to container
1232          * @param $params Array
1233          */
1234         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1235                 $this->backend = $backend;
1236                 $this->container = $fullCont;
1237                 $this->dir = $dir;
1238                 if ( substr( $this->dir, -1 ) === '/' ) {
1239                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1240                 }
1241                 if ( $this->dir == '' ) { // whole container
1242                         $this->suffixStart = 0;
1243                 } else { // dir within container
1244                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1245                 }
1246                 $this->params = $params;
1247         }
1248
1249         /**
1250          * @see Iterator::key()
1251          * @return integer
1252          */
1253         public function key() {
1254                 return $this->pos;
1255         }
1256
1257         /**
1258          * @see Iterator::next()
1259          * @return void
1260          */
1261         public function next() {
1262                 // Advance to the next file in the page
1263                 next( $this->bufferIter );
1264                 ++$this->pos;
1265                 // Check if there are no files left in this page and
1266                 // advance to the next page if this page was not empty.
1267                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1268                         $this->bufferIter = $this->pageFromList(
1269                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1270                         ); // updates $this->bufferAfter
1271                 }
1272         }
1273
1274         /**
1275          * @see Iterator::rewind()
1276          * @return void
1277          */
1278         public function rewind() {
1279                 $this->pos = 0;
1280                 $this->bufferAfter = null;
1281                 $this->bufferIter = $this->pageFromList(
1282                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1283                 ); // updates $this->bufferAfter
1284         }
1285
1286         /**
1287          * @see Iterator::valid()
1288          * @return bool
1289          */
1290         public function valid() {
1291                 if ( $this->bufferIter === null ) {
1292                         return false; // some failure?
1293                 } else {
1294                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1295                 }
1296         }
1297
1298         /**
1299          * Get the given list portion (page)
1300          *
1301          * @param $container string Resolved container name
1302          * @param $dir string Resolved path relative to container
1303          * @param $after string|null
1304          * @param $limit integer
1305          * @param $params Array
1306          * @return Traversable|Array|null Returns null on failure
1307          */
1308         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1309 }
1310
1311 /**
1312  * Iterator for listing directories
1313  */
1314 class SwiftFileBackendDirList extends SwiftFileBackendList {
1315         /**
1316          * @see Iterator::current()
1317          * @return string|bool String (relative path) or false
1318          */
1319         public function current() {
1320                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1321         }
1322
1323         /**
1324          * @see SwiftFileBackendList::pageFromList()
1325          * @return Array|null
1326          */
1327         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1328                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1329         }
1330 }
1331
1332 /**
1333  * Iterator for listing regular files
1334  */
1335 class SwiftFileBackendFileList extends SwiftFileBackendList {
1336         /**
1337          * @see Iterator::current()
1338          * @return string|bool String (relative path) or false
1339          */
1340         public function current() {
1341                 return substr( current( $this->bufferIter ), $this->suffixStart );
1342         }
1343
1344         /**
1345          * @see SwiftFileBackendList::pageFromList()
1346          * @return Array|null
1347          */
1348         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1349                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1350         }
1351 }