includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         protected $maxContCacheSize = 300; // integer; max containers with entries
  49
  50         /** @var CF_Connection */
  51         protected $conn; // Swift connection handle
  52         protected $connStarted = 0; // integer UNIX timestamp
  53         protected $connContainers = array(); // container object cache
  54         protected $connException; // CloudFiles exception
  55
  56         /**
  57          * @see FileBackendStore::__construct()
  58          * Additional $config params include:
  59          *    swiftAuthUrl       : Swift authentication server URL
  60          *    swiftUser          : Swift user used by MediaWiki (account:username)
  61          *    swiftKey           : Swift authentication key for the above user
  62          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  63          *    swiftAnonUser      : Swift user used for end-user requests (account:username).
  64          *                         If set, then views of public containers are assumed to go
  65          *                         through this user. If not set, then public containers are
  66          *                         accessible to unauthenticated requests via ".r:*" in the ACL.
  67          *    swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  68          *    swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  69          *                         If files may likely change, this should probably not exceed
  70          *                         a few days. For example, deletions may take this long to apply.
  71          *                         If object purging is enabled, however, this is not an issue.
  72          *    swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  73          *    shardViaHashLevels : Map of container names to sharding config with:
  74          *                         'base'   : base of hash characters, 16 or 36
  75          *                         'levels' : the number of hash levels (and digits)
  76          *                         'repeat' : hash subdirectories are prefixed with all the
  77          *                                    parent hash directory names (e.g. "a/ab/abc")
  78          */
  79         public function __construct( array $config ) {
  80                 parent::__construct( $config );
  81                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  82                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  83                 }
  84                 // Required settings
  85                 $this->auth = new CF_Authentication(
  86                         $config['swiftUser'],
  87                         $config['swiftKey'],
  88                         null, // account; unused
  89                         $config['swiftAuthUrl']
  90                 );
  91                 // Optional settings
  92                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  93                         ? $config['swiftAuthTTL']
  94                         : 5 * 60; // some sane number
  95                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  96                         ? $config['swiftAnonUser']
  97                         : '';
  98                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  99                         ? $config['shardViaHashLevels']
 100                         : '';
 101                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 102                         ? $config['swiftUseCDN']
 103                         : false;
 104                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 105                         ? $config['swiftCDNExpiry']
 106                         : 3600; // hour
 107                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 108                         ? $config['swiftCDNPurgable']
 109                         : true;
 110                 // Cache container info to mask latency
 111                 $this->memCache = wfGetMainCache();
 112         }
 113
 114         /**
 115          * @see FileBackendStore::resolveContainerPath()
 116          * @return null
 117          */
 118         protected function resolveContainerPath( $container, $relStoragePath ) {
 119                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 120                         return null; // too long for Swift
 121                 }
 122                 return $relStoragePath;
 123         }
 124
 125         /**
 126          * @see FileBackendStore::isPathUsableInternal()
 127          * @return bool
 128          */
 129         public function isPathUsableInternal( $storagePath ) {
 130                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 131                 if ( $rel === null ) {
 132                         return false; // invalid
 133                 }
 134
 135                 try {
 136                         $this->getContainer( $container );
 137                         return true; // container exists
 138                 } catch ( NoSuchContainerException $e ) {
 139                 } catch ( CloudFilesException $e ) { // some other exception?
 140                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 141                 }
 142
 143                 return false;
 144         }
 145
 146         /**
 147          * @see FileBackendStore::doCreateInternal()
 148          * @return Status
 149          */
 150         protected function doCreateInternal( array $params ) {
 151                 $status = Status::newGood();
 152
 153                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 154                 if ( $dstRel === null ) {
 155                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 156                         return $status;
 157                 }
 158
 159                 // (a) Check the destination container and object
 160                 try {
 161                         $dContObj = $this->getContainer( $dstCont );
 162                         if ( empty( $params['overwrite'] ) &&
 163                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 164                         {
 165                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 166                                 return $status;
 167                         }
 168                 } catch ( NoSuchContainerException $e ) {
 169                         $status->fatal( 'backend-fail-create', $params['dst'] );
 170                         return $status;
 171                 } catch ( CloudFilesException $e ) { // some other exception?
 172                         $this->handleException( $e, $status, __METHOD__, $params );
 173                         return $status;
 174                 }
 175
 176                 // (b) Get a SHA-1 hash of the object
 177                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 178
 179                 // (c) Actually create the object
 180                 try {
 181                         // Create a fresh CF_Object with no fields preloaded.
 182                         // We don't want to preserve headers, metadata, and such.
 183                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 184                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 185                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 186                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 187                         // The MD5 here will be checked within Swift against its own MD5.
 188                         $obj->set_etag( md5( $params['content'] ) );
 189                         // Use the same content type as StreamFile for security
 190                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 191                         if ( !empty( $params['async'] ) ) { // deferred
 192                                 $handle = $obj->write_async( $params['content'] );
 193                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 194                                 $status->value->affectedObjects[] = $obj;
 195                         } else { // actually write the object in Swift
 196                                 $obj->write( $params['content'] );
 197                                 $this->purgeCDNCache( array( $obj ) );
 198                         }
 199                 } catch ( CDNNotEnabledException $e ) {
 200                         // CDN not enabled; nothing to see here
 201                 } catch ( BadContentTypeException $e ) {
 202                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 203                 } catch ( CloudFilesException $e ) { // some other exception?
 204                         $this->handleException( $e, $status, __METHOD__, $params );
 205                 }
 206
 207                 return $status;
 208         }
 209
 210         /**
 211          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 212          */
 213         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 214                 try {
 215                         $cfOp->getLastResponse();
 216                 } catch ( BadContentTypeException $e ) {
 217                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 218                 }
 219         }
 220
 221         /**
 222          * @see FileBackendStore::doStoreInternal()
 223          * @return Status
 224          */
 225         protected function doStoreInternal( array $params ) {
 226                 $status = Status::newGood();
 227
 228                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 229                 if ( $dstRel === null ) {
 230                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 231                         return $status;
 232                 }
 233
 234                 // (a) Check the destination container and object
 235                 try {
 236                         $dContObj = $this->getContainer( $dstCont );
 237                         if ( empty( $params['overwrite'] ) &&
 238                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 239                         {
 240                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 241                                 return $status;
 242                         }
 243                 } catch ( NoSuchContainerException $e ) {
 244                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 245                         return $status;
 246                 } catch ( CloudFilesException $e ) { // some other exception?
 247                         $this->handleException( $e, $status, __METHOD__, $params );
 248                         return $status;
 249                 }
 250
 251                 // (b) Get a SHA-1 hash of the object
 252                 $sha1Hash = sha1_file( $params['src'] );
 253                 if ( $sha1Hash === false ) { // source doesn't exist?
 254                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 255                         return $status;
 256                 }
 257                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 258
 259                 // (c) Actually store the object
 260                 try {
 261                         // Create a fresh CF_Object with no fields preloaded.
 262                         // We don't want to preserve headers, metadata, and such.
 263                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 264                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 265                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 266                         // The MD5 here will be checked within Swift against its own MD5.
 267                         $obj->set_etag( md5_file( $params['src'] ) );
 268                         // Use the same content type as StreamFile for security
 269                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 270                         if ( !empty( $params['async'] ) ) { // deferred
 271                                 wfSuppressWarnings();
 272                                 $fp = fopen( $params['src'], 'rb' );
 273                                 wfRestoreWarnings();
 274                                 if ( !$fp ) {
 275                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 276                                 } else {
 277                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 278                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 279                                         $status->value->resourcesToClose[] = $fp;
 280                                         $status->value->affectedObjects[] = $obj;
 281                                 }
 282                         } else { // actually write the object in Swift
 283                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 284                                 $this->purgeCDNCache( array( $obj ) );
 285                         }
 286                 } catch ( CDNNotEnabledException $e ) {
 287                         // CDN not enabled; nothing to see here
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 } catch ( IOException $e ) {
 291                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 292                 } catch ( CloudFilesException $e ) { // some other exception?
 293                         $this->handleException( $e, $status, __METHOD__, $params );
 294                 }
 295
 296                 return $status;
 297         }
 298
 299         /**
 300          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 301          */
 302         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 303                 try {
 304                         $cfOp->getLastResponse();
 305                 } catch ( BadContentTypeException $e ) {
 306                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 307                 } catch ( IOException $e ) {
 308                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 309                 }
 310         }
 311
 312         /**
 313          * @see FileBackendStore::doCopyInternal()
 314          * @return Status
 315          */
 316         protected function doCopyInternal( array $params ) {
 317                 $status = Status::newGood();
 318
 319                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 320                 if ( $srcRel === null ) {
 321                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 322                         return $status;
 323                 }
 324
 325                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 326                 if ( $dstRel === null ) {
 327                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 328                         return $status;
 329                 }
 330
 331                 // (a) Check the source/destination containers and destination object
 332                 try {
 333                         $sContObj = $this->getContainer( $srcCont );
 334                         $dContObj = $this->getContainer( $dstCont );
 335                         if ( empty( $params['overwrite'] ) &&
 336                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 337                         {
 338                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 339                                 return $status;
 340                         }
 341                 } catch ( NoSuchContainerException $e ) {
 342                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 343                         return $status;
 344                 } catch ( CloudFilesException $e ) { // some other exception?
 345                         $this->handleException( $e, $status, __METHOD__, $params );
 346                         return $status;
 347                 }
 348
 349                 // (b) Actually copy the file to the destination
 350                 try {
 351                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 352                         if ( !empty( $params['async'] ) ) { // deferred
 353                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 354                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 355                                 $status->value->affectedObjects[] = $dstObj;
 356                         } else { // actually write the object in Swift
 357                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 358                                 $this->purgeCDNCache( array( $dstObj ) );
 359                         }
 360                 } catch ( CDNNotEnabledException $e ) {
 361                         // CDN not enabled; nothing to see here
 362                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 363                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 364                 } catch ( CloudFilesException $e ) { // some other exception?
 365                         $this->handleException( $e, $status, __METHOD__, $params );
 366                 }
 367
 368                 return $status;
 369         }
 370
 371         /**
 372          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 373          */
 374         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 375                 try {
 376                         $cfOp->getLastResponse();
 377                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 378                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 379                 }
 380         }
 381
 382         /**
 383          * @see FileBackendStore::doMoveInternal()
 384          * @return Status
 385          */
 386         protected function doMoveInternal( array $params ) {
 387                 $status = Status::newGood();
 388
 389                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 390                 if ( $srcRel === null ) {
 391                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 392                         return $status;
 393                 }
 394
 395                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 396                 if ( $dstRel === null ) {
 397                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 398                         return $status;
 399                 }
 400
 401                 // (a) Check the source/destination containers and destination object
 402                 try {
 403                         $sContObj = $this->getContainer( $srcCont );
 404                         $dContObj = $this->getContainer( $dstCont );
 405                         if ( empty( $params['overwrite'] ) &&
 406                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 407                         {
 408                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 409                                 return $status;
 410                         }
 411                 } catch ( NoSuchContainerException $e ) {
 412                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 413                         return $status;
 414                 } catch ( CloudFilesException $e ) { // some other exception?
 415                         $this->handleException( $e, $status, __METHOD__, $params );
 416                         return $status;
 417                 }
 418
 419                 // (b) Actually move the file to the destination
 420                 try {
 421                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 422                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 423                         if ( !empty( $params['async'] ) ) { // deferred
 424                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 425                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 426                                 $status->value->affectedObjects[] = $srcObj;
 427                                 $status->value->affectedObjects[] = $dstObj;
 428                         } else { // actually write the object in Swift
 429                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 430                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 431                         }
 432                 } catch ( CDNNotEnabledException $e ) {
 433                         // CDN not enabled; nothing to see here
 434                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 435                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 436                 } catch ( CloudFilesException $e ) { // some other exception?
 437                         $this->handleException( $e, $status, __METHOD__, $params );
 438                 }
 439
 440                 return $status;
 441         }
 442
 443         /**
 444          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 445          */
 446         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 447                 try {
 448                         $cfOp->getLastResponse();
 449                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 450                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 451                 }
 452         }
 453
 454         /**
 455          * @see FileBackendStore::doDeleteInternal()
 456          * @return Status
 457          */
 458         protected function doDeleteInternal( array $params ) {
 459                 $status = Status::newGood();
 460
 461                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 462                 if ( $srcRel === null ) {
 463                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 464                         return $status;
 465                 }
 466
 467                 try {
 468                         $sContObj = $this->getContainer( $srcCont );
 469                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 470                         if ( !empty( $params['async'] ) ) { // deferred
 471                                 $handle = $sContObj->delete_object_async( $srcRel );
 472                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 473                                 $status->value->affectedObjects[] = $srcObj;
 474                         } else { // actually write the object in Swift
 475                                 $sContObj->delete_object( $srcRel );
 476                                 $this->purgeCDNCache( array( $srcObj ) );
 477                         }
 478                 } catch ( CDNNotEnabledException $e ) {
 479                         // CDN not enabled; nothing to see here
 480                 } catch ( NoSuchContainerException $e ) {
 481                         $status->fatal( 'backend-fail-delete', $params['src'] );
 482                 } catch ( NoSuchObjectException $e ) {
 483                         if ( empty( $params['ignoreMissingSource'] ) ) {
 484                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 485                         }
 486                 } catch ( CloudFilesException $e ) { // some other exception?
 487                         $this->handleException( $e, $status, __METHOD__, $params );
 488                 }
 489
 490                 return $status;
 491         }
 492
 493         /**
 494          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 495          */
 496         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 497                 try {
 498                         $cfOp->getLastResponse();
 499                 } catch ( NoSuchContainerException $e ) {
 500                         $status->fatal( 'backend-fail-delete', $params['src'] );
 501                 } catch ( NoSuchObjectException $e ) {
 502                         if ( empty( $params['ignoreMissingSource'] ) ) {
 503                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 504                         }
 505                 }
 506         }
 507
 508         /**
 509          * @see FileBackendStore::doPrepareInternal()
 510          * @return Status
 511          */
 512         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 513                 $status = Status::newGood();
 514
 515                 // (a) Check if container already exists
 516                 try {
 517                         $contObj = $this->getContainer( $fullCont );
 518                         // NoSuchContainerException not thrown: container must exist
 519                         return $status; // already exists
 520                 } catch ( NoSuchContainerException $e ) {
 521                         // NoSuchContainerException thrown: container does not exist
 522                 } catch ( CloudFilesException $e ) { // some other exception?
 523                         $this->handleException( $e, $status, __METHOD__, $params );
 524                         return $status;
 525                 }
 526
 527                 // (b) Create container as needed
 528                 try {
 529                         $contObj = $this->createContainer( $fullCont );
 530                         if ( !empty( $params['noAccess'] ) ) {
 531                                 // Make container private to end-users...
 532                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 533                         } else {
 534                                 // Make container public to end-users...
 535                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 536                         }
 537                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 538                                 $contObj->make_public( $this->swiftCDNExpiry );
 539                         }
 540                 } catch ( CDNNotEnabledException $e ) {
 541                         // CDN not enabled; nothing to see here
 542                 } catch ( CloudFilesException $e ) { // some other exception?
 543                         $this->handleException( $e, $status, __METHOD__, $params );
 544                         return $status;
 545                 }
 546
 547                 return $status;
 548         }
 549
 550         /**
 551          * @see FileBackendStore::doSecureInternal()
 552          * @return Status
 553          */
 554         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 555                 $status = Status::newGood();
 556                 if ( empty( $params['noAccess'] ) ) {
 557                         return $status; // nothing to do
 558                 }
 559
 560                 // Restrict container from end-users...
 561                 try {
 562                         // doPrepareInternal() should have been called,
 563                         // so the Swift container should already exist...
 564                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 565                         // NoSuchContainerException not thrown: container must exist
 566
 567                         // Make container private to end-users...
 568                         $status->merge( $this->setContainerAccess(
 569                                 $contObj,
 570                                 array( $this->auth->username ), // read
 571                                 array( $this->auth->username ) // write
 572                         ) );
 573                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 574                                 $contObj->make_private();
 575                         }
 576                 } catch ( CDNNotEnabledException $e ) {
 577                         // CDN not enabled; nothing to see here
 578                 } catch ( CloudFilesException $e ) { // some other exception?
 579                         $this->handleException( $e, $status, __METHOD__, $params );
 580                 }
 581
 582                 return $status;
 583         }
 584
 585         /**
 586          * @see FileBackendStore::doPublishInternal()
 587          * @return Status
 588          */
 589         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 590                 $status = Status::newGood();
 591
 592                 // Unrestrict container from end-users...
 593                 try {
 594                         // doPrepareInternal() should have been called,
 595                         // so the Swift container should already exist...
 596                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 597                         // NoSuchContainerException not thrown: container must exist
 598
 599                         // Make container public to end-users...
 600                         if ( $this->swiftAnonUser != '' ) {
 601                                 $status->merge( $this->setContainerAccess(
 602                                         $contObj,
 603                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 604                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 605                                 ) );
 606                         } else {
 607                                 $status->merge( $this->setContainerAccess(
 608                                         $contObj,
 609                                         array( $this->auth->username, '.r:*' ), // read
 610                                         array( $this->auth->username ) // write
 611                                 ) );
 612                         }
 613                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 614                                 $contObj->make_public();
 615                         }
 616                 } catch ( CDNNotEnabledException $e ) {
 617                         // CDN not enabled; nothing to see here
 618                 } catch ( CloudFilesException $e ) { // some other exception?
 619                         $this->handleException( $e, $status, __METHOD__, $params );
 620                 }
 621
 622                 return $status;
 623         }
 624
 625         /**
 626          * @see FileBackendStore::doCleanInternal()
 627          * @return Status
 628          */
 629         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 630                 $status = Status::newGood();
 631
 632                 // Only containers themselves can be removed, all else is virtual
 633                 if ( $dir != '' ) {
 634                         return $status; // nothing to do
 635                 }
 636
 637                 // (a) Check the container
 638                 try {
 639                         $contObj = $this->getContainer( $fullCont, true );
 640                 } catch ( NoSuchContainerException $e ) {
 641                         return $status; // ok, nothing to do
 642                 } catch ( CloudFilesException $e ) { // some other exception?
 643                         $this->handleException( $e, $status, __METHOD__, $params );
 644                         return $status;
 645                 }
 646
 647                 // (b) Delete the container if empty
 648                 if ( $contObj->object_count == 0 ) {
 649                         try {
 650                                 $this->deleteContainer( $fullCont );
 651                         } catch ( NoSuchContainerException $e ) {
 652                                 return $status; // race?
 653                         } catch ( NonEmptyContainerException $e ) {
 654                                 return $status; // race? consistency delay?
 655                         } catch ( CloudFilesException $e ) { // some other exception?
 656                                 $this->handleException( $e, $status, __METHOD__, $params );
 657                                 return $status;
 658                         }
 659                 }
 660
 661                 return $status;
 662         }
 663
 664         /**
 665          * @see FileBackendStore::doFileExists()
 666          * @return array|bool|null
 667          */
 668         protected function doGetFileStat( array $params ) {
 669                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 670                 if ( $srcRel === null ) {
 671                         return false; // invalid storage path
 672                 }
 673
 674                 $stat = false;
 675                 try {
 676                         $contObj = $this->getContainer( $srcCont );
 677                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 678                         $this->addMissingMetadata( $srcObj, $params['src'] );
 679                         $stat = array(
 680                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 681                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 682                                 'size'  => (int)$srcObj->content_length,
 683                                 'sha1'  => $srcObj->metadata['Sha1base36']
 684                         );
 685                 } catch ( NoSuchContainerException $e ) {
 686                 } catch ( NoSuchObjectException $e ) {
 687                 } catch ( CloudFilesException $e ) { // some other exception?
 688                         $stat = null;
 689                         $this->handleException( $e, null, __METHOD__, $params );
 690                 }
 691
 692                 return $stat;
 693         }
 694
 695         /**
 696          * Fill in any missing object metadata and save it to Swift
 697          *
 698          * @param $obj CF_Object
 699          * @param $path string Storage path to object
 700          * @return bool Success
 701          * @throws Exception cloudfiles exceptions
 702          */
 703         protected function addMissingMetadata( CF_Object $obj, $path ) {
 704                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 705                         return true; // nothing to do
 706                 }
 707                 $status = Status::newGood();
 708                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 709                 if ( $status->isOK() ) {
 710                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 711                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 712                         if ( $tmpFile ) {
 713                                 $hash = $tmpFile->getSha1Base36();
 714                                 if ( $hash !== false ) {
 715                                         $obj->metadata['Sha1base36'] = $hash;
 716                                         $obj->sync_metadata(); // save to Swift
 717                                         return true; // success
 718                                 }
 719                         }
 720                 }
 721                 $obj->metadata['Sha1base36'] = false;
 722                 return false; // failed
 723         }
 724
 725         /**
 726          * @see FileBackend::getFileContents()
 727          * @return bool|null|string
 728          */
 729         public function getFileContents( array $params ) {
 730                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 731                 if ( $srcRel === null ) {
 732                         return false; // invalid storage path
 733                 }
 734
 735                 if ( !$this->fileExists( $params ) ) {
 736                         return null;
 737                 }
 738
 739                 $data = false;
 740                 try {
 741                         $sContObj = $this->getContainer( $srcCont );
 742                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 743                         $data = $obj->read( $this->headersFromParams( $params ) );
 744                 } catch ( NoSuchContainerException $e ) {
 745                 } catch ( CloudFilesException $e ) { // some other exception?
 746                         $this->handleException( $e, null, __METHOD__, $params );
 747                 }
 748
 749                 return $data;
 750         }
 751
 752         /**
 753          * @see FileBackendStore::doDirectoryExists()
 754          * @return bool|null
 755          */
 756         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 757                 try {
 758                         $container = $this->getContainer( $fullCont );
 759                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 760                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 761                 } catch ( NoSuchContainerException $e ) {
 762                         return false;
 763                 } catch ( CloudFilesException $e ) { // some other exception?
 764                         $this->handleException( $e, null, __METHOD__,
 765                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 766                 }
 767
 768                 return null; // error
 769         }
 770
 771         /**
 772          * @see FileBackendStore::getDirectoryListInternal()
 773          * @return SwiftFileBackendDirList
 774          */
 775         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 776                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 777         }
 778
 779         /**
 780          * @see FileBackendStore::getFileListInternal()
 781          * @return SwiftFileBackendFileList
 782          */
 783         public function getFileListInternal( $fullCont, $dir, array $params ) {
 784                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 785         }
 786
 787         /**
 788          * Do not call this function outside of SwiftFileBackendFileList
 789          *
 790          * @param $fullCont string Resolved container name
 791          * @param $dir string Resolved storage directory with no trailing slash
 792          * @param $after string|null Storage path of file to list items after
 793          * @param $limit integer Max number of items to list
 794          * @param $params Array Includes flag for 'topOnly'
 795          * @return Array List of relative paths of dirs directly under $dir
 796          */
 797         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 798                 $dirs = array();
 799                 if ( $after === INF ) {
 800                         return $dirs; // nothing more
 801                 }
 802                 wfProfileIn( __METHOD__ . '-' . $this->name );
 803
 804                 try {
 805                         $container = $this->getContainer( $fullCont );
 806                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 807                         // Non-recursive: only list dirs right under $dir
 808                         if ( !empty( $params['topOnly'] ) ) {
 809                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 810                                 foreach ( $objects as $object ) { // files and dirs
 811                                         if ( substr( $object, -1 ) === '/' ) {
 812                                                 $dirs[] = $object; // directories end in '/'
 813                                         }
 814                                 }
 815                         // Recursive: list all dirs under $dir and its subdirs
 816                         } else {
 817                                 // Get directory from last item of prior page
 818                                 $lastDir = $this->getParentDir( $after ); // must be first page
 819                                 $objects = $container->list_objects( $limit, $after, $prefix );
 820                                 foreach ( $objects as $object ) { // files
 821                                         $objectDir = $this->getParentDir( $object ); // directory of object
 822                                         if ( $objectDir !== false ) { // file has a parent dir
 823                                                 // Swift stores paths in UTF-8, using binary sorting.
 824                                                 // See function "create_container_table" in common/db.py.
 825                                                 // If a directory is not "greater" than the last one,
 826                                                 // then it was already listed by the calling iterator.
 827                                                 if ( $objectDir > $lastDir ) {
 828                                                         $pDir = $objectDir;
 829                                                         do { // add dir and all its parent dirs
 830                                                                 $dirs[] = "{$pDir}/";
 831                                                                 $pDir = $this->getParentDir( $pDir );
 832                                                         } while ( $pDir !== false // sanity
 833                                                                 && $pDir > $lastDir // not done already
 834                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 835                                                         );
 836                                                 }
 837                                                 $lastDir = $objectDir;
 838                                         }
 839                                 }
 840                         }
 841                         if ( count( $objects ) < $limit ) {
 842                                 $after = INF; // avoid a second RTT
 843                         } else {
 844                                 $after = end( $objects ); // update last item
 845                         }
 846                 } catch ( NoSuchContainerException $e ) {
 847                 } catch ( CloudFilesException $e ) { // some other exception?
 848                         $this->handleException( $e, null, __METHOD__,
 849                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 850                 }
 851
 852                 wfProfileOut( __METHOD__ . '-' . $this->name );
 853                 return $dirs;
 854         }
 855
 856         protected function getParentDir( $path ) {
 857                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 858         }
 859
 860         /**
 861          * Do not call this function outside of SwiftFileBackendFileList
 862          *
 863          * @param $fullCont string Resolved container name
 864          * @param $dir string Resolved storage directory with no trailing slash
 865          * @param $after string|null Storage path of file to list items after
 866          * @param $limit integer Max number of items to list
 867          * @param $params Array Includes flag for 'topOnly'
 868          * @return Array List of relative paths of files under $dir
 869          */
 870         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 871                 $files = array();
 872                 if ( $after === INF ) {
 873                         return $files; // nothing more
 874                 }
 875                 wfProfileIn( __METHOD__ . '-' . $this->name );
 876
 877                 try {
 878                         $container = $this->getContainer( $fullCont );
 879                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 880                         // Non-recursive: only list files right under $dir
 881                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 882                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 883                                 foreach ( $objects as $object ) {
 884                                         if ( substr( $object, -1 ) !== '/' ) {
 885                                                 $files[] = $object; // directories end in '/'
 886                                         }
 887                                 }
 888                         // Recursive: list all files under $dir and its subdirs
 889                         } else { // files
 890                                 $objects = $container->list_objects( $limit, $after, $prefix );
 891                                 $files = $objects;
 892                         }
 893                         if ( count( $objects ) < $limit ) {
 894                                 $after = INF; // avoid a second RTT
 895                         } else {
 896                                 $after = end( $objects ); // update last item
 897                         }
 898                 } catch ( NoSuchContainerException $e ) {
 899                 } catch ( CloudFilesException $e ) { // some other exception?
 900                         $this->handleException( $e, null, __METHOD__,
 901                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 902                 }
 903
 904                 wfProfileOut( __METHOD__ . '-' . $this->name );
 905                 return $files;
 906         }
 907
 908         /**
 909          * @see FileBackendStore::doGetFileSha1base36()
 910          * @return bool
 911          */
 912         protected function doGetFileSha1base36( array $params ) {
 913                 $stat = $this->getFileStat( $params );
 914                 if ( $stat ) {
 915                         return $stat['sha1'];
 916                 } else {
 917                         return false;
 918                 }
 919         }
 920
 921         /**
 922          * @see FileBackendStore::doStreamFile()
 923          * @return Status
 924          */
 925         protected function doStreamFile( array $params ) {
 926                 $status = Status::newGood();
 927
 928                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 929                 if ( $srcRel === null ) {
 930                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 931                 }
 932
 933                 try {
 934                         $cont = $this->getContainer( $srcCont );
 935                 } catch ( NoSuchContainerException $e ) {
 936                         $status->fatal( 'backend-fail-stream', $params['src'] );
 937                         return $status;
 938                 } catch ( CloudFilesException $e ) { // some other exception?
 939                         $this->handleException( $e, $status, __METHOD__, $params );
 940                         return $status;
 941                 }
 942
 943                 try {
 944                         $output = fopen( 'php://output', 'wb' );
 945                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 946                         $obj->stream( $output, $this->headersFromParams( $params ) );
 947                 } catch ( CloudFilesException $e ) { // some other exception?
 948                         $this->handleException( $e, $status, __METHOD__, $params );
 949                 }
 950
 951                 return $status;
 952         }
 953
 954         /**
 955          * @see FileBackendStore::getLocalCopy()
 956          * @return null|TempFSFile
 957          */
 958         public function getLocalCopy( array $params ) {
 959                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 960                 if ( $srcRel === null ) {
 961                         return null;
 962                 }
 963
 964                 # Check the recursion guard to avoid loops when filling metadata
 965                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 966                         return null;
 967                 }
 968
 969                 $tmpFile = null;
 970                 try {
 971                         $sContObj = $this->getContainer( $srcCont );
 972                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 973                         // Get source file extension
 974                         $ext = FileBackend::extensionFromPath( $srcRel );
 975                         // Create a new temporary file...
 976                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 977                         if ( $tmpFile ) {
 978                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 979                                 if ( $handle ) {
 980                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 981                                         fclose( $handle );
 982                                 } else {
 983                                         $tmpFile = null; // couldn't open temp file
 984                                 }
 985                         }
 986                 } catch ( NoSuchContainerException $e ) {
 987                         $tmpFile = null;
 988                 } catch ( CloudFilesException $e ) { // some other exception?
 989                         $tmpFile = null;
 990                         $this->handleException( $e, null, __METHOD__, $params );
 991                 }
 992
 993                 return $tmpFile;
 994         }
 995
 996         /**
 997          * @see FileBackendStore::directoriesAreVirtual()
 998          * @return bool
 999          */
1000         protected function directoriesAreVirtual() {
1001                 return true;
1002         }
1003
1004         /**
1005          * Get headers to send to Swift when reading a file based
1006          * on a FileBackend params array, e.g. that of getLocalCopy().
1007          * $params is currently only checked for a 'latest' flag.
1008          *
1009          * @param $params Array
1010          * @return Array
1011          */
1012         protected function headersFromParams( array $params ) {
1013                 $hdrs = array();
1014                 if ( !empty( $params['latest'] ) ) {
1015                         $hdrs[] = 'X-Newest: true';
1016                 }
1017                 return $hdrs;
1018         }
1019
1020         /**
1021          * @see FileBackendStore::doExecuteOpHandlesInternal()
1022          * @return Array List of corresponding Status objects
1023          */
1024         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1025                 $statuses = array();
1026
1027                 $cfOps = array(); // list of CF_Async_Op objects
1028                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1029                         $cfOps[$index] = $fileOpHandle->cfOp;
1030                 }
1031                 $batch = new CF_Async_Op_Batch( $cfOps );
1032
1033                 $cfOps = $batch->execute();
1034                 foreach ( $cfOps as $index => $cfOp ) {
1035                         $status = Status::newGood();
1036                         try { // catch exceptions; update status
1037                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1038                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1039                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1040                         } catch ( CloudFilesException $e ) { // some other exception?
1041                                 $this->handleException( $e, $status,
1042                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1043                         }
1044                         $statuses[$index] = $status;
1045                 }
1046
1047                 return $statuses;
1048         }
1049
1050         /**
1051          * Set read/write permissions for a Swift container.
1052          *
1053          * $readGrps is a list of the possible criteria for a request to have
1054          * access to read a container. Each item is one of the following formats:
1055          *   account:user       - Grants access if the request is by the given user
1056          *   .r:<regex>         - Grants access if the request is from a referrer host that
1057          *                        matches the expression and the request is not for a listing.
1058          *                        Setting this to '*' effectively makes a container public.
1059          *   .rlistings:<regex> - Grants access if the request is from a referrer host that
1060          *                        matches the expression and the request for a listing.
1061          * $writeGrps is a list of the possible criteria for a request to have
1062          * access to write to a container. Each item is of the following format:
1063          *   account:user       - Grants access if the request is by the given user
1064          *
1065          * @see http://swift.openstack.org/misc.html#acls
1066          *
1067          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1068          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1069          *
1070          * @param $contObj CF_Container Swift container
1071          * @param $readGrps Array List of read access routes
1072          * @param $writeGrps Array List of write access routes
1073          * @return Status
1074          */
1075         protected function setContainerAccess(
1076                 CF_Container $contObj, array $readGrps, array $writeGrps
1077         ) {
1078                 $creds = $contObj->cfs_auth->export_credentials();
1079
1080                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1081
1082                 // Note: 10 second timeout consistent with php-cloudfiles
1083                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1084                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1085                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1086                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1087
1088                 return $req->execute(); // should return 204
1089         }
1090
1091         /**
1092          * Purge the CDN cache of affected objects if CDN caching is enabled.
1093          * This is for Rackspace/Akamai CDNs.
1094          *
1095          * @param $objects Array List of CF_Object items
1096          * @return void
1097          */
1098         public function purgeCDNCache( array $objects ) {
1099                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1100                         foreach ( $objects as $object ) {
1101                                 try {
1102                                         $object->purge_from_cdn();
1103                                 } catch ( CDNNotEnabledException $e ) {
1104                                         // CDN not enabled; nothing to see here
1105                                 } catch ( CloudFilesException $e ) {
1106                                         $this->handleException( $e, null, __METHOD__,
1107                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1108                                 }
1109                         }
1110                 }
1111         }
1112
1113         /**
1114          * Get a connection to the Swift proxy
1115          *
1116          * @return CF_Connection|bool False on failure
1117          * @throws CloudFilesException
1118          */
1119         protected function getConnection() {
1120                 if ( $this->connException instanceof Exception ) {
1121                         throw $this->connException; // failed last attempt
1122                 }
1123                 // Session keys expire after a while, so we renew them periodically
1124                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1125                         $this->conn->close(); // close active cURL connections
1126                         $this->conn = null;
1127                 }
1128                 // Authenticate with proxy and get a session key...
1129                 if ( !$this->conn ) {
1130                         $this->connStarted = 0;
1131                         $this->connContainers = array();
1132                         try {
1133                                 $this->auth->authenticate();
1134                                 $this->conn = new CF_Connection( $this->auth );
1135                                 $this->connStarted = time();
1136                         } catch ( CloudFilesException $e ) {
1137                                 $this->connException = $e; // don't keep re-trying
1138                                 throw $e; // throw it back
1139                         }
1140                 }
1141                 return $this->conn;
1142         }
1143
1144         /**
1145          * @see FileBackendStore::doClearCache()
1146          */
1147         protected function doClearCache( array $paths = null ) {
1148                 $this->connContainers = array(); // clear container object cache
1149         }
1150
1151         /**
1152          * Get a Swift container object, possibly from process cache.
1153          * Use $reCache if the file count or byte count is needed.
1154          *
1155          * @param $container string Container name
1156          * @param $bypassCache bool Bypass all caches and load from Swift
1157          * @return CF_Container
1158          * @throws CloudFilesException
1159          */
1160         protected function getContainer( $container, $bypassCache = false ) {
1161                 $conn = $this->getConnection(); // Swift proxy connection
1162                 if ( $bypassCache ) { // purge cache
1163                         unset( $this->connContainers[$container] );
1164                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1165                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1166                 }
1167                 if ( !isset( $this->connContainers[$container] ) ) {
1168                         $contObj = $conn->get_container( $container );
1169                         // NoSuchContainerException not thrown: container must exist
1170                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1171                                 reset( $this->connContainers );
1172                                 unset( $this->connContainers[key( $this->connContainers )] );
1173                         }
1174                         $this->connContainers[$container] = $contObj; // cache it
1175                         if ( !$bypassCache ) {
1176                                 $this->setContainerCache( $container, // update persistent cache
1177                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1178                                 );
1179                         }
1180                 }
1181                 return $this->connContainers[$container];
1182         }
1183
1184         /**
1185          * Create a Swift container
1186          *
1187          * @param $container string Container name
1188          * @return CF_Container
1189          * @throws CloudFilesException
1190          */
1191         protected function createContainer( $container ) {
1192                 $conn = $this->getConnection(); // Swift proxy connection
1193                 $contObj = $conn->create_container( $container );
1194                 $this->connContainers[$container] = $contObj; // cache it
1195                 return $contObj;
1196         }
1197
1198         /**
1199          * Delete a Swift container
1200          *
1201          * @param $container string Container name
1202          * @return void
1203          * @throws CloudFilesException
1204          */
1205         protected function deleteContainer( $container ) {
1206                 $conn = $this->getConnection(); // Swift proxy connection
1207                 unset( $this->connContainers[$container] ); // purge cache
1208                 $conn->delete_container( $container );
1209         }
1210
1211         /**
1212          * @see FileBackendStore::doPrimeContainerCache()
1213          * @return void
1214          */
1215         protected function doPrimeContainerCache( array $containerInfo ) {
1216                 try {
1217                         $conn = $this->getConnection(); // Swift proxy connection
1218                         foreach ( $containerInfo as $container => $info ) {
1219                                 $this->connContainers[$container] = new CF_Container(
1220                                         $conn->cfs_auth,
1221                                         $conn->cfs_http,
1222                                         $container,
1223                                         $info['count'],
1224                                         $info['bytes']
1225                                 );
1226                         }
1227                 } catch ( CloudFilesException $e ) { // some other exception?
1228                         $this->handleException( $e, null, __METHOD__, array() );
1229                 }
1230         }
1231
1232         /**
1233          * Log an unexpected exception for this backend.
1234          * This also sets the Status object to have a fatal error.
1235          *
1236          * @param $e Exception
1237          * @param $status Status|null
1238          * @param $func string
1239          * @param $params Array
1240          * @return void
1241          */
1242         protected function handleException( Exception $e, $status, $func, array $params ) {
1243                 if ( $status instanceof Status ) {
1244                         if ( $e instanceof AuthenticationException ) {
1245                                 $status->fatal( 'backend-fail-connect', $this->name );
1246                         } else {
1247                                 $status->fatal( 'backend-fail-internal', $this->name );
1248                         }
1249                 }
1250                 if ( $e->getMessage() ) {
1251                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1252                 }
1253                 wfDebugLog( 'SwiftBackend',
1254                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1255                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1256                 );
1257         }
1258 }
1259
1260 /**
1261  * @see FileBackendStoreOpHandle
1262  */
1263 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1264         /** @var CF_Async_Op */
1265         public $cfOp;
1266         /** @var Array */
1267         public $affectedObjects = array();
1268
1269         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1270                 $this->backend = $backend;
1271                 $this->params = $params;
1272                 $this->call = $call;
1273                 $this->cfOp = $cfOp;
1274         }
1275 }
1276
1277 /**
1278  * SwiftFileBackend helper class to page through listings.
1279  * Swift also has a listing limit of 10,000 objects for sanity.
1280  * Do not use this class from places outside SwiftFileBackend.
1281  *
1282  * @ingroup FileBackend
1283  */
1284 abstract class SwiftFileBackendList implements Iterator {
1285         /** @var Array */
1286         protected $bufferIter = array();
1287         protected $bufferAfter = null; // string; list items *after* this path
1288         protected $pos = 0; // integer
1289         /** @var Array */
1290         protected $params = array();
1291
1292         /** @var SwiftFileBackend */
1293         protected $backend;
1294         protected $container; // string; container name
1295         protected $dir; // string; storage directory
1296         protected $suffixStart; // integer
1297
1298         const PAGE_SIZE = 5000; // file listing buffer size
1299
1300         /**
1301          * @param $backend SwiftFileBackend
1302          * @param $fullCont string Resolved container name
1303          * @param $dir string Resolved directory relative to container
1304          * @param $params Array
1305          */
1306         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1307                 $this->backend = $backend;
1308                 $this->container = $fullCont;
1309                 $this->dir = $dir;
1310                 if ( substr( $this->dir, -1 ) === '/' ) {
1311                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1312                 }
1313                 if ( $this->dir == '' ) { // whole container
1314                         $this->suffixStart = 0;
1315                 } else { // dir within container
1316                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1317                 }
1318                 $this->params = $params;
1319         }
1320
1321         /**
1322          * @see Iterator::key()
1323          * @return integer
1324          */
1325         public function key() {
1326                 return $this->pos;
1327         }
1328
1329         /**
1330          * @see Iterator::next()
1331          * @return void
1332          */
1333         public function next() {
1334                 // Advance to the next file in the page
1335                 next( $this->bufferIter );
1336                 ++$this->pos;
1337                 // Check if there are no files left in this page and
1338                 // advance to the next page if this page was not empty.
1339                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1340                         $this->bufferIter = $this->pageFromList(
1341                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1342                         ); // updates $this->bufferAfter
1343                 }
1344         }
1345
1346         /**
1347          * @see Iterator::rewind()
1348          * @return void
1349          */
1350         public function rewind() {
1351                 $this->pos = 0;
1352                 $this->bufferAfter = null;
1353                 $this->bufferIter = $this->pageFromList(
1354                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1355                 ); // updates $this->bufferAfter
1356         }
1357
1358         /**
1359          * @see Iterator::valid()
1360          * @return bool
1361          */
1362         public function valid() {
1363                 if ( $this->bufferIter === null ) {
1364                         return false; // some failure?
1365                 } else {
1366                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1367                 }
1368         }
1369
1370         /**
1371          * Get the given list portion (page)
1372          *
1373          * @param $container string Resolved container name
1374          * @param $dir string Resolved path relative to container
1375          * @param $after string|null
1376          * @param $limit integer
1377          * @param $params Array
1378          * @return Traversable|Array|null Returns null on failure
1379          */
1380         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1381 }
1382
1383 /**
1384  * Iterator for listing directories
1385  */
1386 class SwiftFileBackendDirList extends SwiftFileBackendList {
1387         /**
1388          * @see Iterator::current()
1389          * @return string|bool String (relative path) or false
1390          */
1391         public function current() {
1392                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1393         }
1394
1395         /**
1396          * @see SwiftFileBackendList::pageFromList()
1397          * @return Array|null
1398          */
1399         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1400                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1401         }
1402 }
1403
1404 /**
1405  * Iterator for listing regular files
1406  */
1407 class SwiftFileBackendFileList extends SwiftFileBackendList {
1408         /**
1409          * @see Iterator::current()
1410          * @return string|bool String (relative path) or false
1411          */
1412         public function current() {
1413                 return substr( current( $this->bufferIter ), $this->suffixStart );
1414         }
1415
1416         /**
1417          * @see SwiftFileBackendList::pageFromList()
1418          * @return Array|null
1419          */
1420         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1421                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1422         }
1423 }