includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         protected $maxContCacheSize = 300; // integer; max containers with entries
  49
  50         /** @var CF_Connection */
  51         protected $conn; // Swift connection handle
  52         protected $connStarted = 0; // integer UNIX timestamp
  53         protected $connContainers = array(); // container object cache
  54         protected $connException; // CloudFiles exception
  55
  56         /**
  57          * @see FileBackendStore::__construct()
  58          * Additional $config params include:
  59          *   - swiftAuthUrl       : Swift authentication server URL
  60          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  61          *   - swiftKey           : Swift authentication key for the above user
  62          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  63          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  64          *                          If set, then views of public containers are assumed to go
  65          *                          through this user. If not set, then public containers are
  66          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  67          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  68          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  69          *                          If files may likely change, this should probably not exceed
  70          *                          a few days. For example, deletions may take this long to apply.
  71          *                          If object purging is enabled, however, this is not an issue.
  72          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  73          *   - shardViaHashLevels : Map of container names to sharding config with:
  74          *                             - base   : base of hash characters, 16 or 36
  75          *                             - levels : the number of hash levels (and digits)
  76          *                             - repeat : hash subdirectories are prefixed with all the
  77          *                                        parent hash directory names (e.g. "a/ab/abc")
  78          */
  79         public function __construct( array $config ) {
  80                 parent::__construct( $config );
  81                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  82                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  83                 }
  84                 // Required settings
  85                 $this->auth = new CF_Authentication(
  86                         $config['swiftUser'],
  87                         $config['swiftKey'],
  88                         null, // account; unused
  89                         $config['swiftAuthUrl']
  90                 );
  91                 // Optional settings
  92                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  93                         ? $config['swiftAuthTTL']
  94                         : 5 * 60; // some sane number
  95                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  96                         ? $config['swiftAnonUser']
  97                         : '';
  98                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  99                         ? $config['shardViaHashLevels']
 100                         : '';
 101                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 102                         ? $config['swiftUseCDN']
 103                         : false;
 104                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 105                         ? $config['swiftCDNExpiry']
 106                         : 3600; // hour
 107                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 108                         ? $config['swiftCDNPurgable']
 109                         : true;
 110                 // Cache container info to mask latency
 111                 $this->memCache = wfGetMainCache();
 112         }
 113
 114         /**
 115          * @see FileBackendStore::resolveContainerPath()
 116          * @return null
 117          */
 118         protected function resolveContainerPath( $container, $relStoragePath ) {
 119                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 120                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 121                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 122                         return null; // too long for Swift
 123                 }
 124                 return $relStoragePath;
 125         }
 126
 127         /**
 128          * @see FileBackendStore::isPathUsableInternal()
 129          * @return bool
 130          */
 131         public function isPathUsableInternal( $storagePath ) {
 132                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 133                 if ( $rel === null ) {
 134                         return false; // invalid
 135                 }
 136
 137                 try {
 138                         $this->getContainer( $container );
 139                         return true; // container exists
 140                 } catch ( NoSuchContainerException $e ) {
 141                 } catch ( CloudFilesException $e ) { // some other exception?
 142                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 143                 }
 144
 145                 return false;
 146         }
 147
 148         /**
 149          * @see FileBackendStore::doCreateInternal()
 150          * @return Status
 151          */
 152         protected function doCreateInternal( array $params ) {
 153                 $status = Status::newGood();
 154
 155                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 156                 if ( $dstRel === null ) {
 157                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 158                         return $status;
 159                 }
 160
 161                 // (a) Check the destination container and object
 162                 try {
 163                         $dContObj = $this->getContainer( $dstCont );
 164                         if ( empty( $params['overwrite'] ) &&
 165                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 166                         {
 167                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 168                                 return $status;
 169                         }
 170                 } catch ( NoSuchContainerException $e ) {
 171                         $status->fatal( 'backend-fail-create', $params['dst'] );
 172                         return $status;
 173                 } catch ( CloudFilesException $e ) { // some other exception?
 174                         $this->handleException( $e, $status, __METHOD__, $params );
 175                         return $status;
 176                 }
 177
 178                 // (b) Get a SHA-1 hash of the object
 179                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 180
 181                 // (c) Actually create the object
 182                 try {
 183                         // Create a fresh CF_Object with no fields preloaded.
 184                         // We don't want to preserve headers, metadata, and such.
 185                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 186                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 187                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 188                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 189                         // The MD5 here will be checked within Swift against its own MD5.
 190                         $obj->set_etag( md5( $params['content'] ) );
 191                         // Use the same content type as StreamFile for security
 192                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 193                         if ( !strlen( $obj->content_type ) ) { // special case
 194                                 $obj->content_type = 'unknown/unknown';
 195                         }
 196                         if ( !empty( $params['async'] ) ) { // deferred
 197                                 $handle = $obj->write_async( $params['content'] );
 198                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 199                                 $status->value->affectedObjects[] = $obj;
 200                         } else { // actually write the object in Swift
 201                                 $obj->write( $params['content'] );
 202                                 $this->purgeCDNCache( array( $obj ) );
 203                         }
 204                 } catch ( CDNNotEnabledException $e ) {
 205                         // CDN not enabled; nothing to see here
 206                 } catch ( BadContentTypeException $e ) {
 207                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 208                 } catch ( CloudFilesException $e ) { // some other exception?
 209                         $this->handleException( $e, $status, __METHOD__, $params );
 210                 }
 211
 212                 return $status;
 213         }
 214
 215         /**
 216          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 217          */
 218         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 219                 try {
 220                         $cfOp->getLastResponse();
 221                 } catch ( BadContentTypeException $e ) {
 222                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 223                 }
 224         }
 225
 226         /**
 227          * @see FileBackendStore::doStoreInternal()
 228          * @return Status
 229          */
 230         protected function doStoreInternal( array $params ) {
 231                 $status = Status::newGood();
 232
 233                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 234                 if ( $dstRel === null ) {
 235                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 236                         return $status;
 237                 }
 238
 239                 // (a) Check the destination container and object
 240                 try {
 241                         $dContObj = $this->getContainer( $dstCont );
 242                         if ( empty( $params['overwrite'] ) &&
 243                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 244                         {
 245                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 246                                 return $status;
 247                         }
 248                 } catch ( NoSuchContainerException $e ) {
 249                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 250                         return $status;
 251                 } catch ( CloudFilesException $e ) { // some other exception?
 252                         $this->handleException( $e, $status, __METHOD__, $params );
 253                         return $status;
 254                 }
 255
 256                 // (b) Get a SHA-1 hash of the object
 257                 $sha1Hash = sha1_file( $params['src'] );
 258                 if ( $sha1Hash === false ) { // source doesn't exist?
 259                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 260                         return $status;
 261                 }
 262                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 263
 264                 // (c) Actually store the object
 265                 try {
 266                         // Create a fresh CF_Object with no fields preloaded.
 267                         // We don't want to preserve headers, metadata, and such.
 268                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 269                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 270                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 271                         // The MD5 here will be checked within Swift against its own MD5.
 272                         $obj->set_etag( md5_file( $params['src'] ) );
 273                         // Use the same content type as StreamFile for security
 274                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 275                         if ( !strlen( $obj->content_type ) ) { // special case
 276                                 $obj->content_type = 'unknown/unknown';
 277                         }
 278                         if ( !empty( $params['async'] ) ) { // deferred
 279                                 wfSuppressWarnings();
 280                                 $fp = fopen( $params['src'], 'rb' );
 281                                 wfRestoreWarnings();
 282                                 if ( !$fp ) {
 283                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 284                                 } else {
 285                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 286                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 287                                         $status->value->resourcesToClose[] = $fp;
 288                                         $status->value->affectedObjects[] = $obj;
 289                                 }
 290                         } else { // actually write the object in Swift
 291                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 292                                 $this->purgeCDNCache( array( $obj ) );
 293                         }
 294                 } catch ( CDNNotEnabledException $e ) {
 295                         // CDN not enabled; nothing to see here
 296                 } catch ( BadContentTypeException $e ) {
 297                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 298                 } catch ( IOException $e ) {
 299                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 300                 } catch ( CloudFilesException $e ) { // some other exception?
 301                         $this->handleException( $e, $status, __METHOD__, $params );
 302                 }
 303
 304                 return $status;
 305         }
 306
 307         /**
 308          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 309          */
 310         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 311                 try {
 312                         $cfOp->getLastResponse();
 313                 } catch ( BadContentTypeException $e ) {
 314                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 315                 } catch ( IOException $e ) {
 316                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 317                 }
 318         }
 319
 320         /**
 321          * @see FileBackendStore::doCopyInternal()
 322          * @return Status
 323          */
 324         protected function doCopyInternal( array $params ) {
 325                 $status = Status::newGood();
 326
 327                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 328                 if ( $srcRel === null ) {
 329                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 330                         return $status;
 331                 }
 332
 333                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 334                 if ( $dstRel === null ) {
 335                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 336                         return $status;
 337                 }
 338
 339                 // (a) Check the source/destination containers and destination object
 340                 try {
 341                         $sContObj = $this->getContainer( $srcCont );
 342                         $dContObj = $this->getContainer( $dstCont );
 343                         if ( empty( $params['overwrite'] ) &&
 344                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 345                         {
 346                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 347                                 return $status;
 348                         }
 349                 } catch ( NoSuchContainerException $e ) {
 350                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 351                         return $status;
 352                 } catch ( CloudFilesException $e ) { // some other exception?
 353                         $this->handleException( $e, $status, __METHOD__, $params );
 354                         return $status;
 355                 }
 356
 357                 // (b) Actually copy the file to the destination
 358                 try {
 359                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 360                         if ( !empty( $params['async'] ) ) { // deferred
 361                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 362                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 363                                 $status->value->affectedObjects[] = $dstObj;
 364                         } else { // actually write the object in Swift
 365                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 366                                 $this->purgeCDNCache( array( $dstObj ) );
 367                         }
 368                 } catch ( CDNNotEnabledException $e ) {
 369                         // CDN not enabled; nothing to see here
 370                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 371                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 372                 } catch ( CloudFilesException $e ) { // some other exception?
 373                         $this->handleException( $e, $status, __METHOD__, $params );
 374                 }
 375
 376                 return $status;
 377         }
 378
 379         /**
 380          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 381          */
 382         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 383                 try {
 384                         $cfOp->getLastResponse();
 385                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 386                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 387                 }
 388         }
 389
 390         /**
 391          * @see FileBackendStore::doMoveInternal()
 392          * @return Status
 393          */
 394         protected function doMoveInternal( array $params ) {
 395                 $status = Status::newGood();
 396
 397                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 398                 if ( $srcRel === null ) {
 399                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 400                         return $status;
 401                 }
 402
 403                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 404                 if ( $dstRel === null ) {
 405                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 406                         return $status;
 407                 }
 408
 409                 // (a) Check the source/destination containers and destination object
 410                 try {
 411                         $sContObj = $this->getContainer( $srcCont );
 412                         $dContObj = $this->getContainer( $dstCont );
 413                         if ( empty( $params['overwrite'] ) &&
 414                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 415                         {
 416                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 417                                 return $status;
 418                         }
 419                 } catch ( NoSuchContainerException $e ) {
 420                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 421                         return $status;
 422                 } catch ( CloudFilesException $e ) { // some other exception?
 423                         $this->handleException( $e, $status, __METHOD__, $params );
 424                         return $status;
 425                 }
 426
 427                 // (b) Actually move the file to the destination
 428                 try {
 429                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 430                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 431                         if ( !empty( $params['async'] ) ) { // deferred
 432                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 433                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 434                                 $status->value->affectedObjects[] = $srcObj;
 435                                 $status->value->affectedObjects[] = $dstObj;
 436                         } else { // actually write the object in Swift
 437                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 438                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 439                         }
 440                 } catch ( CDNNotEnabledException $e ) {
 441                         // CDN not enabled; nothing to see here
 442                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 443                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 444                 } catch ( CloudFilesException $e ) { // some other exception?
 445                         $this->handleException( $e, $status, __METHOD__, $params );
 446                 }
 447
 448                 return $status;
 449         }
 450
 451         /**
 452          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 453          */
 454         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 455                 try {
 456                         $cfOp->getLastResponse();
 457                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 458                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 459                 }
 460         }
 461
 462         /**
 463          * @see FileBackendStore::doDeleteInternal()
 464          * @return Status
 465          */
 466         protected function doDeleteInternal( array $params ) {
 467                 $status = Status::newGood();
 468
 469                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 470                 if ( $srcRel === null ) {
 471                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 472                         return $status;
 473                 }
 474
 475                 try {
 476                         $sContObj = $this->getContainer( $srcCont );
 477                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 478                         if ( !empty( $params['async'] ) ) { // deferred
 479                                 $handle = $sContObj->delete_object_async( $srcRel );
 480                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 481                                 $status->value->affectedObjects[] = $srcObj;
 482                         } else { // actually write the object in Swift
 483                                 $sContObj->delete_object( $srcRel );
 484                                 $this->purgeCDNCache( array( $srcObj ) );
 485                         }
 486                 } catch ( CDNNotEnabledException $e ) {
 487                         // CDN not enabled; nothing to see here
 488                 } catch ( NoSuchContainerException $e ) {
 489                         $status->fatal( 'backend-fail-delete', $params['src'] );
 490                 } catch ( NoSuchObjectException $e ) {
 491                         if ( empty( $params['ignoreMissingSource'] ) ) {
 492                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 493                         }
 494                 } catch ( CloudFilesException $e ) { // some other exception?
 495                         $this->handleException( $e, $status, __METHOD__, $params );
 496                 }
 497
 498                 return $status;
 499         }
 500
 501         /**
 502          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 503          */
 504         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 505                 try {
 506                         $cfOp->getLastResponse();
 507                 } catch ( NoSuchContainerException $e ) {
 508                         $status->fatal( 'backend-fail-delete', $params['src'] );
 509                 } catch ( NoSuchObjectException $e ) {
 510                         if ( empty( $params['ignoreMissingSource'] ) ) {
 511                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 512                         }
 513                 }
 514         }
 515
 516         /**
 517          * @see FileBackendStore::doPrepareInternal()
 518          * @return Status
 519          */
 520         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 521                 $status = Status::newGood();
 522
 523                 // (a) Check if container already exists
 524                 try {
 525                         $contObj = $this->getContainer( $fullCont );
 526                         // NoSuchContainerException not thrown: container must exist
 527                         return $status; // already exists
 528                 } catch ( NoSuchContainerException $e ) {
 529                         // NoSuchContainerException thrown: container does not exist
 530                 } catch ( CloudFilesException $e ) { // some other exception?
 531                         $this->handleException( $e, $status, __METHOD__, $params );
 532                         return $status;
 533                 }
 534
 535                 // (b) Create container as needed
 536                 try {
 537                         $contObj = $this->createContainer( $fullCont );
 538                         if ( !empty( $params['noAccess'] ) ) {
 539                                 // Make container private to end-users...
 540                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 541                         } else {
 542                                 // Make container public to end-users...
 543                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 544                         }
 545                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 546                                 $contObj->make_public( $this->swiftCDNExpiry );
 547                         }
 548                 } catch ( CDNNotEnabledException $e ) {
 549                         // CDN not enabled; nothing to see here
 550                 } catch ( CloudFilesException $e ) { // some other exception?
 551                         $this->handleException( $e, $status, __METHOD__, $params );
 552                         return $status;
 553                 }
 554
 555                 return $status;
 556         }
 557
 558         /**
 559          * @see FileBackendStore::doSecureInternal()
 560          * @return Status
 561          */
 562         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 563                 $status = Status::newGood();
 564                 if ( empty( $params['noAccess'] ) ) {
 565                         return $status; // nothing to do
 566                 }
 567
 568                 // Restrict container from end-users...
 569                 try {
 570                         // doPrepareInternal() should have been called,
 571                         // so the Swift container should already exist...
 572                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 573                         // NoSuchContainerException not thrown: container must exist
 574
 575                         // Make container private to end-users...
 576                         $status->merge( $this->setContainerAccess(
 577                                 $contObj,
 578                                 array( $this->auth->username ), // read
 579                                 array( $this->auth->username ) // write
 580                         ) );
 581                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 582                                 $contObj->make_private();
 583                         }
 584                 } catch ( CDNNotEnabledException $e ) {
 585                         // CDN not enabled; nothing to see here
 586                 } catch ( CloudFilesException $e ) { // some other exception?
 587                         $this->handleException( $e, $status, __METHOD__, $params );
 588                 }
 589
 590                 return $status;
 591         }
 592
 593         /**
 594          * @see FileBackendStore::doPublishInternal()
 595          * @return Status
 596          */
 597         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 598                 $status = Status::newGood();
 599
 600                 // Unrestrict container from end-users...
 601                 try {
 602                         // doPrepareInternal() should have been called,
 603                         // so the Swift container should already exist...
 604                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 605                         // NoSuchContainerException not thrown: container must exist
 606
 607                         // Make container public to end-users...
 608                         if ( $this->swiftAnonUser != '' ) {
 609                                 $status->merge( $this->setContainerAccess(
 610                                         $contObj,
 611                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 612                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 613                                 ) );
 614                         } else {
 615                                 $status->merge( $this->setContainerAccess(
 616                                         $contObj,
 617                                         array( $this->auth->username, '.r:*' ), // read
 618                                         array( $this->auth->username ) // write
 619                                 ) );
 620                         }
 621                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 622                                 $contObj->make_public();
 623                         }
 624                 } catch ( CDNNotEnabledException $e ) {
 625                         // CDN not enabled; nothing to see here
 626                 } catch ( CloudFilesException $e ) { // some other exception?
 627                         $this->handleException( $e, $status, __METHOD__, $params );
 628                 }
 629
 630                 return $status;
 631         }
 632
 633         /**
 634          * @see FileBackendStore::doCleanInternal()
 635          * @return Status
 636          */
 637         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 638                 $status = Status::newGood();
 639
 640                 // Only containers themselves can be removed, all else is virtual
 641                 if ( $dir != '' ) {
 642                         return $status; // nothing to do
 643                 }
 644
 645                 // (a) Check the container
 646                 try {
 647                         $contObj = $this->getContainer( $fullCont, true );
 648                 } catch ( NoSuchContainerException $e ) {
 649                         return $status; // ok, nothing to do
 650                 } catch ( CloudFilesException $e ) { // some other exception?
 651                         $this->handleException( $e, $status, __METHOD__, $params );
 652                         return $status;
 653                 }
 654
 655                 // (b) Delete the container if empty
 656                 if ( $contObj->object_count == 0 ) {
 657                         try {
 658                                 $this->deleteContainer( $fullCont );
 659                         } catch ( NoSuchContainerException $e ) {
 660                                 return $status; // race?
 661                         } catch ( NonEmptyContainerException $e ) {
 662                                 return $status; // race? consistency delay?
 663                         } catch ( CloudFilesException $e ) { // some other exception?
 664                                 $this->handleException( $e, $status, __METHOD__, $params );
 665                                 return $status;
 666                         }
 667                 }
 668
 669                 return $status;
 670         }
 671
 672         /**
 673          * @see FileBackendStore::doFileExists()
 674          * @return array|bool|null
 675          */
 676         protected function doGetFileStat( array $params ) {
 677                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 678                 if ( $srcRel === null ) {
 679                         return false; // invalid storage path
 680                 }
 681
 682                 $stat = false;
 683                 try {
 684                         $contObj = $this->getContainer( $srcCont );
 685                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 686                         $this->addMissingMetadata( $srcObj, $params['src'] );
 687                         $stat = array(
 688                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 689                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 690                                 'size'  => (int)$srcObj->content_length,
 691                                 'sha1'  => $srcObj->metadata['Sha1base36']
 692                         );
 693                 } catch ( NoSuchContainerException $e ) {
 694                 } catch ( NoSuchObjectException $e ) {
 695                 } catch ( CloudFilesException $e ) { // some other exception?
 696                         $stat = null;
 697                         $this->handleException( $e, null, __METHOD__, $params );
 698                 }
 699
 700                 return $stat;
 701         }
 702
 703         /**
 704          * Fill in any missing object metadata and save it to Swift
 705          *
 706          * @param $obj CF_Object
 707          * @param $path string Storage path to object
 708          * @return bool Success
 709          * @throws Exception cloudfiles exceptions
 710          */
 711         protected function addMissingMetadata( CF_Object $obj, $path ) {
 712                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 713                         return true; // nothing to do
 714                 }
 715                 $status = Status::newGood();
 716                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 717                 if ( $status->isOK() ) {
 718                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 719                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 720                         if ( $tmpFile ) {
 721                                 $hash = $tmpFile->getSha1Base36();
 722                                 if ( $hash !== false ) {
 723                                         $obj->metadata['Sha1base36'] = $hash;
 724                                         $obj->sync_metadata(); // save to Swift
 725                                         return true; // success
 726                                 }
 727                         }
 728                 }
 729                 $obj->metadata['Sha1base36'] = false;
 730                 return false; // failed
 731         }
 732
 733         /**
 734          * @see FileBackend::getFileContents()
 735          * @return bool|null|string
 736          */
 737         public function getFileContents( array $params ) {
 738                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 739                 if ( $srcRel === null ) {
 740                         return false; // invalid storage path
 741                 }
 742
 743                 if ( !$this->fileExists( $params ) ) {
 744                         return null;
 745                 }
 746
 747                 $data = false;
 748                 try {
 749                         $sContObj = $this->getContainer( $srcCont );
 750                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 751                         $data = $obj->read( $this->headersFromParams( $params ) );
 752                 } catch ( NoSuchContainerException $e ) {
 753                 } catch ( CloudFilesException $e ) { // some other exception?
 754                         $this->handleException( $e, null, __METHOD__, $params );
 755                 }
 756
 757                 return $data;
 758         }
 759
 760         /**
 761          * @see FileBackendStore::doDirectoryExists()
 762          * @return bool|null
 763          */
 764         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 765                 try {
 766                         $container = $this->getContainer( $fullCont );
 767                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 768                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 769                 } catch ( NoSuchContainerException $e ) {
 770                         return false;
 771                 } catch ( CloudFilesException $e ) { // some other exception?
 772                         $this->handleException( $e, null, __METHOD__,
 773                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 774                 }
 775
 776                 return null; // error
 777         }
 778
 779         /**
 780          * @see FileBackendStore::getDirectoryListInternal()
 781          * @return SwiftFileBackendDirList
 782          */
 783         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 784                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 785         }
 786
 787         /**
 788          * @see FileBackendStore::getFileListInternal()
 789          * @return SwiftFileBackendFileList
 790          */
 791         public function getFileListInternal( $fullCont, $dir, array $params ) {
 792                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 793         }
 794
 795         /**
 796          * Do not call this function outside of SwiftFileBackendFileList
 797          *
 798          * @param $fullCont string Resolved container name
 799          * @param $dir string Resolved storage directory with no trailing slash
 800          * @param $after string|null Storage path of file to list items after
 801          * @param $limit integer Max number of items to list
 802          * @param $params Array Includes flag for 'topOnly'
 803          * @return Array List of relative paths of dirs directly under $dir
 804          */
 805         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 806                 $dirs = array();
 807                 if ( $after === INF ) {
 808                         return $dirs; // nothing more
 809                 }
 810                 wfProfileIn( __METHOD__ . '-' . $this->name );
 811
 812                 try {
 813                         $container = $this->getContainer( $fullCont );
 814                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 815                         // Non-recursive: only list dirs right under $dir
 816                         if ( !empty( $params['topOnly'] ) ) {
 817                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 818                                 foreach ( $objects as $object ) { // files and dirs
 819                                         if ( substr( $object, -1 ) === '/' ) {
 820                                                 $dirs[] = $object; // directories end in '/'
 821                                         }
 822                                 }
 823                         // Recursive: list all dirs under $dir and its subdirs
 824                         } else {
 825                                 // Get directory from last item of prior page
 826                                 $lastDir = $this->getParentDir( $after ); // must be first page
 827                                 $objects = $container->list_objects( $limit, $after, $prefix );
 828                                 foreach ( $objects as $object ) { // files
 829                                         $objectDir = $this->getParentDir( $object ); // directory of object
 830                                         if ( $objectDir !== false ) { // file has a parent dir
 831                                                 // Swift stores paths in UTF-8, using binary sorting.
 832                                                 // See function "create_container_table" in common/db.py.
 833                                                 // If a directory is not "greater" than the last one,
 834                                                 // then it was already listed by the calling iterator.
 835                                                 if ( $objectDir > $lastDir ) {
 836                                                         $pDir = $objectDir;
 837                                                         do { // add dir and all its parent dirs
 838                                                                 $dirs[] = "{$pDir}/";
 839                                                                 $pDir = $this->getParentDir( $pDir );
 840                                                         } while ( $pDir !== false // sanity
 841                                                                 && $pDir > $lastDir // not done already
 842                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 843                                                         );
 844                                                 }
 845                                                 $lastDir = $objectDir;
 846                                         }
 847                                 }
 848                         }
 849                         if ( count( $objects ) < $limit ) {
 850                                 $after = INF; // avoid a second RTT
 851                         } else {
 852                                 $after = end( $objects ); // update last item
 853                         }
 854                 } catch ( NoSuchContainerException $e ) {
 855                 } catch ( CloudFilesException $e ) { // some other exception?
 856                         $this->handleException( $e, null, __METHOD__,
 857                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 858                 }
 859
 860                 wfProfileOut( __METHOD__ . '-' . $this->name );
 861                 return $dirs;
 862         }
 863
 864         protected function getParentDir( $path ) {
 865                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 866         }
 867
 868         /**
 869          * Do not call this function outside of SwiftFileBackendFileList
 870          *
 871          * @param $fullCont string Resolved container name
 872          * @param $dir string Resolved storage directory with no trailing slash
 873          * @param $after string|null Storage path of file to list items after
 874          * @param $limit integer Max number of items to list
 875          * @param $params Array Includes flag for 'topOnly'
 876          * @return Array List of relative paths of files under $dir
 877          */
 878         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 879                 $files = array();
 880                 if ( $after === INF ) {
 881                         return $files; // nothing more
 882                 }
 883                 wfProfileIn( __METHOD__ . '-' . $this->name );
 884
 885                 try {
 886                         $container = $this->getContainer( $fullCont );
 887                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 888                         // Non-recursive: only list files right under $dir
 889                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 890                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 891                                 foreach ( $objects as $object ) {
 892                                         if ( substr( $object, -1 ) !== '/' ) {
 893                                                 $files[] = $object; // directories end in '/'
 894                                         }
 895                                 }
 896                         // Recursive: list all files under $dir and its subdirs
 897                         } else { // files
 898                                 $objects = $container->list_objects( $limit, $after, $prefix );
 899                                 $files = $objects;
 900                         }
 901                         if ( count( $objects ) < $limit ) {
 902                                 $after = INF; // avoid a second RTT
 903                         } else {
 904                                 $after = end( $objects ); // update last item
 905                         }
 906                 } catch ( NoSuchContainerException $e ) {
 907                 } catch ( CloudFilesException $e ) { // some other exception?
 908                         $this->handleException( $e, null, __METHOD__,
 909                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 910                 }
 911
 912                 wfProfileOut( __METHOD__ . '-' . $this->name );
 913                 return $files;
 914         }
 915
 916         /**
 917          * @see FileBackendStore::doGetFileSha1base36()
 918          * @return bool
 919          */
 920         protected function doGetFileSha1base36( array $params ) {
 921                 $stat = $this->getFileStat( $params );
 922                 if ( $stat ) {
 923                         return $stat['sha1'];
 924                 } else {
 925                         return false;
 926                 }
 927         }
 928
 929         /**
 930          * @see FileBackendStore::doStreamFile()
 931          * @return Status
 932          */
 933         protected function doStreamFile( array $params ) {
 934                 $status = Status::newGood();
 935
 936                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 937                 if ( $srcRel === null ) {
 938                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 939                 }
 940
 941                 try {
 942                         $cont = $this->getContainer( $srcCont );
 943                 } catch ( NoSuchContainerException $e ) {
 944                         $status->fatal( 'backend-fail-stream', $params['src'] );
 945                         return $status;
 946                 } catch ( CloudFilesException $e ) { // some other exception?
 947                         $this->handleException( $e, $status, __METHOD__, $params );
 948                         return $status;
 949                 }
 950
 951                 try {
 952                         $output = fopen( 'php://output', 'wb' );
 953                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 954                         $obj->stream( $output, $this->headersFromParams( $params ) );
 955                 } catch ( CloudFilesException $e ) { // some other exception?
 956                         $this->handleException( $e, $status, __METHOD__, $params );
 957                 }
 958
 959                 return $status;
 960         }
 961
 962         /**
 963          * @see FileBackendStore::getLocalCopy()
 964          * @return null|TempFSFile
 965          */
 966         public function getLocalCopy( array $params ) {
 967                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 968                 if ( $srcRel === null ) {
 969                         return null;
 970                 }
 971
 972                 # Check the recursion guard to avoid loops when filling metadata
 973                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 974                         return null;
 975                 }
 976
 977                 $tmpFile = null;
 978                 try {
 979                         $sContObj = $this->getContainer( $srcCont );
 980                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 981                         // Get source file extension
 982                         $ext = FileBackend::extensionFromPath( $srcRel );
 983                         // Create a new temporary file...
 984                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 985                         if ( $tmpFile ) {
 986                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 987                                 if ( $handle ) {
 988                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 989                                         fclose( $handle );
 990                                 } else {
 991                                         $tmpFile = null; // couldn't open temp file
 992                                 }
 993                         }
 994                 } catch ( NoSuchContainerException $e ) {
 995                         $tmpFile = null;
 996                 } catch ( CloudFilesException $e ) { // some other exception?
 997                         $tmpFile = null;
 998                         $this->handleException( $e, null, __METHOD__, $params );
 999                 }
1000
1001                 return $tmpFile;
1002         }
1003
1004         /**
1005          * @see FileBackendStore::directoriesAreVirtual()
1006          * @return bool
1007          */
1008         protected function directoriesAreVirtual() {
1009                 return true;
1010         }
1011
1012         /**
1013          * Get headers to send to Swift when reading a file based
1014          * on a FileBackend params array, e.g. that of getLocalCopy().
1015          * $params is currently only checked for a 'latest' flag.
1016          *
1017          * @param $params Array
1018          * @return Array
1019          */
1020         protected function headersFromParams( array $params ) {
1021                 $hdrs = array();
1022                 if ( !empty( $params['latest'] ) ) {
1023                         $hdrs[] = 'X-Newest: true';
1024                 }
1025                 return $hdrs;
1026         }
1027
1028         /**
1029          * @see FileBackendStore::doExecuteOpHandlesInternal()
1030          * @return Array List of corresponding Status objects
1031          */
1032         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1033                 $statuses = array();
1034
1035                 $cfOps = array(); // list of CF_Async_Op objects
1036                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1037                         $cfOps[$index] = $fileOpHandle->cfOp;
1038                 }
1039                 $batch = new CF_Async_Op_Batch( $cfOps );
1040
1041                 $cfOps = $batch->execute();
1042                 foreach ( $cfOps as $index => $cfOp ) {
1043                         $status = Status::newGood();
1044                         try { // catch exceptions; update status
1045                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1046                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1047                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1048                         } catch ( CloudFilesException $e ) { // some other exception?
1049                                 $this->handleException( $e, $status,
1050                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1051                         }
1052                         $statuses[$index] = $status;
1053                 }
1054
1055                 return $statuses;
1056         }
1057
1058         /**
1059          * Set read/write permissions for a Swift container.
1060          *
1061          * $readGrps is a list of the possible criteria for a request to have
1062          * access to read a container. Each item is one of the following formats:
1063          *   - account:user       : Grants access if the request is by the given user
1064          *   - .r:<regex>         : Grants access if the request is from a referrer host that
1065          *                          matches the expression and the request is not for a listing.
1066          *                          Setting this to '*' effectively makes a container public.
1067          *   - .rlistings:<regex> : Grants access if the request is from a referrer host that
1068          *                          matches the expression and the request for a listing.
1069          *
1070          * $writeGrps is a list of the possible criteria for a request to have
1071          * access to write to a container. Each item is of the following format:
1072          *   - account:user       : Grants access if the request is by the given user
1073          *
1074          * @see http://swift.openstack.org/misc.html#acls
1075          *
1076          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1077          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1078          *
1079          * @param $contObj CF_Container Swift container
1080          * @param $readGrps Array List of read access routes
1081          * @param $writeGrps Array List of write access routes
1082          * @return Status
1083          */
1084         protected function setContainerAccess(
1085                 CF_Container $contObj, array $readGrps, array $writeGrps
1086         ) {
1087                 $creds = $contObj->cfs_auth->export_credentials();
1088
1089                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1090
1091                 // Note: 10 second timeout consistent with php-cloudfiles
1092                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1093                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1094                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1095                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1096
1097                 return $req->execute(); // should return 204
1098         }
1099
1100         /**
1101          * Purge the CDN cache of affected objects if CDN caching is enabled.
1102          * This is for Rackspace/Akamai CDNs.
1103          *
1104          * @param $objects Array List of CF_Object items
1105          * @return void
1106          */
1107         public function purgeCDNCache( array $objects ) {
1108                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1109                         foreach ( $objects as $object ) {
1110                                 try {
1111                                         $object->purge_from_cdn();
1112                                 } catch ( CDNNotEnabledException $e ) {
1113                                         // CDN not enabled; nothing to see here
1114                                 } catch ( CloudFilesException $e ) {
1115                                         $this->handleException( $e, null, __METHOD__,
1116                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1117                                 }
1118                         }
1119                 }
1120         }
1121
1122         /**
1123          * Get a connection to the Swift proxy
1124          *
1125          * @return CF_Connection|bool False on failure
1126          * @throws CloudFilesException
1127          */
1128         protected function getConnection() {
1129                 if ( $this->connException instanceof Exception ) {
1130                         throw $this->connException; // failed last attempt
1131                 }
1132                 // Session keys expire after a while, so we renew them periodically
1133                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1134                         $this->conn->close(); // close active cURL connections
1135                         $this->conn = null;
1136                 }
1137                 // Authenticate with proxy and get a session key...
1138                 if ( !$this->conn ) {
1139                         $this->connStarted = 0;
1140                         $this->connContainers = array();
1141                         try {
1142                                 $this->auth->authenticate();
1143                                 $this->conn = new CF_Connection( $this->auth );
1144                                 $this->connStarted = time();
1145                         } catch ( CloudFilesException $e ) {
1146                                 $this->connException = $e; // don't keep re-trying
1147                                 throw $e; // throw it back
1148                         }
1149                 }
1150                 return $this->conn;
1151         }
1152
1153         /**
1154          * @see FileBackendStore::doClearCache()
1155          */
1156         protected function doClearCache( array $paths = null ) {
1157                 $this->connContainers = array(); // clear container object cache
1158         }
1159
1160         /**
1161          * Get a Swift container object, possibly from process cache.
1162          * Use $reCache if the file count or byte count is needed.
1163          *
1164          * @param $container string Container name
1165          * @param $bypassCache bool Bypass all caches and load from Swift
1166          * @return CF_Container
1167          * @throws CloudFilesException
1168          */
1169         protected function getContainer( $container, $bypassCache = false ) {
1170                 $conn = $this->getConnection(); // Swift proxy connection
1171                 if ( $bypassCache ) { // purge cache
1172                         unset( $this->connContainers[$container] );
1173                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1174                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1175                 }
1176                 if ( !isset( $this->connContainers[$container] ) ) {
1177                         $contObj = $conn->get_container( $container );
1178                         // NoSuchContainerException not thrown: container must exist
1179                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1180                                 reset( $this->connContainers );
1181                                 unset( $this->connContainers[key( $this->connContainers )] );
1182                         }
1183                         $this->connContainers[$container] = $contObj; // cache it
1184                         if ( !$bypassCache ) {
1185                                 $this->setContainerCache( $container, // update persistent cache
1186                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1187                                 );
1188                         }
1189                 }
1190                 return $this->connContainers[$container];
1191         }
1192
1193         /**
1194          * Create a Swift container
1195          *
1196          * @param $container string Container name
1197          * @return CF_Container
1198          * @throws CloudFilesException
1199          */
1200         protected function createContainer( $container ) {
1201                 $conn = $this->getConnection(); // Swift proxy connection
1202                 $contObj = $conn->create_container( $container );
1203                 $this->connContainers[$container] = $contObj; // cache it
1204                 return $contObj;
1205         }
1206
1207         /**
1208          * Delete a Swift container
1209          *
1210          * @param $container string Container name
1211          * @return void
1212          * @throws CloudFilesException
1213          */
1214         protected function deleteContainer( $container ) {
1215                 $conn = $this->getConnection(); // Swift proxy connection
1216                 unset( $this->connContainers[$container] ); // purge cache
1217                 $conn->delete_container( $container );
1218         }
1219
1220         /**
1221          * @see FileBackendStore::doPrimeContainerCache()
1222          * @return void
1223          */
1224         protected function doPrimeContainerCache( array $containerInfo ) {
1225                 try {
1226                         $conn = $this->getConnection(); // Swift proxy connection
1227                         foreach ( $containerInfo as $container => $info ) {
1228                                 $this->connContainers[$container] = new CF_Container(
1229                                         $conn->cfs_auth,
1230                                         $conn->cfs_http,
1231                                         $container,
1232                                         $info['count'],
1233                                         $info['bytes']
1234                                 );
1235                         }
1236                 } catch ( CloudFilesException $e ) { // some other exception?
1237                         $this->handleException( $e, null, __METHOD__, array() );
1238                 }
1239         }
1240
1241         /**
1242          * Log an unexpected exception for this backend.
1243          * This also sets the Status object to have a fatal error.
1244          *
1245          * @param $e Exception
1246          * @param $status Status|null
1247          * @param $func string
1248          * @param $params Array
1249          * @return void
1250          */
1251         protected function handleException( Exception $e, $status, $func, array $params ) {
1252                 if ( $status instanceof Status ) {
1253                         if ( $e instanceof AuthenticationException ) {
1254                                 $status->fatal( 'backend-fail-connect', $this->name );
1255                         } else {
1256                                 $status->fatal( 'backend-fail-internal', $this->name );
1257                         }
1258                 }
1259                 if ( $e->getMessage() ) {
1260                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1261                 }
1262                 wfDebugLog( 'SwiftBackend',
1263                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1264                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1265                 );
1266         }
1267 }
1268
1269 /**
1270  * @see FileBackendStoreOpHandle
1271  */
1272 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1273         /** @var CF_Async_Op */
1274         public $cfOp;
1275         /** @var Array */
1276         public $affectedObjects = array();
1277
1278         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1279                 $this->backend = $backend;
1280                 $this->params = $params;
1281                 $this->call = $call;
1282                 $this->cfOp = $cfOp;
1283         }
1284 }
1285
1286 /**
1287  * SwiftFileBackend helper class to page through listings.
1288  * Swift also has a listing limit of 10,000 objects for sanity.
1289  * Do not use this class from places outside SwiftFileBackend.
1290  *
1291  * @ingroup FileBackend
1292  */
1293 abstract class SwiftFileBackendList implements Iterator {
1294         /** @var Array */
1295         protected $bufferIter = array();
1296         protected $bufferAfter = null; // string; list items *after* this path
1297         protected $pos = 0; // integer
1298         /** @var Array */
1299         protected $params = array();
1300
1301         /** @var SwiftFileBackend */
1302         protected $backend;
1303         protected $container; // string; container name
1304         protected $dir; // string; storage directory
1305         protected $suffixStart; // integer
1306
1307         const PAGE_SIZE = 9000; // file listing buffer size
1308
1309         /**
1310          * @param $backend SwiftFileBackend
1311          * @param $fullCont string Resolved container name
1312          * @param $dir string Resolved directory relative to container
1313          * @param $params Array
1314          */
1315         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1316                 $this->backend = $backend;
1317                 $this->container = $fullCont;
1318                 $this->dir = $dir;
1319                 if ( substr( $this->dir, -1 ) === '/' ) {
1320                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1321                 }
1322                 if ( $this->dir == '' ) { // whole container
1323                         $this->suffixStart = 0;
1324                 } else { // dir within container
1325                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1326                 }
1327                 $this->params = $params;
1328         }
1329
1330         /**
1331          * @see Iterator::key()
1332          * @return integer
1333          */
1334         public function key() {
1335                 return $this->pos;
1336         }
1337
1338         /**
1339          * @see Iterator::next()
1340          * @return void
1341          */
1342         public function next() {
1343                 // Advance to the next file in the page
1344                 next( $this->bufferIter );
1345                 ++$this->pos;
1346                 // Check if there are no files left in this page and
1347                 // advance to the next page if this page was not empty.
1348                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1349                         $this->bufferIter = $this->pageFromList(
1350                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1351                         ); // updates $this->bufferAfter
1352                 }
1353         }
1354
1355         /**
1356          * @see Iterator::rewind()
1357          * @return void
1358          */
1359         public function rewind() {
1360                 $this->pos = 0;
1361                 $this->bufferAfter = null;
1362                 $this->bufferIter = $this->pageFromList(
1363                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1364                 ); // updates $this->bufferAfter
1365         }
1366
1367         /**
1368          * @see Iterator::valid()
1369          * @return bool
1370          */
1371         public function valid() {
1372                 if ( $this->bufferIter === null ) {
1373                         return false; // some failure?
1374                 } else {
1375                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1376                 }
1377         }
1378
1379         /**
1380          * Get the given list portion (page)
1381          *
1382          * @param $container string Resolved container name
1383          * @param $dir string Resolved path relative to container
1384          * @param $after string|null
1385          * @param $limit integer
1386          * @param $params Array
1387          * @return Traversable|Array|null Returns null on failure
1388          */
1389         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1390 }
1391
1392 /**
1393  * Iterator for listing directories
1394  */
1395 class SwiftFileBackendDirList extends SwiftFileBackendList {
1396         /**
1397          * @see Iterator::current()
1398          * @return string|bool String (relative path) or false
1399          */
1400         public function current() {
1401                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1402         }
1403
1404         /**
1405          * @see SwiftFileBackendList::pageFromList()
1406          * @return Array|null
1407          */
1408         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1409                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1410         }
1411 }
1412
1413 /**
1414  * Iterator for listing regular files
1415  */
1416 class SwiftFileBackendFileList extends SwiftFileBackendList {
1417         /**
1418          * @see Iterator::current()
1419          * @return string|bool String (relative path) or false
1420          */
1421         public function current() {
1422                 return substr( current( $this->bufferIter ), $this->suffixStart );
1423         }
1424
1425         /**
1426          * @see SwiftFileBackendList::pageFromList()
1427          * @return Array|null
1428          */
1429         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1430                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1431         }
1432 }