includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         /** @var CF_Connection */
  49         protected $conn; // Swift connection handle
  50         protected $sessionStarted = 0; // integer UNIX timestamp
  51
  52         /** @var CloudFilesException */
  53         protected $connException;
  54         protected $connErrorTime = 0; // UNIX timestamp
  55
  56         /** @var BagOStuff */
  57         protected $srvCache;
  58
  59         /** @var ProcessCacheLRU */
  60         protected $connContainerCache; // container object cache
  61
  62         /**
  63          * @see FileBackendStore::__construct()
  64          * Additional $config params include:
  65          *   - swiftAuthUrl       : Swift authentication server URL
  66          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  67          *   - swiftKey           : Swift authentication key for the above user
  68          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  69          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  70          *                          If set, then views of public containers are assumed to go
  71          *                          through this user. If not set, then public containers are
  72          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  73          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  74          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  75          *                          If files may likely change, this should probably not exceed
  76          *                          a few days. For example, deletions may take this long to apply.
  77          *                          If object purging is enabled, however, this is not an issue.
  78          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  79          *   - shardViaHashLevels : Map of container names to sharding config with:
  80          *                             - base   : base of hash characters, 16 or 36
  81          *                             - levels : the number of hash levels (and digits)
  82          *                             - repeat : hash subdirectories are prefixed with all the
  83          *                                        parent hash directory names (e.g. "a/ab/abc")
  84          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC/XCache.
  85          *                          This is probably insecure in shared hosting environments.
  86          */
  87         public function __construct( array $config ) {
  88                 parent::__construct( $config );
  89                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  90                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  91                 }
  92                 // Required settings
  93                 $this->auth = new CF_Authentication(
  94                         $config['swiftUser'],
  95                         $config['swiftKey'],
  96                         null, // account; unused
  97                         $config['swiftAuthUrl']
  98                 );
  99                 // Optional settings
 100                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 101                         ? $config['swiftAuthTTL']
 102                         : 5 * 60; // some sane number
 103                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 104                         ? $config['swiftAnonUser']
 105                         : '';
 106                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 107                         ? $config['shardViaHashLevels']
 108                         : '';
 109                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 110                         ? $config['swiftUseCDN']
 111                         : false;
 112                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 113                         ? $config['swiftCDNExpiry']
 114                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 115                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 116                         ? $config['swiftCDNPurgable']
 117                         : true;
 118                 // Cache container information to mask latency
 119                 $this->memCache = wfGetMainCache();
 120                 // Process cache for container info
 121                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 122                 // Cache auth token information to avoid RTTs
 123                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 124                         try { // look for APC, XCache, WinCache, ect...
 125                                 $this->srvCache = ObjectCache::newAccelerator( array() );
 126                         } catch ( Exception $e ) {}
 127                 }
 128                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 129         }
 130
 131         /**
 132          * @see FileBackendStore::resolveContainerPath()
 133          * @return null
 134          */
 135         protected function resolveContainerPath( $container, $relStoragePath ) {
 136                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 137                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 138                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 139                         return null; // too long for Swift
 140                 }
 141                 return $relStoragePath;
 142         }
 143
 144         /**
 145          * @see FileBackendStore::isPathUsableInternal()
 146          * @return bool
 147          */
 148         public function isPathUsableInternal( $storagePath ) {
 149                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 150                 if ( $rel === null ) {
 151                         return false; // invalid
 152                 }
 153
 154                 try {
 155                         $this->getContainer( $container );
 156                         return true; // container exists
 157                 } catch ( NoSuchContainerException $e ) {
 158                 } catch ( CloudFilesException $e ) { // some other exception?
 159                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 160                 }
 161
 162                 return false;
 163         }
 164
 165         /**
 166          * @see FileBackendStore::doCreateInternal()
 167          * @return Status
 168          */
 169         protected function doCreateInternal( array $params ) {
 170                 $status = Status::newGood();
 171
 172                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 173                 if ( $dstRel === null ) {
 174                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 175                         return $status;
 176                 }
 177
 178                 // (a) Check the destination container and object
 179                 try {
 180                         $dContObj = $this->getContainer( $dstCont );
 181                         if ( empty( $params['overwrite'] ) &&
 182                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 183                         {
 184                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 185                                 return $status;
 186                         }
 187                 } catch ( NoSuchContainerException $e ) {
 188                         $status->fatal( 'backend-fail-create', $params['dst'] );
 189                         return $status;
 190                 } catch ( CloudFilesException $e ) { // some other exception?
 191                         $this->handleException( $e, $status, __METHOD__, $params );
 192                         return $status;
 193                 }
 194
 195                 // (b) Get a SHA-1 hash of the object
 196                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 197
 198                 // (c) Actually create the object
 199                 try {
 200                         // Create a fresh CF_Object with no fields preloaded.
 201                         // We don't want to preserve headers, metadata, and such.
 202                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 203                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 204                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 205                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 206                         // The MD5 here will be checked within Swift against its own MD5.
 207                         $obj->set_etag( md5( $params['content'] ) );
 208                         // Use the same content type as StreamFile for security
 209                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 210                         if ( !strlen( $obj->content_type ) ) { // special case
 211                                 $obj->content_type = 'unknown/unknown';
 212                         }
 213                         if ( !empty( $params['async'] ) ) { // deferred
 214                                 $handle = $obj->write_async( $params['content'] );
 215                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 216                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 217                                         $status->value->affectedObjects[] = $obj;
 218                                 }
 219                         } else { // actually write the object in Swift
 220                                 $obj->write( $params['content'] );
 221                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 222                                         $this->purgeCDNCache( array( $obj ) );
 223                                 }
 224                         }
 225                 } catch ( CDNNotEnabledException $e ) {
 226                         // CDN not enabled; nothing to see here
 227                 } catch ( BadContentTypeException $e ) {
 228                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 229                 } catch ( CloudFilesException $e ) { // some other exception?
 230                         $this->handleException( $e, $status, __METHOD__, $params );
 231                 }
 232
 233                 return $status;
 234         }
 235
 236         /**
 237          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 238          */
 239         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 240                 try {
 241                         $cfOp->getLastResponse();
 242                 } catch ( BadContentTypeException $e ) {
 243                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 244                 }
 245         }
 246
 247         /**
 248          * @see FileBackendStore::doStoreInternal()
 249          * @return Status
 250          */
 251         protected function doStoreInternal( array $params ) {
 252                 $status = Status::newGood();
 253
 254                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 255                 if ( $dstRel === null ) {
 256                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 257                         return $status;
 258                 }
 259
 260                 // (a) Check the destination container and object
 261                 try {
 262                         $dContObj = $this->getContainer( $dstCont );
 263                         if ( empty( $params['overwrite'] ) &&
 264                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 265                         {
 266                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 267                                 return $status;
 268                         }
 269                 } catch ( NoSuchContainerException $e ) {
 270                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 271                         return $status;
 272                 } catch ( CloudFilesException $e ) { // some other exception?
 273                         $this->handleException( $e, $status, __METHOD__, $params );
 274                         return $status;
 275                 }
 276
 277                 // (b) Get a SHA-1 hash of the object
 278                 $sha1Hash = sha1_file( $params['src'] );
 279                 if ( $sha1Hash === false ) { // source doesn't exist?
 280                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 281                         return $status;
 282                 }
 283                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 284
 285                 // (c) Actually store the object
 286                 try {
 287                         // Create a fresh CF_Object with no fields preloaded.
 288                         // We don't want to preserve headers, metadata, and such.
 289                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 290                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 291                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 292                         // The MD5 here will be checked within Swift against its own MD5.
 293                         $obj->set_etag( md5_file( $params['src'] ) );
 294                         // Use the same content type as StreamFile for security
 295                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 296                         if ( !strlen( $obj->content_type ) ) { // special case
 297                                 $obj->content_type = 'unknown/unknown';
 298                         }
 299                         if ( !empty( $params['async'] ) ) { // deferred
 300                                 wfSuppressWarnings();
 301                                 $fp = fopen( $params['src'], 'rb' );
 302                                 wfRestoreWarnings();
 303                                 if ( !$fp ) {
 304                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 305                                 } else {
 306                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 307                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 308                                         $status->value->resourcesToClose[] = $fp;
 309                                         if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 310                                                 $status->value->affectedObjects[] = $obj;
 311                                         }
 312                                 }
 313                         } else { // actually write the object in Swift
 314                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 315                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 316                                         $this->purgeCDNCache( array( $obj ) );
 317                                 }
 318                         }
 319                 } catch ( CDNNotEnabledException $e ) {
 320                         // CDN not enabled; nothing to see here
 321                 } catch ( BadContentTypeException $e ) {
 322                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 323                 } catch ( IOException $e ) {
 324                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 325                 } catch ( CloudFilesException $e ) { // some other exception?
 326                         $this->handleException( $e, $status, __METHOD__, $params );
 327                 }
 328
 329                 return $status;
 330         }
 331
 332         /**
 333          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 334          */
 335         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 336                 try {
 337                         $cfOp->getLastResponse();
 338                 } catch ( BadContentTypeException $e ) {
 339                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 340                 } catch ( IOException $e ) {
 341                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 342                 }
 343         }
 344
 345         /**
 346          * @see FileBackendStore::doCopyInternal()
 347          * @return Status
 348          */
 349         protected function doCopyInternal( array $params ) {
 350                 $status = Status::newGood();
 351
 352                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 353                 if ( $srcRel === null ) {
 354                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 355                         return $status;
 356                 }
 357
 358                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 359                 if ( $dstRel === null ) {
 360                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 361                         return $status;
 362                 }
 363
 364                 // (a) Check the source/destination containers and destination object
 365                 try {
 366                         $sContObj = $this->getContainer( $srcCont );
 367                         $dContObj = $this->getContainer( $dstCont );
 368                         if ( empty( $params['overwrite'] ) &&
 369                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 370                         {
 371                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 372                                 return $status;
 373                         }
 374                 } catch ( NoSuchContainerException $e ) {
 375                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 376                         return $status;
 377                 } catch ( CloudFilesException $e ) { // some other exception?
 378                         $this->handleException( $e, $status, __METHOD__, $params );
 379                         return $status;
 380                 }
 381
 382                 // (b) Actually copy the file to the destination
 383                 try {
 384                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 385                         if ( !empty( $params['async'] ) ) { // deferred
 386                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 387                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 388                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 389                                         $status->value->affectedObjects[] = $dstObj;
 390                                 }
 391                         } else { // actually write the object in Swift
 392                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 393                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 394                                         $this->purgeCDNCache( array( $dstObj ) );
 395                                 }
 396                         }
 397                 } catch ( CDNNotEnabledException $e ) {
 398                         // CDN not enabled; nothing to see here
 399                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 400                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 401                 } catch ( CloudFilesException $e ) { // some other exception?
 402                         $this->handleException( $e, $status, __METHOD__, $params );
 403                 }
 404
 405                 return $status;
 406         }
 407
 408         /**
 409          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 410          */
 411         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 412                 try {
 413                         $cfOp->getLastResponse();
 414                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 415                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 416                 }
 417         }
 418
 419         /**
 420          * @see FileBackendStore::doMoveInternal()
 421          * @return Status
 422          */
 423         protected function doMoveInternal( array $params ) {
 424                 $status = Status::newGood();
 425
 426                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 427                 if ( $srcRel === null ) {
 428                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 429                         return $status;
 430                 }
 431
 432                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 433                 if ( $dstRel === null ) {
 434                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 435                         return $status;
 436                 }
 437
 438                 // (a) Check the source/destination containers and destination object
 439                 try {
 440                         $sContObj = $this->getContainer( $srcCont );
 441                         $dContObj = $this->getContainer( $dstCont );
 442                         if ( empty( $params['overwrite'] ) &&
 443                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 444                         {
 445                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 446                                 return $status;
 447                         }
 448                 } catch ( NoSuchContainerException $e ) {
 449                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 450                         return $status;
 451                 } catch ( CloudFilesException $e ) { // some other exception?
 452                         $this->handleException( $e, $status, __METHOD__, $params );
 453                         return $status;
 454                 }
 455
 456                 // (b) Actually move the file to the destination
 457                 try {
 458                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 459                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 460                         if ( !empty( $params['async'] ) ) { // deferred
 461                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 462                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 463                                 $status->value->affectedObjects[] = $srcObj;
 464                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 465                                         $status->value->affectedObjects[] = $dstObj;
 466                                 }
 467                         } else { // actually write the object in Swift
 468                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 469                                 $this->purgeCDNCache( array( $srcObj ) );
 470                                 if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
 471                                         $this->purgeCDNCache( array( $dstObj ) );
 472                                 }
 473                         }
 474                 } catch ( CDNNotEnabledException $e ) {
 475                         // CDN not enabled; nothing to see here
 476                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 477                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 478                 } catch ( CloudFilesException $e ) { // some other exception?
 479                         $this->handleException( $e, $status, __METHOD__, $params );
 480                 }
 481
 482                 return $status;
 483         }
 484
 485         /**
 486          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 487          */
 488         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 489                 try {
 490                         $cfOp->getLastResponse();
 491                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 492                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 493                 }
 494         }
 495
 496         /**
 497          * @see FileBackendStore::doDeleteInternal()
 498          * @return Status
 499          */
 500         protected function doDeleteInternal( array $params ) {
 501                 $status = Status::newGood();
 502
 503                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 504                 if ( $srcRel === null ) {
 505                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 506                         return $status;
 507                 }
 508
 509                 try {
 510                         $sContObj = $this->getContainer( $srcCont );
 511                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 512                         if ( !empty( $params['async'] ) ) { // deferred
 513                                 $handle = $sContObj->delete_object_async( $srcRel );
 514                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 515                                 $status->value->affectedObjects[] = $srcObj;
 516                         } else { // actually write the object in Swift
 517                                 $sContObj->delete_object( $srcRel );
 518                                 $this->purgeCDNCache( array( $srcObj ) );
 519                         }
 520                 } catch ( CDNNotEnabledException $e ) {
 521                         // CDN not enabled; nothing to see here
 522                 } catch ( NoSuchContainerException $e ) {
 523                         $status->fatal( 'backend-fail-delete', $params['src'] );
 524                 } catch ( NoSuchObjectException $e ) {
 525                         if ( empty( $params['ignoreMissingSource'] ) ) {
 526                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 527                         }
 528                 } catch ( CloudFilesException $e ) { // some other exception?
 529                         $this->handleException( $e, $status, __METHOD__, $params );
 530                 }
 531
 532                 return $status;
 533         }
 534
 535         /**
 536          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 537          */
 538         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 539                 try {
 540                         $cfOp->getLastResponse();
 541                 } catch ( NoSuchContainerException $e ) {
 542                         $status->fatal( 'backend-fail-delete', $params['src'] );
 543                 } catch ( NoSuchObjectException $e ) {
 544                         if ( empty( $params['ignoreMissingSource'] ) ) {
 545                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 546                         }
 547                 }
 548         }
 549
 550         /**
 551          * @see FileBackendStore::doPrepareInternal()
 552          * @return Status
 553          */
 554         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 555                 $status = Status::newGood();
 556
 557                 // (a) Check if container already exists
 558                 try {
 559                         $contObj = $this->getContainer( $fullCont );
 560                         // NoSuchContainerException not thrown: container must exist
 561                         return $status; // already exists
 562                 } catch ( NoSuchContainerException $e ) {
 563                         // NoSuchContainerException thrown: container does not exist
 564                 } catch ( CloudFilesException $e ) { // some other exception?
 565                         $this->handleException( $e, $status, __METHOD__, $params );
 566                         return $status;
 567                 }
 568
 569                 // (b) Create container as needed
 570                 try {
 571                         $contObj = $this->createContainer( $fullCont );
 572                         if ( !empty( $params['noAccess'] ) ) {
 573                                 // Make container private to end-users...
 574                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 575                         } else {
 576                                 // Make container public to end-users...
 577                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 578                         }
 579                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 580                                 $contObj->make_public( $this->swiftCDNExpiry );
 581                         }
 582                 } catch ( CDNNotEnabledException $e ) {
 583                         // CDN not enabled; nothing to see here
 584                 } catch ( CloudFilesException $e ) { // some other exception?
 585                         $this->handleException( $e, $status, __METHOD__, $params );
 586                         return $status;
 587                 }
 588
 589                 return $status;
 590         }
 591
 592         /**
 593          * @see FileBackendStore::doSecureInternal()
 594          * @return Status
 595          */
 596         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 597                 $status = Status::newGood();
 598                 if ( empty( $params['noAccess'] ) ) {
 599                         return $status; // nothing to do
 600                 }
 601
 602                 // Restrict container from end-users...
 603                 try {
 604                         // doPrepareInternal() should have been called,
 605                         // so the Swift container should already exist...
 606                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 607                         // NoSuchContainerException not thrown: container must exist
 608
 609                         // Make container private to end-users...
 610                         $status->merge( $this->setContainerAccess(
 611                                 $contObj,
 612                                 array( $this->auth->username ), // read
 613                                 array( $this->auth->username ) // write
 614                         ) );
 615                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 616                                 $contObj->make_private();
 617                         }
 618                 } catch ( CDNNotEnabledException $e ) {
 619                         // CDN not enabled; nothing to see here
 620                 } catch ( CloudFilesException $e ) { // some other exception?
 621                         $this->handleException( $e, $status, __METHOD__, $params );
 622                 }
 623
 624                 return $status;
 625         }
 626
 627         /**
 628          * @see FileBackendStore::doPublishInternal()
 629          * @return Status
 630          */
 631         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 632                 $status = Status::newGood();
 633
 634                 // Unrestrict container from end-users...
 635                 try {
 636                         // doPrepareInternal() should have been called,
 637                         // so the Swift container should already exist...
 638                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 639                         // NoSuchContainerException not thrown: container must exist
 640
 641                         // Make container public to end-users...
 642                         if ( $this->swiftAnonUser != '' ) {
 643                                 $status->merge( $this->setContainerAccess(
 644                                         $contObj,
 645                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 646                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 647                                 ) );
 648                         } else {
 649                                 $status->merge( $this->setContainerAccess(
 650                                         $contObj,
 651                                         array( $this->auth->username, '.r:*' ), // read
 652                                         array( $this->auth->username ) // write
 653                                 ) );
 654                         }
 655                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 656                                 $contObj->make_public();
 657                         }
 658                 } catch ( CDNNotEnabledException $e ) {
 659                         // CDN not enabled; nothing to see here
 660                 } catch ( CloudFilesException $e ) { // some other exception?
 661                         $this->handleException( $e, $status, __METHOD__, $params );
 662                 }
 663
 664                 return $status;
 665         }
 666
 667         /**
 668          * @see FileBackendStore::doCleanInternal()
 669          * @return Status
 670          */
 671         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 672                 $status = Status::newGood();
 673
 674                 // Only containers themselves can be removed, all else is virtual
 675                 if ( $dir != '' ) {
 676                         return $status; // nothing to do
 677                 }
 678
 679                 // (a) Check the container
 680                 try {
 681                         $contObj = $this->getContainer( $fullCont, true );
 682                 } catch ( NoSuchContainerException $e ) {
 683                         return $status; // ok, nothing to do
 684                 } catch ( CloudFilesException $e ) { // some other exception?
 685                         $this->handleException( $e, $status, __METHOD__, $params );
 686                         return $status;
 687                 }
 688
 689                 // (b) Delete the container if empty
 690                 if ( $contObj->object_count == 0 ) {
 691                         try {
 692                                 $this->deleteContainer( $fullCont );
 693                         } catch ( NoSuchContainerException $e ) {
 694                                 return $status; // race?
 695                         } catch ( NonEmptyContainerException $e ) {
 696                                 return $status; // race? consistency delay?
 697                         } catch ( CloudFilesException $e ) { // some other exception?
 698                                 $this->handleException( $e, $status, __METHOD__, $params );
 699                                 return $status;
 700                         }
 701                 }
 702
 703                 return $status;
 704         }
 705
 706         /**
 707          * @see FileBackendStore::doFileExists()
 708          * @return array|bool|null
 709          */
 710         protected function doGetFileStat( array $params ) {
 711                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 712                 if ( $srcRel === null ) {
 713                         return false; // invalid storage path
 714                 }
 715
 716                 $stat = false;
 717                 try {
 718                         $contObj = $this->getContainer( $srcCont );
 719                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 720                         $this->addMissingMetadata( $srcObj, $params['src'] );
 721                         $stat = array(
 722                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 723                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 724                                 'size'  => (int)$srcObj->content_length,
 725                                 'sha1'  => $srcObj->metadata['Sha1base36']
 726                         );
 727                 } catch ( NoSuchContainerException $e ) {
 728                 } catch ( NoSuchObjectException $e ) {
 729                 } catch ( CloudFilesException $e ) { // some other exception?
 730                         $stat = null;
 731                         $this->handleException( $e, null, __METHOD__, $params );
 732                 }
 733
 734                 return $stat;
 735         }
 736
 737         /**
 738          * Fill in any missing object metadata and save it to Swift
 739          *
 740          * @param $obj CF_Object
 741          * @param $path string Storage path to object
 742          * @return bool Success
 743          * @throws Exception cloudfiles exceptions
 744          */
 745         protected function addMissingMetadata( CF_Object $obj, $path ) {
 746                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 747                         return true; // nothing to do
 748                 }
 749                 wfProfileIn( __METHOD__ );
 750                 $status = Status::newGood();
 751                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 752                 if ( $status->isOK() ) {
 753                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 754                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 755                         if ( $tmpFile ) {
 756                                 $hash = $tmpFile->getSha1Base36();
 757                                 if ( $hash !== false ) {
 758                                         $obj->metadata['Sha1base36'] = $hash;
 759                                         $obj->sync_metadata(); // save to Swift
 760                                         wfProfileOut( __METHOD__ );
 761                                         return true; // success
 762                                 }
 763                         }
 764                 }
 765                 $obj->metadata['Sha1base36'] = false;
 766                 wfProfileOut( __METHOD__ );
 767                 return false; // failed
 768         }
 769
 770         /**
 771          * @see FileBackend::getFileContents()
 772          * @return bool|null|string
 773          */
 774         public function getFileContents( array $params ) {
 775                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 776                 if ( $srcRel === null ) {
 777                         return false; // invalid storage path
 778                 }
 779
 780                 if ( !$this->fileExists( $params ) ) {
 781                         return null;
 782                 }
 783
 784                 $data = false;
 785                 try {
 786                         $sContObj = $this->getContainer( $srcCont );
 787                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 788                         $data = $obj->read( $this->headersFromParams( $params ) );
 789                 } catch ( NoSuchContainerException $e ) {
 790                 } catch ( CloudFilesException $e ) { // some other exception?
 791                         $this->handleException( $e, null, __METHOD__, $params );
 792                 }
 793
 794                 return $data;
 795         }
 796
 797         /**
 798          * @see FileBackendStore::doDirectoryExists()
 799          * @return bool|null
 800          */
 801         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 802                 try {
 803                         $container = $this->getContainer( $fullCont );
 804                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 805                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 806                 } catch ( NoSuchContainerException $e ) {
 807                         return false;
 808                 } catch ( CloudFilesException $e ) { // some other exception?
 809                         $this->handleException( $e, null, __METHOD__,
 810                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 811                 }
 812
 813                 return null; // error
 814         }
 815
 816         /**
 817          * @see FileBackendStore::getDirectoryListInternal()
 818          * @return SwiftFileBackendDirList
 819          */
 820         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 821                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 822         }
 823
 824         /**
 825          * @see FileBackendStore::getFileListInternal()
 826          * @return SwiftFileBackendFileList
 827          */
 828         public function getFileListInternal( $fullCont, $dir, array $params ) {
 829                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 830         }
 831
 832         /**
 833          * Do not call this function outside of SwiftFileBackendFileList
 834          *
 835          * @param $fullCont string Resolved container name
 836          * @param $dir string Resolved storage directory with no trailing slash
 837          * @param $after string|null Storage path of file to list items after
 838          * @param $limit integer Max number of items to list
 839          * @param $params Array Includes flag for 'topOnly'
 840          * @return Array List of relative paths of dirs directly under $dir
 841          */
 842         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 843                 $dirs = array();
 844                 if ( $after === INF ) {
 845                         return $dirs; // nothing more
 846                 }
 847                 wfProfileIn( __METHOD__ . '-' . $this->name );
 848
 849                 try {
 850                         $container = $this->getContainer( $fullCont );
 851                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 852                         // Non-recursive: only list dirs right under $dir
 853                         if ( !empty( $params['topOnly'] ) ) {
 854                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 855                                 foreach ( $objects as $object ) { // files and dirs
 856                                         if ( substr( $object, -1 ) === '/' ) {
 857                                                 $dirs[] = $object; // directories end in '/'
 858                                         }
 859                                 }
 860                         // Recursive: list all dirs under $dir and its subdirs
 861                         } else {
 862                                 // Get directory from last item of prior page
 863                                 $lastDir = $this->getParentDir( $after ); // must be first page
 864                                 $objects = $container->list_objects( $limit, $after, $prefix );
 865                                 foreach ( $objects as $object ) { // files
 866                                         $objectDir = $this->getParentDir( $object ); // directory of object
 867                                         if ( $objectDir !== false ) { // file has a parent dir
 868                                                 // Swift stores paths in UTF-8, using binary sorting.
 869                                                 // See function "create_container_table" in common/db.py.
 870                                                 // If a directory is not "greater" than the last one,
 871                                                 // then it was already listed by the calling iterator.
 872                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
 873                                                         $pDir = $objectDir;
 874                                                         do { // add dir and all its parent dirs
 875                                                                 $dirs[] = "{$pDir}/";
 876                                                                 $pDir = $this->getParentDir( $pDir );
 877                                                         } while ( $pDir !== false // sanity
 878                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
 879                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 880                                                         );
 881                                                 }
 882                                                 $lastDir = $objectDir;
 883                                         }
 884                                 }
 885                         }
 886                         if ( count( $objects ) < $limit ) {
 887                                 $after = INF; // avoid a second RTT
 888                         } else {
 889                                 $after = end( $objects ); // update last item
 890                         }
 891                 } catch ( NoSuchContainerException $e ) {
 892                 } catch ( CloudFilesException $e ) { // some other exception?
 893                         $this->handleException( $e, null, __METHOD__,
 894                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 895                 }
 896
 897                 wfProfileOut( __METHOD__ . '-' . $this->name );
 898                 return $dirs;
 899         }
 900
 901         protected function getParentDir( $path ) {
 902                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 903         }
 904
 905         /**
 906          * Do not call this function outside of SwiftFileBackendFileList
 907          *
 908          * @param $fullCont string Resolved container name
 909          * @param $dir string Resolved storage directory with no trailing slash
 910          * @param $after string|null Storage path of file to list items after
 911          * @param $limit integer Max number of items to list
 912          * @param $params Array Includes flag for 'topOnly'
 913          * @return Array List of relative paths of files under $dir
 914          */
 915         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 916                 $files = array();
 917                 if ( $after === INF ) {
 918                         return $files; // nothing more
 919                 }
 920                 wfProfileIn( __METHOD__ . '-' . $this->name );
 921
 922                 try {
 923                         $container = $this->getContainer( $fullCont );
 924                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 925                         // Non-recursive: only list files right under $dir
 926                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 927                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 928                                 foreach ( $objects as $object ) {
 929                                         if ( substr( $object, -1 ) !== '/' ) {
 930                                                 $files[] = $object; // directories end in '/'
 931                                         }
 932                                 }
 933                         // Recursive: list all files under $dir and its subdirs
 934                         } else { // files
 935                                 $objects = $container->list_objects( $limit, $after, $prefix );
 936                                 $files = $objects;
 937                         }
 938                         if ( count( $objects ) < $limit ) {
 939                                 $after = INF; // avoid a second RTT
 940                         } else {
 941                                 $after = end( $objects ); // update last item
 942                         }
 943                 } catch ( NoSuchContainerException $e ) {
 944                 } catch ( CloudFilesException $e ) { // some other exception?
 945                         $this->handleException( $e, null, __METHOD__,
 946                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 947                 }
 948
 949                 wfProfileOut( __METHOD__ . '-' . $this->name );
 950                 return $files;
 951         }
 952
 953         /**
 954          * @see FileBackendStore::doGetFileSha1base36()
 955          * @return bool
 956          */
 957         protected function doGetFileSha1base36( array $params ) {
 958                 $stat = $this->getFileStat( $params );
 959                 if ( $stat ) {
 960                         return $stat['sha1'];
 961                 } else {
 962                         return false;
 963                 }
 964         }
 965
 966         /**
 967          * @see FileBackendStore::doStreamFile()
 968          * @return Status
 969          */
 970         protected function doStreamFile( array $params ) {
 971                 $status = Status::newGood();
 972
 973                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 974                 if ( $srcRel === null ) {
 975                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 976                 }
 977
 978                 try {
 979                         $cont = $this->getContainer( $srcCont );
 980                 } catch ( NoSuchContainerException $e ) {
 981                         $status->fatal( 'backend-fail-stream', $params['src'] );
 982                         return $status;
 983                 } catch ( CloudFilesException $e ) { // some other exception?
 984                         $this->handleException( $e, $status, __METHOD__, $params );
 985                         return $status;
 986                 }
 987
 988                 try {
 989                         $output = fopen( 'php://output', 'wb' );
 990                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 991                         $obj->stream( $output, $this->headersFromParams( $params ) );
 992                 } catch ( CloudFilesException $e ) { // some other exception?
 993                         $this->handleException( $e, $status, __METHOD__, $params );
 994                 }
 995
 996                 return $status;
 997         }
 998
 999         /**
1000          * @see FileBackendStore::getLocalCopy()
1001          * @return null|TempFSFile
1002          */
1003         public function getLocalCopy( array $params ) {
1004                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1005                 if ( $srcRel === null ) {
1006                         return null;
1007                 }
1008
1009                 // Blindly create a tmp file and stream to it, catching any exception if the file does
1010                 // not exist. Also, doing a stat here will cause infinite loops when filling metadata.
1011                 $tmpFile = null;
1012                 try {
1013                         $sContObj = $this->getContainer( $srcCont );
1014                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1015                         // Get source file extension
1016                         $ext = FileBackend::extensionFromPath( $srcRel );
1017                         // Create a new temporary file...
1018                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1019                         if ( $tmpFile ) {
1020                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1021                                 if ( $handle ) {
1022                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
1023                                         fclose( $handle );
1024                                 } else {
1025                                         $tmpFile = null; // couldn't open temp file
1026                                 }
1027                         }
1028                 } catch ( NoSuchContainerException $e ) {
1029                         $tmpFile = null;
1030                 } catch ( NoSuchObjectException $e ) {
1031                         $tmpFile = null;
1032                 } catch ( CloudFilesException $e ) { // some other exception?
1033                         $tmpFile = null;
1034                         $this->handleException( $e, null, __METHOD__, $params );
1035                 }
1036
1037                 return $tmpFile;
1038         }
1039
1040         /**
1041          * @see FileBackendStore::directoriesAreVirtual()
1042          * @return bool
1043          */
1044         protected function directoriesAreVirtual() {
1045                 return true;
1046         }
1047
1048         /**
1049          * Get headers to send to Swift when reading a file based
1050          * on a FileBackend params array, e.g. that of getLocalCopy().
1051          * $params is currently only checked for a 'latest' flag.
1052          *
1053          * @param $params Array
1054          * @return Array
1055          */
1056         protected function headersFromParams( array $params ) {
1057                 $hdrs = array();
1058                 if ( !empty( $params['latest'] ) ) {
1059                         $hdrs[] = 'X-Newest: true';
1060                 }
1061                 return $hdrs;
1062         }
1063
1064         /**
1065          * @see FileBackendStore::doExecuteOpHandlesInternal()
1066          * @return Array List of corresponding Status objects
1067          */
1068         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1069                 $statuses = array();
1070
1071                 $cfOps = array(); // list of CF_Async_Op objects
1072                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1073                         $cfOps[$index] = $fileOpHandle->cfOp;
1074                 }
1075                 $batch = new CF_Async_Op_Batch( $cfOps );
1076
1077                 $cfOps = $batch->execute();
1078                 foreach ( $cfOps as $index => $cfOp ) {
1079                         $status = Status::newGood();
1080                         try { // catch exceptions; update status
1081                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1082                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1083                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1084                         } catch ( CloudFilesException $e ) { // some other exception?
1085                                 $this->handleException( $e, $status,
1086                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1087                         }
1088                         $statuses[$index] = $status;
1089                 }
1090
1091                 return $statuses;
1092         }
1093
1094         /**
1095          * Set read/write permissions for a Swift container.
1096          *
1097          * $readGrps is a list of the possible criteria for a request to have
1098          * access to read a container. Each item is one of the following formats:
1099          *   - account:user       : Grants access if the request is by the given user
1100          *   - .r:<regex>         : Grants access if the request is from a referrer host that
1101          *                          matches the expression and the request is not for a listing.
1102          *                          Setting this to '*' effectively makes a container public.
1103          *   - .rlistings:<regex> : Grants access if the request is from a referrer host that
1104          *                          matches the expression and the request for a listing.
1105          *
1106          * $writeGrps is a list of the possible criteria for a request to have
1107          * access to write to a container. Each item is of the following format:
1108          *   - account:user       : Grants access if the request is by the given user
1109          *
1110          * @see http://swift.openstack.org/misc.html#acls
1111          *
1112          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1113          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1114          *
1115          * @param $contObj CF_Container Swift container
1116          * @param $readGrps Array List of read access routes
1117          * @param $writeGrps Array List of write access routes
1118          * @return Status
1119          */
1120         protected function setContainerAccess(
1121                 CF_Container $contObj, array $readGrps, array $writeGrps
1122         ) {
1123                 $creds = $contObj->cfs_auth->export_credentials();
1124
1125                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1126
1127                 // Note: 10 second timeout consistent with php-cloudfiles
1128                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1129                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1130                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1131                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1132
1133                 return $req->execute(); // should return 204
1134         }
1135
1136         /**
1137          * Purge the CDN cache of affected objects if CDN caching is enabled.
1138          * This is for Rackspace/Akamai CDNs.
1139          *
1140          * @param $objects Array List of CF_Object items
1141          * @return void
1142          */
1143         public function purgeCDNCache( array $objects ) {
1144                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1145                         foreach ( $objects as $object ) {
1146                                 try {
1147                                         $object->purge_from_cdn();
1148                                 } catch ( CDNNotEnabledException $e ) {
1149                                         // CDN not enabled; nothing to see here
1150                                 } catch ( CloudFilesException $e ) {
1151                                         $this->handleException( $e, null, __METHOD__,
1152                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1153                                 }
1154                         }
1155                 }
1156         }
1157
1158         /**
1159          * Get an authenticated connection handle to the Swift proxy
1160          *
1161          * @return CF_Connection|bool False on failure
1162          * @throws CloudFilesException
1163          */
1164         protected function getConnection() {
1165                 if ( $this->connException instanceof CloudFilesException ) {
1166                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1167                                 throw $this->connException; // failed last attempt; don't bother
1168                         } else { // actually retry this time
1169                                 $this->connException = null;
1170                                 $this->connErrorTime = 0;
1171                         }
1172                 }
1173                 // Session keys expire after a while, so we renew them periodically
1174                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1175                 // Authenticate with proxy and get a session key...
1176                 if ( !$this->conn || $reAuth ) {
1177                         $this->sessionStarted = 0;
1178                         $this->connContainerCache->clear();
1179                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1180                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1181                         if ( is_array( $creds ) ) { // cache hit
1182                                 $this->auth->load_cached_credentials(
1183                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1184                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1185                         } else { // cache miss
1186                                 try {
1187                                         $this->auth->authenticate();
1188                                         $creds = $this->auth->export_credentials();
1189                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1190                                         $this->sessionStarted = time();
1191                                 } catch ( CloudFilesException $e ) {
1192                                         $this->connException = $e; // don't keep re-trying
1193                                         $this->connErrorTime = time();
1194                                         throw $e; // throw it back
1195                                 }
1196                         }
1197                         if ( $this->conn ) { // re-authorizing?
1198                                 $this->conn->close(); // close active cURL handles in CF_Http object
1199                         }
1200                         $this->conn = new CF_Connection( $this->auth );
1201                 }
1202                 return $this->conn;
1203         }
1204
1205         /**
1206          * Get the cache key for a container
1207          *
1208          * @param $username string
1209          * @return string
1210          */
1211         private function getCredsCacheKey( $username ) {
1212                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1213         }
1214
1215         /**
1216          * @see FileBackendStore::doClearCache()
1217          */
1218         protected function doClearCache( array $paths = null ) {
1219                 $this->connContainerCache->clear(); // clear container object cache
1220         }
1221
1222         /**
1223          * Get a Swift container object, possibly from process cache.
1224          * Use $reCache if the file count or byte count is needed.
1225          *
1226          * @param $container string Container name
1227          * @param $bypassCache bool Bypass all caches and load from Swift
1228          * @return CF_Container
1229          * @throws CloudFilesException
1230          */
1231         protected function getContainer( $container, $bypassCache = false ) {
1232                 $conn = $this->getConnection(); // Swift proxy connection
1233                 if ( $bypassCache ) { // purge cache
1234                         $this->connContainerCache->clear( $container );
1235                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1236                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1237                 }
1238                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1239                         $contObj = $conn->get_container( $container );
1240                         // NoSuchContainerException not thrown: container must exist
1241                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1242                         if ( !$bypassCache ) {
1243                                 $this->setContainerCache( $container, // update persistent cache
1244                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1245                                 );
1246                         }
1247                 }
1248                 return $this->connContainerCache->get( $container, 'obj' );
1249         }
1250
1251         /**
1252          * Create a Swift container
1253          *
1254          * @param $container string Container name
1255          * @return CF_Container
1256          * @throws CloudFilesException
1257          */
1258         protected function createContainer( $container ) {
1259                 $conn = $this->getConnection(); // Swift proxy connection
1260                 $contObj = $conn->create_container( $container );
1261                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1262                 return $contObj;
1263         }
1264
1265         /**
1266          * Delete a Swift container
1267          *
1268          * @param $container string Container name
1269          * @return void
1270          * @throws CloudFilesException
1271          */
1272         protected function deleteContainer( $container ) {
1273                 $conn = $this->getConnection(); // Swift proxy connection
1274                 $this->connContainerCache->clear( $container ); // purge
1275                 $conn->delete_container( $container );
1276         }
1277
1278         /**
1279          * @see FileBackendStore::doPrimeContainerCache()
1280          * @return void
1281          */
1282         protected function doPrimeContainerCache( array $containerInfo ) {
1283                 try {
1284                         $conn = $this->getConnection(); // Swift proxy connection
1285                         foreach ( $containerInfo as $container => $info ) {
1286                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1287                                         $container, $info['count'], $info['bytes'] );
1288                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1289                         }
1290                 } catch ( CloudFilesException $e ) { // some other exception?
1291                         $this->handleException( $e, null, __METHOD__, array() );
1292                 }
1293         }
1294
1295         /**
1296          * Log an unexpected exception for this backend.
1297          * This also sets the Status object to have a fatal error.
1298          *
1299          * @param $e Exception
1300          * @param $status Status|null
1301          * @param $func string
1302          * @param $params Array
1303          * @return void
1304          */
1305         protected function handleException( Exception $e, $status, $func, array $params ) {
1306                 if ( $status instanceof Status ) {
1307                         if ( $e instanceof AuthenticationException ) {
1308                                 $status->fatal( 'backend-fail-connect', $this->name );
1309                         } else {
1310                                 $status->fatal( 'backend-fail-internal', $this->name );
1311                         }
1312                 }
1313                 if ( $e->getMessage() ) {
1314                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1315                 }
1316                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1317                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1318                 }
1319                 wfDebugLog( 'SwiftBackend',
1320                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1321                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1322                 );
1323         }
1324 }
1325
1326 /**
1327  * @see FileBackendStoreOpHandle
1328  */
1329 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1330         /** @var CF_Async_Op */
1331         public $cfOp;
1332         /** @var Array */
1333         public $affectedObjects = array();
1334
1335         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1336                 $this->backend = $backend;
1337                 $this->params = $params;
1338                 $this->call = $call;
1339                 $this->cfOp = $cfOp;
1340         }
1341 }
1342
1343 /**
1344  * SwiftFileBackend helper class to page through listings.
1345  * Swift also has a listing limit of 10,000 objects for sanity.
1346  * Do not use this class from places outside SwiftFileBackend.
1347  *
1348  * @ingroup FileBackend
1349  */
1350 abstract class SwiftFileBackendList implements Iterator {
1351         /** @var Array */
1352         protected $bufferIter = array();
1353         protected $bufferAfter = null; // string; list items *after* this path
1354         protected $pos = 0; // integer
1355         /** @var Array */
1356         protected $params = array();
1357
1358         /** @var SwiftFileBackend */
1359         protected $backend;
1360         protected $container; // string; container name
1361         protected $dir; // string; storage directory
1362         protected $suffixStart; // integer
1363
1364         const PAGE_SIZE = 9000; // file listing buffer size
1365
1366         /**
1367          * @param $backend SwiftFileBackend
1368          * @param $fullCont string Resolved container name
1369          * @param $dir string Resolved directory relative to container
1370          * @param $params Array
1371          */
1372         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1373                 $this->backend = $backend;
1374                 $this->container = $fullCont;
1375                 $this->dir = $dir;
1376                 if ( substr( $this->dir, -1 ) === '/' ) {
1377                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1378                 }
1379                 if ( $this->dir == '' ) { // whole container
1380                         $this->suffixStart = 0;
1381                 } else { // dir within container
1382                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1383                 }
1384                 $this->params = $params;
1385         }
1386
1387         /**
1388          * @see Iterator::key()
1389          * @return integer
1390          */
1391         public function key() {
1392                 return $this->pos;
1393         }
1394
1395         /**
1396          * @see Iterator::next()
1397          * @return void
1398          */
1399         public function next() {
1400                 // Advance to the next file in the page
1401                 next( $this->bufferIter );
1402                 ++$this->pos;
1403                 // Check if there are no files left in this page and
1404                 // advance to the next page if this page was not empty.
1405                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1406                         $this->bufferIter = $this->pageFromList(
1407                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1408                         ); // updates $this->bufferAfter
1409                 }
1410         }
1411
1412         /**
1413          * @see Iterator::rewind()
1414          * @return void
1415          */
1416         public function rewind() {
1417                 $this->pos = 0;
1418                 $this->bufferAfter = null;
1419                 $this->bufferIter = $this->pageFromList(
1420                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1421                 ); // updates $this->bufferAfter
1422         }
1423
1424         /**
1425          * @see Iterator::valid()
1426          * @return bool
1427          */
1428         public function valid() {
1429                 if ( $this->bufferIter === null ) {
1430                         return false; // some failure?
1431                 } else {
1432                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1433                 }
1434         }
1435
1436         /**
1437          * Get the given list portion (page)
1438          *
1439          * @param $container string Resolved container name
1440          * @param $dir string Resolved path relative to container
1441          * @param $after string|null
1442          * @param $limit integer
1443          * @param $params Array
1444          * @return Traversable|Array|null Returns null on failure
1445          */
1446         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1447 }
1448
1449 /**
1450  * Iterator for listing directories
1451  */
1452 class SwiftFileBackendDirList extends SwiftFileBackendList {
1453         /**
1454          * @see Iterator::current()
1455          * @return string|bool String (relative path) or false
1456          */
1457         public function current() {
1458                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1459         }
1460
1461         /**
1462          * @see SwiftFileBackendList::pageFromList()
1463          * @return Array|null
1464          */
1465         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1466                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1467         }
1468 }
1469
1470 /**
1471  * Iterator for listing regular files
1472  */
1473 class SwiftFileBackendFileList extends SwiftFileBackendList {
1474         /**
1475          * @see Iterator::current()
1476          * @return string|bool String (relative path) or false
1477          */
1478         public function current() {
1479                 return substr( current( $this->bufferIter ), $this->suffixStart );
1480         }
1481
1482         /**
1483          * @see SwiftFileBackendList::pageFromList()
1484          * @return Array|null
1485          */
1486         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1487                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1488         }
1489 }