includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Russ Nelson
   6  * @author Aaron Schulz
   7  */
   8
   9 /**
  10  * @brief Class for an OpenStack Swift based file backend.
  11  *
  12  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  13  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  14  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  15  *
  16  * Status messages should avoid mentioning the Swift account name.
  17  * Likewise, error suppression should be used to avoid path disclosure.
  18  *
  19  * @ingroup FileBackend
  20  * @since 1.19
  21  */
  22 class SwiftFileBackend extends FileBackendStore {
  23         /** @var CF_Authentication */
  24         protected $auth; // Swift authentication handler
  25         protected $authTTL; // integer seconds
  26         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  27         protected $maxContCacheSize = 100; // integer; max containers with entries
  28
  29         /** @var CF_Connection */
  30         protected $conn; // Swift connection handle
  31         protected $connStarted = 0; // integer UNIX timestamp
  32         protected $connContainers = array(); // container object cache
  33
  34         /**
  35          * @see FileBackendStore::__construct()
  36          * Additional $config params include:
  37          *    swiftAuthUrl       : Swift authentication server URL
  38          *    swiftUser          : Swift user used by MediaWiki (account:username)
  39          *    swiftKey           : Swift authentication key for the above user
  40          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  41          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  42          *    shardViaHashLevels : Map of container names to sharding config with:
  43          *                         'base'   : base of hash characters, 16 or 36
  44          *                         'levels' : the number of hash levels (and digits)
  45          *                         'repeat' : hash subdirectories are prefixed with all the
  46          *                                    parent hash directory names (e.g. "a/ab/abc")
  47          */
  48         public function __construct( array $config ) {
  49                 parent::__construct( $config );
  50                 // Required settings
  51                 $this->auth = new CF_Authentication(
  52                         $config['swiftUser'],
  53                         $config['swiftKey'],
  54                         null, // account; unused
  55                         $config['swiftAuthUrl']
  56                 );
  57                 // Optional settings
  58                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  59                         ? $config['swiftAuthTTL']
  60                         : 120; // some sane number
  61                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  62                         ? $config['swiftAnonUser']
  63                         : '';
  64                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  65                         ? $config['shardViaHashLevels']
  66                         : '';
  67                 // Cache container info to mask latency
  68                 $this->memCache = wfGetMainCache();
  69         }
  70
  71         /**
  72          * @see FileBackendStore::resolveContainerPath()
  73          * @return null
  74          */
  75         protected function resolveContainerPath( $container, $relStoragePath ) {
  76                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  77                         return null; // too long for Swift
  78                 }
  79                 return $relStoragePath;
  80         }
  81
  82         /**
  83          * @see FileBackendStore::isPathUsableInternal()
  84          * @return bool
  85          */
  86         public function isPathUsableInternal( $storagePath ) {
  87                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
  88                 if ( $rel === null ) {
  89                         return false; // invalid
  90                 }
  91
  92                 try {
  93                         $this->getContainer( $container );
  94                         return true; // container exists
  95                 } catch ( NoSuchContainerException $e ) {
  96                 } catch ( InvalidResponseException $e ) {
  97                 } catch ( Exception $e ) { // some other exception?
  98                         $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
  99                 }
 100
 101                 return false;
 102         }
 103
 104         /**
 105          * @see FileBackendStore::doCreateInternal()
 106          * @return Status
 107          */
 108         protected function doCreateInternal( array $params ) {
 109                 $status = Status::newGood();
 110
 111                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 112                 if ( $dstRel === null ) {
 113                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 114                         return $status;
 115                 }
 116
 117                 // (a) Check the destination container and object
 118                 try {
 119                         $dContObj = $this->getContainer( $dstCont );
 120                         if ( empty( $params['overwrite'] ) &&
 121                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 122                         {
 123                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 124                                 return $status;
 125                         }
 126                 } catch ( NoSuchContainerException $e ) {
 127                         $status->fatal( 'backend-fail-create', $params['dst'] );
 128                         return $status;
 129                 } catch ( InvalidResponseException $e ) {
 130                         $status->fatal( 'backend-fail-connect', $this->name );
 131                         return $status;
 132                 } catch ( Exception $e ) { // some other exception?
 133                         $status->fatal( 'backend-fail-internal', $this->name );
 134                         $this->logException( $e, __METHOD__, $params );
 135                         return $status;
 136                 }
 137
 138                 // (b) Get a SHA-1 hash of the object
 139                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 140
 141                 // (c) Actually create the object
 142                 try {
 143                         // Create a fresh CF_Object with no fields preloaded.
 144                         // We don't want to preserve headers, metadata, and such.
 145                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 146                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 147                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 148                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 149                         // The MD5 here will be checked within Swift against its own MD5.
 150                         $obj->set_etag( md5( $params['content'] ) );
 151                         // Use the same content type as StreamFile for security
 152                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 153                         // Actually write the object in Swift
 154                         $obj->write( $params['content'] );
 155                 } catch ( BadContentTypeException $e ) {
 156                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 157                 } catch ( InvalidResponseException $e ) {
 158                         $status->fatal( 'backend-fail-connect', $this->name );
 159                 } catch ( Exception $e ) { // some other exception?
 160                         $status->fatal( 'backend-fail-internal', $this->name );
 161                         $this->logException( $e, __METHOD__, $params );
 162                 }
 163
 164                 return $status;
 165         }
 166
 167         /**
 168          * @see FileBackendStore::doStoreInternal()
 169          * @return Status
 170          */
 171         protected function doStoreInternal( array $params ) {
 172                 $status = Status::newGood();
 173
 174                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 175                 if ( $dstRel === null ) {
 176                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 177                         return $status;
 178                 }
 179
 180                 // (a) Check the destination container and object
 181                 try {
 182                         $dContObj = $this->getContainer( $dstCont );
 183                         if ( empty( $params['overwrite'] ) &&
 184                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 185                         {
 186                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 187                                 return $status;
 188                         }
 189                 } catch ( NoSuchContainerException $e ) {
 190                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 191                         return $status;
 192                 } catch ( InvalidResponseException $e ) {
 193                         $status->fatal( 'backend-fail-connect', $this->name );
 194                         return $status;
 195                 } catch ( Exception $e ) { // some other exception?
 196                         $status->fatal( 'backend-fail-internal', $this->name );
 197                         $this->logException( $e, __METHOD__, $params );
 198                         return $status;
 199                 }
 200
 201                 // (b) Get a SHA-1 hash of the object
 202                 $sha1Hash = sha1_file( $params['src'] );
 203                 if ( $sha1Hash === false ) { // source doesn't exist?
 204                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 205                         return $status;
 206                 }
 207                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 208
 209                 // (c) Actually store the object
 210                 try {
 211                         // Create a fresh CF_Object with no fields preloaded.
 212                         // We don't want to preserve headers, metadata, and such.
 213                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 214                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 215                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 216                         // The MD5 here will be checked within Swift against its own MD5.
 217                         $obj->set_etag( md5_file( $params['src'] ) );
 218                         // Use the same content type as StreamFile for security
 219                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 220                         // Actually write the object in Swift
 221                         $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
 222                 } catch ( BadContentTypeException $e ) {
 223                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 224                 } catch ( IOException $e ) {
 225                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 226                 } catch ( InvalidResponseException $e ) {
 227                         $status->fatal( 'backend-fail-connect', $this->name );
 228                 } catch ( Exception $e ) { // some other exception?
 229                         $status->fatal( 'backend-fail-internal', $this->name );
 230                         $this->logException( $e, __METHOD__, $params );
 231                 }
 232
 233                 return $status;
 234         }
 235
 236         /**
 237          * @see FileBackendStore::doCopyInternal()
 238          * @return Status
 239          */
 240         protected function doCopyInternal( array $params ) {
 241                 $status = Status::newGood();
 242
 243                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 244                 if ( $srcRel === null ) {
 245                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 246                         return $status;
 247                 }
 248
 249                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 250                 if ( $dstRel === null ) {
 251                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 252                         return $status;
 253                 }
 254
 255                 // (a) Check the source/destination containers and destination object
 256                 try {
 257                         $sContObj = $this->getContainer( $srcCont );
 258                         $dContObj = $this->getContainer( $dstCont );
 259                         if ( empty( $params['overwrite'] ) &&
 260                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 261                         {
 262                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 263                                 return $status;
 264                         }
 265                 } catch ( NoSuchContainerException $e ) {
 266                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 267                         return $status;
 268                 } catch ( InvalidResponseException $e ) {
 269                         $status->fatal( 'backend-fail-connect', $this->name );
 270                         return $status;
 271                 } catch ( Exception $e ) { // some other exception?
 272                         $status->fatal( 'backend-fail-internal', $this->name );
 273                         $this->logException( $e, __METHOD__, $params );
 274                         return $status;
 275                 }
 276
 277                 // (b) Actually copy the file to the destination
 278                 try {
 279                         $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 280                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 281                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 282                 } catch ( InvalidResponseException $e ) {
 283                         $status->fatal( 'backend-fail-connect', $this->name );
 284                 } catch ( Exception $e ) { // some other exception?
 285                         $status->fatal( 'backend-fail-internal', $this->name );
 286                         $this->logException( $e, __METHOD__, $params );
 287                 }
 288
 289                 return $status;
 290         }
 291
 292         /**
 293          * @see FileBackendStore::doDeleteInternal()
 294          * @return Status
 295          */
 296         protected function doDeleteInternal( array $params ) {
 297                 $status = Status::newGood();
 298
 299                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 300                 if ( $srcRel === null ) {
 301                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 302                         return $status;
 303                 }
 304
 305                 try {
 306                         $sContObj = $this->getContainer( $srcCont );
 307                         $sContObj->delete_object( $srcRel );
 308                 } catch ( NoSuchContainerException $e ) {
 309                         $status->fatal( 'backend-fail-delete', $params['src'] );
 310                 } catch ( NoSuchObjectException $e ) {
 311                         if ( empty( $params['ignoreMissingSource'] ) ) {
 312                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 313                         }
 314                 } catch ( InvalidResponseException $e ) {
 315                         $status->fatal( 'backend-fail-connect', $this->name );
 316                 } catch ( Exception $e ) { // some other exception?
 317                         $status->fatal( 'backend-fail-internal', $this->name );
 318                         $this->logException( $e, __METHOD__, $params );
 319                 }
 320
 321                 return $status;
 322         }
 323
 324         /**
 325          * @see FileBackendStore::doPrepareInternal()
 326          * @return Status
 327          */
 328         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 329                 $status = Status::newGood();
 330
 331                 // (a) Check if container already exists
 332                 try {
 333                         $contObj = $this->getContainer( $fullCont );
 334                         // NoSuchContainerException not thrown: container must exist
 335                         return $status; // already exists
 336                 } catch ( NoSuchContainerException $e ) {
 337                         // NoSuchContainerException thrown: container does not exist
 338                 } catch ( InvalidResponseException $e ) {
 339                         $status->fatal( 'backend-fail-connect', $this->name );
 340                         return $status;
 341                 } catch ( Exception $e ) { // some other exception?
 342                         $status->fatal( 'backend-fail-internal', $this->name );
 343                         $this->logException( $e, __METHOD__, $params );
 344                         return $status;
 345                 }
 346
 347                 // (b) Create container as needed
 348                 try {
 349                         $contObj = $this->createContainer( $fullCont );
 350                         if ( $this->swiftAnonUser != '' ) {
 351                                 // Make container public to end-users...
 352                                 $status->merge( $this->setContainerAccess(
 353                                         $contObj,
 354                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 355                                         array( $this->auth->username ) // write
 356                                 ) );
 357                         }
 358                 } catch ( InvalidResponseException $e ) {
 359                         $status->fatal( 'backend-fail-connect', $this->name );
 360                         return $status;
 361                 } catch ( Exception $e ) { // some other exception?
 362                         $status->fatal( 'backend-fail-internal', $this->name );
 363                         $this->logException( $e, __METHOD__, $params );
 364                         return $status;
 365                 }
 366
 367                 return $status;
 368         }
 369
 370         /**
 371          * @see FileBackendStore::doSecureInternal()
 372          * @return Status
 373          */
 374         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 375                 $status = Status::newGood();
 376
 377                 if ( $this->swiftAnonUser != '' ) {
 378                         // Restrict container from end-users...
 379                         try {
 380                                 // doPrepareInternal() should have been called,
 381                                 // so the Swift container should already exist...
 382                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 383                                 // NoSuchContainerException not thrown: container must exist
 384                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 385                                         $status->merge( $this->setContainerAccess(
 386                                                 $contObj,
 387                                                 array( $this->auth->username ), // read
 388                                                 array( $this->auth->username ) // write
 389                                         ) );
 390                                         // @TODO: when php-cloudfiles supports container
 391                                         // metadata, we can make use of that to avoid RTTs
 392                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 393                                 }
 394                         } catch ( InvalidResponseException $e ) {
 395                                 $status->fatal( 'backend-fail-connect', $this->name );
 396                         } catch ( Exception $e ) { // some other exception?
 397                                 $status->fatal( 'backend-fail-internal', $this->name );
 398                                 $this->logException( $e, __METHOD__, $params );
 399                         }
 400                 }
 401
 402                 return $status;
 403         }
 404
 405         /**
 406          * @see FileBackendStore::doCleanInternal()
 407          * @return Status
 408          */
 409         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 410                 $status = Status::newGood();
 411
 412                 // Only containers themselves can be removed, all else is virtual
 413                 if ( $dir != '' ) {
 414                         return $status; // nothing to do
 415                 }
 416
 417                 // (a) Check the container
 418                 try {
 419                         $contObj = $this->getContainer( $fullCont, true );
 420                 } catch ( NoSuchContainerException $e ) {
 421                         return $status; // ok, nothing to do
 422                 } catch ( InvalidResponseException $e ) {
 423                         $status->fatal( 'backend-fail-connect', $this->name );
 424                         return $status;
 425                 } catch ( Exception $e ) { // some other exception?
 426                         $status->fatal( 'backend-fail-internal', $this->name );
 427                         $this->logException( $e, __METHOD__, $params );
 428                         return $status;
 429                 }
 430
 431                 // (b) Delete the container if empty
 432                 if ( $contObj->object_count == 0 ) {
 433                         try {
 434                                 $this->deleteContainer( $fullCont );
 435                         } catch ( NoSuchContainerException $e ) {
 436                                 return $status; // race?
 437                         } catch ( InvalidResponseException $e ) {
 438                                 $status->fatal( 'backend-fail-connect', $this->name );
 439                                 return $status;
 440                         } catch ( Exception $e ) { // some other exception?
 441                                 $status->fatal( 'backend-fail-internal', $this->name );
 442                                 $this->logException( $e, __METHOD__, $params );
 443                                 return $status;
 444                         }
 445                 }
 446
 447                 return $status;
 448         }
 449
 450         /**
 451          * @see FileBackendStore::doFileExists()
 452          * @return array|bool|null
 453          */
 454         protected function doGetFileStat( array $params ) {
 455                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 456                 if ( $srcRel === null ) {
 457                         return false; // invalid storage path
 458                 }
 459
 460                 $stat = false;
 461                 try {
 462                         $contObj = $this->getContainer( $srcCont );
 463                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 464                         $this->addMissingMetadata( $srcObj, $params['src'] );
 465                         $stat = array(
 466                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 467                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 468                                 'size'  => $srcObj->content_length,
 469                                 'sha1'  => $srcObj->metadata['Sha1base36']
 470                         );
 471                 } catch ( NoSuchContainerException $e ) {
 472                 } catch ( NoSuchObjectException $e ) {
 473                 } catch ( InvalidResponseException $e ) {
 474                         $stat = null;
 475                 } catch ( Exception $e ) { // some other exception?
 476                         $stat = null;
 477                         $this->logException( $e, __METHOD__, $params );
 478                 }
 479
 480                 return $stat;
 481         }
 482
 483         /**
 484          * Fill in any missing object metadata and save it to Swift
 485          *
 486          * @param $obj CF_Object
 487          * @param $path string Storage path to object
 488          * @return bool Success
 489          * @throws Exception cloudfiles exceptions
 490          */
 491         protected function addMissingMetadata( CF_Object $obj, $path ) {
 492                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 493                         return true; // nothing to do
 494                 }
 495                 $status = Status::newGood();
 496                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 497                 if ( $status->isOK() ) {
 498                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 499                         if ( $tmpFile ) {
 500                                 $hash = $tmpFile->getSha1Base36();
 501                                 if ( $hash !== false ) {
 502                                         $obj->metadata['Sha1base36'] = $hash;
 503                                         $obj->sync_metadata(); // save to Swift
 504                                         return true; // success
 505                                 }
 506                         }
 507                 }
 508                 $obj->metadata['Sha1base36'] = false;
 509                 return false; // failed
 510         }
 511
 512         /**
 513          * @see FileBackend::getFileContents()
 514          * @return bool|null|string
 515          */
 516         public function getFileContents( array $params ) {
 517                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 518                 if ( $srcRel === null ) {
 519                         return false; // invalid storage path
 520                 }
 521
 522                 if ( !$this->fileExists( $params ) ) {
 523                         return null;
 524                 }
 525
 526                 $data = false;
 527                 try {
 528                         $sContObj = $this->getContainer( $srcCont );
 529                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 530                         $data = $obj->read( $this->headersFromParams( $params ) );
 531                 } catch ( NoSuchContainerException $e ) {
 532                 } catch ( InvalidResponseException $e ) {
 533                 } catch ( Exception $e ) { // some other exception?
 534                         $this->logException( $e, __METHOD__, $params );
 535                 }
 536
 537                 return $data;
 538         }
 539
 540         /**
 541          * @see FileBackendStore::getFileListInternal()
 542          * @return SwiftFileBackendFileList
 543          */
 544         public function getFileListInternal( $fullCont, $dir, array $params ) {
 545                 return new SwiftFileBackendFileList( $this, $fullCont, $dir );
 546         }
 547
 548         /**
 549          * Do not call this function outside of SwiftFileBackendFileList
 550          *
 551          * @param $fullCont string Resolved container name
 552          * @param $dir string Resolved storage directory with no trailing slash
 553          * @param $after string Storage path of file to list items after
 554          * @param $limit integer Max number of items to list
 555          * @return Array
 556          */
 557         public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) {
 558                 $files = array();
 559
 560                 try {
 561                         $container = $this->getContainer( $fullCont );
 562                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 563                         $files = $container->list_objects( $limit, $after, $prefix );
 564                 } catch ( NoSuchContainerException $e ) {
 565                 } catch ( InvalidResponseException $e ) {
 566                 } catch ( Exception $e ) { // some other exception?
 567                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 568                 }
 569
 570                 return $files;
 571         }
 572
 573         /**
 574          * @see FileBackendStore::doGetFileSha1base36()
 575          * @return bool
 576          */
 577         protected function doGetFileSha1base36( array $params ) {
 578                 $stat = $this->getFileStat( $params );
 579                 if ( $stat ) {
 580                         return $stat['sha1'];
 581                 } else {
 582                         return false;
 583                 }
 584         }
 585
 586         /**
 587          * @see FileBackendStore::doStreamFile()
 588          * @return Status
 589          */
 590         protected function doStreamFile( array $params ) {
 591                 $status = Status::newGood();
 592
 593                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 594                 if ( $srcRel === null ) {
 595                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 596                 }
 597
 598                 try {
 599                         $cont = $this->getContainer( $srcCont );
 600                 } catch ( NoSuchContainerException $e ) {
 601                         $status->fatal( 'backend-fail-stream', $params['src'] );
 602                         return $status;
 603                 } catch ( InvalidResponseException $e ) {
 604                         $status->fatal( 'backend-fail-connect', $this->name );
 605                         return $status;
 606                 } catch ( Exception $e ) { // some other exception?
 607                         $status->fatal( 'backend-fail-stream', $params['src'] );
 608                         $this->logException( $e, __METHOD__, $params );
 609                         return $status;
 610                 }
 611
 612                 try {
 613                         $output = fopen( 'php://output', 'wb' );
 614                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 615                         $obj->stream( $output, $this->headersFromParams( $params ) );
 616                 } catch ( InvalidResponseException $e ) { // 404? connection problem?
 617                         $status->fatal( 'backend-fail-stream', $params['src'] );
 618                 } catch ( Exception $e ) { // some other exception?
 619                         $status->fatal( 'backend-fail-stream', $params['src'] );
 620                         $this->logException( $e, __METHOD__, $params );
 621                 }
 622
 623                 return $status;
 624         }
 625
 626         /**
 627          * @see FileBackendStore::getLocalCopy()
 628          * @return null|TempFSFile
 629          */
 630         public function getLocalCopy( array $params ) {
 631                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 632                 if ( $srcRel === null ) {
 633                         return null;
 634                 }
 635
 636                 if ( !$this->fileExists( $params ) ) {
 637                         return null;
 638                 }
 639
 640                 $tmpFile = null;
 641                 try {
 642                         $sContObj = $this->getContainer( $srcCont );
 643                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 644                         // Get source file extension
 645                         $ext = FileBackend::extensionFromPath( $srcRel );
 646                         // Create a new temporary file...
 647                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 648                         if ( $tmpFile ) {
 649                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 650                                 if ( $handle ) {
 651                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 652                                         fclose( $handle );
 653                                 } else {
 654                                         $tmpFile = null; // couldn't open temp file
 655                                 }
 656                         }
 657                 } catch ( NoSuchContainerException $e ) {
 658                         $tmpFile = null;
 659                 } catch ( InvalidResponseException $e ) {
 660                         $tmpFile = null;
 661                 } catch ( Exception $e ) { // some other exception?
 662                         $tmpFile = null;
 663                         $this->logException( $e, __METHOD__, $params );
 664                 }
 665
 666                 return $tmpFile;
 667         }
 668
 669         /**
 670          * Get headers to send to Swift when reading a file based
 671          * on a FileBackend params array, e.g. that of getLocalCopy().
 672          * $params is currently only checked for a 'latest' flag.
 673          *
 674          * @param $params Array
 675          * @return Array
 676          */
 677         protected function headersFromParams( array $params ) {
 678                 $hdrs = array();
 679                 if ( !empty( $params['latest'] ) ) {
 680                         $hdrs[] = 'X-Newest: true';
 681                 }
 682                 return $hdrs;
 683         }
 684
 685         /**
 686          * Set read/write permissions for a Swift container
 687          *
 688          * @param $contObj CF_Container Swift container
 689          * @param $readGrps Array Swift users who can read (account:user)
 690          * @param $writeGrps Array Swift users who can write (account:user)
 691          * @return Status
 692          */
 693         protected function setContainerAccess(
 694                 CF_Container $contObj, array $readGrps, array $writeGrps
 695         ) {
 696                 $creds = $contObj->cfs_auth->export_credentials();
 697
 698                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 699
 700                 // Note: 10 second timeout consistent with php-cloudfiles
 701                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 702                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 703                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 704                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 705
 706                 return $req->execute(); // should return 204
 707         }
 708
 709         /**
 710          * Get a connection to the Swift proxy
 711          *
 712          * @return CF_Connection|bool False on failure
 713          * @throws InvalidResponseException
 714          */
 715         protected function getConnection() {
 716                 if ( $this->conn === false ) {
 717                         throw new InvalidResponseException; // failed last attempt
 718                 }
 719                 // Session keys expire after a while, so we renew them periodically
 720                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 721                         $this->conn->close(); // close active cURL connections
 722                         $this->conn = null;
 723                 }
 724                 // Authenticate with proxy and get a session key...
 725                 if ( $this->conn === null ) {
 726                         $this->connContainers = array();
 727                         try {
 728                                 $this->auth->authenticate();
 729                                 $this->conn = new CF_Connection( $this->auth );
 730                                 $this->connStarted = time();
 731                         } catch ( AuthenticationException $e ) {
 732                                 $this->conn = false; // don't keep re-trying
 733                         } catch ( InvalidResponseException $e ) {
 734                                 $this->conn = false; // don't keep re-trying
 735                         }
 736                 }
 737                 if ( !$this->conn ) {
 738                         throw new InvalidResponseException; // auth/connection problem
 739                 }
 740                 return $this->conn;
 741         }
 742
 743         /**
 744          * @see FileBackendStore::doClearCache()
 745          */
 746         protected function doClearCache( array $paths = null ) {
 747                 $this->connContainers = array(); // clear container object cache
 748         }
 749
 750         /**
 751          * Get a Swift container object, possibly from process cache.
 752          * Use $reCache if the file count or byte count is needed.
 753          *
 754          * @param $container string Container name
 755          * @param $bypassCache bool Bypass all caches and load from Swift
 756          * @return CF_Container
 757          * @throws InvalidResponseException
 758          */
 759         protected function getContainer( $container, $bypassCache = false ) {
 760                 $conn = $this->getConnection(); // Swift proxy connection
 761                 if ( $bypassCache ) { // purge cache
 762                         unset( $this->connContainers[$container] );
 763                 } elseif ( !isset( $this->connContainers[$container] ) ) {
 764                         $this->primeContainerCache( array( $container ) ); // check persistent cache
 765                 }
 766                 if ( !isset( $this->connContainers[$container] ) ) {
 767                         $contObj = $conn->get_container( $container );
 768                         // NoSuchContainerException not thrown: container must exist
 769                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 770                                 reset( $this->connContainers );
 771                                 unset( $this->connContainers[key( $this->connContainers )] );
 772                         }
 773                         $this->connContainers[$container] = $contObj; // cache it
 774                         if ( !$bypassCache ) {
 775                                 $this->setContainerCache( $container, // update persistent cache
 776                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
 777                                 );
 778                         }
 779                 }
 780                 return $this->connContainers[$container];
 781         }
 782
 783         /**
 784          * Create a Swift container
 785          *
 786          * @param $container string Container name
 787          * @return CF_Container
 788          * @throws InvalidResponseException
 789          */
 790         protected function createContainer( $container ) {
 791                 $conn = $this->getConnection(); // Swift proxy connection
 792                 $contObj = $conn->create_container( $container );
 793                 $this->connContainers[$container] = $contObj; // cache it
 794                 return $contObj;
 795         }
 796
 797         /**
 798          * Delete a Swift container
 799          *
 800          * @param $container string Container name
 801          * @return void
 802          * @throws InvalidResponseException
 803          */
 804         protected function deleteContainer( $container ) {
 805                 $conn = $this->getConnection(); // Swift proxy connection
 806                 $conn->delete_container( $container );
 807                 unset( $this->connContainers[$container] ); // purge cache
 808         }
 809
 810         /**
 811          * @see FileBackendStore::doPrimeContainerCache()
 812          * @return void
 813          */
 814         protected function doPrimeContainerCache( array $containerInfo ) {
 815                 try {
 816                         $conn = $this->getConnection(); // Swift proxy connection
 817                         foreach ( $containerInfo as $container => $info ) {
 818                                 $this->connContainers[$container] = new CF_Container(
 819                                         $conn->cfs_auth,
 820                                         $conn->cfs_http,
 821                                         $container,
 822                                         $info['count'],
 823                                         $info['bytes']
 824                                 );
 825                         }
 826                 } catch ( InvalidResponseException $e ) {
 827                 } catch ( Exception $e ) { // some other exception?
 828                         $this->logException( $e, __METHOD__, array() );
 829                 }
 830         }
 831
 832         /**
 833          * Log an unexpected exception for this backend
 834          *
 835          * @param $e Exception
 836          * @param $func string
 837          * @param $params Array
 838          * @return void
 839          */
 840         protected function logException( Exception $e, $func, array $params ) {
 841                 wfDebugLog( 'SwiftBackend',
 842                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
 843                         ( $e instanceof InvalidResponseException
 844                                 ? ": {$e->getMessage()}"
 845                                 : ""
 846                         )
 847                 );
 848         }
 849 }
 850
 851 /**
 852  * SwiftFileBackend helper class to page through object listings.
 853  * Swift also has a listing limit of 10,000 objects for sanity.
 854  * Do not use this class from places outside SwiftFileBackend.
 855  *
 856  * @ingroup FileBackend
 857  */
 858 class SwiftFileBackendFileList implements Iterator {
 859         /** @var Array */
 860         protected $bufferIter = array();
 861         protected $bufferAfter = null; // string; list items *after* this path
 862         protected $pos = 0; // integer
 863
 864         /** @var SwiftFileBackend */
 865         protected $backend;
 866         protected $container; //
 867         protected $dir; // string storage directory
 868         protected $suffixStart; // integer
 869
 870         const PAGE_SIZE = 5000; // file listing buffer size
 871
 872         /**
 873          * @param $backend SwiftFileBackend
 874          * @param $fullCont string Resolved container name
 875          * @param $dir string Resolved directory relative to container
 876          */
 877         public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) {
 878                 $this->backend = $backend;
 879                 $this->container = $fullCont;
 880                 $this->dir = $dir;
 881                 if ( substr( $this->dir, -1 ) === '/' ) {
 882                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
 883                 }
 884                 if ( $this->dir == '' ) { // whole container
 885                         $this->suffixStart = 0;
 886                 } else { // dir within container
 887                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
 888                 }
 889         }
 890
 891         /**
 892          * @see Iterator::current()
 893          * @return string|bool String or false
 894          */
 895         public function current() {
 896                 return substr( current( $this->bufferIter ), $this->suffixStart );
 897         }
 898
 899         /**
 900          * @see Iterator::key()
 901          * @return integer
 902          */
 903         public function key() {
 904                 return $this->pos;
 905         }
 906
 907         /**
 908          * @see Iterator::next()
 909          * @return void
 910          */
 911         public function next() {
 912                 // Advance to the next file in the page
 913                 next( $this->bufferIter );
 914                 ++$this->pos;
 915                 // Check if there are no files left in this page and
 916                 // advance to the next page if this page was not empty.
 917                 if ( !$this->valid() && count( $this->bufferIter ) ) {
 918                         $this->bufferAfter = end( $this->bufferIter );
 919                         $this->bufferIter = $this->backend->getFileListPageInternal(
 920                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 921                         );
 922                 }
 923         }
 924
 925         /**
 926          * @see Iterator::rewind()
 927          * @return void
 928          */
 929         public function rewind() {
 930                 $this->pos = 0;
 931                 $this->bufferAfter = null;
 932                 $this->bufferIter = $this->backend->getFileListPageInternal(
 933                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 934                 );
 935         }
 936
 937         /**
 938          * @see Iterator::valid()
 939          * @return bool
 940          */
 941         public function valid() {
 942                 return ( current( $this->bufferIter ) !== false ); // no paths can have this value
 943         }
 944 }