includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Russ Nelson
   6  * @author Aaron Schulz
   7  */
   8
   9 /**
  10  * @brief Class for an OpenStack Swift based file backend.
  11  *
  12  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  13  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  14  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  15  *
  16  * Status messages should avoid mentioning the Swift account name.
  17  * Likewise, error suppression should be used to avoid path disclosure.
  18  *
  19  * @ingroup FileBackend
  20  * @since 1.19
  21  */
  22 class SwiftFileBackend extends FileBackendStore {
  23         /** @var CF_Authentication */
  24         protected $auth; // Swift authentication handler
  25         protected $authTTL; // integer seconds
  26         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  27         protected $maxContCacheSize = 300; // integer; max containers with entries
  28
  29         /** @var CF_Connection */
  30         protected $conn; // Swift connection handle
  31         protected $connStarted = 0; // integer UNIX timestamp
  32         protected $connContainers = array(); // container object cache
  33
  34         /**
  35          * @see FileBackendStore::__construct()
  36          * Additional $config params include:
  37          *    swiftAuthUrl       : Swift authentication server URL
  38          *    swiftUser          : Swift user used by MediaWiki (account:username)
  39          *    swiftKey           : Swift authentication key for the above user
  40          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  41          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  42          *    shardViaHashLevels : Map of container names to sharding config with:
  43          *                         'base'   : base of hash characters, 16 or 36
  44          *                         'levels' : the number of hash levels (and digits)
  45          *                         'repeat' : hash subdirectories are prefixed with all the
  46          *                                    parent hash directory names (e.g. "a/ab/abc")
  47          */
  48         public function __construct( array $config ) {
  49                 parent::__construct( $config );
  50                 // Required settings
  51                 $this->auth = new CF_Authentication(
  52                         $config['swiftUser'],
  53                         $config['swiftKey'],
  54                         null, // account; unused
  55                         $config['swiftAuthUrl']
  56                 );
  57                 // Optional settings
  58                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  59                         ? $config['swiftAuthTTL']
  60                         : 5 * 60; // some sane number
  61                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  62                         ? $config['swiftAnonUser']
  63                         : '';
  64                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  65                         ? $config['shardViaHashLevels']
  66                         : '';
  67                 // Cache container info to mask latency
  68                 $this->memCache = wfGetMainCache();
  69         }
  70
  71         /**
  72          * @see FileBackendStore::resolveContainerPath()
  73          * @return null
  74          */
  75         protected function resolveContainerPath( $container, $relStoragePath ) {
  76                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  77                         return null; // too long for Swift
  78                 }
  79                 return $relStoragePath;
  80         }
  81
  82         /**
  83          * @see FileBackendStore::isPathUsableInternal()
  84          * @return bool
  85          */
  86         public function isPathUsableInternal( $storagePath ) {
  87                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
  88                 if ( $rel === null ) {
  89                         return false; // invalid
  90                 }
  91
  92                 try {
  93                         $this->getContainer( $container );
  94                         return true; // container exists
  95                 } catch ( NoSuchContainerException $e ) {
  96                 } catch ( InvalidResponseException $e ) {
  97                 } catch ( Exception $e ) { // some other exception?
  98                         $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
  99                 }
 100
 101                 return false;
 102         }
 103
 104         /**
 105          * @see FileBackendStore::doCreateInternal()
 106          * @return Status
 107          */
 108         protected function doCreateInternal( array $params ) {
 109                 $status = Status::newGood();
 110
 111                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 112                 if ( $dstRel === null ) {
 113                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 114                         return $status;
 115                 }
 116
 117                 // (a) Check the destination container and object
 118                 try {
 119                         $dContObj = $this->getContainer( $dstCont );
 120                         if ( empty( $params['overwrite'] ) &&
 121                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 122                         {
 123                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 124                                 return $status;
 125                         }
 126                 } catch ( NoSuchContainerException $e ) {
 127                         $status->fatal( 'backend-fail-create', $params['dst'] );
 128                         return $status;
 129                 } catch ( InvalidResponseException $e ) {
 130                         $status->fatal( 'backend-fail-connect', $this->name );
 131                         return $status;
 132                 } catch ( Exception $e ) { // some other exception?
 133                         $status->fatal( 'backend-fail-internal', $this->name );
 134                         $this->logException( $e, __METHOD__, $params );
 135                         return $status;
 136                 }
 137
 138                 // (b) Get a SHA-1 hash of the object
 139                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 140
 141                 // (c) Actually create the object
 142                 try {
 143                         // Create a fresh CF_Object with no fields preloaded.
 144                         // We don't want to preserve headers, metadata, and such.
 145                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 146                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 147                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 148                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 149                         // The MD5 here will be checked within Swift against its own MD5.
 150                         $obj->set_etag( md5( $params['content'] ) );
 151                         // Use the same content type as StreamFile for security
 152                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 153                         // Actually write the object in Swift
 154                         $obj->write( $params['content'] );
 155                 } catch ( BadContentTypeException $e ) {
 156                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 157                 } catch ( InvalidResponseException $e ) {
 158                         $status->fatal( 'backend-fail-connect', $this->name );
 159                 } catch ( Exception $e ) { // some other exception?
 160                         $status->fatal( 'backend-fail-internal', $this->name );
 161                         $this->logException( $e, __METHOD__, $params );
 162                 }
 163
 164                 return $status;
 165         }
 166
 167         /**
 168          * @see FileBackendStore::doStoreInternal()
 169          * @return Status
 170          */
 171         protected function doStoreInternal( array $params ) {
 172                 $status = Status::newGood();
 173
 174                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 175                 if ( $dstRel === null ) {
 176                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 177                         return $status;
 178                 }
 179
 180                 // (a) Check the destination container and object
 181                 try {
 182                         $dContObj = $this->getContainer( $dstCont );
 183                         if ( empty( $params['overwrite'] ) &&
 184                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 185                         {
 186                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 187                                 return $status;
 188                         }
 189                 } catch ( NoSuchContainerException $e ) {
 190                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 191                         return $status;
 192                 } catch ( InvalidResponseException $e ) {
 193                         $status->fatal( 'backend-fail-connect', $this->name );
 194                         return $status;
 195                 } catch ( Exception $e ) { // some other exception?
 196                         $status->fatal( 'backend-fail-internal', $this->name );
 197                         $this->logException( $e, __METHOD__, $params );
 198                         return $status;
 199                 }
 200
 201                 // (b) Get a SHA-1 hash of the object
 202                 $sha1Hash = sha1_file( $params['src'] );
 203                 if ( $sha1Hash === false ) { // source doesn't exist?
 204                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 205                         return $status;
 206                 }
 207                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 208
 209                 // (c) Actually store the object
 210                 try {
 211                         // Create a fresh CF_Object with no fields preloaded.
 212                         // We don't want to preserve headers, metadata, and such.
 213                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 214                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 215                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 216                         // The MD5 here will be checked within Swift against its own MD5.
 217                         $obj->set_etag( md5_file( $params['src'] ) );
 218                         // Use the same content type as StreamFile for security
 219                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 220                         // Actually write the object in Swift
 221                         $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
 222                 } catch ( BadContentTypeException $e ) {
 223                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 224                 } catch ( IOException $e ) {
 225                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 226                 } catch ( InvalidResponseException $e ) {
 227                         $status->fatal( 'backend-fail-connect', $this->name );
 228                 } catch ( Exception $e ) { // some other exception?
 229                         $status->fatal( 'backend-fail-internal', $this->name );
 230                         $this->logException( $e, __METHOD__, $params );
 231                 }
 232
 233                 return $status;
 234         }
 235
 236         /**
 237          * @see FileBackendStore::doCopyInternal()
 238          * @return Status
 239          */
 240         protected function doCopyInternal( array $params ) {
 241                 $status = Status::newGood();
 242
 243                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 244                 if ( $srcRel === null ) {
 245                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 246                         return $status;
 247                 }
 248
 249                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 250                 if ( $dstRel === null ) {
 251                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 252                         return $status;
 253                 }
 254
 255                 // (a) Check the source/destination containers and destination object
 256                 try {
 257                         $sContObj = $this->getContainer( $srcCont );
 258                         $dContObj = $this->getContainer( $dstCont );
 259                         if ( empty( $params['overwrite'] ) &&
 260                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 261                         {
 262                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 263                                 return $status;
 264                         }
 265                 } catch ( NoSuchContainerException $e ) {
 266                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 267                         return $status;
 268                 } catch ( InvalidResponseException $e ) {
 269                         $status->fatal( 'backend-fail-connect', $this->name );
 270                         return $status;
 271                 } catch ( Exception $e ) { // some other exception?
 272                         $status->fatal( 'backend-fail-internal', $this->name );
 273                         $this->logException( $e, __METHOD__, $params );
 274                         return $status;
 275                 }
 276
 277                 // (b) Actually copy the file to the destination
 278                 try {
 279                         $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 280                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 281                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 282                 } catch ( InvalidResponseException $e ) {
 283                         $status->fatal( 'backend-fail-connect', $this->name );
 284                 } catch ( Exception $e ) { // some other exception?
 285                         $status->fatal( 'backend-fail-internal', $this->name );
 286                         $this->logException( $e, __METHOD__, $params );
 287                 }
 288
 289                 return $status;
 290         }
 291
 292         /**
 293          * @see FileBackendStore::doDeleteInternal()
 294          * @return Status
 295          */
 296         protected function doDeleteInternal( array $params ) {
 297                 $status = Status::newGood();
 298
 299                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 300                 if ( $srcRel === null ) {
 301                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 302                         return $status;
 303                 }
 304
 305                 try {
 306                         $sContObj = $this->getContainer( $srcCont );
 307                         $sContObj->delete_object( $srcRel );
 308                 } catch ( NoSuchContainerException $e ) {
 309                         $status->fatal( 'backend-fail-delete', $params['src'] );
 310                 } catch ( NoSuchObjectException $e ) {
 311                         if ( empty( $params['ignoreMissingSource'] ) ) {
 312                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 313                         }
 314                 } catch ( InvalidResponseException $e ) {
 315                         $status->fatal( 'backend-fail-connect', $this->name );
 316                 } catch ( Exception $e ) { // some other exception?
 317                         $status->fatal( 'backend-fail-internal', $this->name );
 318                         $this->logException( $e, __METHOD__, $params );
 319                 }
 320
 321                 return $status;
 322         }
 323
 324         /**
 325          * @see FileBackendStore::doPrepareInternal()
 326          * @return Status
 327          */
 328         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 329                 $status = Status::newGood();
 330
 331                 // (a) Check if container already exists
 332                 try {
 333                         $contObj = $this->getContainer( $fullCont );
 334                         // NoSuchContainerException not thrown: container must exist
 335                         return $status; // already exists
 336                 } catch ( NoSuchContainerException $e ) {
 337                         // NoSuchContainerException thrown: container does not exist
 338                 } catch ( InvalidResponseException $e ) {
 339                         $status->fatal( 'backend-fail-connect', $this->name );
 340                         return $status;
 341                 } catch ( Exception $e ) { // some other exception?
 342                         $status->fatal( 'backend-fail-internal', $this->name );
 343                         $this->logException( $e, __METHOD__, $params );
 344                         return $status;
 345                 }
 346
 347                 // (b) Create container as needed
 348                 try {
 349                         $contObj = $this->createContainer( $fullCont );
 350                         if ( $this->swiftAnonUser != '' ) {
 351                                 // Make container public to end-users...
 352                                 $status->merge( $this->setContainerAccess(
 353                                         $contObj,
 354                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 355                                         array( $this->auth->username ) // write
 356                                 ) );
 357                         }
 358                 } catch ( InvalidResponseException $e ) {
 359                         $status->fatal( 'backend-fail-connect', $this->name );
 360                         return $status;
 361                 } catch ( Exception $e ) { // some other exception?
 362                         $status->fatal( 'backend-fail-internal', $this->name );
 363                         $this->logException( $e, __METHOD__, $params );
 364                         return $status;
 365                 }
 366
 367                 return $status;
 368         }
 369
 370         /**
 371          * @see FileBackendStore::doSecureInternal()
 372          * @return Status
 373          */
 374         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 375                 $status = Status::newGood();
 376
 377                 if ( $this->swiftAnonUser != '' ) {
 378                         // Restrict container from end-users...
 379                         try {
 380                                 // doPrepareInternal() should have been called,
 381                                 // so the Swift container should already exist...
 382                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 383                                 // NoSuchContainerException not thrown: container must exist
 384                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 385                                         $status->merge( $this->setContainerAccess(
 386                                                 $contObj,
 387                                                 array( $this->auth->username ), // read
 388                                                 array( $this->auth->username ) // write
 389                                         ) );
 390                                         // @TODO: when php-cloudfiles supports container
 391                                         // metadata, we can make use of that to avoid RTTs
 392                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 393                                 }
 394                         } catch ( InvalidResponseException $e ) {
 395                                 $status->fatal( 'backend-fail-connect', $this->name );
 396                         } catch ( Exception $e ) { // some other exception?
 397                                 $status->fatal( 'backend-fail-internal', $this->name );
 398                                 $this->logException( $e, __METHOD__, $params );
 399                         }
 400                 }
 401
 402                 return $status;
 403         }
 404
 405         /**
 406          * @see FileBackendStore::doCleanInternal()
 407          * @return Status
 408          */
 409         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 410                 $status = Status::newGood();
 411
 412                 // Only containers themselves can be removed, all else is virtual
 413                 if ( $dir != '' ) {
 414                         return $status; // nothing to do
 415                 }
 416
 417                 // (a) Check the container
 418                 try {
 419                         $contObj = $this->getContainer( $fullCont, true );
 420                 } catch ( NoSuchContainerException $e ) {
 421                         return $status; // ok, nothing to do
 422                 } catch ( InvalidResponseException $e ) {
 423                         $status->fatal( 'backend-fail-connect', $this->name );
 424                         return $status;
 425                 } catch ( Exception $e ) { // some other exception?
 426                         $status->fatal( 'backend-fail-internal', $this->name );
 427                         $this->logException( $e, __METHOD__, $params );
 428                         return $status;
 429                 }
 430
 431                 // (b) Delete the container if empty
 432                 if ( $contObj->object_count == 0 ) {
 433                         try {
 434                                 $this->deleteContainer( $fullCont );
 435                         } catch ( NoSuchContainerException $e ) {
 436                                 return $status; // race?
 437                         } catch ( InvalidResponseException $e ) {
 438                                 $status->fatal( 'backend-fail-connect', $this->name );
 439                                 return $status;
 440                         } catch ( Exception $e ) { // some other exception?
 441                                 $status->fatal( 'backend-fail-internal', $this->name );
 442                                 $this->logException( $e, __METHOD__, $params );
 443                                 return $status;
 444                         }
 445                 }
 446
 447                 return $status;
 448         }
 449
 450         /**
 451          * @see FileBackendStore::doFileExists()
 452          * @return array|bool|null
 453          */
 454         protected function doGetFileStat( array $params ) {
 455                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 456                 if ( $srcRel === null ) {
 457                         return false; // invalid storage path
 458                 }
 459
 460                 $stat = false;
 461                 try {
 462                         $contObj = $this->getContainer( $srcCont );
 463                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 464                         $this->addMissingMetadata( $srcObj, $params['src'] );
 465                         $stat = array(
 466                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 467                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 468                                 'size'  => $srcObj->content_length,
 469                                 'sha1'  => $srcObj->metadata['Sha1base36']
 470                         );
 471                 } catch ( NoSuchContainerException $e ) {
 472                 } catch ( NoSuchObjectException $e ) {
 473                 } catch ( InvalidResponseException $e ) {
 474                         $stat = null;
 475                 } catch ( Exception $e ) { // some other exception?
 476                         $stat = null;
 477                         $this->logException( $e, __METHOD__, $params );
 478                 }
 479
 480                 return $stat;
 481         }
 482
 483         /**
 484          * Fill in any missing object metadata and save it to Swift
 485          *
 486          * @param $obj CF_Object
 487          * @param $path string Storage path to object
 488          * @return bool Success
 489          * @throws Exception cloudfiles exceptions
 490          */
 491         protected function addMissingMetadata( CF_Object $obj, $path ) {
 492                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 493                         return true; // nothing to do
 494                 }
 495                 $status = Status::newGood();
 496                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 497                 if ( $status->isOK() ) {
 498                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 499                         if ( $tmpFile ) {
 500                                 $hash = $tmpFile->getSha1Base36();
 501                                 if ( $hash !== false ) {
 502                                         $obj->metadata['Sha1base36'] = $hash;
 503                                         $obj->sync_metadata(); // save to Swift
 504                                         return true; // success
 505                                 }
 506                         }
 507                 }
 508                 $obj->metadata['Sha1base36'] = false;
 509                 return false; // failed
 510         }
 511
 512         /**
 513          * @see FileBackend::getFileContents()
 514          * @return bool|null|string
 515          */
 516         public function getFileContents( array $params ) {
 517                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 518                 if ( $srcRel === null ) {
 519                         return false; // invalid storage path
 520                 }
 521
 522                 if ( !$this->fileExists( $params ) ) {
 523                         return null;
 524                 }
 525
 526                 $data = false;
 527                 try {
 528                         $sContObj = $this->getContainer( $srcCont );
 529                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 530                         $data = $obj->read( $this->headersFromParams( $params ) );
 531                 } catch ( NoSuchContainerException $e ) {
 532                 } catch ( InvalidResponseException $e ) {
 533                 } catch ( Exception $e ) { // some other exception?
 534                         $this->logException( $e, __METHOD__, $params );
 535                 }
 536
 537                 return $data;
 538         }
 539
 540         /**
 541          * @see FileBackendStore::doDirectoryExists()
 542          * @return bool|null
 543          */
 544         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 545                 try {
 546                         $container = $this->getContainer( $fullCont );
 547                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 548                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 549                 } catch ( NoSuchContainerException $e ) {
 550                         return false;
 551                 } catch ( InvalidResponseException $e ) {
 552                 } catch ( Exception $e ) { // some other exception?
 553                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 554                 }
 555
 556                 return null; // error
 557         }
 558
 559         /**
 560          * @see FileBackendStore::getDirectoryListInternal()
 561          * @return SwiftFileBackendDirList
 562          */
 563         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 564                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 565         }
 566
 567         /**
 568          * @see FileBackendStore::getFileListInternal()
 569          * @return SwiftFileBackendFileList
 570          */
 571         public function getFileListInternal( $fullCont, $dir, array $params ) {
 572                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 573         }
 574
 575         /**
 576          * Do not call this function outside of SwiftFileBackendFileList
 577          *
 578          * @param $fullCont string Resolved container name
 579          * @param $dir string Resolved storage directory with no trailing slash
 580          * @param $after string|null Storage path of file to list items after
 581          * @param $limit integer Max number of items to list
 582          * @param $params Array Includes flag for 'topOnly'
 583          * @return Array List of relative paths of dirs directly under $dir
 584          */
 585         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 586                 $dirs = array();
 587
 588                 try {
 589                         $container = $this->getContainer( $fullCont );
 590                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 591                         // Non-recursive: only list dirs right under $dir
 592                         if ( !empty( $params['topOnly'] ) ) {
 593                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 594                                 foreach ( $objects as $object ) { // files and dirs
 595                                         if ( substr( $object, -1 ) === '/' ) {
 596                                                 $dirs[] = $object; // directories end in '/'
 597                                         }
 598                                         $after = $object; // update last item
 599                                 }
 600                         // Recursive: list all dirs under $dir and its subdirs
 601                         } else {
 602                                 // Get directory from last item of prior page
 603                                 $lastDir = $this->getParentDir( $after ); // must be first page
 604                                 $objects = $container->list_objects( $limit, $after, $prefix );
 605                                 foreach ( $objects as $object ) { // files
 606                                         $objectDir = $this->getParentDir( $object ); // directory of object
 607                                         if ( $objectDir !== false ) { // file has a parent dir
 608                                                 // Swift stores paths in UTF-8, using binary sorting.
 609                                                 // See function "create_container_table" in common/db.py.
 610                                                 // If a directory is not "greater" than the last one,
 611                                                 // then it was already listed by the calling iterator.
 612                                                 if ( $objectDir > $lastDir ) {
 613                                                         $pDir = $objectDir;
 614                                                         do { // add dir and all its parent dirs
 615                                                                 $dirs[] = "{$pDir}/";
 616                                                                 $pDir = $this->getParentDir( $pDir );
 617                                                         } while ( $pDir !== false // sanity
 618                                                                 && $pDir > $lastDir // not done already
 619                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 620                                                         );
 621                                                 }
 622                                                 $lastDir = $objectDir;
 623                                         }
 624                                         $after = $object; // update last item
 625                                 }
 626                         }
 627                 } catch ( NoSuchContainerException $e ) {
 628                 } catch ( InvalidResponseException $e ) {
 629                 } catch ( Exception $e ) { // some other exception?
 630                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 631                 }
 632
 633                 return $dirs;
 634         }
 635
 636         protected function getParentDir( $path ) {
 637                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 638         }
 639
 640         /**
 641          * Do not call this function outside of SwiftFileBackendFileList
 642          *
 643          * @param $fullCont string Resolved container name
 644          * @param $dir string Resolved storage directory with no trailing slash
 645          * @param $after string|null Storage path of file to list items after
 646          * @param $limit integer Max number of items to list
 647          * @param $params Array Includes flag for 'topOnly'
 648          * @return Array List of relative paths of files under $dir
 649          */
 650         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 651                 $files = array();
 652
 653                 try {
 654                         $container = $this->getContainer( $fullCont );
 655                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 656                         // Non-recursive: only list files right under $dir
 657                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 658                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 659                                 foreach ( $objects as $object ) {
 660                                         if ( substr( $object, -1 ) !== '/' ) {
 661                                                 $files[] = $object; // directories end in '/'
 662                                         }
 663                                 }
 664                         // Recursive: list all files under $dir and its subdirs
 665                         } else { // files
 666                                 $files = $container->list_objects( $limit, $after, $prefix );
 667                         }
 668                         $after = end( $files ); // update last item
 669                         reset( $files ); // reset pointer
 670                 } catch ( NoSuchContainerException $e ) {
 671                 } catch ( InvalidResponseException $e ) {
 672                 } catch ( Exception $e ) { // some other exception?
 673                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 674                 }
 675
 676                 return $files;
 677         }
 678
 679         /**
 680          * @see FileBackendStore::doGetFileSha1base36()
 681          * @return bool
 682          */
 683         protected function doGetFileSha1base36( array $params ) {
 684                 $stat = $this->getFileStat( $params );
 685                 if ( $stat ) {
 686                         return $stat['sha1'];
 687                 } else {
 688                         return false;
 689                 }
 690         }
 691
 692         /**
 693          * @see FileBackendStore::doStreamFile()
 694          * @return Status
 695          */
 696         protected function doStreamFile( array $params ) {
 697                 $status = Status::newGood();
 698
 699                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 700                 if ( $srcRel === null ) {
 701                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 702                 }
 703
 704                 try {
 705                         $cont = $this->getContainer( $srcCont );
 706                 } catch ( NoSuchContainerException $e ) {
 707                         $status->fatal( 'backend-fail-stream', $params['src'] );
 708                         return $status;
 709                 } catch ( InvalidResponseException $e ) {
 710                         $status->fatal( 'backend-fail-connect', $this->name );
 711                         return $status;
 712                 } catch ( Exception $e ) { // some other exception?
 713                         $status->fatal( 'backend-fail-stream', $params['src'] );
 714                         $this->logException( $e, __METHOD__, $params );
 715                         return $status;
 716                 }
 717
 718                 try {
 719                         $output = fopen( 'php://output', 'wb' );
 720                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 721                         $obj->stream( $output, $this->headersFromParams( $params ) );
 722                 } catch ( InvalidResponseException $e ) { // 404? connection problem?
 723                         $status->fatal( 'backend-fail-stream', $params['src'] );
 724                 } catch ( Exception $e ) { // some other exception?
 725                         $status->fatal( 'backend-fail-stream', $params['src'] );
 726                         $this->logException( $e, __METHOD__, $params );
 727                 }
 728
 729                 return $status;
 730         }
 731
 732         /**
 733          * @see FileBackendStore::getLocalCopy()
 734          * @return null|TempFSFile
 735          */
 736         public function getLocalCopy( array $params ) {
 737                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 738                 if ( $srcRel === null ) {
 739                         return null;
 740                 }
 741
 742                 if ( !$this->fileExists( $params ) ) {
 743                         return null;
 744                 }
 745
 746                 $tmpFile = null;
 747                 try {
 748                         $sContObj = $this->getContainer( $srcCont );
 749                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 750                         // Get source file extension
 751                         $ext = FileBackend::extensionFromPath( $srcRel );
 752                         // Create a new temporary file...
 753                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 754                         if ( $tmpFile ) {
 755                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 756                                 if ( $handle ) {
 757                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 758                                         fclose( $handle );
 759                                 } else {
 760                                         $tmpFile = null; // couldn't open temp file
 761                                 }
 762                         }
 763                 } catch ( NoSuchContainerException $e ) {
 764                         $tmpFile = null;
 765                 } catch ( InvalidResponseException $e ) {
 766                         $tmpFile = null;
 767                 } catch ( Exception $e ) { // some other exception?
 768                         $tmpFile = null;
 769                         $this->logException( $e, __METHOD__, $params );
 770                 }
 771
 772                 return $tmpFile;
 773         }
 774
 775         /**
 776          * @see FileBackendStore::directoriesAreVirtual()
 777          * @return bool
 778          */
 779         protected function directoriesAreVirtual() {
 780                 return true;
 781         }
 782
 783         /**
 784          * Get headers to send to Swift when reading a file based
 785          * on a FileBackend params array, e.g. that of getLocalCopy().
 786          * $params is currently only checked for a 'latest' flag.
 787          *
 788          * @param $params Array
 789          * @return Array
 790          */
 791         protected function headersFromParams( array $params ) {
 792                 $hdrs = array();
 793                 if ( !empty( $params['latest'] ) ) {
 794                         $hdrs[] = 'X-Newest: true';
 795                 }
 796                 return $hdrs;
 797         }
 798
 799         /**
 800          * Set read/write permissions for a Swift container
 801          *
 802          * @param $contObj CF_Container Swift container
 803          * @param $readGrps Array Swift users who can read (account:user)
 804          * @param $writeGrps Array Swift users who can write (account:user)
 805          * @return Status
 806          */
 807         protected function setContainerAccess(
 808                 CF_Container $contObj, array $readGrps, array $writeGrps
 809         ) {
 810                 $creds = $contObj->cfs_auth->export_credentials();
 811
 812                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 813
 814                 // Note: 10 second timeout consistent with php-cloudfiles
 815                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 816                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 817                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 818                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 819
 820                 return $req->execute(); // should return 204
 821         }
 822
 823         /**
 824          * Get a connection to the Swift proxy
 825          *
 826          * @return CF_Connection|bool False on failure
 827          * @throws InvalidResponseException
 828          */
 829         protected function getConnection() {
 830                 if ( $this->conn === false ) {
 831                         throw new InvalidResponseException; // failed last attempt
 832                 }
 833                 // Session keys expire after a while, so we renew them periodically
 834                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 835                         $this->conn->close(); // close active cURL connections
 836                         $this->conn = null;
 837                 }
 838                 // Authenticate with proxy and get a session key...
 839                 if ( $this->conn === null ) {
 840                         $this->connContainers = array();
 841                         try {
 842                                 $this->auth->authenticate();
 843                                 $this->conn = new CF_Connection( $this->auth );
 844                                 $this->connStarted = time();
 845                         } catch ( AuthenticationException $e ) {
 846                                 $this->conn = false; // don't keep re-trying
 847                         } catch ( InvalidResponseException $e ) {
 848                                 $this->conn = false; // don't keep re-trying
 849                         }
 850                 }
 851                 if ( !$this->conn ) {
 852                         throw new InvalidResponseException; // auth/connection problem
 853                 }
 854                 return $this->conn;
 855         }
 856
 857         /**
 858          * @see FileBackendStore::doClearCache()
 859          */
 860         protected function doClearCache( array $paths = null ) {
 861                 $this->connContainers = array(); // clear container object cache
 862         }
 863
 864         /**
 865          * Get a Swift container object, possibly from process cache.
 866          * Use $reCache if the file count or byte count is needed.
 867          *
 868          * @param $container string Container name
 869          * @param $bypassCache bool Bypass all caches and load from Swift
 870          * @return CF_Container
 871          * @throws InvalidResponseException
 872          */
 873         protected function getContainer( $container, $bypassCache = false ) {
 874                 $conn = $this->getConnection(); // Swift proxy connection
 875                 if ( $bypassCache ) { // purge cache
 876                         unset( $this->connContainers[$container] );
 877                 } elseif ( !isset( $this->connContainers[$container] ) ) {
 878                         $this->primeContainerCache( array( $container ) ); // check persistent cache
 879                 }
 880                 if ( !isset( $this->connContainers[$container] ) ) {
 881                         $contObj = $conn->get_container( $container );
 882                         // NoSuchContainerException not thrown: container must exist
 883                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 884                                 reset( $this->connContainers );
 885                                 unset( $this->connContainers[key( $this->connContainers )] );
 886                         }
 887                         $this->connContainers[$container] = $contObj; // cache it
 888                         if ( !$bypassCache ) {
 889                                 $this->setContainerCache( $container, // update persistent cache
 890                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
 891                                 );
 892                         }
 893                 }
 894                 return $this->connContainers[$container];
 895         }
 896
 897         /**
 898          * Create a Swift container
 899          *
 900          * @param $container string Container name
 901          * @return CF_Container
 902          * @throws InvalidResponseException
 903          */
 904         protected function createContainer( $container ) {
 905                 $conn = $this->getConnection(); // Swift proxy connection
 906                 $contObj = $conn->create_container( $container );
 907                 $this->connContainers[$container] = $contObj; // cache it
 908                 return $contObj;
 909         }
 910
 911         /**
 912          * Delete a Swift container
 913          *
 914          * @param $container string Container name
 915          * @return void
 916          * @throws InvalidResponseException
 917          */
 918         protected function deleteContainer( $container ) {
 919                 $conn = $this->getConnection(); // Swift proxy connection
 920                 $conn->delete_container( $container );
 921                 unset( $this->connContainers[$container] ); // purge cache
 922         }
 923
 924         /**
 925          * @see FileBackendStore::doPrimeContainerCache()
 926          * @return void
 927          */
 928         protected function doPrimeContainerCache( array $containerInfo ) {
 929                 try {
 930                         $conn = $this->getConnection(); // Swift proxy connection
 931                         foreach ( $containerInfo as $container => $info ) {
 932                                 $this->connContainers[$container] = new CF_Container(
 933                                         $conn->cfs_auth,
 934                                         $conn->cfs_http,
 935                                         $container,
 936                                         $info['count'],
 937                                         $info['bytes']
 938                                 );
 939                         }
 940                 } catch ( InvalidResponseException $e ) {
 941                 } catch ( Exception $e ) { // some other exception?
 942                         $this->logException( $e, __METHOD__, array() );
 943                 }
 944         }
 945
 946         /**
 947          * Log an unexpected exception for this backend
 948          *
 949          * @param $e Exception
 950          * @param $func string
 951          * @param $params Array
 952          * @return void
 953          */
 954         protected function logException( Exception $e, $func, array $params ) {
 955                 wfDebugLog( 'SwiftBackend',
 956                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
 957                         ( $e instanceof InvalidResponseException
 958                                 ? ": {$e->getMessage()}"
 959                                 : ""
 960                         )
 961                 );
 962         }
 963 }
 964
 965 /**
 966  * SwiftFileBackend helper class to page through listings.
 967  * Swift also has a listing limit of 10,000 objects for sanity.
 968  * Do not use this class from places outside SwiftFileBackend.
 969  *
 970  * @ingroup FileBackend
 971  */
 972 abstract class SwiftFileBackendList implements Iterator {
 973         /** @var Array */
 974         protected $bufferIter = array();
 975         protected $bufferAfter = null; // string; list items *after* this path
 976         protected $pos = 0; // integer
 977         /** @var Array */
 978         protected $params = array();
 979
 980         /** @var SwiftFileBackend */
 981         protected $backend;
 982         protected $container; // string; container name
 983         protected $dir; // string; storage directory
 984         protected $suffixStart; // integer
 985
 986         const PAGE_SIZE = 5000; // file listing buffer size
 987
 988         /**
 989          * @param $backend SwiftFileBackend
 990          * @param $fullCont string Resolved container name
 991          * @param $dir string Resolved directory relative to container
 992          * @param $params Array
 993          */
 994         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
 995                 $this->backend = $backend;
 996                 $this->container = $fullCont;
 997                 $this->dir = $dir;
 998                 if ( substr( $this->dir, -1 ) === '/' ) {
 999                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1000                 }
1001                 if ( $this->dir == '' ) { // whole container
1002                         $this->suffixStart = 0;
1003                 } else { // dir within container
1004                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1005                 }
1006                 $this->params = $params;
1007         }
1008
1009         /**
1010          * @see Iterator::key()
1011          * @return integer
1012          */
1013         public function key() {
1014                 return $this->pos;
1015         }
1016
1017         /**
1018          * @see Iterator::next()
1019          * @return void
1020          */
1021         public function next() {
1022                 // Advance to the next file in the page
1023                 next( $this->bufferIter );
1024                 ++$this->pos;
1025                 // Check if there are no files left in this page and
1026                 // advance to the next page if this page was not empty.
1027                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1028                         $this->bufferIter = $this->pageFromList(
1029                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1030                         ); // updates $this->bufferAfter
1031                 }
1032         }
1033
1034         /**
1035          * @see Iterator::rewind()
1036          * @return void
1037          */
1038         public function rewind() {
1039                 $this->pos = 0;
1040                 $this->bufferAfter = null;
1041                 $this->bufferIter = $this->pageFromList(
1042                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1043                 ); // updates $this->bufferAfter
1044         }
1045
1046         /**
1047          * @see Iterator::valid()
1048          * @return bool
1049          */
1050         public function valid() {
1051                 if ( $this->bufferIter === null ) {
1052                         return false; // some failure?
1053                 } else {
1054                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1055                 }
1056         }
1057
1058         /**
1059          * Get the given list portion (page)
1060          *
1061          * @param $container string Resolved container name
1062          * @param $dir string Resolved path relative to container
1063          * @param $after string|null
1064          * @param $limit integer
1065          * @param $params Array
1066          * @return Traversable|Array|null Returns null on failure
1067          */
1068         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1069 }
1070
1071 /**
1072  * Iterator for listing directories
1073  */
1074 class SwiftFileBackendDirList extends SwiftFileBackendList {
1075         /**
1076          * @see Iterator::current()
1077          * @return string|bool String (relative path) or false
1078          */
1079         public function current() {
1080                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1081         }
1082
1083         /**
1084          * @see SwiftFileBackendList::pageFromList()
1085          * @return Array|null
1086          */
1087         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1088                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1089         }
1090 }
1091
1092 /**
1093  * Iterator for listing regular files
1094  */
1095 class SwiftFileBackendFileList extends SwiftFileBackendList {
1096         /**
1097          * @see Iterator::current()
1098          * @return string|bool String (relative path) or false
1099          */
1100         public function current() {
1101                 return substr( current( $this->bufferIter ), $this->suffixStart );
1102         }
1103
1104         /**
1105          * @see SwiftFileBackendList::pageFromList()
1106          * @return Array|null
1107          */
1108         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1109                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1110         }
1111 }