includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $maxContCacheSize = 300; // integer; max containers with entries
  45
  46         /** @var CF_Connection */
  47         protected $conn; // Swift connection handle
  48         protected $connStarted = 0; // integer UNIX timestamp
  49         protected $connContainers = array(); // container object cache
  50
  51         /**
  52          * @see FileBackendStore::__construct()
  53          * Additional $config params include:
  54          *    swiftAuthUrl       : Swift authentication server URL
  55          *    swiftUser          : Swift user used by MediaWiki (account:username)
  56          *    swiftKey           : Swift authentication key for the above user
  57          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  58          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  59          *    shardViaHashLevels : Map of container names to sharding config with:
  60          *                         'base'   : base of hash characters, 16 or 36
  61          *                         'levels' : the number of hash levels (and digits)
  62          *                         'repeat' : hash subdirectories are prefixed with all the
  63          *                                    parent hash directory names (e.g. "a/ab/abc")
  64          */
  65         public function __construct( array $config ) {
  66                 parent::__construct( $config );
  67                 // Required settings
  68                 $this->auth = new CF_Authentication(
  69                         $config['swiftUser'],
  70                         $config['swiftKey'],
  71                         null, // account; unused
  72                         $config['swiftAuthUrl']
  73                 );
  74                 // Optional settings
  75                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  76                         ? $config['swiftAuthTTL']
  77                         : 5 * 60; // some sane number
  78                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  79                         ? $config['swiftAnonUser']
  80                         : '';
  81                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  82                         ? $config['shardViaHashLevels']
  83                         : '';
  84                 // Cache container info to mask latency
  85                 $this->memCache = wfGetMainCache();
  86         }
  87
  88         /**
  89          * @see FileBackendStore::resolveContainerPath()
  90          * @return null
  91          */
  92         protected function resolveContainerPath( $container, $relStoragePath ) {
  93                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  94                         return null; // too long for Swift
  95                 }
  96                 return $relStoragePath;
  97         }
  98
  99         /**
 100          * @see FileBackendStore::isPathUsableInternal()
 101          * @return bool
 102          */
 103         public function isPathUsableInternal( $storagePath ) {
 104                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 105                 if ( $rel === null ) {
 106                         return false; // invalid
 107                 }
 108
 109                 try {
 110                         $this->getContainer( $container );
 111                         return true; // container exists
 112                 } catch ( NoSuchContainerException $e ) {
 113                 } catch ( InvalidResponseException $e ) {
 114                 } catch ( Exception $e ) { // some other exception?
 115                         $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
 116                 }
 117
 118                 return false;
 119         }
 120
 121         /**
 122          * @see FileBackendStore::doCreateInternal()
 123          * @return Status
 124          */
 125         protected function doCreateInternal( array $params ) {
 126                 $status = Status::newGood();
 127
 128                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 129                 if ( $dstRel === null ) {
 130                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 131                         return $status;
 132                 }
 133
 134                 // (a) Check the destination container and object
 135                 try {
 136                         $dContObj = $this->getContainer( $dstCont );
 137                         if ( empty( $params['overwrite'] ) &&
 138                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 139                         {
 140                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 141                                 return $status;
 142                         }
 143                 } catch ( NoSuchContainerException $e ) {
 144                         $status->fatal( 'backend-fail-create', $params['dst'] );
 145                         return $status;
 146                 } catch ( InvalidResponseException $e ) {
 147                         $status->fatal( 'backend-fail-connect', $this->name );
 148                         return $status;
 149                 } catch ( Exception $e ) { // some other exception?
 150                         $status->fatal( 'backend-fail-internal', $this->name );
 151                         $this->logException( $e, __METHOD__, $params );
 152                         return $status;
 153                 }
 154
 155                 // (b) Get a SHA-1 hash of the object
 156                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 157
 158                 // (c) Actually create the object
 159                 try {
 160                         // Create a fresh CF_Object with no fields preloaded.
 161                         // We don't want to preserve headers, metadata, and such.
 162                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 163                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 164                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 165                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 166                         // The MD5 here will be checked within Swift against its own MD5.
 167                         $obj->set_etag( md5( $params['content'] ) );
 168                         // Use the same content type as StreamFile for security
 169                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 170                         // Actually write the object in Swift
 171                         $obj->write( $params['content'] );
 172                 } catch ( BadContentTypeException $e ) {
 173                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 174                 } catch ( InvalidResponseException $e ) {
 175                         $status->fatal( 'backend-fail-connect', $this->name );
 176                 } catch ( Exception $e ) { // some other exception?
 177                         $status->fatal( 'backend-fail-internal', $this->name );
 178                         $this->logException( $e, __METHOD__, $params );
 179                 }
 180
 181                 return $status;
 182         }
 183
 184         /**
 185          * @see FileBackendStore::doStoreInternal()
 186          * @return Status
 187          */
 188         protected function doStoreInternal( array $params ) {
 189                 $status = Status::newGood();
 190
 191                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 192                 if ( $dstRel === null ) {
 193                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 194                         return $status;
 195                 }
 196
 197                 // (a) Check the destination container and object
 198                 try {
 199                         $dContObj = $this->getContainer( $dstCont );
 200                         if ( empty( $params['overwrite'] ) &&
 201                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 202                         {
 203                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 204                                 return $status;
 205                         }
 206                 } catch ( NoSuchContainerException $e ) {
 207                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 208                         return $status;
 209                 } catch ( InvalidResponseException $e ) {
 210                         $status->fatal( 'backend-fail-connect', $this->name );
 211                         return $status;
 212                 } catch ( Exception $e ) { // some other exception?
 213                         $status->fatal( 'backend-fail-internal', $this->name );
 214                         $this->logException( $e, __METHOD__, $params );
 215                         return $status;
 216                 }
 217
 218                 // (b) Get a SHA-1 hash of the object
 219                 $sha1Hash = sha1_file( $params['src'] );
 220                 if ( $sha1Hash === false ) { // source doesn't exist?
 221                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 222                         return $status;
 223                 }
 224                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 225
 226                 // (c) Actually store the object
 227                 try {
 228                         // Create a fresh CF_Object with no fields preloaded.
 229                         // We don't want to preserve headers, metadata, and such.
 230                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 231                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 232                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 233                         // The MD5 here will be checked within Swift against its own MD5.
 234                         $obj->set_etag( md5_file( $params['src'] ) );
 235                         // Use the same content type as StreamFile for security
 236                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 237                         // Actually write the object in Swift
 238                         $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
 239                 } catch ( BadContentTypeException $e ) {
 240                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 241                 } catch ( IOException $e ) {
 242                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 243                 } catch ( InvalidResponseException $e ) {
 244                         $status->fatal( 'backend-fail-connect', $this->name );
 245                 } catch ( Exception $e ) { // some other exception?
 246                         $status->fatal( 'backend-fail-internal', $this->name );
 247                         $this->logException( $e, __METHOD__, $params );
 248                 }
 249
 250                 return $status;
 251         }
 252
 253         /**
 254          * @see FileBackendStore::doCopyInternal()
 255          * @return Status
 256          */
 257         protected function doCopyInternal( array $params ) {
 258                 $status = Status::newGood();
 259
 260                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 261                 if ( $srcRel === null ) {
 262                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 263                         return $status;
 264                 }
 265
 266                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 267                 if ( $dstRel === null ) {
 268                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 269                         return $status;
 270                 }
 271
 272                 // (a) Check the source/destination containers and destination object
 273                 try {
 274                         $sContObj = $this->getContainer( $srcCont );
 275                         $dContObj = $this->getContainer( $dstCont );
 276                         if ( empty( $params['overwrite'] ) &&
 277                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 278                         {
 279                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 280                                 return $status;
 281                         }
 282                 } catch ( NoSuchContainerException $e ) {
 283                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 284                         return $status;
 285                 } catch ( InvalidResponseException $e ) {
 286                         $status->fatal( 'backend-fail-connect', $this->name );
 287                         return $status;
 288                 } catch ( Exception $e ) { // some other exception?
 289                         $status->fatal( 'backend-fail-internal', $this->name );
 290                         $this->logException( $e, __METHOD__, $params );
 291                         return $status;
 292                 }
 293
 294                 // (b) Actually copy the file to the destination
 295                 try {
 296                         $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 297                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 298                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 299                 } catch ( InvalidResponseException $e ) {
 300                         $status->fatal( 'backend-fail-connect', $this->name );
 301                 } catch ( Exception $e ) { // some other exception?
 302                         $status->fatal( 'backend-fail-internal', $this->name );
 303                         $this->logException( $e, __METHOD__, $params );
 304                 }
 305
 306                 return $status;
 307         }
 308
 309         /**
 310          * @see FileBackendStore::doDeleteInternal()
 311          * @return Status
 312          */
 313         protected function doDeleteInternal( array $params ) {
 314                 $status = Status::newGood();
 315
 316                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 317                 if ( $srcRel === null ) {
 318                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 319                         return $status;
 320                 }
 321
 322                 try {
 323                         $sContObj = $this->getContainer( $srcCont );
 324                         $sContObj->delete_object( $srcRel );
 325                 } catch ( NoSuchContainerException $e ) {
 326                         $status->fatal( 'backend-fail-delete', $params['src'] );
 327                 } catch ( NoSuchObjectException $e ) {
 328                         if ( empty( $params['ignoreMissingSource'] ) ) {
 329                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 330                         }
 331                 } catch ( InvalidResponseException $e ) {
 332                         $status->fatal( 'backend-fail-connect', $this->name );
 333                 } catch ( Exception $e ) { // some other exception?
 334                         $status->fatal( 'backend-fail-internal', $this->name );
 335                         $this->logException( $e, __METHOD__, $params );
 336                 }
 337
 338                 return $status;
 339         }
 340
 341         /**
 342          * @see FileBackendStore::doPrepareInternal()
 343          * @return Status
 344          */
 345         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 346                 $status = Status::newGood();
 347
 348                 // (a) Check if container already exists
 349                 try {
 350                         $contObj = $this->getContainer( $fullCont );
 351                         // NoSuchContainerException not thrown: container must exist
 352                         return $status; // already exists
 353                 } catch ( NoSuchContainerException $e ) {
 354                         // NoSuchContainerException thrown: container does not exist
 355                 } catch ( InvalidResponseException $e ) {
 356                         $status->fatal( 'backend-fail-connect', $this->name );
 357                         return $status;
 358                 } catch ( Exception $e ) { // some other exception?
 359                         $status->fatal( 'backend-fail-internal', $this->name );
 360                         $this->logException( $e, __METHOD__, $params );
 361                         return $status;
 362                 }
 363
 364                 // (b) Create container as needed
 365                 try {
 366                         $contObj = $this->createContainer( $fullCont );
 367                         if ( $this->swiftAnonUser != '' ) {
 368                                 // Make container public to end-users...
 369                                 $status->merge( $this->setContainerAccess(
 370                                         $contObj,
 371                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 372                                         array( $this->auth->username ) // write
 373                                 ) );
 374                         }
 375                 } catch ( InvalidResponseException $e ) {
 376                         $status->fatal( 'backend-fail-connect', $this->name );
 377                         return $status;
 378                 } catch ( Exception $e ) { // some other exception?
 379                         $status->fatal( 'backend-fail-internal', $this->name );
 380                         $this->logException( $e, __METHOD__, $params );
 381                         return $status;
 382                 }
 383
 384                 return $status;
 385         }
 386
 387         /**
 388          * @see FileBackendStore::doSecureInternal()
 389          * @return Status
 390          */
 391         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 392                 $status = Status::newGood();
 393
 394                 if ( $this->swiftAnonUser != '' ) {
 395                         // Restrict container from end-users...
 396                         try {
 397                                 // doPrepareInternal() should have been called,
 398                                 // so the Swift container should already exist...
 399                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 400                                 // NoSuchContainerException not thrown: container must exist
 401                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 402                                         $status->merge( $this->setContainerAccess(
 403                                                 $contObj,
 404                                                 array( $this->auth->username ), // read
 405                                                 array( $this->auth->username ) // write
 406                                         ) );
 407                                         // @TODO: when php-cloudfiles supports container
 408                                         // metadata, we can make use of that to avoid RTTs
 409                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 410                                 }
 411                         } catch ( InvalidResponseException $e ) {
 412                                 $status->fatal( 'backend-fail-connect', $this->name );
 413                         } catch ( Exception $e ) { // some other exception?
 414                                 $status->fatal( 'backend-fail-internal', $this->name );
 415                                 $this->logException( $e, __METHOD__, $params );
 416                         }
 417                 }
 418
 419                 return $status;
 420         }
 421
 422         /**
 423          * @see FileBackendStore::doCleanInternal()
 424          * @return Status
 425          */
 426         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 427                 $status = Status::newGood();
 428
 429                 // Only containers themselves can be removed, all else is virtual
 430                 if ( $dir != '' ) {
 431                         return $status; // nothing to do
 432                 }
 433
 434                 // (a) Check the container
 435                 try {
 436                         $contObj = $this->getContainer( $fullCont, true );
 437                 } catch ( NoSuchContainerException $e ) {
 438                         return $status; // ok, nothing to do
 439                 } catch ( InvalidResponseException $e ) {
 440                         $status->fatal( 'backend-fail-connect', $this->name );
 441                         return $status;
 442                 } catch ( Exception $e ) { // some other exception?
 443                         $status->fatal( 'backend-fail-internal', $this->name );
 444                         $this->logException( $e, __METHOD__, $params );
 445                         return $status;
 446                 }
 447
 448                 // (b) Delete the container if empty
 449                 if ( $contObj->object_count == 0 ) {
 450                         try {
 451                                 $this->deleteContainer( $fullCont );
 452                         } catch ( NoSuchContainerException $e ) {
 453                                 return $status; // race?
 454                         } catch ( InvalidResponseException $e ) {
 455                                 $status->fatal( 'backend-fail-connect', $this->name );
 456                                 return $status;
 457                         } catch ( Exception $e ) { // some other exception?
 458                                 $status->fatal( 'backend-fail-internal', $this->name );
 459                                 $this->logException( $e, __METHOD__, $params );
 460                                 return $status;
 461                         }
 462                 }
 463
 464                 return $status;
 465         }
 466
 467         /**
 468          * @see FileBackendStore::doFileExists()
 469          * @return array|bool|null
 470          */
 471         protected function doGetFileStat( array $params ) {
 472                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 473                 if ( $srcRel === null ) {
 474                         return false; // invalid storage path
 475                 }
 476
 477                 $stat = false;
 478                 try {
 479                         $contObj = $this->getContainer( $srcCont );
 480                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 481                         $this->addMissingMetadata( $srcObj, $params['src'] );
 482                         $stat = array(
 483                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 484                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 485                                 'size'  => $srcObj->content_length,
 486                                 'sha1'  => $srcObj->metadata['Sha1base36']
 487                         );
 488                 } catch ( NoSuchContainerException $e ) {
 489                 } catch ( NoSuchObjectException $e ) {
 490                 } catch ( InvalidResponseException $e ) {
 491                         $stat = null;
 492                 } catch ( Exception $e ) { // some other exception?
 493                         $stat = null;
 494                         $this->logException( $e, __METHOD__, $params );
 495                 }
 496
 497                 return $stat;
 498         }
 499
 500         /**
 501          * Fill in any missing object metadata and save it to Swift
 502          *
 503          * @param $obj CF_Object
 504          * @param $path string Storage path to object
 505          * @return bool Success
 506          * @throws Exception cloudfiles exceptions
 507          */
 508         protected function addMissingMetadata( CF_Object $obj, $path ) {
 509                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 510                         return true; // nothing to do
 511                 }
 512                 $status = Status::newGood();
 513                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 514                 if ( $status->isOK() ) {
 515                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 516                         if ( $tmpFile ) {
 517                                 $hash = $tmpFile->getSha1Base36();
 518                                 if ( $hash !== false ) {
 519                                         $obj->metadata['Sha1base36'] = $hash;
 520                                         $obj->sync_metadata(); // save to Swift
 521                                         return true; // success
 522                                 }
 523                         }
 524                 }
 525                 $obj->metadata['Sha1base36'] = false;
 526                 return false; // failed
 527         }
 528
 529         /**
 530          * @see FileBackend::getFileContents()
 531          * @return bool|null|string
 532          */
 533         public function getFileContents( array $params ) {
 534                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 535                 if ( $srcRel === null ) {
 536                         return false; // invalid storage path
 537                 }
 538
 539                 if ( !$this->fileExists( $params ) ) {
 540                         return null;
 541                 }
 542
 543                 $data = false;
 544                 try {
 545                         $sContObj = $this->getContainer( $srcCont );
 546                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 547                         $data = $obj->read( $this->headersFromParams( $params ) );
 548                 } catch ( NoSuchContainerException $e ) {
 549                 } catch ( InvalidResponseException $e ) {
 550                 } catch ( Exception $e ) { // some other exception?
 551                         $this->logException( $e, __METHOD__, $params );
 552                 }
 553
 554                 return $data;
 555         }
 556
 557         /**
 558          * @see FileBackendStore::doDirectoryExists()
 559          * @return bool|null
 560          */
 561         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 562                 try {
 563                         $container = $this->getContainer( $fullCont );
 564                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 565                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 566                 } catch ( NoSuchContainerException $e ) {
 567                         return false;
 568                 } catch ( InvalidResponseException $e ) {
 569                 } catch ( Exception $e ) { // some other exception?
 570                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 571                 }
 572
 573                 return null; // error
 574         }
 575
 576         /**
 577          * @see FileBackendStore::getDirectoryListInternal()
 578          * @return SwiftFileBackendDirList
 579          */
 580         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 581                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 582         }
 583
 584         /**
 585          * @see FileBackendStore::getFileListInternal()
 586          * @return SwiftFileBackendFileList
 587          */
 588         public function getFileListInternal( $fullCont, $dir, array $params ) {
 589                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 590         }
 591
 592         /**
 593          * Do not call this function outside of SwiftFileBackendFileList
 594          *
 595          * @param $fullCont string Resolved container name
 596          * @param $dir string Resolved storage directory with no trailing slash
 597          * @param $after string|null Storage path of file to list items after
 598          * @param $limit integer Max number of items to list
 599          * @param $params Array Includes flag for 'topOnly'
 600          * @return Array List of relative paths of dirs directly under $dir
 601          */
 602         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 603                 $dirs = array();
 604
 605                 try {
 606                         $container = $this->getContainer( $fullCont );
 607                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 608                         // Non-recursive: only list dirs right under $dir
 609                         if ( !empty( $params['topOnly'] ) ) {
 610                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 611                                 foreach ( $objects as $object ) { // files and dirs
 612                                         if ( substr( $object, -1 ) === '/' ) {
 613                                                 $dirs[] = $object; // directories end in '/'
 614                                         }
 615                                         $after = $object; // update last item
 616                                 }
 617                         // Recursive: list all dirs under $dir and its subdirs
 618                         } else {
 619                                 // Get directory from last item of prior page
 620                                 $lastDir = $this->getParentDir( $after ); // must be first page
 621                                 $objects = $container->list_objects( $limit, $after, $prefix );
 622                                 foreach ( $objects as $object ) { // files
 623                                         $objectDir = $this->getParentDir( $object ); // directory of object
 624                                         if ( $objectDir !== false ) { // file has a parent dir
 625                                                 // Swift stores paths in UTF-8, using binary sorting.
 626                                                 // See function "create_container_table" in common/db.py.
 627                                                 // If a directory is not "greater" than the last one,
 628                                                 // then it was already listed by the calling iterator.
 629                                                 if ( $objectDir > $lastDir ) {
 630                                                         $pDir = $objectDir;
 631                                                         do { // add dir and all its parent dirs
 632                                                                 $dirs[] = "{$pDir}/";
 633                                                                 $pDir = $this->getParentDir( $pDir );
 634                                                         } while ( $pDir !== false // sanity
 635                                                                 && $pDir > $lastDir // not done already
 636                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 637                                                         );
 638                                                 }
 639                                                 $lastDir = $objectDir;
 640                                         }
 641                                         $after = $object; // update last item
 642                                 }
 643                         }
 644                 } catch ( NoSuchContainerException $e ) {
 645                 } catch ( InvalidResponseException $e ) {
 646                 } catch ( Exception $e ) { // some other exception?
 647                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 648                 }
 649
 650                 return $dirs;
 651         }
 652
 653         protected function getParentDir( $path ) {
 654                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 655         }
 656
 657         /**
 658          * Do not call this function outside of SwiftFileBackendFileList
 659          *
 660          * @param $fullCont string Resolved container name
 661          * @param $dir string Resolved storage directory with no trailing slash
 662          * @param $after string|null Storage path of file to list items after
 663          * @param $limit integer Max number of items to list
 664          * @param $params Array Includes flag for 'topOnly'
 665          * @return Array List of relative paths of files under $dir
 666          */
 667         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 668                 $files = array();
 669
 670                 try {
 671                         $container = $this->getContainer( $fullCont );
 672                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 673                         // Non-recursive: only list files right under $dir
 674                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 675                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 676                                 foreach ( $objects as $object ) {
 677                                         if ( substr( $object, -1 ) !== '/' ) {
 678                                                 $files[] = $object; // directories end in '/'
 679                                         }
 680                                 }
 681                         // Recursive: list all files under $dir and its subdirs
 682                         } else { // files
 683                                 $files = $container->list_objects( $limit, $after, $prefix );
 684                         }
 685                         $after = end( $files ); // update last item
 686                         reset( $files ); // reset pointer
 687                 } catch ( NoSuchContainerException $e ) {
 688                 } catch ( InvalidResponseException $e ) {
 689                 } catch ( Exception $e ) { // some other exception?
 690                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 691                 }
 692
 693                 return $files;
 694         }
 695
 696         /**
 697          * @see FileBackendStore::doGetFileSha1base36()
 698          * @return bool
 699          */
 700         protected function doGetFileSha1base36( array $params ) {
 701                 $stat = $this->getFileStat( $params );
 702                 if ( $stat ) {
 703                         return $stat['sha1'];
 704                 } else {
 705                         return false;
 706                 }
 707         }
 708
 709         /**
 710          * @see FileBackendStore::doStreamFile()
 711          * @return Status
 712          */
 713         protected function doStreamFile( array $params ) {
 714                 $status = Status::newGood();
 715
 716                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 717                 if ( $srcRel === null ) {
 718                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 719                 }
 720
 721                 try {
 722                         $cont = $this->getContainer( $srcCont );
 723                 } catch ( NoSuchContainerException $e ) {
 724                         $status->fatal( 'backend-fail-stream', $params['src'] );
 725                         return $status;
 726                 } catch ( InvalidResponseException $e ) {
 727                         $status->fatal( 'backend-fail-connect', $this->name );
 728                         return $status;
 729                 } catch ( Exception $e ) { // some other exception?
 730                         $status->fatal( 'backend-fail-stream', $params['src'] );
 731                         $this->logException( $e, __METHOD__, $params );
 732                         return $status;
 733                 }
 734
 735                 try {
 736                         $output = fopen( 'php://output', 'wb' );
 737                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 738                         $obj->stream( $output, $this->headersFromParams( $params ) );
 739                 } catch ( InvalidResponseException $e ) { // 404? connection problem?
 740                         $status->fatal( 'backend-fail-stream', $params['src'] );
 741                 } catch ( Exception $e ) { // some other exception?
 742                         $status->fatal( 'backend-fail-stream', $params['src'] );
 743                         $this->logException( $e, __METHOD__, $params );
 744                 }
 745
 746                 return $status;
 747         }
 748
 749         /**
 750          * @see FileBackendStore::getLocalCopy()
 751          * @return null|TempFSFile
 752          */
 753         public function getLocalCopy( array $params ) {
 754                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 755                 if ( $srcRel === null ) {
 756                         return null;
 757                 }
 758
 759                 if ( !$this->fileExists( $params ) ) {
 760                         return null;
 761                 }
 762
 763                 $tmpFile = null;
 764                 try {
 765                         $sContObj = $this->getContainer( $srcCont );
 766                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 767                         // Get source file extension
 768                         $ext = FileBackend::extensionFromPath( $srcRel );
 769                         // Create a new temporary file...
 770                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 771                         if ( $tmpFile ) {
 772                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 773                                 if ( $handle ) {
 774                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 775                                         fclose( $handle );
 776                                 } else {
 777                                         $tmpFile = null; // couldn't open temp file
 778                                 }
 779                         }
 780                 } catch ( NoSuchContainerException $e ) {
 781                         $tmpFile = null;
 782                 } catch ( InvalidResponseException $e ) {
 783                         $tmpFile = null;
 784                 } catch ( Exception $e ) { // some other exception?
 785                         $tmpFile = null;
 786                         $this->logException( $e, __METHOD__, $params );
 787                 }
 788
 789                 return $tmpFile;
 790         }
 791
 792         /**
 793          * @see FileBackendStore::directoriesAreVirtual()
 794          * @return bool
 795          */
 796         protected function directoriesAreVirtual() {
 797                 return true;
 798         }
 799
 800         /**
 801          * Get headers to send to Swift when reading a file based
 802          * on a FileBackend params array, e.g. that of getLocalCopy().
 803          * $params is currently only checked for a 'latest' flag.
 804          *
 805          * @param $params Array
 806          * @return Array
 807          */
 808         protected function headersFromParams( array $params ) {
 809                 $hdrs = array();
 810                 if ( !empty( $params['latest'] ) ) {
 811                         $hdrs[] = 'X-Newest: true';
 812                 }
 813                 return $hdrs;
 814         }
 815
 816         /**
 817          * Set read/write permissions for a Swift container
 818          *
 819          * @param $contObj CF_Container Swift container
 820          * @param $readGrps Array Swift users who can read (account:user)
 821          * @param $writeGrps Array Swift users who can write (account:user)
 822          * @return Status
 823          */
 824         protected function setContainerAccess(
 825                 CF_Container $contObj, array $readGrps, array $writeGrps
 826         ) {
 827                 $creds = $contObj->cfs_auth->export_credentials();
 828
 829                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 830
 831                 // Note: 10 second timeout consistent with php-cloudfiles
 832                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 833                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 834                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 835                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 836
 837                 return $req->execute(); // should return 204
 838         }
 839
 840         /**
 841          * Get a connection to the Swift proxy
 842          *
 843          * @return CF_Connection|bool False on failure
 844          * @throws InvalidResponseException
 845          */
 846         protected function getConnection() {
 847                 if ( $this->conn === false ) {
 848                         throw new InvalidResponseException; // failed last attempt
 849                 }
 850                 // Session keys expire after a while, so we renew them periodically
 851                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 852                         $this->conn->close(); // close active cURL connections
 853                         $this->conn = null;
 854                 }
 855                 // Authenticate with proxy and get a session key...
 856                 if ( $this->conn === null ) {
 857                         $this->connContainers = array();
 858                         try {
 859                                 $this->auth->authenticate();
 860                                 $this->conn = new CF_Connection( $this->auth );
 861                                 $this->connStarted = time();
 862                         } catch ( AuthenticationException $e ) {
 863                                 $this->conn = false; // don't keep re-trying
 864                         } catch ( InvalidResponseException $e ) {
 865                                 $this->conn = false; // don't keep re-trying
 866                         }
 867                 }
 868                 if ( !$this->conn ) {
 869                         throw new InvalidResponseException; // auth/connection problem
 870                 }
 871                 return $this->conn;
 872         }
 873
 874         /**
 875          * @see FileBackendStore::doClearCache()
 876          */
 877         protected function doClearCache( array $paths = null ) {
 878                 $this->connContainers = array(); // clear container object cache
 879         }
 880
 881         /**
 882          * Get a Swift container object, possibly from process cache.
 883          * Use $reCache if the file count or byte count is needed.
 884          *
 885          * @param $container string Container name
 886          * @param $bypassCache bool Bypass all caches and load from Swift
 887          * @return CF_Container
 888          * @throws InvalidResponseException
 889          */
 890         protected function getContainer( $container, $bypassCache = false ) {
 891                 $conn = $this->getConnection(); // Swift proxy connection
 892                 if ( $bypassCache ) { // purge cache
 893                         unset( $this->connContainers[$container] );
 894                 } elseif ( !isset( $this->connContainers[$container] ) ) {
 895                         $this->primeContainerCache( array( $container ) ); // check persistent cache
 896                 }
 897                 if ( !isset( $this->connContainers[$container] ) ) {
 898                         $contObj = $conn->get_container( $container );
 899                         // NoSuchContainerException not thrown: container must exist
 900                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 901                                 reset( $this->connContainers );
 902                                 unset( $this->connContainers[key( $this->connContainers )] );
 903                         }
 904                         $this->connContainers[$container] = $contObj; // cache it
 905                         if ( !$bypassCache ) {
 906                                 $this->setContainerCache( $container, // update persistent cache
 907                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
 908                                 );
 909                         }
 910                 }
 911                 return $this->connContainers[$container];
 912         }
 913
 914         /**
 915          * Create a Swift container
 916          *
 917          * @param $container string Container name
 918          * @return CF_Container
 919          * @throws InvalidResponseException
 920          */
 921         protected function createContainer( $container ) {
 922                 $conn = $this->getConnection(); // Swift proxy connection
 923                 $contObj = $conn->create_container( $container );
 924                 $this->connContainers[$container] = $contObj; // cache it
 925                 return $contObj;
 926         }
 927
 928         /**
 929          * Delete a Swift container
 930          *
 931          * @param $container string Container name
 932          * @return void
 933          * @throws InvalidResponseException
 934          */
 935         protected function deleteContainer( $container ) {
 936                 $conn = $this->getConnection(); // Swift proxy connection
 937                 $conn->delete_container( $container );
 938                 unset( $this->connContainers[$container] ); // purge cache
 939         }
 940
 941         /**
 942          * @see FileBackendStore::doPrimeContainerCache()
 943          * @return void
 944          */
 945         protected function doPrimeContainerCache( array $containerInfo ) {
 946                 try {
 947                         $conn = $this->getConnection(); // Swift proxy connection
 948                         foreach ( $containerInfo as $container => $info ) {
 949                                 $this->connContainers[$container] = new CF_Container(
 950                                         $conn->cfs_auth,
 951                                         $conn->cfs_http,
 952                                         $container,
 953                                         $info['count'],
 954                                         $info['bytes']
 955                                 );
 956                         }
 957                 } catch ( InvalidResponseException $e ) {
 958                 } catch ( Exception $e ) { // some other exception?
 959                         $this->logException( $e, __METHOD__, array() );
 960                 }
 961         }
 962
 963         /**
 964          * Log an unexpected exception for this backend
 965          *
 966          * @param $e Exception
 967          * @param $func string
 968          * @param $params Array
 969          * @return void
 970          */
 971         protected function logException( Exception $e, $func, array $params ) {
 972                 wfDebugLog( 'SwiftBackend',
 973                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
 974                         ( $e instanceof InvalidResponseException
 975                                 ? ": {$e->getMessage()}"
 976                                 : ""
 977                         )
 978                 );
 979         }
 980 }
 981
 982 /**
 983  * SwiftFileBackend helper class to page through listings.
 984  * Swift also has a listing limit of 10,000 objects for sanity.
 985  * Do not use this class from places outside SwiftFileBackend.
 986  *
 987  * @ingroup FileBackend
 988  */
 989 abstract class SwiftFileBackendList implements Iterator {
 990         /** @var Array */
 991         protected $bufferIter = array();
 992         protected $bufferAfter = null; // string; list items *after* this path
 993         protected $pos = 0; // integer
 994         /** @var Array */
 995         protected $params = array();
 996
 997         /** @var SwiftFileBackend */
 998         protected $backend;
 999         protected $container; // string; container name
1000         protected $dir; // string; storage directory
1001         protected $suffixStart; // integer
1002
1003         const PAGE_SIZE = 5000; // file listing buffer size
1004
1005         /**
1006          * @param $backend SwiftFileBackend
1007          * @param $fullCont string Resolved container name
1008          * @param $dir string Resolved directory relative to container
1009          * @param $params Array
1010          */
1011         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1012                 $this->backend = $backend;
1013                 $this->container = $fullCont;
1014                 $this->dir = $dir;
1015                 if ( substr( $this->dir, -1 ) === '/' ) {
1016                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1017                 }
1018                 if ( $this->dir == '' ) { // whole container
1019                         $this->suffixStart = 0;
1020                 } else { // dir within container
1021                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1022                 }
1023                 $this->params = $params;
1024         }
1025
1026         /**
1027          * @see Iterator::key()
1028          * @return integer
1029          */
1030         public function key() {
1031                 return $this->pos;
1032         }
1033
1034         /**
1035          * @see Iterator::next()
1036          * @return void
1037          */
1038         public function next() {
1039                 // Advance to the next file in the page
1040                 next( $this->bufferIter );
1041                 ++$this->pos;
1042                 // Check if there are no files left in this page and
1043                 // advance to the next page if this page was not empty.
1044                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1045                         $this->bufferIter = $this->pageFromList(
1046                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1047                         ); // updates $this->bufferAfter
1048                 }
1049         }
1050
1051         /**
1052          * @see Iterator::rewind()
1053          * @return void
1054          */
1055         public function rewind() {
1056                 $this->pos = 0;
1057                 $this->bufferAfter = null;
1058                 $this->bufferIter = $this->pageFromList(
1059                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1060                 ); // updates $this->bufferAfter
1061         }
1062
1063         /**
1064          * @see Iterator::valid()
1065          * @return bool
1066          */
1067         public function valid() {
1068                 if ( $this->bufferIter === null ) {
1069                         return false; // some failure?
1070                 } else {
1071                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1072                 }
1073         }
1074
1075         /**
1076          * Get the given list portion (page)
1077          *
1078          * @param $container string Resolved container name
1079          * @param $dir string Resolved path relative to container
1080          * @param $after string|null
1081          * @param $limit integer
1082          * @param $params Array
1083          * @return Traversable|Array|null Returns null on failure
1084          */
1085         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1086 }
1087
1088 /**
1089  * Iterator for listing directories
1090  */
1091 class SwiftFileBackendDirList extends SwiftFileBackendList {
1092         /**
1093          * @see Iterator::current()
1094          * @return string|bool String (relative path) or false
1095          */
1096         public function current() {
1097                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1098         }
1099
1100         /**
1101          * @see SwiftFileBackendList::pageFromList()
1102          * @return Array|null
1103          */
1104         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1105                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1106         }
1107 }
1108
1109 /**
1110  * Iterator for listing regular files
1111  */
1112 class SwiftFileBackendFileList extends SwiftFileBackendList {
1113         /**
1114          * @see Iterator::current()
1115          * @return string|bool String (relative path) or false
1116          */
1117         public function current() {
1118                 return substr( current( $this->bufferIter ), $this->suffixStart );
1119         }
1120
1121         /**
1122          * @see SwiftFileBackendList::pageFromList()
1123          * @return Array|null
1124          */
1125         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1126                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1127         }
1128 }