tests/phpunit/includes/filebackend/FileBackendTest.php

   1 <?php
   2
   3 /**
   4  * @group FileRepo
   5  * @group FileBackend
   6  * @group medium
   7  */
   8 class FileBackendTest extends MediaWikiTestCase {
   9
  10         /** @var FileBackend */
  11         private $backend;
  12         /** @var FileBackendMultiWrite */
  13         private $multiBackend;
  14         /** @var FSFileBackend */
  15         public $singleBackend;
  16         private static $backendToUse;
  17
  18         protected function setUp() {
  19                 global $wgFileBackends;
  20                 parent::setUp();
  21                 $tmpDir = $this->getNewTempDirectory();
  22                 if ( $this->getCliArg( 'use-filebackend' ) ) {
  23                         if ( self::$backendToUse ) {
  24                                 $this->singleBackend = self::$backendToUse;
  25                         } else {
  26                                 $name = $this->getCliArg( 'use-filebackend' );
  27                                 $useConfig = [];
  28                                 foreach ( $wgFileBackends as $conf ) {
  29                                         if ( $conf['name'] == $name ) {
  30                                                 $useConfig = $conf;
  31                                                 break;
  32                                         }
  33                                 }
  34                                 $useConfig['name'] = 'localtesting'; // swap name
  35                                 $useConfig['shardViaHashLevels'] = [ // test sharding
  36                                         'unittest-cont1' => [ 'levels' => 1, 'base' => 16, 'repeat' => 1 ]
  37                                 ];
  38                                 if ( isset( $useConfig['fileJournal'] ) ) {
  39                                         $useConfig['fileJournal'] = FileJournal::factory( $useConfig['fileJournal'], $name );
  40                                 }
  41                                 $useConfig['lockManager'] = LockManagerGroup::singleton()->get( $useConfig['lockManager'] );
  42                                 $class = $useConfig['class'];
  43                                 self::$backendToUse = new $class( $useConfig );
  44                                 $this->singleBackend = self::$backendToUse;
  45                         }
  46                 } else {
  47                         $this->singleBackend = new FSFileBackend( [
  48                                 'name' => 'localtesting',
  49                                 'lockManager' => LockManagerGroup::singleton()->get( 'fsLockManager' ),
  50                                 'wikiId' => wfWikiID(),
  51                                 'containerPaths' => [
  52                                         'unittest-cont1' => "{$tmpDir}/localtesting-cont1",
  53                                         'unittest-cont2' => "{$tmpDir}/localtesting-cont2" ]
  54                         ] );
  55                 }
  56                 $this->multiBackend = new FileBackendMultiWrite( [
  57                         'name' => 'localtesting',
  58                         'lockManager' => LockManagerGroup::singleton()->get( 'fsLockManager' ),
  59                         'parallelize' => 'implicit',
  60                         'wikiId' => wfWikiId() . wfRandomString(),
  61                         'backends' => [
  62                                 [
  63                                         'name' => 'localmultitesting1',
  64                                         'class' => 'FSFileBackend',
  65                                         'containerPaths' => [
  66                                                 'unittest-cont1' => "{$tmpDir}/localtestingmulti1-cont1",
  67                                                 'unittest-cont2' => "{$tmpDir}/localtestingmulti1-cont2" ],
  68                                         'isMultiMaster' => false
  69                                 ],
  70                                 [
  71                                         'name' => 'localmultitesting2',
  72                                         'class' => 'FSFileBackend',
  73                                         'containerPaths' => [
  74                                                 'unittest-cont1' => "{$tmpDir}/localtestingmulti2-cont1",
  75                                                 'unittest-cont2' => "{$tmpDir}/localtestingmulti2-cont2" ],
  76                                         'isMultiMaster' => true
  77                                 ]
  78                         ]
  79                 ] );
  80         }
  81
  82         private static function baseStorePath() {
  83                 return 'mwstore://localtesting';
  84         }
  85
  86         private function backendClass() {
  87                 return get_class( $this->backend );
  88         }
  89
  90         /**
  91          * @dataProvider provider_testIsStoragePath
  92          * @covers FileBackend::isStoragePath
  93          */
  94         public function testIsStoragePath( $path, $isStorePath ) {
  95                 $this->assertEquals( $isStorePath, FileBackend::isStoragePath( $path ),
  96                         "FileBackend::isStoragePath on path '$path'" );
  97         }
  98
  99         public static function provider_testIsStoragePath() {
 100                 return [
 101                         [ 'mwstore://', true ],
 102                         [ 'mwstore://backend', true ],
 103                         [ 'mwstore://backend/container', true ],
 104                         [ 'mwstore://backend/container/', true ],
 105                         [ 'mwstore://backend/container/path', true ],
 106                         [ 'mwstore://backend//container/', true ],
 107                         [ 'mwstore://backend//container//', true ],
 108                         [ 'mwstore://backend//container//path', true ],
 109                         [ 'mwstore:///', true ],
 110                         [ 'mwstore:/', false ],
 111                         [ 'mwstore:', false ],
 112                 ];
 113         }
 114
 115         /**
 116          * @dataProvider provider_testSplitStoragePath
 117          * @covers FileBackend::splitStoragePath
 118          */
 119         public function testSplitStoragePath( $path, $res ) {
 120                 $this->assertEquals( $res, FileBackend::splitStoragePath( $path ),
 121                         "FileBackend::splitStoragePath on path '$path'" );
 122         }
 123
 124         public static function provider_testSplitStoragePath() {
 125                 return [
 126                         [ 'mwstore://backend/container', [ 'backend', 'container', '' ] ],
 127                         [ 'mwstore://backend/container/', [ 'backend', 'container', '' ] ],
 128                         [ 'mwstore://backend/container/path', [ 'backend', 'container', 'path' ] ],
 129                         [ 'mwstore://backend/container//path', [ 'backend', 'container', '/path' ] ],
 130                         [ 'mwstore://backend//container/path', [ null, null, null ] ],
 131                         [ 'mwstore://backend//container//path', [ null, null, null ] ],
 132                         [ 'mwstore://', [ null, null, null ] ],
 133                         [ 'mwstore://backend', [ null, null, null ] ],
 134                         [ 'mwstore:///', [ null, null, null ] ],
 135                         [ 'mwstore:/', [ null, null, null ] ],
 136                         [ 'mwstore:', [ null, null, null ] ]
 137                 ];
 138         }
 139
 140         /**
 141          * @dataProvider provider_normalizeStoragePath
 142          * @covers FileBackend::normalizeStoragePath
 143          */
 144         public function testNormalizeStoragePath( $path, $res ) {
 145                 $this->assertEquals( $res, FileBackend::normalizeStoragePath( $path ),
 146                         "FileBackend::normalizeStoragePath on path '$path'" );
 147         }
 148
 149         public static function provider_normalizeStoragePath() {
 150                 return [
 151                         [ 'mwstore://backend/container', 'mwstore://backend/container' ],
 152                         [ 'mwstore://backend/container/', 'mwstore://backend/container' ],
 153                         [ 'mwstore://backend/container/path', 'mwstore://backend/container/path' ],
 154                         [ 'mwstore://backend/container//path', 'mwstore://backend/container/path' ],
 155                         [ 'mwstore://backend/container///path', 'mwstore://backend/container/path' ],
 156                         [
 157                                 'mwstore://backend/container///path//to///obj',
 158                                 'mwstore://backend/container/path/to/obj'
 159                         ],
 160                         [ 'mwstore://', null ],
 161                         [ 'mwstore://backend', null ],
 162                         [ 'mwstore://backend//container/path', null ],
 163                         [ 'mwstore://backend//container//path', null ],
 164                         [ 'mwstore:///', null ],
 165                         [ 'mwstore:/', null ],
 166                         [ 'mwstore:', null ],
 167                 ];
 168         }
 169
 170         /**
 171          * @dataProvider provider_testParentStoragePath
 172          * @covers FileBackend::parentStoragePath
 173          */
 174         public function testParentStoragePath( $path, $res ) {
 175                 $this->assertEquals( $res, FileBackend::parentStoragePath( $path ),
 176                         "FileBackend::parentStoragePath on path '$path'" );
 177         }
 178
 179         public static function provider_testParentStoragePath() {
 180                 return [
 181                         [ 'mwstore://backend/container/path/to/obj', 'mwstore://backend/container/path/to' ],
 182                         [ 'mwstore://backend/container/path/to', 'mwstore://backend/container/path' ],
 183                         [ 'mwstore://backend/container/path', 'mwstore://backend/container' ],
 184                         [ 'mwstore://backend/container', null ],
 185                         [ 'mwstore://backend/container/path/to/obj/', 'mwstore://backend/container/path/to' ],
 186                         [ 'mwstore://backend/container/path/to/', 'mwstore://backend/container/path' ],
 187                         [ 'mwstore://backend/container/path/', 'mwstore://backend/container' ],
 188                         [ 'mwstore://backend/container/', null ],
 189                 ];
 190         }
 191
 192         /**
 193          * @dataProvider provider_testExtensionFromPath
 194          * @covers FileBackend::extensionFromPath
 195          */
 196         public function testExtensionFromPath( $path, $res ) {
 197                 $this->assertEquals( $res, FileBackend::extensionFromPath( $path ),
 198                         "FileBackend::extensionFromPath on path '$path'" );
 199         }
 200
 201         public static function provider_testExtensionFromPath() {
 202                 return [
 203                         [ 'mwstore://backend/container/path.txt', 'txt' ],
 204                         [ 'mwstore://backend/container/path.svg.png', 'png' ],
 205                         [ 'mwstore://backend/container/path', '' ],
 206                         [ 'mwstore://backend/container/path.', '' ],
 207                 ];
 208         }
 209
 210         /**
 211          * @dataProvider provider_testStore
 212          */
 213         public function testStore( $op ) {
 214                 $this->addTmpFiles( $op['src'] );
 215
 216                 $this->backend = $this->singleBackend;
 217                 $this->tearDownFiles();
 218                 $this->doTestStore( $op );
 219                 $this->tearDownFiles();
 220
 221                 $this->backend = $this->multiBackend;
 222                 $this->tearDownFiles();
 223                 $this->doTestStore( $op );
 224                 $this->tearDownFiles();
 225         }
 226
 227         /**
 228          * @covers FileBackend::doOperation
 229          */
 230         private function doTestStore( $op ) {
 231                 $backendName = $this->backendClass();
 232
 233                 $source = $op['src'];
 234                 $dest = $op['dst'];
 235                 $this->prepare( [ 'dir' => dirname( $dest ) ] );
 236
 237                 file_put_contents( $source, "Unit test file" );
 238
 239                 if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 240                         $this->backend->store( $op );
 241                 }
 242
 243                 $status = $this->backend->doOperation( $op );
 244
 245                 $this->assertGoodStatus( $status,
 246                         "Store from $source to $dest succeeded without warnings ($backendName)." );
 247                 $this->assertEquals( true, $status->isOK(),
 248                         "Store from $source to $dest succeeded ($backendName)." );
 249                 $this->assertEquals( [ 0 => true ], $status->success,
 250                         "Store from $source to $dest has proper 'success' field in Status ($backendName)." );
 251                 $this->assertEquals( true, file_exists( $source ),
 252                         "Source file $source still exists ($backendName)." );
 253                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $dest ] ),
 254                         "Destination file $dest exists ($backendName)." );
 255
 256                 $this->assertEquals( filesize( $source ),
 257                         $this->backend->getFileSize( [ 'src' => $dest ] ),
 258                         "Destination file $dest has correct size ($backendName)." );
 259
 260                 $props1 = FSFile::getPropsFromPath( $source );
 261                 $props2 = $this->backend->getFileProps( [ 'src' => $dest ] );
 262                 $this->assertEquals( $props1, $props2,
 263                         "Source and destination have the same props ($backendName)." );
 264
 265                 $this->assertBackendPathsConsistent( [ $dest ] );
 266         }
 267
 268         public static function provider_testStore() {
 269                 $cases = [];
 270
 271                 $tmpName = TempFSFile::factory( "unittests_", 'txt' )->getPath();
 272                 $toPath = self::baseStorePath() . '/unittest-cont1/e/fun/obj1.txt';
 273                 $op = [ 'op' => 'store', 'src' => $tmpName, 'dst' => $toPath ];
 274                 $cases[] = [ $op ];
 275
 276                 $op2 = $op;
 277                 $op2['overwrite'] = true;
 278                 $cases[] = [ $op2 ];
 279
 280                 $op3 = $op;
 281                 $op3['overwriteSame'] = true;
 282                 $cases[] = [ $op3 ];
 283
 284                 return $cases;
 285         }
 286
 287         /**
 288          * @dataProvider provider_testCopy
 289          * @covers FileBackend::doOperation
 290          */
 291         public function testCopy( $op ) {
 292                 $this->backend = $this->singleBackend;
 293                 $this->tearDownFiles();
 294                 $this->doTestCopy( $op );
 295                 $this->tearDownFiles();
 296
 297                 $this->backend = $this->multiBackend;
 298                 $this->tearDownFiles();
 299                 $this->doTestCopy( $op );
 300                 $this->tearDownFiles();
 301         }
 302
 303         private function doTestCopy( $op ) {
 304                 $backendName = $this->backendClass();
 305
 306                 $source = $op['src'];
 307                 $dest = $op['dst'];
 308                 $this->prepare( [ 'dir' => dirname( $source ) ] );
 309                 $this->prepare( [ 'dir' => dirname( $dest ) ] );
 310
 311                 if ( isset( $op['ignoreMissingSource'] ) ) {
 312                         $status = $this->backend->doOperation( $op );
 313                         $this->assertGoodStatus( $status,
 314                                 "Move from $source to $dest succeeded without warnings ($backendName)." );
 315                         $this->assertEquals( [ 0 => true ], $status->success,
 316                                 "Move from $source to $dest has proper 'success' field in Status ($backendName)." );
 317                         $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $source ] ),
 318                                 "Source file $source does not exist ($backendName)." );
 319                         $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $dest ] ),
 320                                 "Destination file $dest does not exist ($backendName)." );
 321
 322                         return; // done
 323                 }
 324
 325                 $status = $this->backend->doOperation(
 326                         [ 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ] );
 327                 $this->assertGoodStatus( $status,
 328                         "Creation of file at $source succeeded ($backendName)." );
 329
 330                 if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 331                         $this->backend->copy( $op );
 332                 }
 333
 334                 $status = $this->backend->doOperation( $op );
 335
 336                 $this->assertGoodStatus( $status,
 337                         "Copy from $source to $dest succeeded without warnings ($backendName)." );
 338                 $this->assertEquals( true, $status->isOK(),
 339                         "Copy from $source to $dest succeeded ($backendName)." );
 340                 $this->assertEquals( [ 0 => true ], $status->success,
 341                         "Copy from $source to $dest has proper 'success' field in Status ($backendName)." );
 342                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $source ] ),
 343                         "Source file $source still exists ($backendName)." );
 344                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $dest ] ),
 345                         "Destination file $dest exists after copy ($backendName)." );
 346
 347                 $this->assertEquals(
 348                         $this->backend->getFileSize( [ 'src' => $source ] ),
 349                         $this->backend->getFileSize( [ 'src' => $dest ] ),
 350                         "Destination file $dest has correct size ($backendName)." );
 351
 352                 $props1 = $this->backend->getFileProps( [ 'src' => $source ] );
 353                 $props2 = $this->backend->getFileProps( [ 'src' => $dest ] );
 354                 $this->assertEquals( $props1, $props2,
 355                         "Source and destination have the same props ($backendName)." );
 356
 357                 $this->assertBackendPathsConsistent( [ $source, $dest ] );
 358         }
 359
 360         public static function provider_testCopy() {
 361                 $cases = [];
 362
 363                 $source = self::baseStorePath() . '/unittest-cont1/e/file.txt';
 364                 $dest = self::baseStorePath() . '/unittest-cont2/a/fileMoved.txt';
 365
 366                 $op = [ 'op' => 'copy', 'src' => $source, 'dst' => $dest ];
 367                 $cases[] = [
 368                         $op, // operation
 369                         $source, // source
 370                         $dest, // dest
 371                 ];
 372
 373                 $op2 = $op;
 374                 $op2['overwrite'] = true;
 375                 $cases[] = [
 376                         $op2, // operation
 377                         $source, // source
 378                         $dest, // dest
 379                 ];
 380
 381                 $op2 = $op;
 382                 $op2['overwriteSame'] = true;
 383                 $cases[] = [
 384                         $op2, // operation
 385                         $source, // source
 386                         $dest, // dest
 387                 ];
 388
 389                 $op2 = $op;
 390                 $op2['ignoreMissingSource'] = true;
 391                 $cases[] = [
 392                         $op2, // operation
 393                         $source, // source
 394                         $dest, // dest
 395                 ];
 396
 397                 $op2 = $op;
 398                 $op2['ignoreMissingSource'] = true;
 399                 $cases[] = [
 400                         $op2, // operation
 401                         self::baseStorePath() . '/unittest-cont-bad/e/file.txt', // source
 402                         $dest, // dest
 403                 ];
 404
 405                 return $cases;
 406         }
 407
 408         /**
 409          * @dataProvider provider_testMove
 410          * @covers FileBackend::doOperation
 411          */
 412         public function testMove( $op ) {
 413                 $this->backend = $this->singleBackend;
 414                 $this->tearDownFiles();
 415                 $this->doTestMove( $op );
 416                 $this->tearDownFiles();
 417
 418                 $this->backend = $this->multiBackend;
 419                 $this->tearDownFiles();
 420                 $this->doTestMove( $op );
 421                 $this->tearDownFiles();
 422         }
 423
 424         private function doTestMove( $op ) {
 425                 $backendName = $this->backendClass();
 426
 427                 $source = $op['src'];
 428                 $dest = $op['dst'];
 429                 $this->prepare( [ 'dir' => dirname( $source ) ] );
 430                 $this->prepare( [ 'dir' => dirname( $dest ) ] );
 431
 432                 if ( isset( $op['ignoreMissingSource'] ) ) {
 433                         $status = $this->backend->doOperation( $op );
 434                         $this->assertGoodStatus( $status,
 435                                 "Move from $source to $dest succeeded without warnings ($backendName)." );
 436                         $this->assertEquals( [ 0 => true ], $status->success,
 437                                 "Move from $source to $dest has proper 'success' field in Status ($backendName)." );
 438                         $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $source ] ),
 439                                 "Source file $source does not exist ($backendName)." );
 440                         $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $dest ] ),
 441                                 "Destination file $dest does not exist ($backendName)." );
 442
 443                         return; // done
 444                 }
 445
 446                 $status = $this->backend->doOperation(
 447                         [ 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ] );
 448                 $this->assertGoodStatus( $status,
 449                         "Creation of file at $source succeeded ($backendName)." );
 450
 451                 if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 452                         $this->backend->copy( $op );
 453                 }
 454
 455                 $status = $this->backend->doOperation( $op );
 456                 $this->assertGoodStatus( $status,
 457                         "Move from $source to $dest succeeded without warnings ($backendName)." );
 458                 $this->assertEquals( true, $status->isOK(),
 459                         "Move from $source to $dest succeeded ($backendName)." );
 460                 $this->assertEquals( [ 0 => true ], $status->success,
 461                         "Move from $source to $dest has proper 'success' field in Status ($backendName)." );
 462                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $source ] ),
 463                         "Source file $source does not still exists ($backendName)." );
 464                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $dest ] ),
 465                         "Destination file $dest exists after move ($backendName)." );
 466
 467                 $this->assertNotEquals(
 468                         $this->backend->getFileSize( [ 'src' => $source ] ),
 469                         $this->backend->getFileSize( [ 'src' => $dest ] ),
 470                         "Destination file $dest has correct size ($backendName)." );
 471
 472                 $props1 = $this->backend->getFileProps( [ 'src' => $source ] );
 473                 $props2 = $this->backend->getFileProps( [ 'src' => $dest ] );
 474                 $this->assertEquals( false, $props1['fileExists'],
 475                         "Source file does not exist accourding to props ($backendName)." );
 476                 $this->assertEquals( true, $props2['fileExists'],
 477                         "Destination file exists accourding to props ($backendName)." );
 478
 479                 $this->assertBackendPathsConsistent( [ $source, $dest ] );
 480         }
 481
 482         public static function provider_testMove() {
 483                 $cases = [];
 484
 485                 $source = self::baseStorePath() . '/unittest-cont1/e/file.txt';
 486                 $dest = self::baseStorePath() . '/unittest-cont2/a/fileMoved.txt';
 487
 488                 $op = [ 'op' => 'move', 'src' => $source, 'dst' => $dest ];
 489                 $cases[] = [
 490                         $op, // operation
 491                         $source, // source
 492                         $dest, // dest
 493                 ];
 494
 495                 $op2 = $op;
 496                 $op2['overwrite'] = true;
 497                 $cases[] = [
 498                         $op2, // operation
 499                         $source, // source
 500                         $dest, // dest
 501                 ];
 502
 503                 $op2 = $op;
 504                 $op2['overwriteSame'] = true;
 505                 $cases[] = [
 506                         $op2, // operation
 507                         $source, // source
 508                         $dest, // dest
 509                 ];
 510
 511                 $op2 = $op;
 512                 $op2['ignoreMissingSource'] = true;
 513                 $cases[] = [
 514                         $op2, // operation
 515                         $source, // source
 516                         $dest, // dest
 517                 ];
 518
 519                 $op2 = $op;
 520                 $op2['ignoreMissingSource'] = true;
 521                 $cases[] = [
 522                         $op2, // operation
 523                         self::baseStorePath() . '/unittest-cont-bad/e/file.txt', // source
 524                         $dest, // dest
 525                 ];
 526
 527                 return $cases;
 528         }
 529
 530         /**
 531          * @dataProvider provider_testDelete
 532          * @covers FileBackend::doOperation
 533          */
 534         public function testDelete( $op, $withSource, $okStatus ) {
 535                 $this->backend = $this->singleBackend;
 536                 $this->tearDownFiles();
 537                 $this->doTestDelete( $op, $withSource, $okStatus );
 538                 $this->tearDownFiles();
 539
 540                 $this->backend = $this->multiBackend;
 541                 $this->tearDownFiles();
 542                 $this->doTestDelete( $op, $withSource, $okStatus );
 543                 $this->tearDownFiles();
 544         }
 545
 546         private function doTestDelete( $op, $withSource, $okStatus ) {
 547                 $backendName = $this->backendClass();
 548
 549                 $source = $op['src'];
 550                 $this->prepare( [ 'dir' => dirname( $source ) ] );
 551
 552                 if ( $withSource ) {
 553                         $status = $this->backend->doOperation(
 554                                 [ 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ] );
 555                         $this->assertGoodStatus( $status,
 556                                 "Creation of file at $source succeeded ($backendName)." );
 557                 }
 558
 559                 $status = $this->backend->doOperation( $op );
 560                 if ( $okStatus ) {
 561                         $this->assertGoodStatus( $status,
 562                                 "Deletion of file at $source succeeded without warnings ($backendName)." );
 563                         $this->assertEquals( true, $status->isOK(),
 564                                 "Deletion of file at $source succeeded ($backendName)." );
 565                         $this->assertEquals( [ 0 => true ], $status->success,
 566                                 "Deletion of file at $source has proper 'success' field in Status ($backendName)." );
 567                 } else {
 568                         $this->assertEquals( false, $status->isOK(),
 569                                 "Deletion of file at $source failed ($backendName)." );
 570                 }
 571
 572                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $source ] ),
 573                         "Source file $source does not exist after move ($backendName)." );
 574
 575                 $this->assertFalse(
 576                         $this->backend->getFileSize( [ 'src' => $source ] ),
 577                         "Source file $source has correct size (false) ($backendName)." );
 578
 579                 $props1 = $this->backend->getFileProps( [ 'src' => $source ] );
 580                 $this->assertFalse( $props1['fileExists'],
 581                         "Source file $source does not exist according to props ($backendName)." );
 582
 583                 $this->assertBackendPathsConsistent( [ $source ] );
 584         }
 585
 586         public static function provider_testDelete() {
 587                 $cases = [];
 588
 589                 $source = self::baseStorePath() . '/unittest-cont1/e/myfacefile.txt';
 590
 591                 $op = [ 'op' => 'delete', 'src' => $source ];
 592                 $cases[] = [
 593                         $op, // operation
 594                         true, // with source
 595                         true // succeeds
 596                 ];
 597
 598                 $cases[] = [
 599                         $op, // operation
 600                         false, // without source
 601                         false // fails
 602                 ];
 603
 604                 $op['ignoreMissingSource'] = true;
 605                 $cases[] = [
 606                         $op, // operation
 607                         false, // without source
 608                         true // succeeds
 609                 ];
 610
 611                 $op['ignoreMissingSource'] = true;
 612                 $op['src'] = self::baseStorePath() . '/unittest-cont-bad/e/file.txt';
 613                 $cases[] = [
 614                         $op, // operation
 615                         false, // without source
 616                         true // succeeds
 617                 ];
 618
 619                 return $cases;
 620         }
 621
 622         /**
 623          * @dataProvider provider_testDescribe
 624          * @covers FileBackend::doOperation
 625          */
 626         public function testDescribe( $op, $withSource, $okStatus ) {
 627                 $this->backend = $this->singleBackend;
 628                 $this->tearDownFiles();
 629                 $this->doTestDescribe( $op, $withSource, $okStatus );
 630                 $this->tearDownFiles();
 631
 632                 $this->backend = $this->multiBackend;
 633                 $this->tearDownFiles();
 634                 $this->doTestDescribe( $op, $withSource, $okStatus );
 635                 $this->tearDownFiles();
 636         }
 637
 638         private function doTestDescribe( $op, $withSource, $okStatus ) {
 639                 $backendName = $this->backendClass();
 640
 641                 $source = $op['src'];
 642                 $this->prepare( [ 'dir' => dirname( $source ) ] );
 643
 644                 if ( $withSource ) {
 645                         $status = $this->backend->doOperation(
 646                                 [ 'op' => 'create', 'content' => 'blahblah', 'dst' => $source,
 647                                         'headers' => [ 'Content-Disposition' => 'xxx' ] ] );
 648                         $this->assertGoodStatus( $status,
 649                                 "Creation of file at $source succeeded ($backendName)." );
 650                         if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) {
 651                                 $attr = $this->backend->getFileXAttributes( [ 'src' => $source ] );
 652                                 $this->assertHasHeaders( [ 'Content-Disposition' => 'xxx' ], $attr );
 653                         }
 654
 655                         $status = $this->backend->describe( [ 'src' => $source,
 656                                 'headers' => [ 'Content-Disposition' => '' ] ] ); // remove
 657                         $this->assertGoodStatus( $status,
 658                                 "Removal of header for $source succeeded ($backendName)." );
 659
 660                         if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) {
 661                                 $attr = $this->backend->getFileXAttributes( [ 'src' => $source ] );
 662                                 $this->assertFalse( isset( $attr['headers']['content-disposition'] ),
 663                                         "File 'Content-Disposition' header removed." );
 664                         }
 665                 }
 666
 667                 $status = $this->backend->doOperation( $op );
 668                 if ( $okStatus ) {
 669                         $this->assertGoodStatus( $status,
 670                                 "Describe of file at $source succeeded without warnings ($backendName)." );
 671                         $this->assertEquals( true, $status->isOK(),
 672                                 "Describe of file at $source succeeded ($backendName)." );
 673                         $this->assertEquals( [ 0 => true ], $status->success,
 674                                 "Describe of file at $source has proper 'success' field in Status ($backendName)." );
 675                         if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) {
 676                                 $attr = $this->backend->getFileXAttributes( [ 'src' => $source ] );
 677                                 $this->assertHasHeaders( $op['headers'], $attr );
 678                         }
 679                 } else {
 680                         $this->assertEquals( false, $status->isOK(),
 681                                 "Describe of file at $source failed ($backendName)." );
 682                 }
 683
 684                 $this->assertBackendPathsConsistent( [ $source ] );
 685         }
 686
 687         private function assertHasHeaders( array $headers, array $attr ) {
 688                 foreach ( $headers as $n => $v ) {
 689                         if ( $n !== '' ) {
 690                                 $this->assertTrue( isset( $attr['headers'][strtolower( $n )] ),
 691                                         "File has '$n' header." );
 692                                 $this->assertEquals( $v, $attr['headers'][strtolower( $n )],
 693                                         "File has '$n' header value." );
 694                         } else {
 695                                 $this->assertFalse( isset( $attr['headers'][strtolower( $n )] ),
 696                                         "File does not have '$n' header." );
 697                         }
 698                 }
 699         }
 700
 701         public static function provider_testDescribe() {
 702                 $cases = [];
 703
 704                 $source = self::baseStorePath() . '/unittest-cont1/e/myfacefile.txt';
 705
 706                 $op = [ 'op' => 'describe', 'src' => $source,
 707                         'headers' => [ 'Content-Disposition' => 'inline' ], ];
 708                 $cases[] = [
 709                         $op, // operation
 710                         true, // with source
 711                         true // succeeds
 712                 ];
 713
 714                 $cases[] = [
 715                         $op, // operation
 716                         false, // without source
 717                         false // fails
 718                 ];
 719
 720                 return $cases;
 721         }
 722
 723         /**
 724          * @dataProvider provider_testCreate
 725          * @covers FileBackend::doOperation
 726          */
 727         public function testCreate( $op, $alreadyExists, $okStatus, $newSize ) {
 728                 $this->backend = $this->singleBackend;
 729                 $this->tearDownFiles();
 730                 $this->doTestCreate( $op, $alreadyExists, $okStatus, $newSize );
 731                 $this->tearDownFiles();
 732
 733                 $this->backend = $this->multiBackend;
 734                 $this->tearDownFiles();
 735                 $this->doTestCreate( $op, $alreadyExists, $okStatus, $newSize );
 736                 $this->tearDownFiles();
 737         }
 738
 739         private function doTestCreate( $op, $alreadyExists, $okStatus, $newSize ) {
 740                 $backendName = $this->backendClass();
 741
 742                 $dest = $op['dst'];
 743                 $this->prepare( [ 'dir' => dirname( $dest ) ] );
 744
 745                 $oldText = 'blah...blah...waahwaah';
 746                 if ( $alreadyExists ) {
 747                         $status = $this->backend->doOperation(
 748                                 [ 'op' => 'create', 'content' => $oldText, 'dst' => $dest ] );
 749                         $this->assertGoodStatus( $status,
 750                                 "Creation of file at $dest succeeded ($backendName)." );
 751                 }
 752
 753                 $status = $this->backend->doOperation( $op );
 754                 if ( $okStatus ) {
 755                         $this->assertGoodStatus( $status,
 756                                 "Creation of file at $dest succeeded without warnings ($backendName)." );
 757                         $this->assertEquals( true, $status->isOK(),
 758                                 "Creation of file at $dest succeeded ($backendName)." );
 759                         $this->assertEquals( [ 0 => true ], $status->success,
 760                                 "Creation of file at $dest has proper 'success' field in Status ($backendName)." );
 761                 } else {
 762                         $this->assertEquals( false, $status->isOK(),
 763                                 "Creation of file at $dest failed ($backendName)." );
 764                 }
 765
 766                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $dest ] ),
 767                         "Destination file $dest exists after creation ($backendName)." );
 768
 769                 $props1 = $this->backend->getFileProps( [ 'src' => $dest ] );
 770                 $this->assertEquals( true, $props1['fileExists'],
 771                         "Destination file $dest exists according to props ($backendName)." );
 772                 if ( $okStatus ) { // file content is what we saved
 773                         $this->assertEquals( $newSize, $props1['size'],
 774                                 "Destination file $dest has expected size according to props ($backendName)." );
 775                         $this->assertEquals( $newSize,
 776                                 $this->backend->getFileSize( [ 'src' => $dest ] ),
 777                                 "Destination file $dest has correct size ($backendName)." );
 778                 } else { // file content is some other previous text
 779                         $this->assertEquals( strlen( $oldText ), $props1['size'],
 780                                 "Destination file $dest has original size according to props ($backendName)." );
 781                         $this->assertEquals( strlen( $oldText ),
 782                                 $this->backend->getFileSize( [ 'src' => $dest ] ),
 783                                 "Destination file $dest has original size according to props ($backendName)." );
 784                 }
 785
 786                 $this->assertBackendPathsConsistent( [ $dest ] );
 787         }
 788
 789         /**
 790          * @dataProvider provider_testCreate
 791          */
 792         public static function provider_testCreate() {
 793                 $cases = [];
 794
 795                 $dest = self::baseStorePath() . '/unittest-cont2/a/myspacefile.txt';
 796
 797                 $op = [ 'op' => 'create', 'content' => 'test test testing', 'dst' => $dest ];
 798                 $cases[] = [
 799                         $op, // operation
 800                         false, // no dest already exists
 801                         true, // succeeds
 802                         strlen( $op['content'] )
 803                 ];
 804
 805                 $op2 = $op;
 806                 $op2['content'] = "\n";
 807                 $cases[] = [
 808                         $op2, // operation
 809                         false, // no dest already exists
 810                         true, // succeeds
 811                         strlen( $op2['content'] )
 812                 ];
 813
 814                 $op2 = $op;
 815                 $op2['content'] = "fsf\n waf 3kt";
 816                 $cases[] = [
 817                         $op2, // operation
 818                         true, // dest already exists
 819                         false, // fails
 820                         strlen( $op2['content'] )
 821                 ];
 822
 823                 $op2 = $op;
 824                 $op2['content'] = "egm'g gkpe gpqg eqwgwqg";
 825                 $op2['overwrite'] = true;
 826                 $cases[] = [
 827                         $op2, // operation
 828                         true, // dest already exists
 829                         true, // succeeds
 830                         strlen( $op2['content'] )
 831                 ];
 832
 833                 $op2 = $op;
 834                 $op2['content'] = "39qjmg3-qg";
 835                 $op2['overwriteSame'] = true;
 836                 $cases[] = [
 837                         $op2, // operation
 838                         true, // dest already exists
 839                         false, // succeeds
 840                         strlen( $op2['content'] )
 841                 ];
 842
 843                 return $cases;
 844         }
 845
 846         /**
 847          * @covers FileBackend::doQuickOperations
 848          */
 849         public function testDoQuickOperations() {
 850                 $this->backend = $this->singleBackend;
 851                 $this->doTestDoQuickOperations();
 852                 $this->tearDownFiles();
 853
 854                 $this->backend = $this->multiBackend;
 855                 $this->doTestDoQuickOperations();
 856                 $this->tearDownFiles();
 857         }
 858
 859         private function doTestDoQuickOperations() {
 860                 $backendName = $this->backendClass();
 861
 862                 $base = self::baseStorePath();
 863                 $files = [
 864                         "$base/unittest-cont1/e/fileA.a",
 865                         "$base/unittest-cont1/e/fileB.a",
 866                         "$base/unittest-cont1/e/fileC.a"
 867                 ];
 868                 $createOps = [];
 869                 $purgeOps = [];
 870                 foreach ( $files as $path ) {
 871                         $status = $this->prepare( [ 'dir' => dirname( $path ) ] );
 872                         $this->assertGoodStatus( $status,
 873                                 "Preparing $path succeeded without warnings ($backendName)." );
 874                         $createOps[] = [ 'op' => 'create', 'dst' => $path, 'content' => mt_rand( 0, 50000 ) ];
 875                         $copyOps[] = [ 'op' => 'copy', 'src' => $path, 'dst' => "$path-2" ];
 876                         $moveOps[] = [ 'op' => 'move', 'src' => "$path-2", 'dst' => "$path-3" ];
 877                         $purgeOps[] = [ 'op' => 'delete', 'src' => $path ];
 878                         $purgeOps[] = [ 'op' => 'delete', 'src' => "$path-3" ];
 879                 }
 880                 $purgeOps[] = [ 'op' => 'null' ];
 881
 882                 $this->assertGoodStatus(
 883                         $this->backend->doQuickOperations( $createOps ),
 884                         "Creation of source files succeeded ($backendName)." );
 885                 foreach ( $files as $file ) {
 886                         $this->assertTrue( $this->backend->fileExists( [ 'src' => $file ] ),
 887                                 "File $file exists." );
 888                 }
 889
 890                 $this->assertGoodStatus(
 891                         $this->backend->doQuickOperations( $copyOps ),
 892                         "Quick copy of source files succeeded ($backendName)." );
 893                 foreach ( $files as $file ) {
 894                         $this->assertTrue( $this->backend->fileExists( [ 'src' => "$file-2" ] ),
 895                                 "File $file-2 exists." );
 896                 }
 897
 898                 $this->assertGoodStatus(
 899                         $this->backend->doQuickOperations( $moveOps ),
 900                         "Quick move of source files succeeded ($backendName)." );
 901                 foreach ( $files as $file ) {
 902                         $this->assertTrue( $this->backend->fileExists( [ 'src' => "$file-3" ] ),
 903                                 "File $file-3 move in." );
 904                         $this->assertFalse( $this->backend->fileExists( [ 'src' => "$file-2" ] ),
 905                                 "File $file-2 moved away." );
 906                 }
 907
 908                 $this->assertGoodStatus(
 909                         $this->backend->quickCopy( [ 'src' => $files[0], 'dst' => $files[0] ] ),
 910                         "Copy of file {$files[0]} over itself succeeded ($backendName)." );
 911                 $this->assertTrue( $this->backend->fileExists( [ 'src' => $files[0] ] ),
 912                         "File {$files[0]} still exists." );
 913
 914                 $this->assertGoodStatus(
 915                         $this->backend->quickMove( [ 'src' => $files[0], 'dst' => $files[0] ] ),
 916                         "Move of file {$files[0]} over itself succeeded ($backendName)." );
 917                 $this->assertTrue( $this->backend->fileExists( [ 'src' => $files[0] ] ),
 918                         "File {$files[0]} still exists." );
 919
 920                 $this->assertGoodStatus(
 921                         $this->backend->doQuickOperations( $purgeOps ),
 922                         "Quick deletion of source files succeeded ($backendName)." );
 923                 foreach ( $files as $file ) {
 924                         $this->assertFalse( $this->backend->fileExists( [ 'src' => $file ] ),
 925                                 "File $file purged." );
 926                         $this->assertFalse( $this->backend->fileExists( [ 'src' => "$file-3" ] ),
 927                                 "File $file-3 purged." );
 928                 }
 929         }
 930
 931         /**
 932          * @dataProvider provider_testConcatenate
 933          */
 934         public function testConcatenate( $op, $srcs, $srcsContent, $alreadyExists, $okStatus ) {
 935                 $this->backend = $this->singleBackend;
 936                 $this->tearDownFiles();
 937                 $this->doTestConcatenate( $op, $srcs, $srcsContent, $alreadyExists, $okStatus );
 938                 $this->tearDownFiles();
 939
 940                 $this->backend = $this->multiBackend;
 941                 $this->tearDownFiles();
 942                 $this->doTestConcatenate( $op, $srcs, $srcsContent, $alreadyExists, $okStatus );
 943                 $this->tearDownFiles();
 944         }
 945
 946         private function doTestConcatenate( $params, $srcs, $srcsContent, $alreadyExists, $okStatus ) {
 947                 $backendName = $this->backendClass();
 948
 949                 $expContent = '';
 950                 // Create sources
 951                 $ops = [];
 952                 foreach ( $srcs as $i => $source ) {
 953                         $this->prepare( [ 'dir' => dirname( $source ) ] );
 954                         $ops[] = [
 955                                 'op' => 'create', // operation
 956                                 'dst' => $source, // source
 957                                 'content' => $srcsContent[$i]
 958                         ];
 959                         $expContent .= $srcsContent[$i];
 960                 }
 961                 $status = $this->backend->doOperations( $ops );
 962
 963                 $this->assertGoodStatus( $status,
 964                         "Creation of source files succeeded ($backendName)." );
 965
 966                 $dest = $params['dst'] = $this->getNewTempFile();
 967                 if ( $alreadyExists ) {
 968                         $ok = file_put_contents( $dest, 'blah...blah...waahwaah' ) !== false;
 969                         $this->assertEquals( true, $ok,
 970                                 "Creation of file at $dest succeeded ($backendName)." );
 971                 } else {
 972                         $ok = file_put_contents( $dest, '' ) !== false;
 973                         $this->assertEquals( true, $ok,
 974                                 "Creation of 0-byte file at $dest succeeded ($backendName)." );
 975                 }
 976
 977                 // Combine the files into one
 978                 $status = $this->backend->concatenate( $params );
 979                 if ( $okStatus ) {
 980                         $this->assertGoodStatus( $status,
 981                                 "Creation of concat file at $dest succeeded without warnings ($backendName)." );
 982                         $this->assertEquals( true, $status->isOK(),
 983                                 "Creation of concat file at $dest succeeded ($backendName)." );
 984                 } else {
 985                         $this->assertEquals( false, $status->isOK(),
 986                                 "Creation of concat file at $dest failed ($backendName)." );
 987                 }
 988
 989                 if ( $okStatus ) {
 990                         $this->assertEquals( true, is_file( $dest ),
 991                                 "Dest concat file $dest exists after creation ($backendName)." );
 992                 } else {
 993                         $this->assertEquals( true, is_file( $dest ),
 994                                 "Dest concat file $dest exists after failed creation ($backendName)." );
 995                 }
 996
 997                 $contents = file_get_contents( $dest );
 998                 $this->assertNotEquals( false, $contents, "File at $dest exists ($backendName)." );
 999
1000                 if ( $okStatus ) {
1001                         $this->assertEquals( $expContent, $contents,
1002                                 "Concat file at $dest has correct contents ($backendName)." );
1003                 } else {
1004                         $this->assertNotEquals( $expContent, $contents,
1005                                 "Concat file at $dest has correct contents ($backendName)." );
1006                 }
1007         }
1008
1009         public static function provider_testConcatenate() {
1010                 $cases = [];
1011
1012                 $srcs = [
1013                         self::baseStorePath() . '/unittest-cont1/e/file1.txt',
1014                         self::baseStorePath() . '/unittest-cont1/e/file2.txt',
1015                         self::baseStorePath() . '/unittest-cont1/e/file3.txt',
1016                         self::baseStorePath() . '/unittest-cont1/e/file4.txt',
1017                         self::baseStorePath() . '/unittest-cont1/e/file5.txt',
1018                         self::baseStorePath() . '/unittest-cont1/e/file6.txt',
1019                         self::baseStorePath() . '/unittest-cont1/e/file7.txt',
1020                         self::baseStorePath() . '/unittest-cont1/e/file8.txt',
1021                         self::baseStorePath() . '/unittest-cont1/e/file9.txt',
1022                         self::baseStorePath() . '/unittest-cont1/e/file10.txt'
1023                 ];
1024                 $content = [
1025                         'egfage',
1026                         'ageageag',
1027                         'rhokohlr',
1028                         'shgmslkg',
1029                         'kenga',
1030                         'owagmal',
1031                         'kgmae',
1032                         'g eak;g',
1033                         'lkaem;a',
1034                         'legma'
1035                 ];
1036                 $params = [ 'srcs' => $srcs ];
1037
1038                 $cases[] = [
1039                         $params, // operation
1040                         $srcs, // sources
1041                         $content, // content for each source
1042                         false, // no dest already exists
1043                         true, // succeeds
1044                 ];
1045
1046                 $cases[] = [
1047                         $params, // operation
1048                         $srcs, // sources
1049                         $content, // content for each source
1050                         true, // dest already exists
1051                         false, // succeeds
1052                 ];
1053
1054                 return $cases;
1055         }
1056
1057         /**
1058          * @dataProvider provider_testGetFileStat
1059          * @covers FileBackend::getFileStat
1060          */
1061         public function testGetFileStat( $path, $content, $alreadyExists ) {
1062                 $this->backend = $this->singleBackend;
1063                 $this->tearDownFiles();
1064                 $this->doTestGetFileStat( $path, $content, $alreadyExists );
1065                 $this->tearDownFiles();
1066
1067                 $this->backend = $this->multiBackend;
1068                 $this->tearDownFiles();
1069                 $this->doTestGetFileStat( $path, $content, $alreadyExists );
1070                 $this->tearDownFiles();
1071         }
1072
1073         private function doTestGetFileStat( $path, $content, $alreadyExists ) {
1074                 $backendName = $this->backendClass();
1075
1076                 if ( $alreadyExists ) {
1077                         $this->prepare( [ 'dir' => dirname( $path ) ] );
1078                         $status = $this->create( [ 'dst' => $path, 'content' => $content ] );
1079                         $this->assertGoodStatus( $status,
1080                                 "Creation of file at $path succeeded ($backendName)." );
1081
1082                         $size = $this->backend->getFileSize( [ 'src' => $path ] );
1083                         $time = $this->backend->getFileTimestamp( [ 'src' => $path ] );
1084                         $stat = $this->backend->getFileStat( [ 'src' => $path ] );
1085
1086                         $this->assertEquals( strlen( $content ), $size,
1087                                 "Correct file size of '$path'" );
1088                         $this->assertTrue( abs( time() - wfTimestamp( TS_UNIX, $time ) ) < 10,
1089                                 "Correct file timestamp of '$path'" );
1090
1091                         $size = $stat['size'];
1092                         $time = $stat['mtime'];
1093                         $this->assertEquals( strlen( $content ), $size,
1094                                 "Correct file size of '$path'" );
1095                         $this->assertTrue( abs( time() - wfTimestamp( TS_UNIX, $time ) ) < 10,
1096                                 "Correct file timestamp of '$path'" );
1097
1098                         $this->backend->clearCache( [ $path ] );
1099
1100                         $size = $this->backend->getFileSize( [ 'src' => $path ] );
1101
1102                         $this->assertEquals( strlen( $content ), $size,
1103                                 "Correct file size of '$path'" );
1104
1105                         $this->backend->preloadCache( [ $path ] );
1106
1107                         $size = $this->backend->getFileSize( [ 'src' => $path ] );
1108
1109                         $this->assertEquals( strlen( $content ), $size,
1110                                 "Correct file size of '$path'" );
1111                 } else {
1112                         $size = $this->backend->getFileSize( [ 'src' => $path ] );
1113                         $time = $this->backend->getFileTimestamp( [ 'src' => $path ] );
1114                         $stat = $this->backend->getFileStat( [ 'src' => $path ] );
1115
1116                         $this->assertFalse( $size, "Correct file size of '$path'" );
1117                         $this->assertFalse( $time, "Correct file timestamp of '$path'" );
1118                         $this->assertFalse( $stat, "Correct file stat of '$path'" );
1119                 }
1120         }
1121
1122         public static function provider_testGetFileStat() {
1123                 $cases = [];
1124
1125                 $base = self::baseStorePath();
1126                 $cases[] = [ "$base/unittest-cont1/e/b/z/some_file.txt", "some file contents", true ];
1127                 $cases[] = [ "$base/unittest-cont1/e/b/some-other_file.txt", "", true ];
1128                 $cases[] = [ "$base/unittest-cont1/e/b/some-diff_file.txt", null, false ];
1129
1130                 return $cases;
1131         }
1132
1133         /**
1134          * @dataProvider provider_testGetFileStat
1135          * @covers FileBackend::streamFile
1136          */
1137         public function testStreamFile( $path, $content, $alreadyExists ) {
1138                 $this->backend = $this->singleBackend;
1139                 $this->tearDownFiles();
1140                 $this->doTestStreamFile( $path, $content, $alreadyExists );
1141                 $this->tearDownFiles();
1142         }
1143
1144         private function doTestStreamFile( $path, $content ) {
1145                 $backendName = $this->backendClass();
1146
1147                 // Test doStreamFile() directly to avoid header madness
1148                 $class = new ReflectionClass( $this->backend );
1149                 $method = $class->getMethod( 'doStreamFile' );
1150                 $method->setAccessible( true );
1151
1152                 if ( $content !== null ) {
1153                         $this->prepare( [ 'dir' => dirname( $path ) ] );
1154                         $status = $this->create( [ 'dst' => $path, 'content' => $content ] );
1155                         $this->assertGoodStatus( $status,
1156                                 "Creation of file at $path succeeded ($backendName)." );
1157
1158                         ob_start();
1159                         $method->invokeArgs( $this->backend, [ [ 'src' => $path ] ] );
1160                         $data = ob_get_contents();
1161                         ob_end_clean();
1162
1163                         $this->assertEquals( $content, $data, "Correct content streamed from '$path'" );
1164                 } else { // 404 case
1165                         ob_start();
1166                         $method->invokeArgs( $this->backend, [ [ 'src' => $path ] ] );
1167                         $data = ob_get_contents();
1168                         ob_end_clean();
1169
1170                         $this->assertEquals( '', $data, "Correct content streamed from '$path' ($backendName)" );
1171                 }
1172         }
1173
1174         public static function provider_testStreamFile() {
1175                 $cases = [];
1176
1177                 $base = self::baseStorePath();
1178                 $cases[] = [ "$base/unittest-cont1/e/b/z/some_file.txt", "some file contents" ];
1179                 $cases[] = [ "$base/unittest-cont1/e/b/some-other_file.txt", null ];
1180
1181                 return $cases;
1182         }
1183
1184         /**
1185          * @dataProvider provider_testGetFileContents
1186          * @covers FileBackend::getFileContents
1187          * @covers FileBackend::getFileContentsMulti
1188          */
1189         public function testGetFileContents( $source, $content ) {
1190                 $this->backend = $this->singleBackend;
1191                 $this->tearDownFiles();
1192                 $this->doTestGetFileContents( $source, $content );
1193                 $this->tearDownFiles();
1194
1195                 $this->backend = $this->multiBackend;
1196                 $this->tearDownFiles();
1197                 $this->doTestGetFileContents( $source, $content );
1198                 $this->tearDownFiles();
1199         }
1200
1201         private function doTestGetFileContents( $source, $content ) {
1202                 $backendName = $this->backendClass();
1203
1204                 $srcs = (array)$source;
1205                 $content = (array)$content;
1206                 foreach ( $srcs as $i => $src ) {
1207                         $this->prepare( [ 'dir' => dirname( $src ) ] );
1208                         $status = $this->backend->doOperation(
1209                                 [ 'op' => 'create', 'content' => $content[$i], 'dst' => $src ] );
1210                         $this->assertGoodStatus( $status,
1211                                 "Creation of file at $src succeeded ($backendName)." );
1212                 }
1213
1214                 if ( is_array( $source ) ) {
1215                         $contents = $this->backend->getFileContentsMulti( [ 'srcs' => $source ] );
1216                         foreach ( $contents as $path => $data ) {
1217                                 $this->assertNotEquals( false, $data, "Contents of $path exists ($backendName)." );
1218                                 $this->assertEquals(
1219                                         current( $content ),
1220                                         $data,
1221                                         "Contents of $path is correct ($backendName)."
1222                                 );
1223                                 next( $content );
1224                         }
1225                         $this->assertEquals(
1226                                 $source,
1227                                 array_keys( $contents ),
1228                                 "Contents in right order ($backendName)."
1229                         );
1230                         $this->assertEquals(
1231                                 count( $source ),
1232                                 count( $contents ),
1233                                 "Contents array size correct ($backendName)."
1234                         );
1235                 } else {
1236                         $data = $this->backend->getFileContents( [ 'src' => $source ] );
1237                         $this->assertNotEquals( false, $data, "Contents of $source exists ($backendName)." );
1238                         $this->assertEquals( $content[0], $data, "Contents of $source is correct ($backendName)." );
1239                 }
1240         }
1241
1242         public static function provider_testGetFileContents() {
1243                 $cases = [];
1244
1245                 $base = self::baseStorePath();
1246                 $cases[] = [ "$base/unittest-cont1/e/b/z/some_file.txt", "some file contents" ];
1247                 $cases[] = [ "$base/unittest-cont1/e/b/some-other_file.txt", "more file contents" ];
1248                 $cases[] = [
1249                         [ "$base/unittest-cont1/e/a/x.txt", "$base/unittest-cont1/e/a/y.txt",
1250                                 "$base/unittest-cont1/e/a/z.txt" ],
1251                         [ "contents xx", "contents xy", "contents xz" ]
1252                 ];
1253
1254                 return $cases;
1255         }
1256
1257         /**
1258          * @dataProvider provider_testGetLocalCopy
1259          * @covers FileBackend::getLocalCopy
1260          */
1261         public function testGetLocalCopy( $source, $content ) {
1262                 $this->backend = $this->singleBackend;
1263                 $this->tearDownFiles();
1264                 $this->doTestGetLocalCopy( $source, $content );
1265                 $this->tearDownFiles();
1266
1267                 $this->backend = $this->multiBackend;
1268                 $this->tearDownFiles();
1269                 $this->doTestGetLocalCopy( $source, $content );
1270                 $this->tearDownFiles();
1271         }
1272
1273         private function doTestGetLocalCopy( $source, $content ) {
1274                 $backendName = $this->backendClass();
1275
1276                 $srcs = (array)$source;
1277                 $content = (array)$content;
1278                 foreach ( $srcs as $i => $src ) {
1279                         $this->prepare( [ 'dir' => dirname( $src ) ] );
1280                         $status = $this->backend->doOperation(
1281                                 [ 'op' => 'create', 'content' => $content[$i], 'dst' => $src ] );
1282                         $this->assertGoodStatus( $status,
1283                                 "Creation of file at $src succeeded ($backendName)." );
1284                 }
1285
1286                 if ( is_array( $source ) ) {
1287                         $tmpFiles = $this->backend->getLocalCopyMulti( [ 'srcs' => $source ] );
1288                         foreach ( $tmpFiles as $path => $tmpFile ) {
1289                                 $this->assertNotNull( $tmpFile,
1290                                         "Creation of local copy of $path succeeded ($backendName)." );
1291                                 $contents = file_get_contents( $tmpFile->getPath() );
1292                                 $this->assertNotEquals( false, $contents, "Local copy of $path exists ($backendName)." );
1293                                 $this->assertEquals(
1294                                         current( $content ),
1295                                         $contents,
1296                                         "Local copy of $path is correct ($backendName)."
1297                                 );
1298                                 next( $content );
1299                         }
1300                         $this->assertEquals(
1301                                 $source,
1302                                 array_keys( $tmpFiles ),
1303                                 "Local copies in right order ($backendName)."
1304                         );
1305                         $this->assertEquals(
1306                                 count( $source ),
1307                                 count( $tmpFiles ),
1308                                 "Local copies array size correct ($backendName)."
1309                         );
1310                 } else {
1311                         $tmpFile = $this->backend->getLocalCopy( [ 'src' => $source ] );
1312                         $this->assertNotNull( $tmpFile,
1313                                 "Creation of local copy of $source succeeded ($backendName)." );
1314                         $contents = file_get_contents( $tmpFile->getPath() );
1315                         $this->assertNotEquals( false, $contents, "Local copy of $source exists ($backendName)." );
1316                         $this->assertEquals(
1317                                 $content[0],
1318                                 $contents,
1319                                 "Local copy of $source is correct ($backendName)."
1320                         );
1321                 }
1322
1323                 $obj = new stdClass();
1324                 $tmpFile->bind( $obj );
1325         }
1326
1327         public static function provider_testGetLocalCopy() {
1328                 $cases = [];
1329
1330                 $base = self::baseStorePath();
1331                 $cases[] = [ "$base/unittest-cont1/e/a/z/some_file.txt", "some file contents" ];
1332                 $cases[] = [ "$base/unittest-cont1/e/a/some-other_file.txt", "more file contents" ];
1333                 $cases[] = [ "$base/unittest-cont1/e/a/\$odd&.txt", "test file contents" ];
1334                 $cases[] = [
1335                         [ "$base/unittest-cont1/e/a/x.txt", "$base/unittest-cont1/e/a/y.txt",
1336                                 "$base/unittest-cont1/e/a/z.txt" ],
1337                         [ "contents xx $", "contents xy 111", "contents xz" ]
1338                 ];
1339
1340                 return $cases;
1341         }
1342
1343         /**
1344          * @dataProvider provider_testGetLocalReference
1345          * @covers FileBackend::getLocalReference
1346          */
1347         public function testGetLocalReference( $source, $content ) {
1348                 $this->backend = $this->singleBackend;
1349                 $this->tearDownFiles();
1350                 $this->doTestGetLocalReference( $source, $content );
1351                 $this->tearDownFiles();
1352
1353                 $this->backend = $this->multiBackend;
1354                 $this->tearDownFiles();
1355                 $this->doTestGetLocalReference( $source, $content );
1356                 $this->tearDownFiles();
1357         }
1358
1359         private function doTestGetLocalReference( $source, $content ) {
1360                 $backendName = $this->backendClass();
1361
1362                 $srcs = (array)$source;
1363                 $content = (array)$content;
1364                 foreach ( $srcs as $i => $src ) {
1365                         $this->prepare( [ 'dir' => dirname( $src ) ] );
1366                         $status = $this->backend->doOperation(
1367                                 [ 'op' => 'create', 'content' => $content[$i], 'dst' => $src ] );
1368                         $this->assertGoodStatus( $status,
1369                                 "Creation of file at $src succeeded ($backendName)." );
1370                 }
1371
1372                 if ( is_array( $source ) ) {
1373                         $tmpFiles = $this->backend->getLocalReferenceMulti( [ 'srcs' => $source ] );
1374                         foreach ( $tmpFiles as $path => $tmpFile ) {
1375                                 $this->assertNotNull( $tmpFile,
1376                                         "Creation of local copy of $path succeeded ($backendName)." );
1377                                 $contents = file_get_contents( $tmpFile->getPath() );
1378                                 $this->assertNotEquals( false, $contents, "Local ref of $path exists ($backendName)." );
1379                                 $this->assertEquals(
1380                                         current( $content ),
1381                                         $contents,
1382                                         "Local ref of $path is correct ($backendName)."
1383                                 );
1384                                 next( $content );
1385                         }
1386                         $this->assertEquals(
1387                                 $source,
1388                                 array_keys( $tmpFiles ),
1389                                 "Local refs in right order ($backendName)."
1390                         );
1391                         $this->assertEquals(
1392                                 count( $source ),
1393                                 count( $tmpFiles ),
1394                                 "Local refs array size correct ($backendName)."
1395                         );
1396                 } else {
1397                         $tmpFile = $this->backend->getLocalReference( [ 'src' => $source ] );
1398                         $this->assertNotNull( $tmpFile,
1399                                 "Creation of local copy of $source succeeded ($backendName)." );
1400                         $contents = file_get_contents( $tmpFile->getPath() );
1401                         $this->assertNotEquals( false, $contents, "Local ref of $source exists ($backendName)." );
1402                         $this->assertEquals( $content[0], $contents, "Local ref of $source is correct ($backendName)." );
1403                 }
1404         }
1405
1406         public static function provider_testGetLocalReference() {
1407                 $cases = [];
1408
1409                 $base = self::baseStorePath();
1410                 $cases[] = [ "$base/unittest-cont1/e/a/z/some_file.txt", "some file contents" ];
1411                 $cases[] = [ "$base/unittest-cont1/e/a/some-other_file.txt", "more file contents" ];
1412                 $cases[] = [ "$base/unittest-cont1/e/a/\$odd&.txt", "test file contents" ];
1413                 $cases[] = [
1414                         [ "$base/unittest-cont1/e/a/x.txt", "$base/unittest-cont1/e/a/y.txt",
1415                                 "$base/unittest-cont1/e/a/z.txt" ],
1416                         [ "contents xx 1111", "contents xy %", "contents xz $" ]
1417                 ];
1418
1419                 return $cases;
1420         }
1421
1422         /**
1423          * @covers FileBackend::getLocalCopy
1424          * @covers FileBackend::getLocalReference
1425          */
1426         public function testGetLocalCopyAndReference404() {
1427                 $this->backend = $this->singleBackend;
1428                 $this->tearDownFiles();
1429                 $this->doTestGetLocalCopyAndReference404();
1430                 $this->tearDownFiles();
1431
1432                 $this->backend = $this->multiBackend;
1433                 $this->tearDownFiles();
1434                 $this->doTestGetLocalCopyAndReference404();
1435                 $this->tearDownFiles();
1436         }
1437
1438         public function doTestGetLocalCopyAndReference404() {
1439                 $backendName = $this->backendClass();
1440
1441                 $base = self::baseStorePath();
1442
1443                 $tmpFile = $this->backend->getLocalCopy( [
1444                         'src' => "$base/unittest-cont1/not-there" ] );
1445                 $this->assertEquals( null, $tmpFile, "Local copy of not existing file is null ($backendName)." );
1446
1447                 $tmpFile = $this->backend->getLocalReference( [
1448                         'src' => "$base/unittest-cont1/not-there" ] );
1449                 $this->assertEquals( null, $tmpFile, "Local ref of not existing file is null ($backendName)." );
1450         }
1451
1452         /**
1453          * @dataProvider provider_testGetFileHttpUrl
1454          * @covers FileBackend::getFileHttpUrl
1455          */
1456         public function testGetFileHttpUrl( $source, $content ) {
1457                 $this->backend = $this->singleBackend;
1458                 $this->tearDownFiles();
1459                 $this->doTestGetFileHttpUrl( $source, $content );
1460                 $this->tearDownFiles();
1461
1462                 $this->backend = $this->multiBackend;
1463                 $this->tearDownFiles();
1464                 $this->doTestGetFileHttpUrl( $source, $content );
1465                 $this->tearDownFiles();
1466         }
1467
1468         private function doTestGetFileHttpUrl( $source, $content ) {
1469                 $backendName = $this->backendClass();
1470
1471                 $this->prepare( [ 'dir' => dirname( $source ) ] );
1472                 $status = $this->backend->doOperation(
1473                         [ 'op' => 'create', 'content' => $content, 'dst' => $source ] );
1474                 $this->assertGoodStatus( $status,
1475                         "Creation of file at $source succeeded ($backendName)." );
1476
1477                 $url = $this->backend->getFileHttpUrl( [ 'src' => $source ] );
1478
1479                 if ( $url !== null ) { // supported
1480                         $data = Http::request( "GET", $url, [], __METHOD__ );
1481                         $this->assertEquals( $content, $data,
1482                                 "HTTP GET of URL has right contents ($backendName)." );
1483                 }
1484         }
1485
1486         public static function provider_testGetFileHttpUrl() {
1487                 $cases = [];
1488
1489                 $base = self::baseStorePath();
1490                 $cases[] = [ "$base/unittest-cont1/e/a/z/some_file.txt", "some file contents" ];
1491                 $cases[] = [ "$base/unittest-cont1/e/a/some-other_file.txt", "more file contents" ];
1492                 $cases[] = [ "$base/unittest-cont1/e/a/\$odd&.txt", "test file contents" ];
1493
1494                 return $cases;
1495         }
1496
1497         /**
1498          * @dataProvider provider_testPrepareAndClean
1499          * @covers FileBackend::prepare
1500          * @covers FileBackend::clean
1501          */
1502         public function testPrepareAndClean( $path, $isOK ) {
1503                 $this->backend = $this->singleBackend;
1504                 $this->doTestPrepareAndClean( $path, $isOK );
1505                 $this->tearDownFiles();
1506
1507                 $this->backend = $this->multiBackend;
1508                 $this->doTestPrepareAndClean( $path, $isOK );
1509                 $this->tearDownFiles();
1510         }
1511
1512         public static function provider_testPrepareAndClean() {
1513                 $base = self::baseStorePath();
1514
1515                 return [
1516                         [ "$base/unittest-cont1/e/a/z/some_file1.txt", true ],
1517                         [ "$base/unittest-cont2/a/z/some_file2.txt", true ],
1518                         # Specific to FS backend with no basePath field set
1519                         # array( "$base/unittest-cont3/a/z/some_file3.txt", false ),
1520                 ];
1521         }
1522
1523         private function doTestPrepareAndClean( $path, $isOK ) {
1524                 $backendName = $this->backendClass();
1525
1526                 $status = $this->prepare( [ 'dir' => dirname( $path ) ] );
1527                 if ( $isOK ) {
1528                         $this->assertGoodStatus( $status,
1529                                 "Preparing dir $path succeeded without warnings ($backendName)." );
1530                         $this->assertEquals( true, $status->isOK(),
1531                                 "Preparing dir $path succeeded ($backendName)." );
1532                 } else {
1533                         $this->assertEquals( false, $status->isOK(),
1534                                 "Preparing dir $path failed ($backendName)." );
1535                 }
1536
1537                 $status = $this->backend->secure( [ 'dir' => dirname( $path ) ] );
1538                 if ( $isOK ) {
1539                         $this->assertGoodStatus( $status,
1540                                 "Securing dir $path succeeded without warnings ($backendName)." );
1541                         $this->assertEquals( true, $status->isOK(),
1542                                 "Securing dir $path succeeded ($backendName)." );
1543                 } else {
1544                         $this->assertEquals( false, $status->isOK(),
1545                                 "Securing dir $path failed ($backendName)." );
1546                 }
1547
1548                 $status = $this->backend->publish( [ 'dir' => dirname( $path ) ] );
1549                 if ( $isOK ) {
1550                         $this->assertGoodStatus( $status,
1551                                 "Publishing dir $path succeeded without warnings ($backendName)." );
1552                         $this->assertEquals( true, $status->isOK(),
1553                                 "Publishing dir $path succeeded ($backendName)." );
1554                 } else {
1555                         $this->assertEquals( false, $status->isOK(),
1556                                 "Publishing dir $path failed ($backendName)." );
1557                 }
1558
1559                 $status = $this->backend->clean( [ 'dir' => dirname( $path ) ] );
1560                 if ( $isOK ) {
1561                         $this->assertGoodStatus( $status,
1562                                 "Cleaning dir $path succeeded without warnings ($backendName)." );
1563                         $this->assertEquals( true, $status->isOK(),
1564                                 "Cleaning dir $path succeeded ($backendName)." );
1565                 } else {
1566                         $this->assertEquals( false, $status->isOK(),
1567                                 "Cleaning dir $path failed ($backendName)." );
1568                 }
1569         }
1570
1571         public function testRecursiveClean() {
1572                 $this->backend = $this->singleBackend;
1573                 $this->doTestRecursiveClean();
1574                 $this->tearDownFiles();
1575
1576                 $this->backend = $this->multiBackend;
1577                 $this->doTestRecursiveClean();
1578                 $this->tearDownFiles();
1579         }
1580
1581         /**
1582          * @covers FileBackend::clean
1583          */
1584         private function doTestRecursiveClean() {
1585                 $backendName = $this->backendClass();
1586
1587                 $base = self::baseStorePath();
1588                 $dirs = [
1589                         "$base/unittest-cont1",
1590                         "$base/unittest-cont1/e",
1591                         "$base/unittest-cont1/e/a",
1592                         "$base/unittest-cont1/e/a/b",
1593                         "$base/unittest-cont1/e/a/b/c",
1594                         "$base/unittest-cont1/e/a/b/c/d0",
1595                         "$base/unittest-cont1/e/a/b/c/d1",
1596                         "$base/unittest-cont1/e/a/b/c/d2",
1597                         "$base/unittest-cont1/e/a/b/c/d0/1",
1598                         "$base/unittest-cont1/e/a/b/c/d0/2",
1599                         "$base/unittest-cont1/e/a/b/c/d1/3",
1600                         "$base/unittest-cont1/e/a/b/c/d1/4",
1601                         "$base/unittest-cont1/e/a/b/c/d2/5",
1602                         "$base/unittest-cont1/e/a/b/c/d2/6"
1603                 ];
1604                 foreach ( $dirs as $dir ) {
1605                         $status = $this->prepare( [ 'dir' => $dir ] );
1606                         $this->assertGoodStatus( $status,
1607                                 "Preparing dir $dir succeeded without warnings ($backendName)." );
1608                 }
1609
1610                 if ( $this->backend instanceof FSFileBackend ) {
1611                         foreach ( $dirs as $dir ) {
1612                                 $this->assertEquals( true, $this->backend->directoryExists( [ 'dir' => $dir ] ),
1613                                         "Dir $dir exists ($backendName)." );
1614                         }
1615                 }
1616
1617                 $status = $this->backend->clean(
1618                         [ 'dir' => "$base/unittest-cont1", 'recursive' => 1 ] );
1619                 $this->assertGoodStatus( $status,
1620                         "Recursive cleaning of dir $dir succeeded without warnings ($backendName)." );
1621
1622                 foreach ( $dirs as $dir ) {
1623                         $this->assertEquals( false, $this->backend->directoryExists( [ 'dir' => $dir ] ),
1624                                 "Dir $dir no longer exists ($backendName)." );
1625                 }
1626         }
1627
1628         /**
1629          * @covers FileBackend::doOperations
1630          */
1631         public function testDoOperations() {
1632                 $this->backend = $this->singleBackend;
1633                 $this->tearDownFiles();
1634                 $this->doTestDoOperations();
1635                 $this->tearDownFiles();
1636
1637                 $this->backend = $this->multiBackend;
1638                 $this->tearDownFiles();
1639                 $this->doTestDoOperations();
1640                 $this->tearDownFiles();
1641         }
1642
1643         private function doTestDoOperations() {
1644                 $base = self::baseStorePath();
1645
1646                 $fileA = "$base/unittest-cont1/e/a/b/fileA.txt";
1647                 $fileAContents = '3tqtmoeatmn4wg4qe-mg3qt3 tq';
1648                 $fileB = "$base/unittest-cont1/e/a/b/fileB.txt";
1649                 $fileBContents = 'g-jmq3gpqgt3qtg q3GT ';
1650                 $fileC = "$base/unittest-cont1/e/a/b/fileC.txt";
1651                 $fileCContents = 'eigna[ogmewt 3qt g3qg flew[ag';
1652                 $fileD = "$base/unittest-cont1/e/a/b/fileD.txt";
1653
1654                 $this->prepare( [ 'dir' => dirname( $fileA ) ] );
1655                 $this->create( [ 'dst' => $fileA, 'content' => $fileAContents ] );
1656                 $this->prepare( [ 'dir' => dirname( $fileB ) ] );
1657                 $this->create( [ 'dst' => $fileB, 'content' => $fileBContents ] );
1658                 $this->prepare( [ 'dir' => dirname( $fileC ) ] );
1659                 $this->create( [ 'dst' => $fileC, 'content' => $fileCContents ] );
1660                 $this->prepare( [ 'dir' => dirname( $fileD ) ] );
1661
1662                 $status = $this->backend->doOperations( [
1663                         [ 'op' => 'describe', 'src' => $fileA,
1664                                 'headers' => [ 'X-Content-Length' => '91.3' ], 'disposition' => 'inline' ],
1665                         [ 'op' => 'copy', 'src' => $fileA, 'dst' => $fileC, 'overwrite' => 1 ],
1666                         // Now: A:<A>, B:<B>, C:<A>, D:<empty> (file:<orginal contents>)
1667                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileA, 'overwriteSame' => 1 ],
1668                         // Now: A:<A>, B:<B>, C:<A>, D:<empty>
1669                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileD, 'overwrite' => 1 ],
1670                         // Now: A:<A>, B:<B>, C:<empty>, D:<A>
1671                         [ 'op' => 'move', 'src' => $fileB, 'dst' => $fileC ],
1672                         // Now: A:<A>, B:<empty>, C:<B>, D:<A>
1673                         [ 'op' => 'move', 'src' => $fileD, 'dst' => $fileA, 'overwriteSame' => 1 ],
1674                         // Now: A:<A>, B:<empty>, C:<B>, D:<empty>
1675                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileA, 'overwrite' => 1 ],
1676                         // Now: A:<B>, B:<empty>, C:<empty>, D:<empty>
1677                         [ 'op' => 'copy', 'src' => $fileA, 'dst' => $fileC ],
1678                         // Now: A:<B>, B:<empty>, C:<B>, D:<empty>
1679                         [ 'op' => 'move', 'src' => $fileA, 'dst' => $fileC, 'overwriteSame' => 1 ],
1680                         // Now: A:<empty>, B:<empty>, C:<B>, D:<empty>
1681                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileC, 'overwrite' => 1 ],
1682                         // Does nothing
1683                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileC, 'overwriteSame' => 1 ],
1684                         // Does nothing
1685                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileC, 'overwrite' => 1 ],
1686                         // Does nothing
1687                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileC, 'overwriteSame' => 1 ],
1688                         // Does nothing
1689                         [ 'op' => 'null' ],
1690                         // Does nothing
1691                 ] );
1692
1693                 $this->assertGoodStatus( $status, "Operation batch succeeded" );
1694                 $this->assertEquals( true, $status->isOK(), "Operation batch succeeded" );
1695                 $this->assertEquals( 14, count( $status->success ),
1696                         "Operation batch has correct success array" );
1697
1698                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileA ] ),
1699                         "File does not exist at $fileA" );
1700                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileB ] ),
1701                         "File does not exist at $fileB" );
1702                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileD ] ),
1703                         "File does not exist at $fileD" );
1704
1705                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $fileC ] ),
1706                         "File exists at $fileC" );
1707                 $this->assertEquals( $fileBContents,
1708                         $this->backend->getFileContents( [ 'src' => $fileC ] ),
1709                         "Correct file contents of $fileC" );
1710                 $this->assertEquals( strlen( $fileBContents ),
1711                         $this->backend->getFileSize( [ 'src' => $fileC ] ),
1712                         "Correct file size of $fileC" );
1713                 $this->assertEquals( Wikimedia\base_convert( sha1( $fileBContents ), 16, 36, 31 ),
1714                         $this->backend->getFileSha1Base36( [ 'src' => $fileC ] ),
1715                         "Correct file SHA-1 of $fileC" );
1716         }
1717
1718         /**
1719          * @covers FileBackend::doOperations
1720          */
1721         public function testDoOperationsPipeline() {
1722                 $this->backend = $this->singleBackend;
1723                 $this->tearDownFiles();
1724                 $this->doTestDoOperationsPipeline();
1725                 $this->tearDownFiles();
1726
1727                 $this->backend = $this->multiBackend;
1728                 $this->tearDownFiles();
1729                 $this->doTestDoOperationsPipeline();
1730                 $this->tearDownFiles();
1731         }
1732
1733         // concurrency orientated
1734         private function doTestDoOperationsPipeline() {
1735                 $base = self::baseStorePath();
1736
1737                 $fileAContents = '3tqtmoeatmn4wg4qe-mg3qt3 tq';
1738                 $fileBContents = 'g-jmq3gpqgt3qtg q3GT ';
1739                 $fileCContents = 'eigna[ogmewt 3qt g3qg flew[ag';
1740
1741                 $tmpNameA = TempFSFile::factory( "unittests_", 'txt' )->getPath();
1742                 $tmpNameB = TempFSFile::factory( "unittests_", 'txt' )->getPath();
1743                 $tmpNameC = TempFSFile::factory( "unittests_", 'txt' )->getPath();
1744                 $this->addTmpFiles( [ $tmpNameA, $tmpNameB, $tmpNameC ] );
1745                 file_put_contents( $tmpNameA, $fileAContents );
1746                 file_put_contents( $tmpNameB, $fileBContents );
1747                 file_put_contents( $tmpNameC, $fileCContents );
1748
1749                 $fileA = "$base/unittest-cont1/e/a/b/fileA.txt";
1750                 $fileB = "$base/unittest-cont1/e/a/b/fileB.txt";
1751                 $fileC = "$base/unittest-cont1/e/a/b/fileC.txt";
1752                 $fileD = "$base/unittest-cont1/e/a/b/fileD.txt";
1753
1754                 $this->prepare( [ 'dir' => dirname( $fileA ) ] );
1755                 $this->create( [ 'dst' => $fileA, 'content' => $fileAContents ] );
1756                 $this->prepare( [ 'dir' => dirname( $fileB ) ] );
1757                 $this->prepare( [ 'dir' => dirname( $fileC ) ] );
1758                 $this->prepare( [ 'dir' => dirname( $fileD ) ] );
1759
1760                 $status = $this->backend->doOperations( [
1761                         [ 'op' => 'store', 'src' => $tmpNameA, 'dst' => $fileA, 'overwriteSame' => 1 ],
1762                         [ 'op' => 'store', 'src' => $tmpNameB, 'dst' => $fileB, 'overwrite' => 1 ],
1763                         [ 'op' => 'store', 'src' => $tmpNameC, 'dst' => $fileC, 'overwrite' => 1 ],
1764                         [ 'op' => 'copy', 'src' => $fileA, 'dst' => $fileC, 'overwrite' => 1 ],
1765                         // Now: A:<A>, B:<B>, C:<A>, D:<empty> (file:<orginal contents>)
1766                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileA, 'overwriteSame' => 1 ],
1767                         // Now: A:<A>, B:<B>, C:<A>, D:<empty>
1768                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileD, 'overwrite' => 1 ],
1769                         // Now: A:<A>, B:<B>, C:<empty>, D:<A>
1770                         [ 'op' => 'move', 'src' => $fileB, 'dst' => $fileC ],
1771                         // Now: A:<A>, B:<empty>, C:<B>, D:<A>
1772                         [ 'op' => 'move', 'src' => $fileD, 'dst' => $fileA, 'overwriteSame' => 1 ],
1773                         // Now: A:<A>, B:<empty>, C:<B>, D:<empty>
1774                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileA, 'overwrite' => 1 ],
1775                         // Now: A:<B>, B:<empty>, C:<empty>, D:<empty>
1776                         [ 'op' => 'copy', 'src' => $fileA, 'dst' => $fileC ],
1777                         // Now: A:<B>, B:<empty>, C:<B>, D:<empty>
1778                         [ 'op' => 'move', 'src' => $fileA, 'dst' => $fileC, 'overwriteSame' => 1 ],
1779                         // Now: A:<empty>, B:<empty>, C:<B>, D:<empty>
1780                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileC, 'overwrite' => 1 ],
1781                         // Does nothing
1782                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileC, 'overwriteSame' => 1 ],
1783                         // Does nothing
1784                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileC, 'overwrite' => 1 ],
1785                         // Does nothing
1786                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileC, 'overwriteSame' => 1 ],
1787                         // Does nothing
1788                         [ 'op' => 'null' ],
1789                         // Does nothing
1790                 ] );
1791
1792                 $this->assertGoodStatus( $status, "Operation batch succeeded" );
1793                 $this->assertEquals( true, $status->isOK(), "Operation batch succeeded" );
1794                 $this->assertEquals( 16, count( $status->success ),
1795                         "Operation batch has correct success array" );
1796
1797                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileA ] ),
1798                         "File does not exist at $fileA" );
1799                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileB ] ),
1800                         "File does not exist at $fileB" );
1801                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileD ] ),
1802                         "File does not exist at $fileD" );
1803
1804                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $fileC ] ),
1805                         "File exists at $fileC" );
1806                 $this->assertEquals( $fileBContents,
1807                         $this->backend->getFileContents( [ 'src' => $fileC ] ),
1808                         "Correct file contents of $fileC" );
1809                 $this->assertEquals( strlen( $fileBContents ),
1810                         $this->backend->getFileSize( [ 'src' => $fileC ] ),
1811                         "Correct file size of $fileC" );
1812                 $this->assertEquals( Wikimedia\base_convert( sha1( $fileBContents ), 16, 36, 31 ),
1813                         $this->backend->getFileSha1Base36( [ 'src' => $fileC ] ),
1814                         "Correct file SHA-1 of $fileC" );
1815         }
1816
1817         /**
1818          * @covers FileBackend::doOperations
1819          */
1820         public function testDoOperationsFailing() {
1821                 $this->backend = $this->singleBackend;
1822                 $this->tearDownFiles();
1823                 $this->doTestDoOperationsFailing();
1824                 $this->tearDownFiles();
1825
1826                 $this->backend = $this->multiBackend;
1827                 $this->tearDownFiles();
1828                 $this->doTestDoOperationsFailing();
1829                 $this->tearDownFiles();
1830         }
1831
1832         private function doTestDoOperationsFailing() {
1833                 $base = self::baseStorePath();
1834
1835                 $fileA = "$base/unittest-cont2/a/b/fileA.txt";
1836                 $fileAContents = '3tqtmoeatmn4wg4qe-mg3qt3 tq';
1837                 $fileB = "$base/unittest-cont2/a/b/fileB.txt";
1838                 $fileBContents = 'g-jmq3gpqgt3qtg q3GT ';
1839                 $fileC = "$base/unittest-cont2/a/b/fileC.txt";
1840                 $fileCContents = 'eigna[ogmewt 3qt g3qg flew[ag';
1841                 $fileD = "$base/unittest-cont2/a/b/fileD.txt";
1842
1843                 $this->prepare( [ 'dir' => dirname( $fileA ) ] );
1844                 $this->create( [ 'dst' => $fileA, 'content' => $fileAContents ] );
1845                 $this->prepare( [ 'dir' => dirname( $fileB ) ] );
1846                 $this->create( [ 'dst' => $fileB, 'content' => $fileBContents ] );
1847                 $this->prepare( [ 'dir' => dirname( $fileC ) ] );
1848                 $this->create( [ 'dst' => $fileC, 'content' => $fileCContents ] );
1849
1850                 $status = $this->backend->doOperations( [
1851                         [ 'op' => 'copy', 'src' => $fileA, 'dst' => $fileC, 'overwrite' => 1 ],
1852                         // Now: A:<A>, B:<B>, C:<A>, D:<empty> (file:<orginal contents>)
1853                         [ 'op' => 'copy', 'src' => $fileC, 'dst' => $fileA, 'overwriteSame' => 1 ],
1854                         // Now: A:<A>, B:<B>, C:<A>, D:<empty>
1855                         [ 'op' => 'copy', 'src' => $fileB, 'dst' => $fileD, 'overwrite' => 1 ],
1856                         // Now: A:<A>, B:<B>, C:<A>, D:<B>
1857                         [ 'op' => 'move', 'src' => $fileC, 'dst' => $fileD ],
1858                         // Now: A:<A>, B:<B>, C:<A>, D:<empty> (failed)
1859                         [ 'op' => 'move', 'src' => $fileB, 'dst' => $fileC, 'overwriteSame' => 1 ],
1860                         // Now: A:<A>, B:<B>, C:<A>, D:<empty> (failed)
1861                         [ 'op' => 'move', 'src' => $fileB, 'dst' => $fileA, 'overwrite' => 1 ],
1862                         // Now: A:<B>, B:<empty>, C:<A>, D:<empty>
1863                         [ 'op' => 'delete', 'src' => $fileD ],
1864                         // Now: A:<B>, B:<empty>, C:<A>, D:<empty>
1865                         [ 'op' => 'null' ],
1866                         // Does nothing
1867                 ], [ 'force' => 1 ] );
1868
1869                 $this->assertNotEquals( [], $status->errors, "Operation had warnings" );
1870                 $this->assertEquals( true, $status->isOK(), "Operation batch succeeded" );
1871                 $this->assertEquals( 8, count( $status->success ),
1872                         "Operation batch has correct success array" );
1873
1874                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileB ] ),
1875                         "File does not exist at $fileB" );
1876                 $this->assertEquals( false, $this->backend->fileExists( [ 'src' => $fileD ] ),
1877                         "File does not exist at $fileD" );
1878
1879                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $fileA ] ),
1880                         "File does not exist at $fileA" );
1881                 $this->assertEquals( true, $this->backend->fileExists( [ 'src' => $fileC ] ),
1882                         "File exists at $fileC" );
1883                 $this->assertEquals( $fileBContents,
1884                         $this->backend->getFileContents( [ 'src' => $fileA ] ),
1885                         "Correct file contents of $fileA" );
1886                 $this->assertEquals( strlen( $fileBContents ),
1887                         $this->backend->getFileSize( [ 'src' => $fileA ] ),
1888                         "Correct file size of $fileA" );
1889                 $this->assertEquals( Wikimedia\base_convert( sha1( $fileBContents ), 16, 36, 31 ),
1890                         $this->backend->getFileSha1Base36( [ 'src' => $fileA ] ),
1891                         "Correct file SHA-1 of $fileA" );
1892         }
1893
1894         /**
1895          * @covers FileBackend::getFileList
1896          */
1897         public function testGetFileList() {
1898                 $this->backend = $this->singleBackend;
1899                 $this->tearDownFiles();
1900                 $this->doTestGetFileList();
1901                 $this->tearDownFiles();
1902
1903                 $this->backend = $this->multiBackend;
1904                 $this->tearDownFiles();
1905                 $this->doTestGetFileList();
1906                 $this->tearDownFiles();
1907         }
1908
1909         private function doTestGetFileList() {
1910                 $backendName = $this->backendClass();
1911                 $base = self::baseStorePath();
1912
1913                 // Should have no errors
1914                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont-notexists" ] );
1915
1916                 $files = [
1917                         "$base/unittest-cont1/e/test1.txt",
1918                         "$base/unittest-cont1/e/test2.txt",
1919                         "$base/unittest-cont1/e/test3.txt",
1920                         "$base/unittest-cont1/e/subdir1/test1.txt",
1921                         "$base/unittest-cont1/e/subdir1/test2.txt",
1922                         "$base/unittest-cont1/e/subdir2/test3.txt",
1923                         "$base/unittest-cont1/e/subdir2/test4.txt",
1924                         "$base/unittest-cont1/e/subdir2/subdir/test1.txt",
1925                         "$base/unittest-cont1/e/subdir2/subdir/test2.txt",
1926                         "$base/unittest-cont1/e/subdir2/subdir/test3.txt",
1927                         "$base/unittest-cont1/e/subdir2/subdir/test4.txt",
1928                         "$base/unittest-cont1/e/subdir2/subdir/test5.txt",
1929                         "$base/unittest-cont1/e/subdir2/subdir/sub/test0.txt",
1930                         "$base/unittest-cont1/e/subdir2/subdir/sub/120-px-file.txt",
1931                 ];
1932
1933                 // Add the files
1934                 $ops = [];
1935                 foreach ( $files as $file ) {
1936                         $this->prepare( [ 'dir' => dirname( $file ) ] );
1937                         $ops[] = [ 'op' => 'create', 'content' => 'xxy', 'dst' => $file ];
1938                 }
1939                 $status = $this->backend->doQuickOperations( $ops );
1940                 $this->assertGoodStatus( $status,
1941                         "Creation of files succeeded ($backendName)." );
1942                 $this->assertEquals( true, $status->isOK(),
1943                         "Creation of files succeeded with OK status ($backendName)." );
1944
1945                 // Expected listing at root
1946                 $expected = [
1947                         "e/test1.txt",
1948                         "e/test2.txt",
1949                         "e/test3.txt",
1950                         "e/subdir1/test1.txt",
1951                         "e/subdir1/test2.txt",
1952                         "e/subdir2/test3.txt",
1953                         "e/subdir2/test4.txt",
1954                         "e/subdir2/subdir/test1.txt",
1955                         "e/subdir2/subdir/test2.txt",
1956                         "e/subdir2/subdir/test3.txt",
1957                         "e/subdir2/subdir/test4.txt",
1958                         "e/subdir2/subdir/test5.txt",
1959                         "e/subdir2/subdir/sub/test0.txt",
1960                         "e/subdir2/subdir/sub/120-px-file.txt",
1961                 ];
1962                 sort( $expected );
1963
1964                 // Actual listing (no trailing slash) at root
1965                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont1" ] );
1966                 $list = $this->listToArray( $iter );
1967                 sort( $list );
1968                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
1969
1970                 // Actual listing (no trailing slash) at root with advise
1971                 $iter = $this->backend->getFileList( [
1972                         'dir' => "$base/unittest-cont1",
1973                         'adviseStat' => 1
1974                 ] );
1975                 $list = $this->listToArray( $iter );
1976                 sort( $list );
1977                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
1978
1979                 // Actual listing (with trailing slash) at root
1980                 $list = [];
1981                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont1/" ] );
1982                 foreach ( $iter as $file ) {
1983                         $list[] = $file;
1984                 }
1985                 sort( $list );
1986                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
1987
1988                 // Expected listing at subdir
1989                 $expected = [
1990                         "test1.txt",
1991                         "test2.txt",
1992                         "test3.txt",
1993                         "test4.txt",
1994                         "test5.txt",
1995                         "sub/test0.txt",
1996                         "sub/120-px-file.txt",
1997                 ];
1998                 sort( $expected );
1999
2000                 // Actual listing (no trailing slash) at subdir
2001                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont1/e/subdir2/subdir" ] );
2002                 $list = $this->listToArray( $iter );
2003                 sort( $list );
2004                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
2005
2006                 // Actual listing (no trailing slash) at subdir with advise
2007                 $iter = $this->backend->getFileList( [
2008                         'dir' => "$base/unittest-cont1/e/subdir2/subdir",
2009                         'adviseStat' => 1
2010                 ] );
2011                 $list = $this->listToArray( $iter );
2012                 sort( $list );
2013                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
2014
2015                 // Actual listing (with trailing slash) at subdir
2016                 $list = [];
2017                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont1/e/subdir2/subdir/" ] );
2018                 foreach ( $iter as $file ) {
2019                         $list[] = $file;
2020                 }
2021                 sort( $list );
2022                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName)." );
2023
2024                 // Actual listing (using iterator second time)
2025                 $list = $this->listToArray( $iter );
2026                 sort( $list );
2027                 $this->assertEquals( $expected, $list, "Correct file listing ($backendName), second iteration." );
2028
2029                 // Actual listing (top files only) at root
2030                 $iter = $this->backend->getTopFileList( [ 'dir' => "$base/unittest-cont1" ] );
2031                 $list = $this->listToArray( $iter );
2032                 sort( $list );
2033                 $this->assertEquals( [], $list, "Correct top file listing ($backendName)." );
2034
2035                 // Expected listing (top files only) at subdir
2036                 $expected = [
2037                         "test1.txt",
2038                         "test2.txt",
2039                         "test3.txt",
2040                         "test4.txt",
2041                         "test5.txt"
2042                 ];
2043                 sort( $expected );
2044
2045                 // Actual listing (top files only) at subdir
2046                 $iter = $this->backend->getTopFileList(
2047                         [ 'dir' => "$base/unittest-cont1/e/subdir2/subdir" ]
2048                 );
2049                 $list = $this->listToArray( $iter );
2050                 sort( $list );
2051                 $this->assertEquals( $expected, $list, "Correct top file listing ($backendName)." );
2052
2053                 // Actual listing (top files only) at subdir with advise
2054                 $iter = $this->backend->getTopFileList( [
2055                         'dir' => "$base/unittest-cont1/e/subdir2/subdir",
2056                         'adviseStat' => 1
2057                 ] );
2058                 $list = $this->listToArray( $iter );
2059                 sort( $list );
2060                 $this->assertEquals( $expected, $list, "Correct top file listing ($backendName)." );
2061
2062                 foreach ( $files as $file ) { // clean up
2063                         $this->backend->doOperation( [ 'op' => 'delete', 'src' => $file ] );
2064                 }
2065
2066                 $iter = $this->backend->getFileList( [ 'dir' => "$base/unittest-cont1/not/exists" ] );
2067                 foreach ( $iter as $iter ) {
2068                         // no errors
2069                 }
2070         }
2071
2072         /**
2073          * @covers FileBackend::getTopDirectoryList
2074          * @covers FileBackend::getDirectoryList
2075          */
2076         public function testGetDirectoryList() {
2077                 $this->backend = $this->singleBackend;
2078                 $this->tearDownFiles();
2079                 $this->doTestGetDirectoryList();
2080                 $this->tearDownFiles();
2081
2082                 $this->backend = $this->multiBackend;
2083                 $this->tearDownFiles();
2084                 $this->doTestGetDirectoryList();
2085                 $this->tearDownFiles();
2086         }
2087
2088         private function doTestGetDirectoryList() {
2089                 $backendName = $this->backendClass();
2090
2091                 $base = self::baseStorePath();
2092                 $files = [
2093                         "$base/unittest-cont1/e/test1.txt",
2094                         "$base/unittest-cont1/e/test2.txt",
2095                         "$base/unittest-cont1/e/test3.txt",
2096                         "$base/unittest-cont1/e/subdir1/test1.txt",
2097                         "$base/unittest-cont1/e/subdir1/test2.txt",
2098                         "$base/unittest-cont1/e/subdir2/test3.txt",
2099                         "$base/unittest-cont1/e/subdir2/test4.txt",
2100                         "$base/unittest-cont1/e/subdir2/subdir/test1.txt",
2101                         "$base/unittest-cont1/e/subdir3/subdir/test2.txt",
2102                         "$base/unittest-cont1/e/subdir4/subdir/test3.txt",
2103                         "$base/unittest-cont1/e/subdir4/subdir/test4.txt",
2104                         "$base/unittest-cont1/e/subdir4/subdir/test5.txt",
2105                         "$base/unittest-cont1/e/subdir4/subdir/sub/test0.txt",
2106                         "$base/unittest-cont1/e/subdir4/subdir/sub/120-px-file.txt",
2107                 ];
2108
2109                 // Add the files
2110                 $ops = [];
2111                 foreach ( $files as $file ) {
2112                         $this->prepare( [ 'dir' => dirname( $file ) ] );
2113                         $ops[] = [ 'op' => 'create', 'content' => 'xxy', 'dst' => $file ];
2114                 }
2115                 $status = $this->backend->doQuickOperations( $ops );
2116                 $this->assertGoodStatus( $status,
2117                         "Creation of files succeeded ($backendName)." );
2118                 $this->assertEquals( true, $status->isOK(),
2119                         "Creation of files succeeded with OK status ($backendName)." );
2120
2121                 $this->assertEquals( true,
2122                         $this->backend->directoryExists( [ 'dir' => "$base/unittest-cont1/e/subdir1" ] ),
2123                         "Directory exists in ($backendName)." );
2124                 $this->assertEquals( true,
2125                         $this->backend->directoryExists( [ 'dir' => "$base/unittest-cont1/e/subdir2/subdir" ] ),
2126                         "Directory exists in ($backendName)." );
2127                 $this->assertEquals( false,
2128                         $this->backend->directoryExists( [ 'dir' => "$base/unittest-cont1/e/subdir2/test1.txt" ] ),
2129                         "Directory does not exists in ($backendName)." );
2130
2131                 // Expected listing
2132                 $expected = [
2133                         "e",
2134                 ];
2135                 sort( $expected );
2136
2137                 // Actual listing (no trailing slash)
2138                 $list = [];
2139                 $iter = $this->backend->getTopDirectoryList( [ 'dir' => "$base/unittest-cont1" ] );
2140                 foreach ( $iter as $file ) {
2141                         $list[] = $file;
2142                 }
2143                 sort( $list );
2144
2145                 $this->assertEquals( $expected, $list, "Correct top dir listing ($backendName)." );
2146
2147                 // Expected listing
2148                 $expected = [
2149                         "subdir1",
2150                         "subdir2",
2151                         "subdir3",
2152                         "subdir4",
2153                 ];
2154                 sort( $expected );
2155
2156                 // Actual listing (no trailing slash)
2157                 $list = [];
2158                 $iter = $this->backend->getTopDirectoryList( [ 'dir' => "$base/unittest-cont1/e" ] );
2159                 foreach ( $iter as $file ) {
2160                         $list[] = $file;
2161                 }
2162                 sort( $list );
2163
2164                 $this->assertEquals( $expected, $list, "Correct top dir listing ($backendName)." );
2165
2166                 // Actual listing (with trailing slash)
2167                 $list = [];
2168                 $iter = $this->backend->getTopDirectoryList( [ 'dir' => "$base/unittest-cont1/e/" ] );
2169                 foreach ( $iter as $file ) {
2170                         $list[] = $file;
2171                 }
2172                 sort( $list );
2173
2174                 $this->assertEquals( $expected, $list, "Correct top dir listing ($backendName)." );
2175
2176                 // Expected listing
2177                 $expected = [
2178                         "subdir",
2179                 ];
2180                 sort( $expected );
2181
2182                 // Actual listing (no trailing slash)
2183                 $list = [];
2184                 $iter = $this->backend->getTopDirectoryList( [ 'dir' => "$base/unittest-cont1/e/subdir2" ] );
2185                 foreach ( $iter as $file ) {
2186                         $list[] = $file;
2187                 }
2188                 sort( $list );
2189
2190                 $this->assertEquals( $expected, $list, "Correct top dir listing ($backendName)." );
2191
2192                 // Actual listing (with trailing slash)
2193                 $list = [];
2194                 $iter = $this->backend->getTopDirectoryList(
2195                         [ 'dir' => "$base/unittest-cont1/e/subdir2/" ]
2196                 );
2197
2198                 foreach ( $iter as $file ) {
2199                         $list[] = $file;
2200                 }
2201                 sort( $list );
2202
2203                 $this->assertEquals( $expected, $list, "Correct top dir listing ($backendName)." );
2204
2205                 // Actual listing (using iterator second time)
2206                 $list = [];
2207                 foreach ( $iter as $file ) {
2208                         $list[] = $file;
2209                 }
2210                 sort( $list );
2211
2212                 $this->assertEquals(
2213                         $expected,
2214                         $list,
2215                         "Correct top dir listing ($backendName), second iteration."
2216                 );
2217
2218                 // Expected listing (recursive)
2219                 $expected = [
2220                         "e",
2221                         "e/subdir1",
2222                         "e/subdir2",
2223                         "e/subdir3",
2224                         "e/subdir4",
2225                         "e/subdir2/subdir",
2226                         "e/subdir3/subdir",
2227                         "e/subdir4/subdir",
2228                         "e/subdir4/subdir/sub",
2229                 ];
2230                 sort( $expected );
2231
2232                 // Actual listing (recursive)
2233                 $list = [];
2234                 $iter = $this->backend->getDirectoryList( [ 'dir' => "$base/unittest-cont1/" ] );
2235                 foreach ( $iter as $file ) {
2236                         $list[] = $file;
2237                 }
2238                 sort( $list );
2239
2240                 $this->assertEquals( $expected, $list, "Correct dir listing ($backendName)." );
2241
2242                 // Expected listing (recursive)
2243                 $expected = [
2244                         "subdir",
2245                         "subdir/sub",
2246                 ];
2247                 sort( $expected );
2248
2249                 // Actual listing (recursive)
2250                 $list = [];
2251                 $iter = $this->backend->getDirectoryList( [ 'dir' => "$base/unittest-cont1/e/subdir4" ] );
2252                 foreach ( $iter as $file ) {
2253                         $list[] = $file;
2254                 }
2255                 sort( $list );
2256
2257                 $this->assertEquals( $expected, $list, "Correct dir listing ($backendName)." );
2258
2259                 // Actual listing (recursive, second time)
2260                 $list = [];
2261                 foreach ( $iter as $file ) {
2262                         $list[] = $file;
2263                 }
2264                 sort( $list );
2265
2266                 $this->assertEquals( $expected, $list, "Correct dir listing ($backendName)." );
2267
2268                 $iter = $this->backend->getDirectoryList( [ 'dir' => "$base/unittest-cont1/e/subdir1" ] );
2269                 $items = $this->listToArray( $iter );
2270                 $this->assertEquals( [], $items, "Directory listing is empty." );
2271
2272                 foreach ( $files as $file ) { // clean up
2273                         $this->backend->doOperation( [ 'op' => 'delete', 'src' => $file ] );
2274                 }
2275
2276                 $iter = $this->backend->getDirectoryList( [ 'dir' => "$base/unittest-cont1/not/exists" ] );
2277                 foreach ( $iter as $file ) {
2278                         // no errors
2279                 }
2280
2281                 $items = $this->listToArray( $iter );
2282                 $this->assertEquals( [], $items, "Directory listing is empty." );
2283
2284                 $iter = $this->backend->getDirectoryList( [ 'dir' => "$base/unittest-cont1/e/not/exists" ] );
2285                 $items = $this->listToArray( $iter );
2286                 $this->assertEquals( [], $items, "Directory listing is empty." );
2287         }
2288
2289         /**
2290          * @covers FileBackend::lockFiles
2291          * @covers FileBackend::unlockFiles
2292          */
2293         public function testLockCalls() {
2294                 $this->backend = $this->singleBackend;
2295                 $this->doTestLockCalls();
2296         }
2297
2298         private function doTestLockCalls() {
2299                 $backendName = $this->backendClass();
2300
2301                 $paths = [
2302                         "test1.txt",
2303                         "test2.txt",
2304                         "test3.txt",
2305                         "subdir1",
2306                         "subdir1", // duplicate
2307                         "subdir1/test1.txt",
2308                         "subdir1/test2.txt",
2309                         "subdir2",
2310                         "subdir2", // duplicate
2311                         "subdir2/test3.txt",
2312                         "subdir2/test4.txt",
2313                         "subdir2/subdir",
2314                         "subdir2/subdir/test1.txt",
2315                         "subdir2/subdir/test2.txt",
2316                         "subdir2/subdir/test3.txt",
2317                         "subdir2/subdir/test4.txt",
2318                         "subdir2/subdir/test5.txt",
2319                         "subdir2/subdir/sub",
2320                         "subdir2/subdir/sub/test0.txt",
2321                         "subdir2/subdir/sub/120-px-file.txt",
2322                 ];
2323
2324                 for ( $i = 0; $i < 25; $i++ ) {
2325                         $status = $this->backend->lockFiles( $paths, LockManager::LOCK_EX );
2326                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2327                                 "Locking of files succeeded ($backendName) ($i)." );
2328                         $this->assertEquals( true, $status->isOK(),
2329                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2330
2331                         $status = $this->backend->lockFiles( $paths, LockManager::LOCK_SH );
2332                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2333                                 "Locking of files succeeded ($backendName) ($i)." );
2334                         $this->assertEquals( true, $status->isOK(),
2335                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2336
2337                         $status = $this->backend->unlockFiles( $paths, LockManager::LOCK_SH );
2338                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2339                                 "Locking of files succeeded ($backendName) ($i)." );
2340                         $this->assertEquals( true, $status->isOK(),
2341                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2342
2343                         $status = $this->backend->unlockFiles( $paths, LockManager::LOCK_EX );
2344                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2345                                 "Locking of files succeeded ($backendName). ($i)" );
2346                         $this->assertEquals( true, $status->isOK(),
2347                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2348
2349                         # # Flip the acquire/release ordering around ##
2350
2351                         $status = $this->backend->lockFiles( $paths, LockManager::LOCK_SH );
2352                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2353                                 "Locking of files succeeded ($backendName) ($i)." );
2354                         $this->assertEquals( true, $status->isOK(),
2355                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2356
2357                         $status = $this->backend->lockFiles( $paths, LockManager::LOCK_EX );
2358                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2359                                 "Locking of files succeeded ($backendName) ($i)." );
2360                         $this->assertEquals( true, $status->isOK(),
2361                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2362
2363                         $status = $this->backend->unlockFiles( $paths, LockManager::LOCK_EX );
2364                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2365                                 "Locking of files succeeded ($backendName). ($i)" );
2366                         $this->assertEquals( true, $status->isOK(),
2367                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2368
2369                         $status = $this->backend->unlockFiles( $paths, LockManager::LOCK_SH );
2370                         $this->assertEquals( print_r( [], true ), print_r( $status->errors, true ),
2371                                 "Locking of files succeeded ($backendName) ($i)." );
2372                         $this->assertEquals( true, $status->isOK(),
2373                                 "Locking of files succeeded with OK status ($backendName) ($i)." );
2374                 }
2375
2376                 $status = Status::newGood();
2377                 $sl = $this->backend->getScopedFileLocks( $paths, LockManager::LOCK_EX, $status );
2378                 $this->assertInstanceOf( 'ScopedLock', $sl,
2379                         "Scoped locking of files succeeded ($backendName)." );
2380                 $this->assertEquals( [], $status->errors,
2381                         "Scoped locking of files succeeded ($backendName)." );
2382                 $this->assertEquals( true, $status->isOK(),
2383                         "Scoped locking of files succeeded with OK status ($backendName)." );
2384
2385                 ScopedLock::release( $sl );
2386                 $this->assertEquals( null, $sl,
2387                         "Scoped unlocking of files succeeded ($backendName)." );
2388                 $this->assertEquals( [], $status->errors,
2389                         "Scoped unlocking of files succeeded ($backendName)." );
2390                 $this->assertEquals( true, $status->isOK(),
2391                         "Scoped unlocking of files succeeded with OK status ($backendName)." );
2392         }
2393
2394         /**
2395          * @dataProvider provider_testGetContentType
2396          */
2397         public function testGetContentType( $mimeCallback, $mimeFromString ) {
2398                 global $IP;
2399
2400                 $be = TestingAccessWrapper::newFromObject( new MemoryFileBackend(
2401                         [
2402                                 'name' => 'testing',
2403                                 'class' => 'MemoryFileBackend',
2404                                 'wikiId' => 'meow',
2405                                 'mimeCallback' => $mimeCallback
2406                         ]
2407                 ) );
2408
2409                 $dst = 'mwstore://testing/container/path/to/file_no_ext';
2410                 $src = "$IP/tests/phpunit/data/media/srgb.jpg";
2411                 $this->assertEquals( 'image/jpeg', $be->getContentType( $dst, null, $src ) );
2412                 $this->assertEquals(
2413                         $mimeFromString ? 'image/jpeg' : 'unknown/unknown',
2414                         $be->getContentType( $dst, file_get_contents( $src ), null ) );
2415
2416                 $src = "$IP/tests/phpunit/data/media/Png-native-test.png";
2417                 $this->assertEquals( 'image/png', $be->getContentType( $dst, null, $src ) );
2418                 $this->assertEquals(
2419                         $mimeFromString ? 'image/png' : 'unknown/unknown',
2420                         $be->getContentType( $dst, file_get_contents( $src ), null ) );
2421         }
2422
2423         public static function provider_testGetContentType() {
2424                 return [
2425                         [ null, false ],
2426                         [ [ FileBackendGroup::singleton(), 'guessMimeInternal' ], true ]
2427                 ];
2428         }
2429
2430         public function testReadAffinity() {
2431                 $be = TestingAccessWrapper::newFromObject(
2432                         new FileBackendMultiWrite( [
2433                                 'name' => 'localtesting',
2434                                 'wikiId' => wfWikiId() . mt_rand(),
2435                                 'backends' => [
2436                                         [ // backend 0
2437                                                 'name' => 'multitesting0',
2438                                                 'class' => 'MemoryFileBackend',
2439                                                 'isMultiMaster' => false,
2440                                                 'readAffinity' => true
2441                                         ],
2442                                         [ // backend 1
2443                                                 'name' => 'multitesting1',
2444                                                 'class' => 'MemoryFileBackend',
2445                                                 'isMultiMaster' => true
2446                                         ]
2447                                 ]
2448                         ] )
2449                 );
2450
2451                 $this->assertEquals(
2452                         1,
2453                         $be->getReadIndexFromParams( [ 'latest' => 1 ] ),
2454                         'Reads with "latest" flag use backend 1'
2455                 );
2456                 $this->assertEquals(
2457                         0,
2458                         $be->getReadIndexFromParams( [ 'latest' => 0 ] ),
2459                         'Reads without "latest" flag use backend 0'
2460                 );
2461
2462                 $p = 'container/test-cont/file.txt';
2463                 $be->backends[0]->quickCreate( [
2464                         'dst' => "mwstore://multitesting0/$p", 'content' => 'cattitude' ] );
2465                 $be->backends[1]->quickCreate( [
2466                         'dst' => "mwstore://multitesting1/$p", 'content' => 'princess of power' ] );
2467
2468                 $this->assertEquals(
2469                         'cattitude',
2470                         $be->getFileContents( [ 'src' => "mwstore://localtesting/$p" ] ),
2471                         "Non-latest read came from backend 0"
2472                 );
2473                 $this->assertEquals(
2474                         'princess of power',
2475                         $be->getFileContents( [ 'src' => "mwstore://localtesting/$p", 'latest' => 1 ] ),
2476                         "Latest read came from backend1"
2477                 );
2478         }
2479
2480         public function testAsyncWrites() {
2481                 $be = TestingAccessWrapper::newFromObject(
2482                         new FileBackendMultiWrite( [
2483                                 'name' => 'localtesting',
2484                                 'wikiId' => wfWikiId() . mt_rand(),
2485                                 'backends' => [
2486                                         [ // backend 0
2487                                                 'name' => 'multitesting0',
2488                                                 'class' => 'MemoryFileBackend',
2489                                                 'isMultiMaster' => false
2490                                         ],
2491                                         [ // backend 1
2492                                                 'name' => 'multitesting1',
2493                                                 'class' => 'MemoryFileBackend',
2494                                                 'isMultiMaster' => true
2495                                         ]
2496                                 ],
2497                                 'replication' => 'async'
2498                         ] )
2499                 );
2500
2501                 $this->setMwGlobals( 'wgCommandLineMode', false );
2502
2503                 $p = 'container/test-cont/file.txt';
2504                 $be->quickCreate( [
2505                         'dst' => "mwstore://localtesting/$p", 'content' => 'cattitude' ] );
2506
2507                 $this->assertEquals(
2508                         false,
2509                         $be->backends[0]->getFileContents( [ 'src' => "mwstore://multitesting0/$p" ] ),
2510                         "File not yet written to backend 0"
2511                 );
2512                 $this->assertEquals(
2513                         'cattitude',
2514                         $be->backends[1]->getFileContents( [ 'src' => "mwstore://multitesting1/$p" ] ),
2515                         "File already written to backend 1"
2516                 );
2517
2518                 DeferredUpdates::doUpdates();
2519
2520                 $this->assertEquals(
2521                         'cattitude',
2522                         $be->backends[0]->getFileContents( [ 'src' => "mwstore://multitesting0/$p" ] ),
2523                         "File now written to backend 0"
2524                 );
2525         }
2526
2527         public function testSanitizeOpHeaders() {
2528                 $be = TestingAccessWrapper::newFromObject( new MemoryFileBackend( [
2529                         'name' => 'localtesting',
2530                         'wikiId' => wfWikiID()
2531                 ] ) );
2532
2533                 $name = wfRandomString( 300 );
2534
2535                 $input = [
2536                         'headers' => [
2537                                 'content-Disposition' => FileBackend::makeContentDisposition( 'inline', $name ),
2538                                 'Content-dUration' => 25.6,
2539                                 'X-LONG-VALUE' => str_pad( '0', 300 ),
2540                                 'CONTENT-LENGTH' => 855055,
2541                         ]
2542                 ];
2543                 $expected = [
2544                         'headers' => [
2545                                 'content-disposition' => FileBackend::makeContentDisposition( 'inline', $name ),
2546                                 'content-duration' => 25.6,
2547                                 'content-length' => 855055
2548                         ]
2549                 ];
2550
2551                 MediaWiki\suppressWarnings();
2552                 $actual = $be->sanitizeOpHeaders( $input );
2553                 MediaWiki\restoreWarnings();
2554
2555                 $this->assertEquals( $expected, $actual, "Header sanitized properly" );
2556         }
2557
2558         // helper function
2559         private function listToArray( $iter ) {
2560                 return is_array( $iter ) ? $iter : iterator_to_array( $iter );
2561         }
2562
2563         // test helper wrapper for backend prepare() function
2564         private function prepare( array $params ) {
2565                 return $this->backend->prepare( $params );
2566         }
2567
2568         // test helper wrapper for backend prepare() function
2569         private function create( array $params ) {
2570                 $params['op'] = 'create';
2571
2572                 return $this->backend->doQuickOperations( [ $params ] );
2573         }
2574
2575         function tearDownFiles() {
2576                 $containers = [ 'unittest-cont1', 'unittest-cont2', 'unittest-cont-bad' ];
2577                 foreach ( $containers as $container ) {
2578                         $this->deleteFiles( $container );
2579                 }
2580         }
2581
2582         private function deleteFiles( $container ) {
2583                 $base = self::baseStorePath();
2584                 $iter = $this->backend->getFileList( [ 'dir' => "$base/$container" ] );
2585                 if ( $iter ) {
2586                         foreach ( $iter as $file ) {
2587                                 $this->backend->quickDelete( [ 'src' => "$base/$container/$file" ] );
2588                         }
2589                         // free the directory, to avoid Permission denied under windows on rmdir
2590                         unset( $iter );
2591                 }
2592                 $this->backend->clean( [ 'dir' => "$base/$container", 'recursive' => 1 ] );
2593         }
2594
2595         function assertBackendPathsConsistent( array $paths ) {
2596                 if ( $this->backend instanceof FileBackendMultiWrite ) {
2597                         $status = $this->backend->consistencyCheck( $paths );
2598                         $this->assertGoodStatus( $status, "Files synced: " . implode( ',', $paths ) );
2599                 }
2600         }
2601
2602         function assertGoodStatus( $status, $msg ) {
2603                 $this->assertEquals( print_r( [], 1 ), print_r( $status->errors, 1 ), $msg );
2604         }
2605 }