Merge "[FileBackend] Added a script to copy files from one backend to another. Useful...
[lhc/web/wiklou.git] / includes / filerepo / backend / FSFileBackend.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * @brief Class for a file system (FS) based file backend.
10 *
11 * All "containers" each map to a directory under the backend's base directory.
12 * For backwards-compatibility, some container paths can be set to custom paths.
13 * The wiki ID will not be used in any custom paths, so this should be avoided.
14 *
15 * Having directories with thousands of files will diminish performance.
16 * Sharding can be accomplished by using FileRepo-style hash paths.
17 *
18 * Status messages should avoid mentioning the internal FS paths.
19 * PHP warnings are assumed to be logged rather than output.
20 *
21 * @ingroup FileBackend
22 * @since 1.19
23 */
24 class FSFileBackend extends FileBackendStore {
25 protected $basePath; // string; directory holding the container directories
26 /** @var Array Map of container names to root paths */
27 protected $containerPaths = array(); // for custom container paths
28 protected $fileMode; // integer; file permission mode
29
30 protected $hadWarningErrors = array();
31
32 /**
33 * @see FileBackendStore::__construct()
34 * Additional $config params include:
35 * basePath : File system directory that holds containers.
36 * containerPaths : Map of container names to custom file system directories.
37 * This should only be used for backwards-compatibility.
38 * fileMode : Octal UNIX file permissions to use on files stored.
39 */
40 public function __construct( array $config ) {
41 parent::__construct( $config );
42
43 // Remove any possible trailing slash from directories
44 if ( isset( $config['basePath'] ) ) {
45 $this->basePath = rtrim( $config['basePath'], '/' ); // remove trailing slash
46 } else {
47 $this->basePath = null; // none; containers must have explicit paths
48 }
49
50 if ( isset( $config['containerPaths'] ) ) {
51 $this->containerPaths = (array)$config['containerPaths'];
52 foreach ( $this->containerPaths as &$path ) {
53 $path = rtrim( $path, '/' ); // remove trailing slash
54 }
55 }
56
57 $this->fileMode = isset( $config['fileMode'] )
58 ? $config['fileMode']
59 : 0644;
60 }
61
62 /**
63 * @see FileBackendStore::resolveContainerPath()
64 * @return null|string
65 */
66 protected function resolveContainerPath( $container, $relStoragePath ) {
67 // Check that container has a root directory
68 if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
69 // Check for sane relative paths (assume the base paths are OK)
70 if ( $this->isLegalRelPath( $relStoragePath ) ) {
71 return $relStoragePath;
72 }
73 }
74 return null;
75 }
76
77 /**
78 * Sanity check a relative file system path for validity
79 *
80 * @param $path string Normalized relative path
81 * @return bool
82 */
83 protected function isLegalRelPath( $path ) {
84 // Check for file names longer than 255 chars
85 if ( preg_match( '![^/]{256}!', $path ) ) { // ext3/NTFS
86 return false;
87 }
88 if ( wfIsWindows() ) { // NTFS
89 return !preg_match( '![:*?"<>|]!', $path );
90 } else {
91 return true;
92 }
93 }
94
95 /**
96 * Given the short (unresolved) and full (resolved) name of
97 * a container, return the file system path of the container.
98 *
99 * @param $shortCont string
100 * @param $fullCont string
101 * @return string|null
102 */
103 protected function containerFSRoot( $shortCont, $fullCont ) {
104 if ( isset( $this->containerPaths[$shortCont] ) ) {
105 return $this->containerPaths[$shortCont];
106 } elseif ( isset( $this->basePath ) ) {
107 return "{$this->basePath}/{$fullCont}";
108 }
109 return null; // no container base path defined
110 }
111
112 /**
113 * Get the absolute file system path for a storage path
114 *
115 * @param $storagePath string Storage path
116 * @return string|null
117 */
118 protected function resolveToFSPath( $storagePath ) {
119 list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath );
120 if ( $relPath === null ) {
121 return null; // invalid
122 }
123 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $storagePath );
124 $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
125 if ( $relPath != '' ) {
126 $fsPath .= "/{$relPath}";
127 }
128 return $fsPath;
129 }
130
131 /**
132 * @see FileBackendStore::isPathUsableInternal()
133 * @return bool
134 */
135 public function isPathUsableInternal( $storagePath ) {
136 $fsPath = $this->resolveToFSPath( $storagePath );
137 if ( $fsPath === null ) {
138 return false; // invalid
139 }
140 $parentDir = dirname( $fsPath );
141
142 if ( file_exists( $fsPath ) ) {
143 $ok = is_file( $fsPath ) && is_writable( $fsPath );
144 } else {
145 $ok = is_dir( $parentDir ) && is_writable( $parentDir );
146 }
147
148 return $ok;
149 }
150
151 /**
152 * @see FileBackendStore::doStoreInternal()
153 * @return Status
154 */
155 protected function doStoreInternal( array $params ) {
156 $status = Status::newGood();
157
158 $dest = $this->resolveToFSPath( $params['dst'] );
159 if ( $dest === null ) {
160 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
161 return $status;
162 }
163
164 if ( file_exists( $dest ) ) {
165 if ( !empty( $params['overwrite'] ) ) {
166 $ok = unlink( $dest );
167 if ( !$ok ) {
168 $status->fatal( 'backend-fail-delete', $params['dst'] );
169 return $status;
170 }
171 } else {
172 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
173 return $status;
174 }
175 }
176
177 $ok = copy( $params['src'], $dest );
178 if ( !$ok ) {
179 $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
180 return $status;
181 }
182
183 $this->chmod( $dest );
184
185 return $status;
186 }
187
188 /**
189 * @see FileBackendStore::doCopyInternal()
190 * @return Status
191 */
192 protected function doCopyInternal( array $params ) {
193 $status = Status::newGood();
194
195 $source = $this->resolveToFSPath( $params['src'] );
196 if ( $source === null ) {
197 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
198 return $status;
199 }
200
201 $dest = $this->resolveToFSPath( $params['dst'] );
202 if ( $dest === null ) {
203 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
204 return $status;
205 }
206
207 if ( file_exists( $dest ) ) {
208 if ( !empty( $params['overwrite'] ) ) {
209 $ok = unlink( $dest );
210 if ( !$ok ) {
211 $status->fatal( 'backend-fail-delete', $params['dst'] );
212 return $status;
213 }
214 } else {
215 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
216 return $status;
217 }
218 }
219
220 $ok = copy( $source, $dest );
221 if ( !$ok ) {
222 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
223 return $status;
224 }
225
226 $this->chmod( $dest );
227
228 return $status;
229 }
230
231 /**
232 * @see FileBackendStore::doMoveInternal()
233 * @return Status
234 */
235 protected function doMoveInternal( array $params ) {
236 $status = Status::newGood();
237
238 $source = $this->resolveToFSPath( $params['src'] );
239 if ( $source === null ) {
240 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
241 return $status;
242 }
243
244 $dest = $this->resolveToFSPath( $params['dst'] );
245 if ( $dest === null ) {
246 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
247 return $status;
248 }
249
250 if ( file_exists( $dest ) ) {
251 if ( !empty( $params['overwrite'] ) ) {
252 // Windows does not support moving over existing files
253 if ( wfIsWindows() ) {
254 $ok = unlink( $dest );
255 if ( !$ok ) {
256 $status->fatal( 'backend-fail-delete', $params['dst'] );
257 return $status;
258 }
259 }
260 } else {
261 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
262 return $status;
263 }
264 }
265
266 $ok = rename( $source, $dest );
267 clearstatcache(); // file no longer at source
268 if ( !$ok ) {
269 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
270 return $status;
271 }
272
273 return $status;
274 }
275
276 /**
277 * @see FileBackendStore::doDeleteInternal()
278 * @return Status
279 */
280 protected function doDeleteInternal( array $params ) {
281 $status = Status::newGood();
282
283 $source = $this->resolveToFSPath( $params['src'] );
284 if ( $source === null ) {
285 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
286 return $status;
287 }
288
289 if ( !is_file( $source ) ) {
290 if ( empty( $params['ignoreMissingSource'] ) ) {
291 $status->fatal( 'backend-fail-delete', $params['src'] );
292 }
293 return $status; // do nothing; either OK or bad status
294 }
295
296 $ok = unlink( $source );
297 if ( !$ok ) {
298 $status->fatal( 'backend-fail-delete', $params['src'] );
299 return $status;
300 }
301
302 return $status;
303 }
304
305 /**
306 * @see FileBackendStore::doCreateInternal()
307 * @return Status
308 */
309 protected function doCreateInternal( array $params ) {
310 $status = Status::newGood();
311
312 $dest = $this->resolveToFSPath( $params['dst'] );
313 if ( $dest === null ) {
314 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
315 return $status;
316 }
317
318 if ( file_exists( $dest ) ) {
319 if ( !empty( $params['overwrite'] ) ) {
320 $ok = unlink( $dest );
321 if ( !$ok ) {
322 $status->fatal( 'backend-fail-delete', $params['dst'] );
323 return $status;
324 }
325 } else {
326 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
327 return $status;
328 }
329 }
330
331 $bytes = file_put_contents( $dest, $params['content'] );
332 if ( $bytes === false ) {
333 $status->fatal( 'backend-fail-create', $params['dst'] );
334 return $status;
335 }
336
337 $this->chmod( $dest );
338
339 return $status;
340 }
341
342 /**
343 * @see FileBackendStore::doPrepareInternal()
344 * @return Status
345 */
346 protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
347 $status = Status::newGood();
348 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
349 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
350 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
351 if ( !wfMkdirParents( $dir ) ) { // make directory and its parents
352 $status->fatal( 'directorycreateerror', $params['dir'] );
353 } elseif ( !is_writable( $dir ) ) {
354 $status->fatal( 'directoryreadonlyerror', $params['dir'] );
355 } elseif ( !is_readable( $dir ) ) {
356 $status->fatal( 'directorynotreadableerror', $params['dir'] );
357 }
358 return $status;
359 }
360
361 /**
362 * @see FileBackendStore::doSecureInternal()
363 * @return Status
364 */
365 protected function doSecureInternal( $fullCont, $dirRel, array $params ) {
366 $status = Status::newGood();
367 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
368 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
369 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
370 // Seed new directories with a blank index.html, to prevent crawling...
371 if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) {
372 $bytes = file_put_contents( "{$dir}/index.html", '' );
373 if ( !$bytes ) {
374 $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
375 return $status;
376 }
377 }
378 // Add a .htaccess file to the root of the container...
379 if ( !empty( $params['noAccess'] ) ) {
380 if ( !file_exists( "{$contRoot}/.htaccess" ) ) {
381 $bytes = file_put_contents( "{$contRoot}/.htaccess", "Deny from all\n" );
382 if ( !$bytes ) {
383 $storeDir = "mwstore://{$this->name}/{$shortCont}";
384 $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
385 return $status;
386 }
387 }
388 }
389 return $status;
390 }
391
392 /**
393 * @see FileBackendStore::doCleanInternal()
394 * @return Status
395 */
396 protected function doCleanInternal( $fullCont, $dirRel, array $params ) {
397 $status = Status::newGood();
398 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
399 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
400 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
401 wfSuppressWarnings();
402 if ( is_dir( $dir ) ) {
403 rmdir( $dir ); // remove directory if empty
404 }
405 wfRestoreWarnings();
406 return $status;
407 }
408
409 /**
410 * @see FileBackendStore::doFileExists()
411 * @return array|bool|null
412 */
413 protected function doGetFileStat( array $params ) {
414 $source = $this->resolveToFSPath( $params['src'] );
415 if ( $source === null ) {
416 return false; // invalid storage path
417 }
418
419 $this->trapWarnings(); // don't trust 'false' if there were errors
420 $stat = is_file( $source ) ? stat( $source ) : false; // regular files only
421 $hadError = $this->untrapWarnings();
422
423 if ( $stat ) {
424 return array(
425 'mtime' => wfTimestamp( TS_MW, $stat['mtime'] ),
426 'size' => $stat['size']
427 );
428 } elseif ( !$hadError ) {
429 return false; // file does not exist
430 } else {
431 return null; // failure
432 }
433 }
434
435 /**
436 * @see FileBackendStore::doClearCache()
437 */
438 protected function doClearCache( array $paths = null ) {
439 clearstatcache(); // clear the PHP file stat cache
440 }
441
442 /**
443 * @see FileBackendStore::doDirectoryExists()
444 * @return bool|null
445 */
446 protected function doDirectoryExists( $fullCont, $dirRel, array $params ) {
447 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
448 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
449 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
450
451 $this->trapWarnings(); // don't trust 'false' if there were errors
452 $exists = is_dir( $dir );
453 $hadError = $this->untrapWarnings();
454
455 return $hadError ? null : $exists;
456 }
457
458 /**
459 * @see FileBackendStore::getDirectoryListInternal()
460 * @return Array|null
461 */
462 public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
463 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
464 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
465 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
466 $exists = is_dir( $dir );
467 if ( !$exists ) {
468 wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" );
469 return array(); // nothing under this dir
470 } elseif ( !is_readable( $dir ) ) {
471 wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" );
472 return null; // bad permissions?
473 }
474 return new FSFileBackendDirList( $dir, $params );
475 }
476
477 /**
478 * @see FileBackendStore::getFileListInternal()
479 * @return array|FSFileBackendFileList|null
480 */
481 public function getFileListInternal( $fullCont, $dirRel, array $params ) {
482 list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] );
483 $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid
484 $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot;
485 $exists = is_dir( $dir );
486 if ( !$exists ) {
487 wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" );
488 return array(); // nothing under this dir
489 } elseif ( !is_readable( $dir ) ) {
490 wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" );
491 return null; // bad permissions?
492 }
493 return new FSFileBackendFileList( $dir, $params );
494 }
495
496 /**
497 * @see FileBackendStore::getLocalReference()
498 * @return FSFile|null
499 */
500 public function getLocalReference( array $params ) {
501 $source = $this->resolveToFSPath( $params['src'] );
502 if ( $source === null ) {
503 return null;
504 }
505 return new FSFile( $source );
506 }
507
508 /**
509 * @see FileBackendStore::getLocalCopy()
510 * @return null|TempFSFile
511 */
512 public function getLocalCopy( array $params ) {
513 $source = $this->resolveToFSPath( $params['src'] );
514 if ( $source === null ) {
515 return null;
516 }
517
518 // Create a new temporary file with the same extension...
519 $ext = FileBackend::extensionFromPath( $params['src'] );
520 $tmpFile = TempFSFile::factory( wfBaseName( $source ) . '_', $ext );
521 if ( !$tmpFile ) {
522 return null;
523 }
524 $tmpPath = $tmpFile->getPath();
525
526 // Copy the source file over the temp file
527 $ok = copy( $source, $tmpPath );
528 if ( !$ok ) {
529 return null;
530 }
531
532 $this->chmod( $tmpPath );
533
534 return $tmpFile;
535 }
536
537 /**
538 * @see FileBackendStore::directoriesAreVirtual()
539 * @return bool
540 */
541 protected function directoriesAreVirtual() {
542 return false;
543 }
544
545 /**
546 * Chmod a file, suppressing the warnings
547 *
548 * @param $path string Absolute file system path
549 * @return bool Success
550 */
551 protected function chmod( $path ) {
552 wfSuppressWarnings();
553 $ok = chmod( $path, $this->fileMode );
554 wfRestoreWarnings();
555
556 return $ok;
557 }
558
559 /**
560 * Listen for E_WARNING errors and track whether any happen
561 *
562 * @return bool
563 */
564 protected function trapWarnings() {
565 $this->hadWarningErrors[] = false; // push to stack
566 set_error_handler( array( $this, 'handleWarning' ), E_WARNING );
567 return false; // invoke normal PHP error handler
568 }
569
570 /**
571 * Stop listening for E_WARNING errors and return true if any happened
572 *
573 * @return bool
574 */
575 protected function untrapWarnings() {
576 restore_error_handler(); // restore previous handler
577 return array_pop( $this->hadWarningErrors ); // pop from stack
578 }
579
580 private function handleWarning() {
581 $this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true;
582 return true; // suppress from PHP handler
583 }
584 }
585
586 /**
587 * Wrapper around RecursiveDirectoryIterator/DirectoryIterator that
588 * catches exception or does any custom behavoir that we may want.
589 * Do not use this class from places outside FSFileBackend.
590 *
591 * @ingroup FileBackend
592 */
593 abstract class FSFileBackendList implements Iterator {
594 /** @var Iterator */
595 protected $iter;
596 protected $suffixStart; // integer
597 protected $pos = 0; // integer
598 /** @var Array */
599 protected $params = array();
600
601 /**
602 * @param $dir string file system directory
603 */
604 public function __construct( $dir, array $params ) {
605 $dir = realpath( $dir ); // normalize
606 $this->suffixStart = strlen( $dir ) + 1; // size of "path/to/dir/"
607 $this->params = $params;
608
609 try {
610 $this->iter = $this->initIterator( $dir );
611 } catch ( UnexpectedValueException $e ) {
612 $this->iter = null; // bad permissions? deleted?
613 }
614 }
615
616 /**
617 * Return an appropriate iterator object to wrap
618 *
619 * @param $dir string file system directory
620 * @return Iterator
621 */
622 protected function initIterator( $dir ) {
623 if ( !empty( $this->params['topOnly'] ) ) { // non-recursive
624 # Get an iterator that will get direct sub-nodes
625 return new DirectoryIterator( $dir );
626 } else { // recursive
627 # Get an iterator that will return leaf nodes (non-directories)
628 if ( MWInit::classExists( 'FilesystemIterator' ) ) { // PHP >= 5.3
629 # RecursiveDirectoryIterator extends FilesystemIterator.
630 # FilesystemIterator::SKIP_DOTS default is inconsistent in PHP 5.3.x.
631 $flags = FilesystemIterator::CURRENT_AS_SELF | FilesystemIterator::SKIP_DOTS;
632 return new RecursiveIteratorIterator(
633 new RecursiveDirectoryIterator( $dir, $flags ),
634 RecursiveIteratorIterator::CHILD_FIRST // include dirs
635 );
636 } else { // PHP < 5.3
637 # RecursiveDirectoryIterator extends DirectoryIterator
638 return new RecursiveIteratorIterator(
639 new RecursiveDirectoryIterator( $dir ),
640 RecursiveIteratorIterator::CHILD_FIRST // include dirs
641 );
642 }
643 }
644 }
645
646 /**
647 * @see Iterator::key()
648 * @return integer
649 */
650 public function key() {
651 return $this->pos;
652 }
653
654 /**
655 * @see Iterator::current()
656 * @return string|bool String or false
657 */
658 public function current() {
659 return $this->getRelPath( $this->iter->current()->getPathname() );
660 }
661
662 /**
663 * @see Iterator::next()
664 * @return void
665 */
666 public function next() {
667 try {
668 $this->iter->next();
669 $this->filterViaNext();
670 } catch ( UnexpectedValueException $e ) {
671 $this->iter = null;
672 }
673 ++$this->pos;
674 }
675
676 /**
677 * @see Iterator::rewind()
678 * @return void
679 */
680 public function rewind() {
681 $this->pos = 0;
682 try {
683 $this->iter->rewind();
684 $this->filterViaNext();
685 } catch ( UnexpectedValueException $e ) {
686 $this->iter = null;
687 }
688 }
689
690 /**
691 * @see Iterator::valid()
692 * @return bool
693 */
694 public function valid() {
695 return $this->iter && $this->iter->valid();
696 }
697
698 /**
699 * Filter out items by advancing to the next ones
700 */
701 protected function filterViaNext() {}
702
703 /**
704 * Return only the relative path and normalize slashes to FileBackend-style.
705 * Uses the "real path" since the suffix is based upon that.
706 *
707 * @param $path string
708 * @return string
709 */
710 protected function getRelPath( $path ) {
711 return strtr( substr( realpath( $path ), $this->suffixStart ), '\\', '/' );
712 }
713 }
714
715 class FSFileBackendDirList extends FSFileBackendList {
716 protected function filterViaNext() {
717 while ( $this->iter->valid() ) {
718 if ( $this->iter->current()->isDot() || !$this->iter->current()->isDir() ) {
719 $this->iter->next(); // skip non-directories and dot files
720 } else {
721 break;
722 }
723 }
724 }
725 }
726
727 class FSFileBackendFileList extends FSFileBackendList {
728 protected function filterViaNext() {
729 while ( $this->iter->valid() ) {
730 if ( !$this->iter->current()->isFile() ) {
731 $this->iter->next(); // skip non-files and dot files
732 } else {
733 break;
734 }
735 }
736 }
737 }