return;
}
- // Get the full file path
- $filename = realpath( $wgUploadDirectory . $path );
- $realUpload = realpath( $wgUploadDirectory );
+ // Get the local file repository
+ $repo = RepoGroup::singleton()->getRepo( 'local' );
- // Basic directory traversal check
- if ( substr( $filename, 0, strlen( $realUpload ) ) != $realUpload ) {
- wfForbidden( 'img-auth-accessdenied', 'img-auth-notindir' );
- return;
+ // Get the full file storage path and extract the source file name.
+ // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png).
+ // This only applies to thumbnails, and all thumbnails should
+ // be under a folder that has the source file name.
+ if ( strpos( $path, '/thumb/' ) === 0 ) {
+ $name = wfBaseName( dirname( $path ) ); // file is a thumbnail
+ $filename = $repo->getZonePath( 'thumb' ) . substr( $path, 6 ); // strip "/thumb"
+ } else {
+ $name = wfBaseName( $path ); // file is a source file
+ $filename = $repo->getZonePath( 'public' ) . $path;
}
// Check to see if the file exists
- if ( !file_exists( $filename ) ) {
+ if ( !$repo->fileExists( $filename, FileRepo::FILES_ONLY ) ) {
wfForbidden( 'img-auth-accessdenied','img-auth-nofile', $filename );
return;
}
-
- // Check to see if tried to access a directory
- if ( is_dir( $filename ) ) {
- wfForbidden( 'img-auth-accessdenied','img-auth-isdir', $filename );
- return;
- }
-
- // Extract the file name and chop off the size specifier.
- // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png).
- // This only applies to thumbnails, and all thumbnails should
- // be under a folder that has the source file name.
- $name = wfBaseName( $path );
- if ( strpos( $path, '/thumb/' ) === 0 ) {
- $name = wfBaseName( dirname( $path ) ); // this file is a thumbnail
- }
$title = Title::makeTitleSafe( NS_FILE, $name );
if ( !$title instanceof Title ) { // files have valid titles
// Stream the requested file
wfDebugLog( 'img_auth', "Streaming `".$filename."`." );
- StreamFile::stream( $filename, array( 'Cache-Control: private', 'Vary: Cookie' ) );
+ $repo->streamFile( $filename, array( 'Cache-Control: private', 'Vary: Cookie' ) );
}
/**
'LocalFileRestoreBatch' => 'includes/filerepo/file/LocalFile.php',
'OldLocalFile' => 'includes/filerepo/file/OldLocalFile.php',
'UnregisteredLocalFile' => 'includes/filerepo/file/UnregisteredLocalFile.php',
+ 'FSFile' => 'includes/filerepo/file/FSFile.php',
+ 'TempFSFile' => 'includes/filerepo/file/TempFSFile.php',
+
+ # includes/filerepo/backend
+ 'FileBackendGroup' => 'includes/filerepo/backend/FileBackendGroup.php',
+ 'FileBackendBase' => 'includes/filerepo/backend/FileBackend.php',
+ 'FileBackend' => 'includes/filerepo/backend/FileBackend.php',
+ 'FileBackendMultiWrite' => 'includes/filerepo/backend/FileBackendMultiWrite.php',
+ 'FSFileBackend' => 'includes/filerepo/backend/FSFileBackend.php',
+ 'FSFileIterator' => 'includes/filerepo/backend/FSFileBackend.php',
+ 'LockManagerGroup' => 'includes/filerepo/backend/lockmanager/LockManagerGroup.php',
+ 'LockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php',
+ 'ScopedLock' => 'includes/filerepo/backend/lockmanager/LockManager.php',
+ 'FSLockManager' => 'includes/filerepo/backend/lockmanager/FSLockManager.php',
+ 'DBLockManager' => 'includes/filerepo/backend/lockmanager/DBLockManager.php',
+ 'LSLockManager' => 'includes/filerepo/backend/lockmanager/LSLockManager.php',
+ 'MySqlLockManager'=> 'includes/filerepo/backend/lockmanager/DBLockManager.php',
+ 'NullLockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php',
+ 'FileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'StoreFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'CopyFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'MoveFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'DeleteFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'ConcatenateFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'CreateFileOp' => 'includes/filerepo/backend/FileOp.php',
+ 'NullFileOp' => 'includes/filerepo/backend/FileOp.php',
# includes/installer
'CliInstaller' => 'includes/installer/CliInstaller.php',
*/
$wgUseInstantCommons = false;
+/**
+ * File backend structure configuration.
+ * This is an array of file backend configuration arrays.
+ * Each backend configuration has the following parameters:
+ * 'name' : A unique name for the backend
+ * 'class' : The file backend class to use
+ * 'wikiId' : A unique string that identifies the wiki (container prefix)
+ * 'lockManager' : The name of a lock manager (see $wgFileLockManagers)
+ * Additional parameters are specific to the class used.
+ */
+$wgFileBackends = array();
+
+/**
+ * Array of configuration arrays for each lock manager.
+ * Each backend configuration has the following parameters:
+ * 'name' : A unique name for the lock manger
+ * 'class' : The lock manger class to use
+ * Additional parameters are specific to the class used.
+ */
+$wgFileLockManagers = array();
+
/**
* Show EXIF data, on by default if available.
* Requires PHP's EXIF extension: http://www.php.net/manual/en/ref.exif.php
function wfMkdirParents( $dir, $mode = null, $caller = null ) {
global $wgDirectoryMode;
+ if ( FileBackend::isStoragePath( $dir ) ) { // sanity
+ throw new MWException( __FUNCTION__ . " given storage path `$dir`.");
+ }
+
if ( !is_null( $caller ) ) {
wfDebug( "$caller: called wfMkdirParents($dir)\n" );
}
$wgNamespaceAliases['Image'] = NS_FILE;
$wgNamespaceAliases['Image_talk'] = NS_FILE_TALK;
+/**
+ * Initialise $wgLockManagers to include basic FS version
+ */
+$wgLockManagers[] = array(
+ 'name' => 'fsLockManager',
+ 'class' => 'FSLockManager',
+ 'lockDirectory' => "{$wgUploadDirectory}/lockdir",
+);
+$wgLockManagers[] = array(
+ 'name' => 'nullLockManager',
+ 'class' => 'NullLockManager',
+);
+
/**
* Initialise $wgLocalFileRepo from backwards-compatible settings
*/
$wgForeignFileRepos[] = array(
'class' => 'ForeignAPIRepo',
'name' => 'wikimediacommons',
+ 'directory' => $wgUploadDirectory,
'apibase' => 'http://commons.wikimedia.org/w/api.php',
'hashLevels' => 2,
'fetchDescription' => true,
'apiThumbCacheExpiry' => 86400,
);
}
+/*
+ * Add on default file backend config for repos to $wgFileBackends
+ */
+if ( !isset( $wgLocalFileRepo['backend'] ) ) {
+ $wgFileBackends[] = wfBackendForLegacyRepoConf( $wgLocalFileRepo );
+}
+foreach ( $wgForeignFileRepos as &$repo ) {
+ if ( !isset( $repo['backend'] ) ) {
+ $wgFileBackends[] = wfBackendForLegacyRepoConf( $repo );
+ }
+}
+unset( $repo ); // no global pollution; destroy reference
+/*
+ * Get file backend configuration for a given repo
+ * configuration that lacks a backend parameter.
+ * Also updates the repo config to use the backend.
+ */
+function wfBackendForLegacyRepoConf( &$info ) {
+ // Local vars that used to be FSRepo members...
+ $directory = $info['directory'];
+ $deletedDir = isset( $info['deletedDir'] )
+ ? $info['deletedDir']
+ : false;
+ $thumbDir = isset( $info['thumbDir'] )
+ ? $info['thumbDir']
+ : "{$directory}/thumb";
+ $fileMode = isset( $info['fileMode'] )
+ ? $info['fileMode']
+ : 0644;
+
+ // Make a backend name (based on repo name)
+ $backendName = $info['name'] . '-backend';
+ // Update repo config to use this backend
+ $info['backend'] = $backendName;
+ // Disable "deleted" zone in repo config if deleted dir not set
+ if ( $deletedDir !== false ) {
+ $info['zones']['deleted'] = array(
+ 'container' => 'images-deleted', 'directory' => '' );
+ }
+ // Get the FS backend configuration
+ return array(
+ 'name' => $backendName,
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'fsLockManager',
+ 'containerPaths' => array(
+ "images-public" => "{$directory}",
+ "images-temp" => "{$directory}/temp",
+ "images-thumb" => $thumbDir,
+ "images-deleted" => $deletedDir
+ ),
+ 'fileMode' => $fileMode,
+ );
+}
if ( is_null( $wgEnableAutoRotation ) ) {
// Only enable auto-rotation when the bitmap handler can rotate
wfRunHooks( 'AuthPluginSetup', array( &$wgAuth ) );
}
+# Register file lock managers
+LockManagerGroup::singleton()->register( $wgLockManagers );
+# Register file backends
+FileBackendGroup::singleton()->register( $wgFileBackends );
+
# Placeholders in case of DB error
$wgTitle = null;
* A repository for files accessible via the local filesystem. Does not support
* database access or registration.
* @ingroup FileRepo
+ * @deprecated since 1.19
*/
class FSRepo extends FileRepo {
- var $directory, $deletedDir, $deletedHashLevels, $fileMode;
- var $fileFactory = array( 'UnregisteredLocalFile', 'newFromTitle' );
- var $oldFileFactory = false;
- var $pathDisclosureProtection = 'simple';
-
function __construct( $info ) {
parent::__construct( $info );
-
- // Required settings
- $this->directory = $info['directory'];
- $this->url = $info['url'];
-
- // Optional settings
- $this->hashLevels = isset( $info['hashLevels'] ) ? $info['hashLevels'] : 2;
- $this->deletedHashLevels = isset( $info['deletedHashLevels'] ) ?
- $info['deletedHashLevels'] : $this->hashLevels;
- $this->deletedDir = isset( $info['deletedDir'] ) ? $info['deletedDir'] : false;
- $this->fileMode = isset( $info['fileMode'] ) ? $info['fileMode'] : 0644;
- if ( isset( $info['thumbDir'] ) ) {
- $this->thumbDir = $info['thumbDir'];
- } else {
- $this->thumbDir = "{$this->directory}/thumb";
- }
- if ( isset( $info['thumbUrl'] ) ) {
- $this->thumbUrl = $info['thumbUrl'];
- } else {
- $this->thumbUrl = "{$this->url}/thumb";
- }
- }
-
- /**
- * Get the public root directory of the repository.
- * @return string
- */
- function getRootDirectory() {
- return $this->directory;
- }
-
- /**
- * Get the public root URL of the repository
- * @return string
- */
- function getRootUrl() {
- return $this->url;
- }
-
- /**
- * Returns true if the repository uses a multi-level directory structure
- * @return string
- */
- function isHashed() {
- return (bool)$this->hashLevels;
- }
-
- /**
- * Get the local directory corresponding to one of the three basic zones
- *
- * @param $zone string
- *
- * @return string
- */
- function getZonePath( $zone ) {
- switch ( $zone ) {
- case 'public':
- return $this->directory;
- case 'temp':
- return "{$this->directory}/temp";
- case 'deleted':
- return $this->deletedDir;
- case 'thumb':
- return $this->thumbDir;
- default:
- return false;
- }
- }
-
- /**
- * @see FileRepo::getZoneUrl()
- *
- * @param $zone string
- *
- * @return string url
- */
- function getZoneUrl( $zone ) {
- switch ( $zone ) {
- case 'public':
- return $this->url;
- case 'temp':
- return "{$this->url}/temp";
- case 'deleted':
- return parent::getZoneUrl( $zone ); // no public URL
- case 'thumb':
- return $this->thumbUrl;
- default:
- return parent::getZoneUrl( $zone );
+ if ( !( $this->backend instanceof FSFileBackend ) ) {
+ throw new MWException( "FSRepo only supports FSFileBackend." );
}
}
-
- /**
- * Get a URL referring to this repository, with the private mwrepo protocol.
- * The suffix, if supplied, is considered to be unencoded, and will be
- * URL-encoded before being returned.
- *
- * @param $suffix string
- *
- * @return string
- */
- function getVirtualUrl( $suffix = false ) {
- $path = 'mwrepo://' . $this->name;
- if ( $suffix !== false ) {
- $path .= '/' . rawurlencode( $suffix );
- }
- return $path;
- }
-
- /**
- * Get the local path corresponding to a virtual URL
- *
- * @param $url string
- *
- * @return string
- */
- function resolveVirtualUrl( $url ) {
- if ( substr( $url, 0, 9 ) != 'mwrepo://' ) {
- throw new MWException( __METHOD__.': unknown protocol' );
- }
-
- $bits = explode( '/', substr( $url, 9 ), 3 );
- if ( count( $bits ) != 3 ) {
- throw new MWException( __METHOD__.": invalid mwrepo URL: $url" );
- }
- list( $repo, $zone, $rel ) = $bits;
- if ( $repo !== $this->name ) {
- throw new MWException( __METHOD__.": fetching from a foreign repo is not supported" );
- }
- $base = $this->getZonePath( $zone );
- if ( !$base ) {
- throw new MWException( __METHOD__.": invalid zone: $zone" );
- }
- return $base . '/' . rawurldecode( $rel );
- }
-
- /**
- * Store a batch of files
- *
- * @param $triplets Array: (src,zone,dest) triplets as per store()
- * @param $flags Integer: bitwise combination of the following flags:
- * self::DELETE_SOURCE Delete the source file after upload
- * self::OVERWRITE Overwrite an existing destination file instead of failing
- * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the
- * same contents as the source
- * @return Status
- */
- function storeBatch( $triplets, $flags = 0 ) {
- wfDebug( __METHOD__ . ': Storing ' . count( $triplets ) .
- " triplets; flags: {$flags}\n" );
-
- // Try creating directories
- if ( !wfMkdirParents( $this->directory, null, __METHOD__ ) ) {
- return $this->newFatal( 'upload_directory_missing', $this->directory );
- }
- if ( !is_writable( $this->directory ) ) {
- return $this->newFatal( 'upload_directory_read_only', $this->directory );
- }
-
- // Validate each triplet
- $status = $this->newGood();
- foreach ( $triplets as $i => $triplet ) {
- list( $srcPath, $dstZone, $dstRel ) = $triplet;
-
- // Resolve destination path
- $root = $this->getZonePath( $dstZone );
- if ( !$root ) {
- throw new MWException( "Invalid zone: $dstZone" );
- }
- if ( !$this->validateFilename( $dstRel ) ) {
- throw new MWException( 'Validation error in $dstRel' );
- }
- $dstPath = "$root/$dstRel";
- $dstDir = dirname( $dstPath );
-
- // Create destination directories for this triplet
- if ( !is_dir( $dstDir ) ) {
- if ( !wfMkdirParents( $dstDir, null, __METHOD__ ) ) {
- return $this->newFatal( 'directorycreateerror', $dstDir );
- }
- if ( $dstZone == 'deleted' ) {
- $this->initDeletedDir( $dstDir );
- }
- }
-
- // Resolve source
- if ( self::isVirtualUrl( $srcPath ) ) {
- $srcPath = $triplets[$i][0] = $this->resolveVirtualUrl( $srcPath );
- }
- if ( !is_file( $srcPath ) ) {
- // Make a list of files that don't exist for return to the caller
- $status->fatal( 'filenotfound', $srcPath );
- continue;
- }
-
- // Check overwriting
- if ( !( $flags & self::OVERWRITE ) && file_exists( $dstPath ) ) {
- if ( $flags & self::OVERWRITE_SAME ) {
- $hashSource = sha1_file( $srcPath );
- $hashDest = sha1_file( $dstPath );
- if ( $hashSource != $hashDest ) {
- $status->fatal( 'fileexistserror', $dstPath );
- $status->failCount++;
- }
- } else {
- $status->fatal( 'fileexistserror', $dstPath );
- $status->failCount++;
- }
- }
- }
-
- // Windows does not support moving over existing files, so explicitly delete them
- $deleteDest = wfIsWindows() && ( $flags & self::OVERWRITE );
-
- // Abort now on failure
- if ( !$status->ok ) {
- return $status;
- }
-
- // Execute the store operation for each triplet
- foreach ( $triplets as $i => $triplet ) {
- list( $srcPath, $dstZone, $dstRel ) = $triplet;
- $root = $this->getZonePath( $dstZone );
- $dstPath = "$root/$dstRel";
- $good = true;
-
- if ( $flags & self::DELETE_SOURCE ) {
- if ( $deleteDest ) {
- unlink( $dstPath );
- }
- if ( !rename( $srcPath, $dstPath ) ) {
- $status->error( 'filerenameerror', $srcPath, $dstPath );
- $good = false;
- }
- } else {
- if ( !copy( $srcPath, $dstPath ) ) {
- $status->error( 'filecopyerror', $srcPath, $dstPath );
- $good = false;
- }
- if ( !( $flags & self::SKIP_VALIDATION ) ) {
- wfSuppressWarnings();
- $hashSource = sha1_file( $srcPath );
- $hashDest = sha1_file( $dstPath );
- wfRestoreWarnings();
-
- if ( $hashDest === false || $hashSource !== $hashDest ) {
- wfDebug( __METHOD__ . ': File copy validation failed: ' .
- "$srcPath ($hashSource) to $dstPath ($hashDest)\n" );
-
- $status->error( 'filecopyerror', $srcPath, $dstPath );
- $good = false;
- }
- }
- }
- if ( $good ) {
- $this->chmod( $dstPath );
- $status->successCount++;
- } else {
- $status->failCount++;
- }
- $status->success[$i] = $good;
- }
- return $status;
- }
-
- /**
- * Deletes a batch of files. Each file can be a (zone, rel) pairs, a
- * virtual url or a real path. It will try to delete each file, but
- * ignores any errors that may occur
- *
- * @param $pairs array List of files to delete
- * @return void
- */
- function cleanupBatch( $files ) {
- foreach ( $files as $file ) {
- if ( is_array( $file ) ) {
- // This is a pair, extract it
- list( $zone, $rel ) = $file;
- $root = $this->getZonePath( $zone );
- $path = "$root/$rel";
- } else {
- if ( self::isVirtualUrl( $file ) ) {
- // This is a virtual url, resolve it
- $path = $this->resolveVirtualUrl( $file );
- } else {
- // This is a full file name
- $path = $file;
- }
- }
-
- wfSuppressWarnings();
- unlink( $path );
- wfRestoreWarnings();
- }
- }
- /**
- * Concatenate a list of files into a target file location.
- *
- * @param $fileList array of files
- * @param $targetFile String target path
- * @param $flags Integer: bitwise combination of the following flags:
- * self::FILES_ONLY Mark file as existing only if it is a file (not directory)
- */
- function concatenate( $fileList, $targetPath, $flags = 0 ){
- $status = $this->newGood();
- // Resolve the virtual URL for taget:
- if ( self::isVirtualUrl( $targetPath ) ) {
- $targetPath = $this->resolveVirtualUrl( $targetPath );
- // empty out the target file:
- if ( is_file( $targetPath ) ){
- unlink( $targetPath );
- }
- }
- foreach( $fileList as $sourcePath ){
- // Resolve the virtual URL for source:
- if ( self::isVirtualUrl( $sourcePath ) ) {
- $sourcePath = $this->resolveVirtualUrl( $sourcePath );
- }
- if ( !is_file( $sourcePath ) )
- $status->fatal( 'filenotfound', $sourcePath );
-
- if ( !$status->isOk() ){
- return $status;
- }
-
- // Do the append
- $chunk = file_get_contents( $sourcePath );
- if( $chunk === false ) {
- $status->fatal( 'fileconcatenateerrorread', $sourcePath );
- return $status;
- }
- if( $status->isOk() ) {
- if ( file_put_contents( $targetPath, $chunk, FILE_APPEND ) ) {
- $status->value = $targetPath;
- } else {
- $status->fatal( 'fileconcatenateerror', $sourcePath, $targetPath);
- }
- }
- if ( $flags & self::DELETE_SOURCE ) {
- unlink( $sourcePath );
- }
- }
- return $status;
- }
- /**
- * @deprecated 1.19
- *
- * @return Status
- */
- function append( $srcPath, $toAppendPath, $flags = 0 ) {
- wfDeprecated( __METHOD__, '1.19' );
-
- $status = $this->newGood();
-
- // Resolve the virtual URL
- if ( self::isVirtualUrl( $toAppendPath ) ) {
- $toAppendPath = $this->resolveVirtualUrl( $toAppendPath );
- }
- // Make sure the files are there
- if ( !is_file( $toAppendPath ) )
- $status->fatal( 'filenotfound', $toAppendPath );
-
- if ( !is_file( $srcPath ) )
- $status->fatal( 'filenotfound', $srcPath );
-
- if ( !$status->isOk() ) return $status;
-
- // Do the append
- $chunk = file_get_contents( $srcPath );
- if( $chunk === false ) {
- $status->fatal( 'fileappenderrorread', $srcPath );
- }
-
- if( $status->isOk() ) {
- if ( file_put_contents( $toAppendPath, $chunk, FILE_APPEND ) ) {
- $status->value = $toAppendPath;
- } else {
- $status->fatal( 'fileappenderror', $srcPath, $toAppendPath);
- }
- }
-
- if ( $flags & self::DELETE_SOURCE ) {
- unlink( $srcPath );
- }
-
- return $status;
- }
-
- /* We can actually append to the files, so no-op needed here. */
- function appendFinish( $toAppendPath ) {}
-
- /**
- * Checks existence of specified array of files.
- *
- * @param $files Array: URLs of files to check
- * @param $flags Integer: bitwise combination of the following flags:
- * self::FILES_ONLY Mark file as existing only if it is a file (not directory)
- * @return Either array of files and existence flags, or false
- */
- function fileExistsBatch( $files, $flags = 0 ) {
- if ( !file_exists( $this->directory ) || !is_readable( $this->directory ) ) {
- return false;
- }
- $result = array();
- foreach ( $files as $key => $file ) {
- if ( self::isVirtualUrl( $file ) ) {
- $file = $this->resolveVirtualUrl( $file );
- }
- if( $flags & self::FILES_ONLY ) {
- $result[$key] = is_file( $file );
- } else {
- $result[$key] = file_exists( $file );
- }
- }
-
- return $result;
- }
-
- /**
- * Take all available measures to prevent web accessibility of new deleted
- * directories, in case the user has not configured offline storage
- * @return void
- */
- protected function initDeletedDir( $dir ) {
- // Add a .htaccess file to the root of the deleted zone
- $root = $this->getZonePath( 'deleted' );
- if ( !file_exists( "$root/.htaccess" ) ) {
- file_put_contents( "$root/.htaccess", "Deny from all\n" );
- }
- // Seed new directories with a blank index.html, to prevent crawling
- file_put_contents( "$dir/index.html", '' );
- }
-
- /**
- * Pick a random name in the temp zone and store a file to it.
- * @param $originalName String: the base name of the file as specified
- * by the user. The file extension will be maintained.
- * @param $srcPath String: the current location of the file.
- * @return FileRepoStatus object with the URL in the value.
- */
- function storeTemp( $originalName, $srcPath ) {
- $date = gmdate( "YmdHis" );
- $hashPath = $this->getHashPath( $originalName );
- $dstRel = "$hashPath$date!$originalName";
- $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName );
-
- $result = $this->store( $srcPath, 'temp', $dstRel );
- $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel;
- return $result;
- }
-
- /**
- * Remove a temporary file or mark it for garbage collection
- * @param $virtualUrl String: the virtual URL returned by storeTemp
- * @return Boolean: true on success, false on failure
- */
- function freeTemp( $virtualUrl ) {
- $temp = "mwrepo://{$this->name}/temp";
- if ( substr( $virtualUrl, 0, strlen( $temp ) ) != $temp ) {
- wfDebug( __METHOD__.": Invalid virtual URL\n" );
- return false;
- }
- $path = $this->resolveVirtualUrl( $virtualUrl );
- wfSuppressWarnings();
- $success = unlink( $path );
- wfRestoreWarnings();
- return $success;
- }
-
- /**
- * Publish a batch of files
- * @param $triplets Array: (source,dest,archive) triplets as per publish()
- * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate
- * that the source files should be deleted if possible
- * @return Status
- */
- function publishBatch( $triplets, $flags = 0 ) {
- // Perform initial checks
- if ( !wfMkdirParents( $this->directory, null, __METHOD__ ) ) {
- return $this->newFatal( 'upload_directory_missing', $this->directory );
- }
- if ( !is_writable( $this->directory ) ) {
- return $this->newFatal( 'upload_directory_read_only', $this->directory );
- }
- $status = $this->newGood( array() );
- foreach ( $triplets as $i => $triplet ) {
- list( $srcPath, $dstRel, $archiveRel ) = $triplet;
-
- if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) {
- $triplets[$i][0] = $srcPath = $this->resolveVirtualUrl( $srcPath );
- }
- if ( !$this->validateFilename( $dstRel ) ) {
- throw new MWException( 'Validation error in $dstRel' );
- }
- if ( !$this->validateFilename( $archiveRel ) ) {
- throw new MWException( 'Validation error in $archiveRel' );
- }
- $dstPath = "{$this->directory}/$dstRel";
- $archivePath = "{$this->directory}/$archiveRel";
-
- $dstDir = dirname( $dstPath );
- $archiveDir = dirname( $archivePath );
- // Abort immediately on directory creation errors since they're likely to be repetitive
- if ( !is_dir( $dstDir ) && !wfMkdirParents( $dstDir, null, __METHOD__ ) ) {
- return $this->newFatal( 'directorycreateerror', $dstDir );
- }
- if ( !is_dir( $archiveDir ) && !wfMkdirParents( $archiveDir, null, __METHOD__ ) ) {
- return $this->newFatal( 'directorycreateerror', $archiveDir );
- }
- if ( !is_file( $srcPath ) ) {
- // Make a list of files that don't exist for return to the caller
- $status->fatal( 'filenotfound', $srcPath );
- }
- }
-
- if ( !$status->ok ) {
- return $status;
- }
-
- foreach ( $triplets as $i => $triplet ) {
- list( $srcPath, $dstRel, $archiveRel ) = $triplet;
- $dstPath = "{$this->directory}/$dstRel";
- $archivePath = "{$this->directory}/$archiveRel";
-
- // Archive destination file if it exists
- if( is_file( $dstPath ) ) {
- // Check if the archive file exists
- // This is a sanity check to avoid data loss. In UNIX, the rename primitive
- // unlinks the destination file if it exists. DB-based synchronisation in
- // publishBatch's caller should prevent races. In Windows there's no
- // problem because the rename primitive fails if the destination exists.
- if ( is_file( $archivePath ) ) {
- $success = false;
- } else {
- wfSuppressWarnings();
- $success = rename( $dstPath, $archivePath );
- wfRestoreWarnings();
- }
-
- if( !$success ) {
- $status->error( 'filerenameerror',$dstPath, $archivePath );
- $status->failCount++;
- continue;
- } else {
- wfDebug(__METHOD__.": moved file $dstPath to $archivePath\n");
- }
- $status->value[$i] = 'archived';
- } else {
- $status->value[$i] = 'new';
- }
-
- $good = true;
- wfSuppressWarnings();
- if ( $flags & self::DELETE_SOURCE ) {
- if ( !rename( $srcPath, $dstPath ) ) {
- $status->error( 'filerenameerror', $srcPath, $dstPath );
- $good = false;
- }
- } else {
- if ( !copy( $srcPath, $dstPath ) ) {
- $status->error( 'filecopyerror', $srcPath, $dstPath );
- $good = false;
- }
- }
- wfRestoreWarnings();
-
- if ( $good ) {
- $status->successCount++;
- wfDebug(__METHOD__.": wrote tempfile $srcPath to $dstPath\n");
- // Thread-safe override for umask
- $this->chmod( $dstPath );
- } else {
- $status->failCount++;
- }
- }
- return $status;
- }
-
- /**
- * Move a group of files to the deletion archive.
- * If no valid deletion archive is configured, this may either delete the
- * file or throw an exception, depending on the preference of the repository.
- *
- * @param $sourceDestPairs Array of source/destination pairs. Each element
- * is a two-element array containing the source file path relative to the
- * public root in the first element, and the archive file path relative
- * to the deleted zone root in the second element.
- * @return FileRepoStatus
- */
- function deleteBatch( $sourceDestPairs ) {
- $status = $this->newGood();
- if ( !$this->deletedDir ) {
- throw new MWException( __METHOD__.': no valid deletion archive directory' );
- }
-
- /**
- * Validate filenames and create archive directories
- */
- foreach ( $sourceDestPairs as $pair ) {
- list( $srcRel, $archiveRel ) = $pair;
- if ( !$this->validateFilename( $srcRel ) ) {
- throw new MWException( __METHOD__.':Validation error in $srcRel' );
- }
- if ( !$this->validateFilename( $archiveRel ) ) {
- throw new MWException( __METHOD__.':Validation error in $archiveRel' );
- }
- $archivePath = "{$this->deletedDir}/$archiveRel";
- $archiveDir = dirname( $archivePath );
- if ( !is_dir( $archiveDir ) ) {
- if ( !wfMkdirParents( $archiveDir, null, __METHOD__ ) ) {
- $status->fatal( 'directorycreateerror', $archiveDir );
- continue;
- }
- $this->initDeletedDir( $archiveDir );
- }
- // Check if the archive directory is writable
- // This doesn't appear to work on NTFS
- if ( !is_writable( $archiveDir ) ) {
- $status->fatal( 'filedelete-archive-read-only', $archiveDir );
- }
- }
- if ( !$status->ok ) {
- // Abort early
- return $status;
- }
-
- /**
- * Move the files
- * We're now committed to returning an OK result, which will lead to
- * the files being moved in the DB also.
- */
- foreach ( $sourceDestPairs as $pair ) {
- list( $srcRel, $archiveRel ) = $pair;
- $srcPath = "{$this->directory}/$srcRel";
- $archivePath = "{$this->deletedDir}/$archiveRel";
- if ( file_exists( $archivePath ) ) {
- # A file with this content hash is already archived
- wfSuppressWarnings();
- $good = unlink( $srcPath );
- wfRestoreWarnings();
- if ( !$good ) {
- $status->error( 'filedeleteerror', $srcPath );
- }
- } else{
- wfSuppressWarnings();
- $good = rename( $srcPath, $archivePath );
- wfRestoreWarnings();
- if ( !$good ) {
- $status->error( 'filerenameerror', $srcPath, $archivePath );
- } else {
- $this->chmod( $archivePath );
- }
- }
- if ( $good ) {
- $status->successCount++;
- } else {
- $status->failCount++;
- }
- }
- return $status;
- }
-
- /**
- * Get a relative path for a deletion archive key,
- * e.g. s/z/a/ for sza251lrxrc1jad41h5mgilp8nysje52.jpg
- * @return string
- */
- function getDeletedHashPath( $key ) {
- $path = '';
- for ( $i = 0; $i < $this->deletedHashLevels; $i++ ) {
- $path .= $key[$i] . '/';
- }
- return $path;
- }
-
- /**
- * Call a callback function for every file in the repository.
- * Uses the filesystem even in child classes.
- * @return void
- */
- function enumFilesInFS( $callback ) {
- $numDirs = 1 << ( $this->hashLevels * 4 );
- for ( $flatIndex = 0; $flatIndex < $numDirs; $flatIndex++ ) {
- $hexString = sprintf( "%0{$this->hashLevels}x", $flatIndex );
- $path = $this->directory;
- for ( $hexPos = 0; $hexPos < $this->hashLevels; $hexPos++ ) {
- $path .= '/' . substr( $hexString, 0, $hexPos + 1 );
- }
- if ( !file_exists( $path ) || !is_dir( $path ) ) {
- continue;
- }
- $dir = opendir( $path );
- if ($dir) {
- while ( false !== ( $name = readdir( $dir ) ) ) {
- call_user_func( $callback, $path . '/' . $name );
- }
- closedir( $dir );
- }
- }
- }
-
- /**
- * Call a callback function for every file in the repository
- * May use either the database or the filesystem
- * @return void
- */
- function enumFiles( $callback ) {
- $this->enumFilesInFS( $callback );
- }
-
- /**
- * Get properties of a file with a given virtual URL
- * The virtual URL must refer to this repo
- * @return array
- */
- function getFileProps( $virtualUrl ) {
- $path = $this->resolveVirtualUrl( $virtualUrl );
- return File::getPropsFromPath( $path );
- }
-
- /**
- * Path disclosure protection functions
- *
- * Get a callback function to use for cleaning error message parameters
- */
- function getErrorCleanupFunction() {
- switch ( $this->pathDisclosureProtection ) {
- case 'simple':
- $callback = array( $this, 'simpleClean' );
- break;
- default:
- $callback = parent::getErrorCleanupFunction();
- }
- return $callback;
- }
-
- function simpleClean( $param ) {
- if ( !isset( $this->simpleCleanPairs ) ) {
- global $IP;
- $this->simpleCleanPairs = array(
- $this->directory => 'public',
- "{$this->directory}/temp" => 'temp',
- $IP => '$IP',
- dirname( __FILE__ ) => '$IP/extensions/WebStore',
- );
- if ( $this->deletedDir ) {
- $this->simpleCleanPairs[$this->deletedDir] = 'deleted';
- }
- }
- return strtr( $param, $this->simpleCleanPairs );
- }
-
- /**
- * Chmod a file, supressing the warnings.
- * @param $path String: the path to change
- * @return void
- */
- protected function chmod( $path ) {
- wfSuppressWarnings();
- chmod( $path, $this->fileMode );
- wfRestoreWarnings();
- }
-
}
*/
/**
- * Base class for file repositories.
- * Do not instantiate, use a derived class.
+ * Base class for file repositories
*
* @ingroup FileRepo
*/
-abstract class FileRepo {
+class FileRepo {
const FILES_ONLY = 1;
const DELETE_SOURCE = 1;
const OVERWRITE = 2;
const OVERWRITE_SAME = 4;
const SKIP_VALIDATION = 8;
+ /** @var FileBackendBase */
+ protected $backend;
+ /** @var Array Map of zones to config */
+ protected $zones = array();
+
var $thumbScriptUrl, $transformVia404;
var $descBaseUrl, $scriptDirUrl, $scriptExtension, $articleUrl;
var $fetchDescription, $initialCapital;
- var $pathDisclosureProtection = 'paranoid';
- var $descriptionCacheExpiry, $hashLevels, $url, $thumbUrl;
+ var $pathDisclosureProtection = 'simple'; // 'paranoid'
+ var $descriptionCacheExpiry, $url, $thumbUrl;
+ var $hashLevels, $deletedHashLevels;
/**
* Factory functions for creating new files
* Override these in the base class
*/
- var $fileFactory = false, $oldFileFactory = false;
+ var $fileFactory = array( 'UnregisteredLocalFile', 'newFromTitle' );
+ var $oldFileFactory = false;
var $fileFactoryKey = false, $oldFileFactoryKey = false;
function __construct( $info ) {
// Required settings
$this->name = $info['name'];
+ $this->url = isset( $info['url'] )
+ ? $info['url']
+ : false; // a subclass will need to set the URL (e.g. ForeignAPIRepo)
+ if ( $info['backend'] instanceof FileBackendBase ) {
+ $this->backend = $info['backend']; // useful for testing
+ } else {
+ $this->backend = FileBackendGroup::singleton()->get( $info['backend'] );
+ }
- // Optional settings
- $this->initialCapital = MWNamespace::isCapitalized( NS_FILE );
- foreach ( array( 'descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription',
- 'thumbScriptUrl', 'initialCapital', 'pathDisclosureProtection',
- 'descriptionCacheExpiry', 'hashLevels', 'url', 'thumbUrl', 'scriptExtension' )
- as $var )
- {
+ // Optional settings that can have no value
+ $optionalSettings = array(
+ 'descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription',
+ 'thumbScriptUrl', 'pathDisclosureProtection', 'descriptionCacheExpiry',
+ 'scriptExtension'
+ );
+ foreach ( $optionalSettings as $var ) {
if ( isset( $info[$var] ) ) {
$this->$var = $info[$var];
}
}
+
+ // Optional settings that have a default
+ $this->initialCapital = isset( $info['initialCapital'] )
+ ? $info['initialCapital']
+ : MWNamespace::isCapitalized( NS_FILE );
+ $this->thumbUrl = isset( $info['thumbUrl'] )
+ ? $info['thumbUrl']
+ : "{$this->url}/thumb";
+ $this->hashLevels = isset( $info['hashLevels'] )
+ ? $info['hashLevels']
+ : 2;
+ $this->deletedHashLevels = isset( $info['deletedHashLevels'] )
+ ? $info['deletedHashLevels']
+ : $this->hashLevels;
$this->transformVia404 = !empty( $info['transformVia404'] );
+ $this->zones = isset( $info['zones'] )
+ ? $info['zones']
+ : array();
+ // Give defaults for the basic zones...
+ foreach ( array( 'public', 'thumb', 'temp', 'deleted' ) as $zone ) {
+ if ( !isset( $this->zones[$zone] ) ) {
+ if ( $zone === 'deleted' ) {
+ $this->zones[$zone] = array(
+ 'container' => null, // user must set this up
+ 'directory' => '' // container root
+ );
+ } else {
+ $this->zones[$zone] = array(
+ 'container' => "images-$zone",
+ 'directory' => '' // container root
+ );
+ }
+ }
+ }
+ }
+
+ /**
+ * Get the file backend instance
+ *
+ * @return FileBackendBase
+ */
+ public function getBackend() {
+ return $this->backend;
+ }
+
+ /**
+ * Prepare all the zones for basic usage.
+ * See initDeletedDir() for additional setup needed for the 'deleted' zone.
+ *
+ * @param $doZones Array Only do a particular zones
+ * @return Status
+ */
+ protected function initZones( $doZones = array() ) {
+ $status = $this->newGood();
+ $doZones = (array)$doZones; // string => array
+ foreach ( $this->zones as $zone => $info ) {
+ if ( $doZones && !in_array( $zone, $doZones ) ) {
+ continue;
+ }
+ $root = $this->getZonePath( $zone );
+ if ( $root !== null ) {
+ $params = array( 'dir' => $this->getZonePath( $zone ) );
+ $status->merge( $this->backend->prepare( $params ) );
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Take all available measures to prevent web accessibility of new deleted
+ * directories, in case the user has not configured offline storage
+ *
+ * @return void
+ */
+ protected function initDeletedDir( $dir ) {
+ // Add a .htaccess file to the root of the deleted zone
+ $root = $this->getZonePath( 'deleted' );
+ $this->backend->secure( array( 'dir' => $root, 'noAccess' => true ) );
+ // Seed new directories with a blank index.html, to prevent crawling
+ $this->backend->secure( array( 'dir' => $dir, 'noListing' => true ) );
}
/**
* Determine if a string is an mwrepo:// URL
*
* @param $url string
- *
* @return bool
*/
- static function isVirtualUrl( $url ) {
+ public static function isVirtualUrl( $url ) {
return substr( $url, 0, 9 ) == 'mwrepo://';
}
+ /**
+ * Get a URL referring to this repository, with the private mwrepo protocol.
+ * The suffix, if supplied, is considered to be unencoded, and will be
+ * URL-encoded before being returned.
+ *
+ * @param $suffix string
+ * @return string
+ */
+ public function getVirtualUrl( $suffix = false ) {
+ $path = 'mwrepo://' . $this->name;
+ if ( $suffix !== false ) {
+ $path .= '/' . rawurlencode( $suffix );
+ }
+ return $path;
+ }
+
+ /**
+ * Get the URL corresponding to one of the four basic zones
+ *
+ * @param $zone String: one of: public, deleted, temp, thumb
+ * @return String or false
+ */
+ public function getZoneUrl( $zone ) {
+ switch ( $zone ) {
+ case 'public':
+ return $this->url;
+ case 'temp':
+ return "{$this->url}/temp";
+ case 'deleted':
+ return false; // no public URL
+ case 'thumb':
+ return $this->thumbUrl;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * Get the backend storage path corresponding to a virtual URL
+ *
+ * @param $url string
+ * @return string
+ */
+ function resolveVirtualUrl( $url ) {
+ if ( substr( $url, 0, 9 ) != 'mwrepo://' ) {
+ throw new MWException( __METHOD__.': unknown protocol' );
+ }
+ $bits = explode( '/', substr( $url, 9 ), 3 );
+ if ( count( $bits ) != 3 ) {
+ throw new MWException( __METHOD__.": invalid mwrepo URL: $url" );
+ }
+ list( $repo, $zone, $rel ) = $bits;
+ if ( $repo !== $this->name ) {
+ throw new MWException( __METHOD__.": fetching from a foreign repo is not supported" );
+ }
+ $base = $this->getZonePath( $zone );
+ if ( !$base ) {
+ throw new MWException( __METHOD__.": invalid zone: $zone" );
+ }
+ return $base . '/' . rawurldecode( $rel );
+ }
+
+ /**
+ * The the storage container and base path of a zone
+ *
+ * @param $zone string
+ * @return Array (container, base path) or (null, null)
+ */
+ protected function getZoneLocation( $zone ) {
+ if ( !isset( $this->zones[$zone] ) ) {
+ return array( null, null ); // bogus
+ }
+ return array( $this->zones[$zone]['container'], $this->zones[$zone]['directory'] );
+ }
+
+ /**
+ * Get the storage path corresponding to one of the zones
+ *
+ * @param $zone string
+ * @return string|null
+ */
+ public function getZonePath( $zone ) {
+ list( $container, $base ) = $this->getZoneLocation( $zone );
+ if ( $container === null || $base === null ) {
+ return null;
+ }
+ $backendName = $this->backend->getName();
+ if ( $base != '' ) { // may not be set
+ $base = "/{$base}";
+ }
+ return "mwstore://$backendName/{$container}{$base}";
+ }
+
/**
* Create a new File object from the local repository
*
* instance of the repository's old file class instead of a
* current file. Repositories not supporting version control
* should return false if this parameter is set.
- *
* @return File|null A File, or null if passed an invalid Title
*/
- function newFile( $title, $time = false ) {
+ public function newFile( $title, $time = false ) {
$title = File::normalizeTitle( $title );
if ( !$title ) {
return null;
* private: If true, return restricted (deleted) files if the current
* user is allowed to view them. Otherwise, such files will not
* be found.
- *
* @return File|false
*/
- function findFile( $title, $options = array() ) {
+ public function findFile( $title, $options = array() ) {
$title = File::normalizeTitle( $title );
if ( !$title ) {
return false;
return false;
}
$redir = $this->checkRedirect( $title );
- if( $redir && $title->getNamespace() == NS_FILE) {
+ if ( $redir && $title->getNamespace() == NS_FILE) {
$img = $this->newFile( $redir );
- if( !$img ) {
+ if ( !$img ) {
return false;
}
- if( $img->exists() ) {
+ if ( $img->exists() ) {
$img->redirectedFrom( $title->getDBkey() );
return $img;
}
/**
* Find many files at once.
+ *
* @param $items An array of titles, or an array of findFile() options with
* the "title" option giving the title. Example:
*
* $findItem = array( 'title' => $title, 'private' => true );
* $findBatch = array( $findItem );
* $repo->findFiles( $findBatch );
- *
* @return array
*/
- function findFiles( $items ) {
+ public function findFiles( $items ) {
$result = array();
foreach ( $items as $item ) {
if ( is_array( $item ) ) {
*
* @param $sha1 String base 36 SHA-1 hash
* @param $options Option array, same as findFile().
+ * @return File|false
*/
- function findFileFromKey( $sha1, $options = array() ) {
+ public function findFileFromKey( $sha1, $options = array() ) {
$time = isset( $options['time'] ) ? $options['time'] : false;
# First try to find a matching current version of a file...
}
/**
- * Get the URL of thumb.php
+ * Get an array or iterator of file objects for files that have a given
+ * SHA-1 content hash.
+ *
+ * STUB
*/
- function getThumbScriptUrl() {
- return $this->thumbScriptUrl;
+ public function findBySha1( $hash ) {
+ return array();
}
/**
- * Get the URL corresponding to one of the four basic zones
- * @param $zone String: one of: public, deleted, temp, thumb
- * @return String or false
+ * Get the public root URL of the repository
+ *
+ * @return string
*/
- function getZoneUrl( $zone ) {
- return false;
+ public function getRootUrl() {
+ return $this->url;
+ }
+
+ /**
+ * Returns true if the repository uses a multi-level directory structure
+ *
+ * @return string
+ */
+ public function isHashed() {
+ return (bool)$this->hashLevels;
+ }
+
+ /**
+ * Get the URL of thumb.php
+ *
+ * @return string
+ */
+ public function getThumbScriptUrl() {
+ return $this->thumbScriptUrl;
}
/**
*
* @return bool
*/
- function canTransformVia404() {
+ public function canTransformVia404() {
return $this->transformVia404;
}
/**
* Get the name of an image from its title object
+ *
* @param $title Title
*/
- function getNameFromTitle( Title $title ) {
+ public function getNameFromTitle( Title $title ) {
+ global $wgContLang;
if ( $this->initialCapital != MWNamespace::isCapitalized( NS_FILE ) ) {
- global $wgContLang;
$name = $title->getUserCaseDBKey();
if ( $this->initialCapital ) {
$name = $wgContLang->ucfirst( $name );
return $name;
}
+ /**
+ * Get the public zone root storage directory of the repository
+ *
+ * @return string
+ */
+ public function getRootDirectory() {
+ return $this->getZonePath( 'public' );
+ }
+
+ /**
+ * Get a relative path including trailing slash, e.g. f/fa/
+ * If the repo is not hashed, returns an empty string
+ *
+ * @param $name string
+ * @return string
+ */
+ public function getHashPath( $name ) {
+ return self::getHashPathForLevel( $name, $this->hashLevels );
+ }
+
/**
* @param $name
* @param $levels
*
* @return integer
*/
- function getHashLevels() {
+ public function getHashLevels() {
return $this->hashLevels;
}
/**
- * Get a relative path including trailing slash, e.g. f/fa/
- * If the repo is not hashed, returns an empty string
- *
- * @param $name string
+ * Get the name of this repository, as specified by $info['name]' to the constructor
*
* @return string
*/
- function getHashPath( $name ) {
- return self::getHashPathForLevel( $name, $this->hashLevels );
- }
-
- /**
- * Get the name of this repository, as specified by $info['name]' to the constructor
- */
- function getName() {
+ public function getName() {
return $this->name;
}
* @param $entry string Entry point; defaults to index
* @return string
*/
- function makeUrl( $query = '', $entry = 'index' ) {
+ public function makeUrl( $query = '', $entry = 'index' ) {
$ext = isset( $this->scriptExtension ) ? $this->scriptExtension : '.php';
return wfAppendQuery( "{$this->scriptDirUrl}/{$entry}{$ext}", $query );
}
*
* In particular, it uses the article paths as specified to the repository
* constructor, whereas local repositories use the local Title functions.
+ *
+ * @param $name string
+ * @return string
*/
- function getDescriptionUrl( $name ) {
+ public function getDescriptionUrl( $name ) {
$encName = wfUrlencode( $name );
if ( !is_null( $this->descBaseUrl ) ) {
# "http://example.com/wiki/Image:"
* MediaWiki this means action=render. This should only be called by the
* repository's file class, since it may return invalid results. User code
* should use File::getDescriptionText().
+ *
* @param $name String: name of image to fetch
* @param $lang String: language to fetch it in, if any.
+ * @return string
*/
- function getDescriptionRenderUrl( $name, $lang = null ) {
+ public function getDescriptionRenderUrl( $name, $lang = null ) {
$query = 'action=render';
if ( !is_null( $lang ) ) {
$query .= '&uselang=' . $lang;
/**
* Get the URL of the stylesheet to apply to description pages
+ *
* @return string
*/
- function getDescriptionStylesheetUrl() {
+ public function getDescriptionStylesheetUrl() {
if ( $this->scriptDirUrl ) {
return $this->makeUrl( 'title=MediaWiki:Filepage.css&' .
wfArrayToCGI( Skin::getDynamicStylesheetQuery() ) );
/**
* Store a file to a given destination.
*
- * @param $srcPath String: source path or virtual URL
+ * @param $srcPath String: source FS path, storage path, or virtual URL
* @param $dstZone String: destination zone
* @param $dstRel String: destination relative path
* @param $flags Integer: bitwise combination of the following flags:
* same contents as the source
* @return FileRepoStatus
*/
- function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
+ public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
$status = $this->storeBatch( array( array( $srcPath, $dstZone, $dstRel ) ), $flags );
if ( $status->successCount == 0 ) {
$status->ok = false;
/**
* Store a batch of files
*
- * @param $triplets Array: (src,zone,dest) triplets as per store()
- * @param $flags Integer: flags as per store
+ * @param $triplets Array: (src, dest zone, dest rel) triplets as per store()
+ * @param $flags Integer: bitwise combination of the following flags:
+ * self::DELETE_SOURCE Delete the source file after upload
+ * self::OVERWRITE Overwrite an existing destination file instead of failing
+ * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the
+ * same contents as the source
+ * @return FileRepoStatus
*/
- abstract function storeBatch( $triplets, $flags = 0 );
+ public function storeBatch( $triplets, $flags = 0 ) {
+ $backend = $this->backend; // convenience
+
+ // Try creating directories
+ $status = $this->initZones();
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+
+ $status = $this->newGood();
+
+ $operations = array();
+ $sourceFSFilesToDelete = array(); // cleanup for disk source files
+ // Validate each triplet and get the store operation...
+ foreach ( $triplets as $i => $triplet ) {
+ list( $srcPath, $dstZone, $dstRel ) = $triplet;
+
+ // Resolve destination path
+ $root = $this->getZonePath( $dstZone );
+ if ( !$root ) {
+ throw new MWException( "Invalid zone: $dstZone" );
+ }
+ if ( !$this->validateFilename( $dstRel ) ) {
+ throw new MWException( 'Validation error in $dstRel' );
+ }
+ $dstPath = "$root/$dstRel";
+ $dstDir = dirname( $dstPath );
+
+ // Create destination directories for this triplet
+ if ( !$backend->prepare( array( 'dir' => $dstDir ) )->isOK() ) {
+ return $this->newFatal( 'directorycreateerror', $dstDir );
+ }
+
+ if ( $dstZone == 'deleted' ) {
+ $this->initDeletedDir( $dstDir );
+ }
+
+ // Resolve source to a storage path if virtual
+ if ( self::isVirtualUrl( $srcPath ) ) {
+ $srcPath = $this->resolveVirtualUrl( $srcPath );
+ }
+
+ // Get the appropriate file operation
+ if ( FileBackend::isStoragePath( $srcPath ) ) {
+ $opName = ( $flags & self::DELETE_SOURCE ) ? 'move' : 'copy';
+ } else {
+ $opName = 'store';
+ if ( $flags & self::DELETE_SOURCE ) {
+ $sourceFSFilesToDelete[] = $srcPath;
+ }
+ }
+ $operations[] = array(
+ 'op' => $opName,
+ 'src' => $srcPath,
+ 'dst' => $dstPath,
+ 'overwriteDest' => $flags & self::OVERWRITE,
+ 'overwriteSame' => $flags & self::OVERWRITE_SAME,
+ );
+ }
+
+ // Execute the store operation for each triplet
+ $opts = array( 'ignoreErrors' => true );
+ $status->merge( $backend->doOperations( $operations, $opts ) );
+ // Cleanup for disk source files...
+ foreach ( $sourceFSFilesToDelete as $file ) {
+ wfSuppressWarnings();
+ unlink( $file ); // FS cleanup
+ wfRestoreWarnings();
+ }
+
+ return $status;
+ }
+
+ /**
+ * Deletes a batch of files.
+ * Each file can be a (zone, rel) pair, virtual url, storage path, or FS path.
+ * It will try to delete each file, but ignores any errors that may occur.
+ *
+ * @param $pairs array List of files to delete
+ * @return void
+ */
+ public function cleanupBatch( $files ) {
+ $operations = array();
+ $sourceFSFilesToDelete = array(); // cleanup for disk source files
+ foreach ( $files as $file ) {
+ if ( is_array( $file ) ) {
+ // This is a pair, extract it
+ list( $zone, $rel ) = $file;
+ $root = $this->getZonePath( $zone );
+ $path = "$root/$rel";
+ } else {
+ if ( self::isVirtualUrl( $file ) ) {
+ // This is a virtual url, resolve it
+ $path = $this->resolveVirtualUrl( $file );
+ } else {
+ // This is a full file name
+ $path = $file;
+ }
+ }
+ // Get a file operation if needed
+ if ( FileBackend::isStoragePath( $path ) ) {
+ $operations[] = array(
+ 'op' => 'delete',
+ 'src' => $path,
+ );
+ } else {
+ $sourceFSFilesToDelete[] = $path;
+ }
+ }
+ // Actually delete files from storage...
+ $opts = array( 'ignoreErrors' => true );
+ $this->backend->doOperations( $operations, $opts );
+ // Cleanup for disk source files...
+ foreach ( $sourceFSFilesToDelete as $file ) {
+ wfSuppressWarnings();
+ unlink( $path ); // FS cleanup
+ wfRestoreWarnings();
+ }
+ }
/**
* Pick a random name in the temp zone and store a file to it.
* @param $originalName String: the base name of the file as specified
* by the user. The file extension will be maintained.
* @param $srcPath String: the current location of the file.
+ * @return FileRepoStatus object with the URL in the value.
*/
- abstract function storeTemp( $originalName, $srcPath );
+ public function storeTemp( $originalName, $srcPath ) {
+ $date = gmdate( "YmdHis" );
+ $hashPath = $this->getHashPath( $originalName );
+ $dstRel = "{$hashPath}{$date}!{$originalName}";
+ $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName );
+ $result = $this->store( $srcPath, 'temp', $dstRel );
+ $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel;
+ return $result;
+ }
/**
- * Concatenate and array of file sources.
- * @param $fileList Array of file sources
- * @param $targetPath String target destination for file.
- * @throws MWException
- */
- abstract function concatenate( $fileList, $targetPath, $flags = 0 );
-
- /**
- * Append the contents of the source path to the given file, OR queue
- * the appending operation in anticipation of a later appendFinish() call.
- * @param $srcPath String: location of the source file
- * @param $toAppendPath String: path to append to.
- * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate
- * that the source file should be deleted if possible
- * @return mixed Status or false
+ * Concatenate a list of files into a target file location.
+ *
+ * @param $srcPaths Array Ordered list of source virtual URLs/storage paths
+ * @param $dstPath String Target virtual URL/storage path
+ * @param $flags Integer: bitwise combination of the following flags:
+ * self::DELETE_SOURCE Delete the source files
+ * @return FileRepoStatus
*/
- abstract function append( $srcPath, $toAppendPath, $flags = 0 );
+ function concatenate( $srcPaths, $dstPath, $flags = 0 ) {
+ $status = $this->newGood();
+ // Resolve target to a storage path if virtual
+ $dest = $this->resolveToStoragePath( $dstPath );
- /**
- * Finish the append operation.
- * @param $toAppendPath String: path to append to.
- * @return mixed Status or false
- */
- abstract function appendFinish( $toAppendPath );
+ $sources = array();
+ $deleteOperations = array(); // post-concatenate ops
+ foreach ( $srcPaths as $srcPath ) {
+ // Resolve source to a storage path if virtual
+ $source = $this->resolveToStoragePath( $srcPath );
+ $sources[] = $source; // chunk to merge
+ if ( $flags & self::DELETE_SOURCE ) {
+ $deleteOperations[] = array( 'op' => 'delete', 'src' => $source );
+ }
+ }
+
+ // Concatenate the chunks into one file
+ $op = array( 'op' => 'concatenate',
+ 'srcs' => $sources, 'dst' => $dest, 'overwriteDest' => true );
+ $status->merge( $this->backend->doOperation( $op ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+
+ // Delete the sources if required
+ if ( $deleteOperations ) {
+ $opts = array( 'ignoreErrors' => true );
+ $status->merge( $this->backend->doOperations( $deleteOperations, $opts ) );
+ }
+
+ // Make sure status is OK, despite any $deleteOperations fatals
+ $status->setResult( true );
+
+ return $status;
+ }
/**
* Remove a temporary file or mark it for garbage collection
+ *
* @param $virtualUrl String: the virtual URL returned by storeTemp
* @return Boolean: true on success, false on failure
- * STUB
*/
- function freeTemp( $virtualUrl ) {
- return true;
+ public function freeTemp( $virtualUrl ) {
+ $temp = "mwrepo://{$this->name}/temp";
+ if ( substr( $virtualUrl, 0, strlen( $temp ) ) != $temp ) {
+ wfDebug( __METHOD__.": Invalid temp virtual URL\n" );
+ return false;
+ }
+ $path = $this->resolveVirtualUrl( $virtualUrl );
+ $op = array( 'op' => 'delete', 'src' => $path );
+ $status = $this->backend->doOperation( $op );
+ return $status->isOK();
}
/**
- * Copy or move a file either from the local filesystem or from an mwrepo://
- * virtual URL, into this repository at the specified destination location.
+ * Copy or move a file either from a storage path, virtual URL,
+ * or FS path, into this repository at the specified destination location.
*
* Returns a FileRepoStatus object. On success, the value contains "new" or
* "archived", to indicate whether the file was new with that name.
*
- * @param $srcPath String: the source path or URL
+ * @param $srcPath String: the source FS path, storage path, or URL
* @param $dstRel String: the destination relative path
* @param $archiveRel String: the relative path where the existing file is to
* be archived, if there is one. Relative to the public zone root.
* @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate
* that the source file should be deleted if possible
*/
- function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) {
+ public function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) {
$status = $this->publishBatch( array( array( $srcPath, $dstRel, $archiveRel ) ), $flags );
if ( $status->successCount == 0 ) {
$status->ok = false;
/**
* Publish a batch of files
- * @param $triplets Array: (source,dest,archive) triplets as per publish()
+ *
+ * @param $triplets Array: (source, dest, archive) triplets as per publish()
* @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate
* that the source files should be deleted if possible
+ * @return FileRepoStatus
*/
- abstract function publishBatch( $triplets, $flags = 0 );
+ public function publishBatch( $triplets, $flags = 0 ) {
+ $backend = $this->backend; // convenience
+
+ // Try creating directories
+ $status = $this->initZones( 'public' );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+
+ $status = $this->newGood( array() );
+
+ $operations = array();
+ $sourceFSFilesToDelete = array(); // cleanup for disk source files
+ // Validate each triplet and get the store operation...
+ foreach ( $triplets as $i => $triplet ) {
+ list( $srcPath, $dstRel, $archiveRel ) = $triplet;
+ // Resolve source to a storage path if virtual
+ if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) {
+ $srcPath = $this->resolveVirtualUrl( $srcPath );
+ }
+ if ( !$this->validateFilename( $dstRel ) ) {
+ throw new MWException( 'Validation error in $dstRel' );
+ }
+ if ( !$this->validateFilename( $archiveRel ) ) {
+ throw new MWException( 'Validation error in $archiveRel' );
+ }
+
+ $publicRoot = $this->getZonePath( 'public' );
+ $dstPath = "$publicRoot/$dstRel";
+ $archivePath = "$publicRoot/$archiveRel";
+
+ $dstDir = dirname( $dstPath );
+ $archiveDir = dirname( $archivePath );
+ // Abort immediately on directory creation errors since they're likely to be repetitive
+ if ( !$backend->prepare( array( 'dir' => $dstDir ) )->isOK() ) {
+ return $this->newFatal( 'directorycreateerror', $dstDir );
+ }
+ if ( !$backend->prepare( array( 'dir' => $archiveDir ) )->isOK() ) {
+ return $this->newFatal( 'directorycreateerror', $archiveDir );
+ }
+
+ // Archive destination file if it exists
+ if ( $backend->fileExists( array( 'src' => $dstPath ) ) ) {
+ // Check if the archive file exists
+ // This is a sanity check to avoid data loss. In UNIX, the rename primitive
+ // unlinks the destination file if it exists. DB-based synchronisation in
+ // publishBatch's caller should prevent races. In Windows there's no
+ // problem because the rename primitive fails if the destination exists.
+ if ( $backend->fileExists( array( 'src' => $archivePath ) ) ) {
+ $operations[] = array( 'op' => 'null' );
+ continue;
+ } else {
+ $operations[] = array(
+ 'op' => 'move',
+ 'src' => $dstPath,
+ 'dst' => $archivePath
+ );
+ }
+ $status->value[$i] = 'archived';
+ } else {
+ $status->value[$i] = 'new';
+ }
+ // Copy (or move) the source file to the destination
+ if ( FileBackend::isStoragePath( $srcPath ) ) {
+ if ( $flags & self::DELETE_SOURCE ) {
+ $operations[] = array(
+ 'op' => 'move',
+ 'src' => $srcPath,
+ 'dst' => $dstPath
+ );
+ } else {
+ $operations[] = array(
+ 'op' => 'copy',
+ 'src' => $srcPath,
+ 'dst' => $dstPath
+ );
+ }
+ } else { // FS source path
+ $operations[] = array(
+ 'op' => 'store',
+ 'src' => $srcPath,
+ 'dst' => $dstPath
+ );
+ if ( $flags & self::DELETE_SOURCE ) {
+ $sourceFSFilesToDelete[] = $srcPath;
+ }
+ }
+ }
+
+ // Execute the operations for each triplet
+ $opts = array( 'ignoreErrors' => true );
+ $status->merge( $backend->doOperations( $operations, $opts ) );
+ // Cleanup for disk source files...
+ foreach ( $sourceFSFilesToDelete as $file ) {
+ wfSuppressWarnings();
+ unlink( $file ); // FS cleanup
+ wfRestoreWarnings();
+ }
+
+ return $status;
+ }
/**
- * @param $file
- * @param int $flags
+ * Checks existence of a a file
+ *
+ * @param $file Virtual URL (or storage path) of file to check
+ * @param $flags Integer: bitwise combination of the following flags:
+ * self::FILES_ONLY Mark file as existing only if it is a file (not directory)
* @return bool
*/
- function fileExists( $file, $flags = 0 ) {
+ public function fileExists( $file, $flags = 0 ) {
$result = $this->fileExistsBatch( array( $file ), $flags );
return $result[0];
}
/**
* Checks existence of an array of files.
*
- * @param $files Array: URLs (or paths) of files to check
+ * @param $files Array: Virtual URLs (or storage paths) of files to check
* @param $flags Integer: bitwise combination of the following flags:
* self::FILES_ONLY Mark file as existing only if it is a file (not directory)
* @return Either array of files and existence flags, or false
*/
- abstract function fileExistsBatch( $files, $flags = 0 );
+ public function fileExistsBatch( $files, $flags = 0 ) {
+ if ( !$this->initZones() ) {
+ return false;
+ }
+ $result = array();
+ foreach ( $files as $key => $file ) {
+ if ( self::isVirtualUrl( $file ) ) {
+ $file = $this->resolveVirtualUrl( $file );
+ }
+ if ( FileBackend::isStoragePath( $file ) ) {
+ $result[$key] = $this->backend->fileExists( array( 'src' => $file ) );
+ } else {
+ if ( $flags & self::FILES_ONLY ) {
+ $result[$key] = is_file( $file ); // FS only
+ } else {
+ $result[$key] = file_exists( $file ); // FS only
+ }
+ }
+ }
+
+ return $result;
+ }
+
+ /**
+ * Move a file to the deletion archive.
+ * If no valid deletion archive exists, this may either delete the file
+ * or throw an exception, depending on the preference of the repository
+ *
+ * @param $srcRel Mixed: relative path for the file to be deleted
+ * @param $archiveRel Mixed: relative path for the archive location.
+ * Relative to a private archive directory.
+ * @return FileRepoStatus object
+ */
+ public function delete( $srcRel, $archiveRel ) {
+ return $this->deleteBatch( array( array( $srcRel, $archiveRel ) ) );
+ }
/**
* Move a group of files to the deletion archive.
* to the deleted zone root in the second element.
* @return FileRepoStatus
*/
- abstract function deleteBatch( $sourceDestPairs );
+ public function deleteBatch( $sourceDestPairs ) {
+ $backend = $this->backend; // convenience
+
+ if ( !isset( $this->zones['deleted']['container'] ) ) {
+ throw new MWException( __METHOD__.': no valid deletion archive directory' );
+ }
+
+ // Try creating directories
+ $status = $this->initZones( array( 'public', 'deleted' ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+
+ $status = $this->newGood();
+
+ $operations = array();
+ // Validate filenames and create archive directories
+ foreach ( $sourceDestPairs as $pair ) {
+ list( $srcRel, $archiveRel ) = $pair;
+ if ( !$this->validateFilename( $srcRel ) ) {
+ throw new MWException( __METHOD__.':Validation error in $srcRel' );
+ }
+ if ( !$this->validateFilename( $archiveRel ) ) {
+ throw new MWException( __METHOD__.':Validation error in $archiveRel' );
+ }
+
+ $publicRoot = $this->getZonePath( 'public' );
+ $srcPath = "{$publicRoot}/$srcRel";
+
+ $deletedRoot = $this->getZonePath( 'deleted' );
+ $archivePath = "{$deletedRoot}/{$archiveRel}";
+ $archiveDir = dirname( $archivePath ); // does not touch FS
+
+ // Create destination directories
+ if ( !$backend->prepare( array( 'dir' => $archiveDir ) )->isOK() ) {
+ return $this->newFatal( 'directorycreateerror', $archiveDir );
+ }
+ $this->initDeletedDir( $archiveDir );
+
+ if ( $backend->fileExists( array( 'src' => $archivePath ) ) ) {
+ $operations[] = array(
+ 'op' => 'delete',
+ 'src' => $srcPath
+ );
+ } else {
+ $operations[] = array(
+ 'op' => 'move',
+ 'src' => $srcPath,
+ 'dst' => $archivePath
+ );
+ }
+ }
+
+ // Move the files by execute the operations for each pair.
+ // We're now committed to returning an OK result, which will
+ // lead to the files being moved in the DB also.
+ $opts = array( 'ignoreErrors' => true );
+ $status->merge( $backend->doOperations( $operations, $opts ) );
+
+ return $status;
+ }
/**
- * Move a file to the deletion archive.
- * If no valid deletion archive exists, this may either delete the file
- * or throw an exception, depending on the preference of the repository
- * @param $srcRel Mixed: relative path for the file to be deleted
- * @param $archiveRel Mixed: relative path for the archive location.
- * Relative to a private archive directory.
- * @return FileRepoStatus object
+ * Get a relative path for a deletion archive key,
+ * e.g. s/z/a/ for sza251lrxrc1jad41h5mgilp8nysje52.jpg
+ *
+ * @return string
*/
- function delete( $srcRel, $archiveRel ) {
- return $this->deleteBatch( array( array( $srcRel, $archiveRel ) ) );
+ public function getDeletedHashPath( $key ) {
+ $path = '';
+ for ( $i = 0; $i < $this->deletedHashLevels; $i++ ) {
+ $path .= $key[$i] . '/';
+ }
+ return $path;
}
/**
- * Get properties of a file with a given virtual URL
- * The virtual URL must refer to this repo
- * Properties should ultimately be obtained via File::getPropsFromPath()
+ * If a path is a virtual URL, resolve it to a storage path.
+ * Otherwise, just return the path as it is.
*
+ * @param $path string
+ * @return string
+ * @throws MWException
+ */
+ protected function resolveToStoragePath( $path ) {
+ if ( $this->isVirtualUrl( $path ) ) {
+ return $this->resolveVirtualUrl( $path );
+ }
+ return $path;
+ }
+
+ /**
+ * Get a local FS copy of a file with a given virtual URL/storage path.
+ * Temporary files may be purged when the file object falls out of scope.
+ *
+ * @param $virtualUrl string
+ * @return TempFSFile|null Returns null on failure
+ */
+ public function getLocalCopy( $virtualUrl ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ return $this->backend->getLocalCopy( array( 'src' => $path ) );
+ }
+
+ /**
+ * Get a local FS file with a given virtual URL/storage path.
+ * The file is either an original or a copy. It should not be changed.
+ * Temporary files may be purged when the file object falls out of scope.
+ *
* @param $virtualUrl string
+ * @return FSFile|null Returns null on failure.
*/
- abstract function getFileProps( $virtualUrl );
+ public function getLocalReference( $virtualUrl ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ return $this->backend->getLocalReference( array( 'src' => $path ) );
+ }
/**
- * Call a callback function for every file in the repository
- * May use either the database or the filesystem
- * STUB
+ * Get properties of a file with a given virtual URL/storage path.
+ * Properties should ultimately be obtained via FSFile::getProps().
+ *
+ * @param $virtualUrl string
+ * @return Array
*/
- function enumFiles( $callback ) {
- throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) );
+ public function getFileProps( $virtualUrl ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ return $this->backend->getFileProps( array( 'src' => $path ) );
+ }
+
+ /**
+ * Get the timestamp of a file with a given virtual URL/storage path
+ *
+ * @param $virtualUrl string
+ * @return string|false
+ */
+ public function getFileTimestamp( $virtualUrl ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ return $this->backend->getFileTimestamp( array( 'src' => $path ) );
+ }
+
+ /**
+ * Get the sha1 of a file with a given virtual URL/storage path
+ *
+ * @param $virtualUrl string
+ * @return string|false
+ */
+ public function getFileSha1( $virtualUrl ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ $tmpFile = $this->backend->getLocalReference( array( 'src' => $path ) );
+ if ( !$tmpFile ) {
+ return false;
+ }
+ return $tmpFile->getSha1Base36();
+ }
+
+ /**
+ * Attempt to stream a file with the given virtual URL/storage path
+ *
+ * @param $virtualUrl string
+ * @param $headers Array Additional HTTP headers to send on success
+ * @return bool Success
+ */
+ public function streamFile( $virtualUrl, $headers = array() ) {
+ $path = $this->resolveToStoragePath( $virtualUrl );
+ $params = array( 'src' => $path, 'headers' => $headers );
+ return $this->backend->streamFile( $params )->isOK();
+ }
+
+ /**
+ * Call a callback function for every public file in the repository.
+ * May use either the database or the filesystem.
+ *
+ * @param $callback Array|string
+ * @return void
+ */
+ public function enumFiles( $callback ) {
+ return $this->enumFilesInStorage( $callback );
+ }
+
+ /**
+ * Call a callback function for every public file in the repository.
+ * May use either the database or the filesystem.
+ *
+ * @param $callback Array|string
+ * @return void
+ */
+ protected function enumFilesInStorage( $callback ) {
+ $publicRoot = $this->getZonePath( 'public' );
+ $numDirs = 1 << ( $this->hashLevels * 4 );
+ // Use a priori assumptions about directory structure
+ // to reduce the tree height of the scanning process.
+ for ( $flatIndex = 0; $flatIndex < $numDirs; $flatIndex++ ) {
+ $hexString = sprintf( "%0{$this->hashLevels}x", $flatIndex );
+ $path = $publicRoot;
+ for ( $hexPos = 0; $hexPos < $this->hashLevels; $hexPos++ ) {
+ $path .= '/' . substr( $hexString, 0, $hexPos + 1 );
+ }
+ $iterator = $this->backend->getFileList( array( 'dir' => $path ) );
+ foreach ( $iterator as $name ) {
+ // Each item returned is a public file
+ call_user_func( $callback, "{$path}/{$name}" );
+ }
+ }
}
/**
* Determine if a relative path is valid, i.e. not blank or involving directory traveral
*
* @param $filename string
- *
* @return bool
*/
- function validateFilename( $filename ) {
+ public function validateFilename( $filename ) {
if ( strval( $filename ) == '' ) {
return false;
}
}
}
- /**#@+
- * Path disclosure protection functions
- */
- function paranoidClean( $param ) { return '[hidden]'; }
-
- /**
- * @param $param
- * @return
- */
- function passThrough( $param ) { return $param; }
-
/**
* Get a callback function to use for cleaning error message parameters
+ *
+ * @return Array
*/
function getErrorCleanupFunction() {
switch ( $this->pathDisclosureProtection ) {
case 'none':
$callback = array( $this, 'passThrough' );
break;
+ case 'simple':
+ $callback = array( $this, 'simpleClean' );
+ break;
default: // 'paranoid'
$callback = array( $this, 'paranoidClean' );
}
return $callback;
}
- /**#@-*/
+
+ /**
+ * Path disclosure protection function
+ *
+ * @param $param string
+ * @return string
+ */
+ function paranoidClean( $param ) {
+ return '[hidden]';
+ }
+
+ /**
+ * Path disclosure protection function
+ *
+ * @param $param string
+ * @return string
+ */
+ function simpleClean( $param ) {
+ global $IP;
+ if ( !isset( $this->simpleCleanPairs ) ) {
+ $this->simpleCleanPairs = array(
+ $IP => '$IP', // sanity
+ );
+ }
+ return strtr( $param, $this->simpleCleanPairs );
+ }
+
+ /**
+ * Path disclosure protection function
+ *
+ * @param $param string
+ * @return string
+ */
+ function passThrough( $param ) {
+ return $param;
+ }
/**
* Create a new fatal error
+ *
+ * @return FileRepoStatus
*/
function newFatal( $message /*, parameters...*/ ) {
$params = func_get_args();
/**
* Delete files in the deleted directory if they are not referenced in the filearchive table
+ *
* STUB
*/
- function cleanupDeletedBatch( $storageKeys ) {}
+ public function cleanupDeletedBatch( $storageKeys ) {}
/**
* Checks if there is a redirect named as $title. If there is, return the
* @param $title Title of image
* @return Bool
*/
- function checkRedirect( Title $title ) {
+ public function checkRedirect( Title $title ) {
return false;
}
* STUB
* @param $title Title of image
*/
- function invalidateImageRedirect( Title $title ) {}
+ public function invalidateImageRedirect( Title $title ) {}
/**
- * Get an array or iterator of file objects for files that have a given
- * SHA-1 content hash.
+ * Get the human-readable name of the repo
*
- * STUB
- */
- function findBySha1( $hash ) {
- return array();
- }
-
- /**
- * Get the human-readable name of the repo.
* @return string
*/
public function getDisplayName() {
*
* @return bool
*/
- function isLocal() {
+ public function isLocal() {
return $this->getName() == 'local';
}
* Get a key for this repo in the local cache domain. These cache keys are
* not shared with remote instances of the repo.
* The parameters are the parts of the key, as for wfMemcKey().
+ *
+ * @return string
*/
function getLocalCacheKey( /*...*/ ) {
$args = func_get_args();
*
* @return UploadStash
*/
- function getUploadStash() {
+ public function getUploadStash() {
return new UploadStash( $this );
}
}
var $apiThumbCacheExpiry = 86400; /* 24*60*60 */
/* Redownload thumbnail files after a month */
var $fileCacheExpiry = 2592000; /* 86400*30 */
- /* Local image directory */
- var $directory;
- var $thumbDir;
protected $mQueryCache = array();
protected $mFileExists = array();
function __construct( $info ) {
+ global $wgLocalFileRepo, $wgUploadDirectory;
+ if ( !isset( $info['directory'] ) ) { // b/c
+ $info['directory'] = $wgUploadDirectory; // Local image directory
+ }
parent::__construct( $info );
- global $wgUploadDirectory;
// http://commons.wikimedia.org/w/api.php
$this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null;
- $this->directory = isset( $info['directory'] ) ? $info['directory'] : $wgUploadDirectory;
if( isset( $info['apiThumbCacheExpiry'] ) ) {
$this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
}
// If we can cache thumbs we can guess sane defaults for these
if( $this->canCacheThumbs() && !$this->url ) {
- global $wgLocalFileRepo;
$this->url = $wgLocalFileRepo['url'];
}
if( $this->canCacheThumbs() && !$this->thumbUrl ) {
$this->thumbUrl = $this->url . '/thumb';
}
- if ( isset( $info['thumbDir'] ) ) {
- $this->thumbDir = $info['thumbDir'];
- } else {
- $this->thumbDir = "{$this->directory}/thumb";
- }
}
/**
$localFilename = $localPath . "/" . $fileName;
$localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) . rawurlencode( $name ) . "/" . rawurlencode( $fileName );
- if( file_exists( $localFilename ) && isset( $metadata['timestamp'] ) ) {
+ if( $this->fileExists( $localFilename ) && isset( $metadata['timestamp'] ) ) {
wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" );
- $modified = filemtime( $localFilename );
+ $modified = $this->getFileTimestamp( $localFilename );
$remoteModified = strtotime( $metadata['timestamp'] );
$current = time();
$diff = abs( $modified - $current );
wfDebug( __METHOD__ . " Could not download thumb\n" );
return false;
}
- if ( !is_dir($localPath) ) {
- if( !wfMkdirParents( $localPath, null, __METHOD__ ) ) {
- wfDebug( __METHOD__ . " could not create directory $localPath for thumb\n" );
- return $foreignUrl;
- }
- }
# @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
wfSuppressWarnings();
- if( !file_put_contents( $localFilename, $thumb ) ) {
+ $backend = $this->getBackend();
+ $op = array( 'op' => 'create', 'dst' => $localFilename, 'content' => $thumb );
+ if( !$backend->doOperation( $op )->isOK() ) {
wfRestoreWarnings();
wfDebug( __METHOD__ . " could not write to thumb path\n" );
return $foreignUrl;
}
/**
- * Get the local directory corresponding to one of the three basic zones
+ * Get the local directory corresponding to one of the basic zones
*/
function getZonePath( $zone ) {
- switch ( $zone ) {
- case 'public':
- return $this->directory;
- case 'thumb':
- return $this->thumbDir;
- default:
- return false;
+ $supported = array( 'public', 'thumb' );
+ if ( in_array( $zone, $supported ) ) {
+ return parent::getZonePath( $zone );
}
+ return false;
}
/**
return false;
}
}
+
+ function enumFiles( $callback ) {
+ throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) );
+ }
}
function getSlaveDB() {
return wfGetDB( DB_SLAVE, array(), $this->wiki );
}
+
function hasSharedCache() {
return $this->hasSharedCache;
}
/**
* A repository that stores files in the local filesystem and registers them
* in the wiki's own database. This is the most commonly used repository class.
+ *
* @ingroup FileRepo
*/
-class LocalRepo extends FSRepo {
+class LocalRepo extends FileRepo {
var $fileFactory = array( 'LocalFile', 'newFromTitle' );
var $fileFactoryKey = array( 'LocalFile', 'newFromKey' );
var $oldFileFactory = array( 'OldLocalFile', 'newFromTitle' );
/**
* @throws MWException
- * @param $row
+ * @param $row
* @return File
*/
function newFileFromRow( $row ) {
* @return FileRepoStatus
*/
function cleanupDeletedBatch( $storageKeys ) {
+ $backend = $this->backend; // convenience
$root = $this->getZonePath( 'deleted' );
$dbw = $this->getMasterDB();
$status = $this->newGood();
$hidden = $this->hiddenFileHasKey( $key, 'lock' );
if ( !$deleted && !$hidden ) { // not in use now
wfDebug( __METHOD__ . ": deleting $key\n" );
- wfSuppressWarnings();
- $unlink = unlink( $path );
- wfRestoreWarnings();
- if ( !$unlink ) {
+ $op = array( 'op' => 'delete', 'src' => $path );
+ if ( !$backend->doOperation( $op )->isOK() ) {
$status->error( 'undelete-cleanup-error', $path );
$status->failCount++;
}
/**
* Check if a deleted (filearchive) file has this sha1 key
+ *
* @param $key String File storage key (base-36 sha1 key with file extension)
* @param $lock String|null Use "lock" to lock the row via FOR UPDATE
* @return bool File with this key is in use
/**
* Check if a hidden (revision delete) file has this sha1 key
+ *
* @param $key String File storage key (base-36 sha1 key with file extension)
* @param $lock String|null Use "lock" to lock the row via FOR UPDATE
* @return bool File with this key is in use
/**
* Function link Title::getArticleID().
* We can't say Title object, what database it should use, so we duplicate that function here.
+ *
* @param $title Title
*/
protected function getArticleID( $title ) {
}
$dbr = $this->getSlaveDB();
$id = $dbr->selectField(
- 'page', // Table
- 'page_id', //Field
- array( //Conditions
+ 'page', // Table
+ 'page_id', //Field
+ array( //Conditions
'page_namespace' => $title->getNamespace(),
'page_title' => $title->getDBkey(),
),
- __METHOD__ //Function name
+ __METHOD__ //Function name
);
return $id;
}
/**
* Get an array or iterator of file objects for files that have a given
* SHA-1 content hash.
+ *
+ * @param string
* @return Array
*/
function findBySha1( $hash ) {
* Get a key on the primary cache for this repository.
* Returns false if the repository's cache is not accessible at this site.
* The parameters are the parts of the key, as for wfMemcKey().
+ *
* @return string
*/
function getSharedCacheKey( /*...*/ ) {
$repo = $this->getRepo( $repoName );
return $repo->getFileProps( $fileName );
} else {
- return File::getPropsFromPath( $fileName );
+ return FSFile::getPropsFromPath( $fileName );
}
}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * Class for a file system based file backend.
+ * Status messages should avoid mentioning the internal FS paths.
+ * Likewise, error suppression should be used to avoid path disclosure.
+ *
+ * @ingroup FileBackend
+ */
+class FSFileBackend extends FileBackend {
+ /** @var Array Map of container names to paths */
+ protected $containerPaths = array();
+ protected $fileMode; // file permission mode
+
+ /**
+ * @see FileBackend::__construct()
+ * Additional $config params include:
+ * containerPaths : Map of container names to absolute file system paths
+ * fileMode : Octal UNIX file permissions to use on files stored
+ */
+ function __construct( array $config ) {
+ parent::__construct( $config );
+ $this->containerPaths = (array)$config['containerPaths'];
+ foreach ( $this->containerPaths as $container => &$path ) {
+ if ( substr( $path, -1 ) === '/' ) {
+ $path = substr( $path, 0, -1 ); // remove trailing slash
+ }
+ }
+ $this->fileMode = isset( $config['fileMode'] )
+ ? $config['fileMode']
+ : 0644;
+ }
+
+ /**
+ * @see FileBackend::resolveContainerPath()
+ */
+ protected function resolveContainerPath( $container, $relStoragePath ) {
+ // Get absolute path given the container base dir
+ if ( isset( $this->containerPaths[$container] ) ) {
+ return $this->containerPaths[$container] . "/{$relStoragePath}";
+ }
+ return null;
+ }
+
+ /**
+ * @see FileBackend::doStore()
+ */
+ protected function doStore( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
+ }
+ if ( file_exists( $dest ) ) {
+ if ( !empty( $params['overwriteDest'] ) ) {
+ wfSuppressWarnings();
+ $ok = unlink( $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['dst'] );
+ return $status;
+ }
+ } else {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ } else {
+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
+ $status->fatal( 'directorycreateerror', $params['dst'] );
+ return $status;
+ }
+ }
+
+ wfSuppressWarnings();
+ $ok = copy( $params['src'], $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ return $status;
+ }
+
+ $this->chmod( $dest );
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doCopy()
+ */
+ protected function doCopy( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+ return $status;
+ }
+
+ list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
+ }
+
+ if ( file_exists( $dest ) ) {
+ if ( !empty( $params['overwriteDest'] ) ) {
+ wfSuppressWarnings();
+ $ok = unlink( $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['dst'] );
+ return $status;
+ }
+ } else {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ } else {
+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
+ $status->fatal( 'directorycreateerror', $params['dst'] );
+ return $status;
+ }
+ }
+
+ wfSuppressWarnings();
+ $ok = copy( $source, $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ return $status;
+ }
+
+ $this->chmod( $dest );
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doMove()
+ */
+ protected function doMove( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+ return $status;
+ }
+ list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
+ }
+
+ if ( file_exists( $dest ) ) {
+ if ( !empty( $params['overwriteDest'] ) ) {
+ // Windows does not support moving over existing files
+ if ( wfIsWindows() ) {
+ wfSuppressWarnings();
+ $ok = unlink( $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['dst'] );
+ return $status;
+ }
+ }
+ } else {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ } else {
+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
+ $status->fatal( 'directorycreateerror', $params['dst'] );
+ return $status;
+ }
+ }
+
+ wfSuppressWarnings();
+ $ok = rename( $source, $dest );
+ clearstatcache(); // file no longer at source
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ return $status;
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doDelete()
+ */
+ protected function doDelete( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+ return $status;
+ }
+
+ if ( !is_file( $source ) ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ }
+ return $status; // do nothing; either OK or bad status
+ }
+
+ wfSuppressWarnings();
+ $ok = unlink( $source );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ return $status;
+ }
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doConcatenate()
+ */
+ protected function doConcatenate( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
+ }
+
+ // Check if the destination file exists and we can't handle that
+ $destExists = file_exists( $dest );
+ if ( $destExists && empty( $params['overwriteDest'] ) ) {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+
+ // Create a new temporary file...
+ wfSuppressWarnings();
+ $tmpPath = tempnam( wfTempDir(), 'concatenate' );
+ wfRestoreWarnings();
+ if ( $tmpPath === false ) {
+ $status->fatal( 'backend-fail-createtemp' );
+ return $status;
+ }
+
+ // Build up that file using the source chunks (in order)...
+ wfSuppressWarnings();
+ $tmpHandle = fopen( $tmpPath, 'a' );
+ wfRestoreWarnings();
+ if ( $tmpHandle === false ) {
+ $status->fatal( 'backend-fail-opentemp', $tmpPath );
+ return $status;
+ }
+ foreach ( $params['srcs'] as $virtualSource ) {
+ list( $c, $source ) = $this->resolveStoragePath( $virtualSource );
+ if ( $source === null ) {
+ fclose( $tmpHandle );
+ $status->fatal( 'backend-fail-invalidpath', $virtualSource );
+ return $status;
+ }
+ // Load chunk into memory (it should be a small file)
+ $sourceHandle = fopen( $source, 'r' );
+ if ( $sourceHandle === false ) {
+ fclose( $tmpHandle );
+ $status->fatal( 'backend-fail-read', $virtualSource );
+ return $status;
+ }
+ // Append chunk to file (pass chunk size to avoid magic quotes)
+ if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
+ fclose( $sourceHandle );
+ fclose( $tmpHandle );
+ $status->fatal( 'backend-fail-writetemp', $tmpPath );
+ return $status;
+ }
+ fclose( $sourceHandle );
+ }
+ wfSuppressWarnings();
+ if ( !fclose( $tmpHandle ) ) {
+ $status->fatal( 'backend-fail-closetemp', $tmpPath );
+ return $status;
+ }
+ wfRestoreWarnings();
+
+ // Handle overwrite behavior of file destination if applicable.
+ // Note that we already checked if no overwrite params were set above.
+ if ( $destExists ) {
+ // Windows does not support moving over existing files
+ if ( wfIsWindows() ) {
+ wfSuppressWarnings();
+ $ok = unlink( $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['dst'] );
+ return $status;
+ }
+ }
+ } else {
+ // Make sure destination directory exists
+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
+ $status->fatal( 'directorycreateerror', $params['dst'] );
+ return $status;
+ }
+ }
+
+ // Rename the temporary file to the destination path
+ wfSuppressWarnings();
+ $ok = rename( $tmpPath, $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-move', $tmpPath, $params['dst'] );
+ return $status;
+ }
+
+ $this->chmod( $dest );
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::doCreate()
+ */
+ protected function doCreate( array $params ) {
+ $status = Status::newGood();
+
+ list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] );
+ if ( $dest === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+ return $status;
+ }
+
+ if ( file_exists( $dest ) ) {
+ if ( !empty( $params['overwriteDest'] ) ) {
+ wfSuppressWarnings();
+ $ok = unlink( $dest );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-delete', $params['dst'] );
+ return $status;
+ }
+ } else {
+ $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
+ return $status;
+ }
+ } else {
+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
+ $status->fatal( 'directorycreateerror', $params['dst'] );
+ return $status;
+ }
+ }
+
+ wfSuppressWarnings();
+ $ok = file_put_contents( $dest, $params['content'] );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ return $status;
+ }
+
+ $this->chmod( $dest );
+
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::prepare()
+ */
+ function prepare( array $params ) {
+ $status = Status::newGood();
+ list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ return $status; // invalid storage path
+ }
+ if ( !wfMkdirParents( $dir ) ) {
+ $status->fatal( 'directorycreateerror', $params['dir'] );
+ return $status;
+ } elseif ( !is_writable( $dir ) ) {
+ $status->fatal( 'directoryreadonlyerror', $params['dir'] );
+ return $status;
+ } elseif ( !is_readable( $dir ) ) {
+ $status->fatal( 'directorynotreadableerror', $params['dir'] );
+ return $status;
+ }
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::secure()
+ */
+ function secure( array $params ) {
+ $status = Status::newGood();
+ list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ return $status; // invalid storage path
+ }
+ if ( !wfMkdirParents( $dir ) ) {
+ $status->fatal( 'directorycreateerror', $params['dir'] );
+ return $status;
+ }
+ // Add a .htaccess file to the root of the deleted zone
+ if ( !empty( $params['noAccess'] ) && !file_exists( "{$dir}/.htaccess" ) ) {
+ wfSuppressWarnings();
+ $ok = file_put_contents( "{$dir}/.htaccess", "Deny from all\n" );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-create', $params['dir'] . '/.htaccess' );
+ return $status;
+ }
+ }
+ // Seed new directories with a blank index.html, to prevent crawling
+ if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) {
+ wfSuppressWarnings();
+ $ok = file_put_contents( "{$dir}/index.html", '' );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' );
+ return $status;
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::clean()
+ */
+ function clean( array $params ) {
+ $status = Status::newGood();
+ list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+ return $status; // invalid storage path
+ }
+ wfSuppressWarnings();
+ if ( is_dir( $dir ) ) {
+ rmdir( $dir ); // remove directory if empty
+ }
+ wfRestoreWarnings();
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::fileExists()
+ */
+ function fileExists( array $params ) {
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ return false; // invalid storage path
+ }
+ wfSuppressWarnings();
+ $exists = is_file( $source );
+ wfRestoreWarnings();
+ return $exists;
+ }
+
+ /**
+ * @see FileBackend::getFileTimestamp()
+ */
+ function getFileTimestamp( array $params ) {
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ return false; // invalid storage path
+ }
+ $fsFile = new FSFile( $source );
+ return $fsFile->getTimestamp();
+ }
+
+ /**
+ * @see FileBackend::getFileList()
+ */
+ function getFileList( array $params ) {
+ list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] );
+ if ( $dir === null ) { // invalid storage path
+ return null;
+ }
+ wfSuppressWarnings();
+ $exists = is_dir( $dir );
+ wfRestoreWarnings();
+ if ( !$exists ) {
+ return array(); // nothing under this dir
+ }
+ wfSuppressWarnings();
+ $readable = is_readable( $dir );
+ wfRestoreWarnings();
+ if ( !$readable ) {
+ return null; // bad permissions?
+ }
+ return new FSFileIterator( $dir );
+ }
+
+ /**
+ * @see FileBackend::getLocalReference()
+ */
+ function getLocalReference( array $params ) {
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ return null;
+ }
+ return new FSFile( $source );
+ }
+
+ /**
+ * @see FileBackend::getLocalCopy()
+ */
+ function getLocalCopy( array $params ) {
+ list( $c, $source ) = $this->resolveStoragePath( $params['src'] );
+ if ( $source === null ) {
+ return null;
+ }
+
+ // Get source file extension
+ $i = strrpos( $source, '.' );
+ $ext = strtolower( $i ? substr( $source, $i + 1 ) : '' );
+ // Create a new temporary file...
+ $tmpFile = TempFSFile::factory( wfBaseName( $source ) . '_', $ext );
+ if ( !$tmpFile ) {
+ return null;
+ }
+ $tmpPath = $tmpFile->getPath();
+
+ // Copy the source file over the temp file
+ wfSuppressWarnings();
+ $ok = copy( $source, $tmpPath );
+ wfRestoreWarnings();
+ if ( !$ok ) {
+ return null;
+ }
+
+ $this->chmod( $tmpPath );
+
+ return $tmpFile;
+ }
+
+ /**
+ * Chmod a file, suppressing the warnings
+ *
+ * @param $path string Absolute file system path
+ * @return bool Success
+ */
+ protected function chmod( $path ) {
+ wfSuppressWarnings();
+ $ok = chmod( $path, $this->fileMode );
+ wfRestoreWarnings();
+
+ return $ok;
+ }
+}
+
+/**
+ * Wrapper around RecursiveDirectoryIterator that catches
+ * exception or does any custom behavoir that we may want.
+ *
+ * @ingroup FileBackend
+ */
+class FSFileIterator implements Iterator {
+ /** @var RecursiveIteratorIterator */
+ protected $iter;
+
+ /**
+ * Get an FSFileIterator from a file system directory
+ *
+ * @param $dir string
+ */
+ public function __construct( $dir ) {
+ try {
+ $this->iter = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( $dir ) );
+ } catch ( UnexpectedValueException $e ) {
+ $this->iter = null; // bad permissions? deleted?
+ }
+ }
+
+ public function current() {
+ return $this->iter->current();
+ }
+
+ public function key() {
+ return $this->iter->key();
+ }
+
+ public function next() {
+ try {
+ $this->iter->next();
+ } catch ( UnexpectedValueException $e ) {
+ $this->iter = null;
+ }
+ }
+
+ public function rewind() {
+ try {
+ $this->iter->rewind();
+ } catch ( UnexpectedValueException $e ) {
+ $this->iter = null;
+ }
+ }
+
+ public function valid() {
+ return $this->iter && $this->iter->valid();
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * Base class for all file backend classes (including multi-write backends).
+ * This class defines the methods as abstract that subclasses must implement.
+ * Outside callers can assume that all backends will have these functions.
+ *
+ * All "storage paths" are of the format "mwstore://backend/container/path".
+ * The paths use typical file system (FS) notation, though any particular backend may
+ * not actually be using a local filesystem. Therefore, the paths are only virtual.
+ *
+ * FS-based backends are somewhat more restrictive due to the existence of real
+ * directory files; a regular file cannot have the same name as a directory. Other
+ * backends with virtual directories may not have this limitation.
+ *
+ * Methods should avoid throwing exceptions at all costs.
+ * As a corollary, external dependencies should be kept to a minimum.
+ *
+ * @ingroup FileBackend
+ * @since 1.19
+ */
+abstract class FileBackendBase {
+ protected $name; // unique backend name
+ protected $wikiId; // unique wiki name
+ /** @var LockManager */
+ protected $lockManager;
+
+ /**
+ * Build a new object from configuration.
+ * This should only be called from within FileBackendGroup.
+ *
+ * $config includes:
+ * 'name' : The name of this backend
+ * 'wikiId' : Prefix to container names that is unique to this wiki
+ * 'lockManager' : Registered name of the file lock manager to use
+ *
+ * @param $config Array
+ */
+ public function __construct( array $config ) {
+ $this->name = $config['name'];
+ $this->wikiId = isset( $config['wikiId'] )
+ ? $config['wikiId']
+ : wfWikiID();
+ $this->lockManager = LockManagerGroup::singleton()->get( $config['lockManager'] );
+ }
+
+ /**
+ * Get the unique backend name.
+ *
+ * We may have multiple different backends of the same type.
+ * For example, we can have two Swift backends using different proxies.
+ *
+ * @return string
+ */
+ final public function getName() {
+ return $this->name;
+ }
+
+ /**
+ * This is the main entry point into the backend for write operations.
+ * Callers supply an ordered list of operations to perform as a transaction.
+ * If any serious errors occur, all attempted operations will be rolled back.
+ *
+ * $ops is an array of arrays. The outer array holds a list of operations.
+ * Each inner array is a set of key value pairs that specify an operation.
+ *
+ * Supported operations and their parameters:
+ * a) Create a new file in storage with the contents of a string
+ * array(
+ * 'op' => 'create',
+ * 'dst' => <storage path>,
+ * 'content' => <string of new file contents>,
+ * 'overwriteDest' => <boolean>,
+ * 'overwriteSame' => <boolean>
+ * )
+ * b) Copy a file system file into storage
+ * array(
+ * 'op' => 'store',
+ * 'src' => <file system path>,
+ * 'dst' => <storage path>,
+ * 'overwriteDest' => <boolean>,
+ * 'overwriteSame' => <boolean>
+ * )
+ * c) Copy a file within storage
+ * array(
+ * 'op' => 'copy',
+ * 'src' => <storage path>,
+ * 'dst' => <storage path>,
+ * 'overwriteDest' => <boolean>,
+ * 'overwriteSame' => <boolean>
+ * )
+ * d) Move a file within storage
+ * array(
+ * 'op' => 'move',
+ * 'src' => <storage path>,
+ * 'dst' => <storage path>,
+ * 'overwriteDest' => <boolean>,
+ * 'overwriteSame' => <boolean>
+ * )
+ * e) Delete a file within storage
+ * array(
+ * 'op' => 'delete',
+ * 'src' => <storage path>,
+ * 'ignoreMissingSource' => <boolean>
+ * )
+ * f) Concatenate a list of files into a single file within storage
+ * array(
+ * 'op' => 'concatenate',
+ * 'srcs' => <ordered array of storage paths>,
+ * 'dst' => <storage path>,
+ * 'overwriteDest' => <boolean>
+ * )
+ * g) Do nothing (no-op)
+ * array(
+ * 'op' => 'null',
+ * )
+ *
+ * Boolean flags for operations (operation-specific):
+ * 'ignoreMissingSource' : The operation will simply succeed and do
+ * nothing if the source file does not exist.
+ * 'overwriteDest' : Any destination file will be overwritten.
+ * 'overwriteSame' : An error will not be given if a file already
+ * exists at the destination that has the same
+ * contents as the new contents to be written there.
+ *
+ * $opts is an associative of options, including:
+ * 'nonLocking' : No locks are acquired for the operations.
+ * This can increase performance for non-critical writes.
+ * 'ignoreErrors' : Serious errors that would normally cause a rollback
+ * do not. The remaining operations are still attempted.
+ *
+ * Return value:
+ * This returns a Status, which contains all warnings and fatals that occured
+ * during the operation. The 'failCount', 'successCount', and 'success' members
+ * will reflect each operation attempted. The status will be "OK" unless any
+ * of the operations failed and the 'ignoreErrors' parameter was not set.
+ *
+ * @param $ops Array List of operations to execute in order
+ * @param $opts Array Batch operation options
+ * @return Status
+ */
+ abstract public function doOperations( array $ops, array $opts = array() );
+
+ /**
+ * Same as doOperations() except it takes a single operation array
+ *
+ * @param $op Array
+ * @param $opts Array
+ * @return Status
+ */
+ final public function doOperation( array $op, array $opts = array() ) {
+ return $this->doOperations( array( $op ), $opts );
+ }
+
+ /**
+ * Prepare a storage path for usage. This will create containers
+ * that don't yet exist or, on FS backends, create parent directories.
+ *
+ * $params include:
+ * dir : storage directory
+ *
+ * @param $params Array
+ * @return Status
+ */
+ abstract public function prepare( array $params );
+
+ /**
+ * Take measures to block web access to a directory and
+ * the container it belongs to. FS backends might add .htaccess
+ * files wheras backends like Swift this might restrict container
+ * access to backend user that represents end-users in web request.
+ * This is not guaranteed to actually do anything.
+ *
+ * $params include:
+ * dir : storage directory
+ * noAccess : try to deny file access
+ * noListing : try to deny file listing
+ *
+ * @param $params Array
+ * @return Status
+ */
+ abstract public function secure( array $params );
+
+ /**
+ * Clean up an empty storage directory.
+ * On FS backends, the directory will be deleted. Others may do nothing.
+ *
+ * $params include:
+ * dir : storage directory
+ *
+ * @param $params Array
+ * @return Status
+ */
+ abstract public function clean( array $params );
+
+ /**
+ * Check if a file exists at a storage path in the backend.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return bool
+ */
+ abstract public function fileExists( array $params );
+
+ /**
+ * Get a SHA-1 hash of the file at a storage path in the backend.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return string|false Hash string or false on failure
+ */
+ abstract public function getFileSha1Base36( array $params );
+
+ /**
+ * Get the last-modified timestamp of the file at a storage path.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return string|false TS_MW timestamp or false on failure
+ */
+ abstract public function getFileTimestamp( array $params );
+
+ /**
+ * Get the properties of the file at a storage path in the backend.
+ * Returns FSFile::placeholderProps() on failure.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return Array
+ */
+ abstract public function getFileProps( array $params );
+
+ /**
+ * Stream the file at a storage path in the backend.
+ * Appropriate HTTP headers (Status, Content-Type, Content-Length)
+ * must be sent if streaming began, while none should be sent otherwise.
+ * Implementations should flush the output buffer before sending data.
+ *
+ * $params include:
+ * src : source storage path
+ * headers : additional HTTP headers to send on success
+ *
+ * @param $params Array
+ * @return Status
+ */
+ abstract public function streamFile( array $params );
+
+ /**
+ * Get an iterator to list out all object files under a storage directory.
+ * If the directory is of the form "mwstore://container", then all items in
+ * the container should be listed. If of the form "mwstore://container/dir",
+ * then all items under that container directory should be listed.
+ * Results should be storage paths relative to the given directory.
+ *
+ * $params include:
+ * dir : storage path directory
+ *
+ * @return Traversable|Array|null Returns null on failure
+ */
+ abstract public function getFileList( array $params );
+
+ /**
+ * Returns a file system file, identical to the file at a storage path.
+ * The file returned is either:
+ * a) A local copy of the file at a storage path in the backend.
+ * The temporary copy will have the same extension as the source.
+ * b) An original of the file at a storage path in the backend.
+ * Temporary files may be purged when the file object falls out of scope.
+ *
+ * Write operations should *never* be done on this file as some backends
+ * may do internal tracking or may be instances of FileBackendMultiWrite.
+ * In that later case, there are copies of the file that must stay in sync.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return FSFile|null Returns null on failure
+ */
+ abstract public function getLocalReference( array $params );
+
+ /**
+ * Get a local copy on disk of the file at a storage path in the backend.
+ * The temporary copy will have the same file extension as the source.
+ * Temporary files may be purged when the file object falls out of scope.
+ *
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return TempFSFile|null Returns null on failure
+ */
+ abstract public function getLocalCopy( array $params );
+
+ /**
+ * Lock the files at the given storage paths in the backend.
+ * This will either lock all the files or none (on failure).
+ *
+ * Callers should consider using getScopedFileLocks() instead.
+ *
+ * @param $paths Array Storage paths
+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH
+ * @return Status
+ */
+ final public function lockFiles( array $paths, $type ) {
+ return $this->lockManager->lock( $paths, $type );
+ }
+
+ /**
+ * Unlock the files at the given storage paths in the backend.
+ *
+ * @param $paths Array Storage paths
+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH
+ * @return Status
+ */
+ final public function unlockFiles( array $paths, $type ) {
+ return $this->lockManager->unlock( $paths, $type );
+ }
+
+ /**
+ * Lock the files at the given storage paths in the backend.
+ * This will either lock all the files or none (on failure).
+ * On failure, the status object will be updated with errors.
+ *
+ * Once the return value goes out scope, the locks will be released and
+ * the status updated. Unlock fatals will not change the status "OK" value.
+ *
+ * @param $paths Array Storage paths
+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH
+ * @param $status Status Status to update on lock/unlock
+ * @return ScopedLock|null Returns null on failure
+ */
+ final public function getScopedFileLocks( array $paths, $type, Status $status ) {
+ return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
+ }
+}
+
+/**
+ * Base class for all single-write backends.
+ * This class defines the methods as abstract that subclasses must implement.
+ *
+ * @ingroup FileBackend
+ * @since 1.19
+ */
+abstract class FileBackend extends FileBackendBase {
+ /** @var Array */
+ protected $cache = array(); // (storage path => key => value)
+ protected $maxCacheSize = 50; // integer; max paths with entries
+
+ /**
+ * Store a file into the backend from a file on disk.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * src : source path on disk
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function store( array $params ) {
+ $status = $this->doStore( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::store()
+ */
+ abstract protected function doStore( array $params );
+
+ /**
+ * Copy a file from one storage path to another in the backend.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * src : source storage path
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function copy( array $params ) {
+ $status = $this->doCopy( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::copy()
+ */
+ abstract protected function doCopy( array $params );
+
+ /**
+ * Delete a file at the storage path.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * src : source storage path
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function delete( array $params ) {
+ $status = $this->doDelete( $params );
+ $this->clearCache( array( $params['src'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::delete()
+ */
+ abstract protected function doDelete( array $params );
+
+ /**
+ * Move a file from one storage path to another in the backend.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * src : source storage path
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function move( array $params ) {
+ $status = $this->doMove( $params );
+ $this->clearCache( array( $params['src'], $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::move()
+ */
+ protected function doMove( array $params ) {
+ // Copy source to dest
+ $status = $this->backend->copy( $params );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Delete source (only fails due to races or medium going down)
+ $status->merge( $this->backend->delete( array( 'src' => $params['src'] ) ) );
+ $status->setResult( true, $status->value ); // ignore delete() errors
+ return $status;
+ }
+
+ /**
+ * Combines files from several storage paths into a new file in the backend.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function concatenate( array $params ) {
+ $status = $this->doConcatenate( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::concatenate()
+ */
+ abstract protected function doConcatenate( array $params );
+
+ /**
+ * Create a file in the backend with the given contents.
+ * Do not call this function from places outside FileBackend and FileOp.
+ * $params include:
+ * content : the raw file contents
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function create( array $params ) {
+ $status = $this->doCreate( $params );
+ $this->clearCache( array( $params['dst'] ) );
+ return $status;
+ }
+
+ /**
+ * @see FileBackend::create()
+ */
+ abstract protected function doCreate( array $params );
+
+ /**
+ * @see FileBackendBase::prepare()
+ */
+ public function prepare( array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackendBase::secure()
+ */
+ public function secure( array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackendBase::clean()
+ */
+ public function clean( array $params ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @see FileBackendBase::getFileSha1Base36()
+ */
+ public function getFileSha1Base36( array $params ) {
+ $path = $params['src'];
+ if ( isset( $this->cache[$path]['sha1'] ) ) {
+ return $this->cache[$path]['sha1'];
+ }
+ $fsFile = $this->getLocalReference( array( 'src' => $path ) );
+ if ( !$fsFile ) {
+ return false;
+ } else {
+ $sha1 = $fsFile->getSha1Base36();
+ if ( $sha1 !== false ) { // don't cache negatives
+ $this->trimCache(); // limit memory
+ $this->cache[$path]['sha1'] = $sha1;
+ }
+ return $sha1;
+ }
+ }
+
+ /**
+ * @see FileBackendBase::getFileProps()
+ */
+ public function getFileProps( array $params ) {
+ $fsFile = $this->getLocalReference( array( 'src' => $params['src'] ) );
+ if ( !$fsFile ) {
+ return FSFile::placeholderProps();
+ } else {
+ return $fsFile->getProps();
+ }
+ }
+
+ /**
+ * @see FileBackendBase::getLocalReference()
+ */
+ public function getLocalReference( array $params ) {
+ return $this->getLocalCopy( $params );
+ }
+
+ /**
+ * @see FileBackendBase::streamFile()
+ */
+ function streamFile( array $params ) {
+ $status = Status::newGood();
+
+ $fsFile = $this->getLocalReference( array( 'src' => $params['src'] ) );
+ if ( !$fsFile ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+ return $status;
+ }
+
+ $extraHeaders = isset( $params['headers'] )
+ ? $params['headers']
+ : array();
+
+ $ok = StreamFile::stream( $fsFile->getPath(), $extraHeaders, false );
+ if ( !$ok ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+ return $status;
+ }
+
+ return $status;
+ }
+
+ /**
+ * Get the list of supported operations and their corresponding FileOp classes.
+ *
+ * @return Array
+ */
+ protected function supportedOperations() {
+ return array(
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'concatenate' => 'ConcatenateFileOp',
+ 'create' => 'CreateFileOp',
+ 'null' => 'NullFileOp'
+ );
+ }
+
+ /**
+ * Return a list of FileOp objects from a list of operations.
+ * The result must have the same number of items as the input.
+ * An exception is thrown if an unsupported operation is requested.
+ *
+ * @param $ops Array Same format as doOperations()
+ * @return Array List of FileOp objects
+ * @throws MWException
+ */
+ final public function getOperations( array $ops ) {
+ $supportedOps = $this->supportedOperations();
+
+ $performOps = array(); // array of FileOp objects
+ // Build up ordered array of FileOps...
+ foreach ( $ops as $operation ) {
+ $opName = $operation['op'];
+ if ( isset( $supportedOps[$opName] ) ) {
+ $class = $supportedOps[$opName];
+ // Get params for this operation
+ $params = $operation;
+ // Append the FileOp class
+ $performOps[] = new $class( $this, $params );
+ } else {
+ throw new MWException( "Operation `$opName` is not supported." );
+ }
+ }
+
+ return $performOps;
+ }
+
+ /**
+ * @see FileBackendBase::doOperations()
+ */
+ final public function doOperations( array $ops, array $opts = array() ) {
+ $status = Status::newGood();
+
+ // Build up a list of FileOps...
+ $performOps = $this->getOperations( $ops );
+
+ if ( empty( $opts['nonLocking'] ) ) {
+ // Build up a list of files to lock...
+ $filesLockEx = $filesLockSh = array();
+ foreach ( $performOps as $index => $fileOp ) {
+ $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
+ $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
+ }
+ // Try to lock those files for the scope of this function...
+ $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
+ $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
+ if ( !$status->isOK() ) {
+ return $status; // abort
+ }
+ }
+
+ // Clear any cache entries (after locks acquired)
+ $this->clearCache();
+ // Actually attempt the operation batch...
+ $status->merge( FileOp::attemptBatch( $performOps, $opts ) );
+
+ return $status;
+ }
+
+ /**
+ * Invalidate the file existence and property cache
+ *
+ * @param $paths Array Clear cache for specific files
+ * @return void
+ */
+ final public function clearCache( array $paths = null ) {
+ if ( $paths === null ) {
+ $this->cache = array();
+ } else {
+ foreach ( $paths as $path ) {
+ unset( $this->cache[$path] );
+ }
+ }
+ }
+
+ /**
+ * Prune the cache if it is too big to add an item
+ *
+ * @return void
+ */
+ protected function trimCache() {
+ if ( count( $this->cache ) >= $this->maxCacheSize ) {
+ reset( $this->cache );
+ $key = key( $this->cache );
+ unset( $this->cache[$key] );
+ }
+ }
+
+ /**
+ * Check if a given path is a mwstore:// path.
+ * This does not do any actual validation or existence checks.
+ *
+ * @param $path string
+ * @return bool
+ */
+ final public static function isStoragePath( $path ) {
+ return ( strpos( $path, 'mwstore://' ) === 0 );
+ }
+
+ /**
+ * Split a storage path (e.g. "mwstore://backend/container/path/to/object")
+ * into a backend name, a container name, and a relative object path.
+ *
+ * @param $storagePath string
+ * @return Array (backend, container, rel object) or (null, null, null)
+ */
+ final public static function splitStoragePath( $storagePath ) {
+ if ( self::isStoragePath( $storagePath ) ) {
+ // Note: strlen( 'mwstore://' ) = 10
+ $parts = explode( '/', substr( $storagePath, 10 ), 3 );
+ if ( count( $parts ) == 3 ) {
+ return $parts; // e.g. "backend/container/path"
+ } elseif ( count( $parts ) == 2 ) {
+ return array( $parts[0], $parts[1], '' ); // e.g. "backend/container"
+ }
+ }
+ return array( null, null, null );
+ }
+
+ /**
+ * Validate a container name.
+ * Null is returned if the name has illegal characters.
+ *
+ * @param $container string
+ * @return bool
+ */
+ final protected static function isValidContainerName( $container ) {
+ // This accounts for Swift and S3 restrictions. Also note
+ // that these urlencode to the same string, which is useful
+ // since the Swift size limit is *after* URL encoding.
+ return preg_match( '/^[a-zA-Z0-9._-]{1,256}$/u', $container );
+ }
+
+ /**
+ * Validate and normalize a relative storage path.
+ * Null is returned if the path involves directory traversal.
+ * Traversal is insecure for FS backends and broken for others.
+ *
+ * @param $path string
+ * @return string|null
+ */
+ final protected static function normalizeStoragePath( $path ) {
+ // Normalize directory separators
+ $path = strtr( $path, '\\', '/' );
+ // Use the same traversal protection as Title::secureAndSplit()
+ if ( strpos( $path, '.' ) !== false ) {
+ if (
+ $path === '.' ||
+ $path === '..' ||
+ strpos( $path, './' ) === 0 ||
+ strpos( $path, '../' ) === 0 ||
+ strpos( $path, '/./' ) !== false ||
+ strpos( $path, '/../' ) !== false
+ ) {
+ return null;
+ }
+ }
+ return $path;
+ }
+
+ /**
+ * Split a storage path (e.g. "mwstore://backend/container/path/to/object")
+ * into an internal container name and an internal relative object name.
+ * This also checks that the storage path is valid and is within this backend.
+ *
+ * @param $storagePath string
+ * @return Array (container, object name) or (null, null) if path is invalid
+ */
+ final protected function resolveStoragePath( $storagePath ) {
+ list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
+ if ( $backend === $this->name ) { // must be for this backend
+ $relPath = self::normalizeStoragePath( $relPath );
+ if ( $relPath !== null ) {
+ $relPath = $this->resolveContainerPath( $container, $relPath );
+ if ( $relPath !== null ) {
+ $container = $this->fullContainerName( $container );
+ if ( self::isValidContainerName( $container ) ) {
+ $container = $this->resolveContainerName( $container );
+ if ( $container !== null ) {
+ return array( $container, $relPath );
+ }
+ }
+ }
+ }
+ }
+ return array( null, null );
+ }
+
+ /**
+ * Get the full container name, including the wiki ID prefix
+ *
+ * @param $container string
+ * @return string
+ */
+ final protected function fullContainerName( $container ) {
+ if ( $this->wikiId != '' ) {
+ return "{$this->wikiId}-$container";
+ } else {
+ return $container;
+ }
+ }
+
+ /**
+ * Resolve a container name, checking if it's allowed by the backend.
+ * This is intended for internal use, such as encoding illegal chars.
+ * Subclasses can override this to be more restrictive.
+ *
+ * @param $container string
+ * @return string|null
+ */
+ protected function resolveContainerName( $container ) {
+ return $container;
+ }
+
+ /**
+ * Resolve a relative storage path, checking if it's allowed by the backend.
+ * This is intended for internal use, such as encoding illegal chars
+ * or perhaps getting absolute paths (e.g. FS based backends).
+ *
+ * @param $container string Container the path is relative to
+ * @param $relStoragePath string Relative storage path
+ * @return string|null Path or null if not valid
+ */
+ protected function resolveContainerPath( $container, $relStoragePath ) {
+ return $relStoragePath;
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileBackend
+ */
+
+/**
+ * Class to handle file backend registration
+ *
+ * @ingroup FileBackend
+ */
+class FileBackendGroup {
+ protected static $instance = null;
+
+ /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */
+ protected $backends = array();
+
+ protected function __construct() {}
+ protected function __clone() {}
+
+ public static function singleton() {
+ if ( self::$instance == null ) {
+ self::$instance = new self();
+ }
+ return self::$instance;
+ }
+
+ /**
+ * Destroy the singleton instance, so that a new one will be created next
+ * time singleton() is called.
+ */
+ public static function destroySingleton() {
+ self::$instance = null;
+ }
+
+ /**
+ * Register an array of file backend configurations
+ *
+ * @param $configs Array
+ * @return void
+ * @throws MWException
+ */
+ public function register( array $configs ) {
+ foreach ( $configs as $config ) {
+ if ( !isset( $config['name'] ) ) {
+ throw new MWException( "Cannot register a backend with no name." );
+ }
+ $name = $config['name'];
+ if ( !isset( $config['class'] ) ) {
+ throw new MWException( "Cannot register backend `{$name}` with no class." );
+ }
+ $class = $config['class'];
+
+ unset( $config['class'] ); // backend won't need this
+ $this->backends[$name] = array(
+ 'class' => $class,
+ 'config' => $config,
+ 'instance' => null
+ );
+ }
+ }
+
+ /**
+ * Get the backend object with a given name
+ *
+ * @param $name string
+ * @return FileBackendBase
+ * @throws MWException
+ */
+ public function get( $name ) {
+ if ( !isset( $this->backends[$name] ) ) {
+ throw new MWException( "No backend defined with the name `$name`." );
+ }
+ // Lazy-load the actual backend instance
+ if ( !isset( $this->backends[$name]['instance'] ) ) {
+ $class = $this->backends[$name]['class'];
+ $config = $this->backends[$name]['config'];
+ $this->backends[$name]['instance'] = new $class( $config );
+ }
+ return $this->backends[$name]['instance'];
+ }
+
+ /**
+ * Get an appropriate backend object from a storage path
+ *
+ * @param $storagePath string
+ * @return FileBackendBase|null Backend or null on failure
+ */
+ public function backendFromPath( $storagePath ) {
+ list( $backend, $c, $p ) = FileBackend::splitStoragePath( $storagePath );
+ if ( $backend !== null && isset( $this->backends[$backend] ) ) {
+ return $this->get( $backend );
+ }
+ return null;
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * This class defines a multi-write backend. Multiple backends can be
+ * registered to this proxy backend and it will act as a single backend.
+ * Use this when all access to those backends is through this proxy backend.
+ * At least one of the backends must be declared the "master" backend.
+ *
+ * The order that the backends are defined sets the priority of which
+ * backend is read from or written to first. Functions like fileExists()
+ * and getFileProps() will return information based on the first backend
+ * that has the file. Special cases are listed below:
+ * a) getFileTimestamp() will always check only the master backend to
+ * avoid confusing and inconsistent results.
+ *
+ * All write operations are performed on all backends.
+ * If an operation fails on one backend it will be rolled back from the others.
+ *
+ * @ingroup FileBackend
+ */
+class FileBackendMultiWrite extends FileBackendBase {
+ /** @var Array Prioritized list of FileBackend objects */
+ protected $fileBackends = array(); // array of (backend index => backends)
+ protected $masterIndex = -1; // index of master backend
+
+ /**
+ * Construct a proxy backend that consists of several internal backends.
+ * $config contains:
+ * 'name' : The name of the proxy backend
+ * 'lockManager' : Registered name of the file lock manager to use
+ * 'backends' : Array of backend config and multi-backend settings.
+ * Each value is the config used in the constructor of a
+ * FileBackend class, but with these additional settings:
+ * 'class' : The name of the backend class
+ * 'isMultiMaster' : This must be set for one backend.
+ * @param $config Array
+ */
+ public function __construct( array $config ) {
+ parent::__construct( $config );
+ // Construct backends here rather than via registration
+ // to keep these backends hidden from outside the proxy.
+ foreach ( $config['backends'] as $index => $config ) {
+ if ( !isset( $config['class'] ) ) {
+ throw new MWException( 'No class given for a backend config.' );
+ }
+ $class = $config['class'];
+ $this->fileBackends[$index] = new $class( $config );
+ if ( !empty( $config['isMultiMaster'] ) ) {
+ if ( $this->masterIndex >= 0 ) {
+ throw new MWException( 'More than one master backend defined.' );
+ }
+ $this->masterIndex = $index;
+ }
+ }
+ if ( $this->masterIndex < 0 ) { // need backends and must have a master
+ throw new MWException( 'No master backend defined.' );
+ }
+ }
+
+ final public function doOperations( array $ops, array $opts = array() ) {
+ $status = Status::newGood();
+
+ $performOps = array(); // list of FileOp objects
+ $filesLockEx = $filesLockSh = array(); // storage paths to lock
+ // Build up a list of FileOps. The list will have all the ops
+ // for one backend, then all the ops for the next, and so on.
+ // These batches of ops are all part of a continuous array.
+ // Also build up a list of files to lock...
+ foreach ( $this->fileBackends as $index => $backend ) {
+ $backendOps = $this->substOpPaths( $ops, $backend );
+ $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
+ if ( $index == 0 && empty( $opts['nonLocking'] ) ) {
+ // Set "files to lock" from the first batch so we don't try to set all
+ // locks two or three times over (depending on the number of backends).
+ // A lock on one storage path is a lock on all the backends.
+ foreach ( $performOps as $index => $fileOp ) {
+ $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
+ $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
+ }
+ // Lock the paths under the proxy backend's name
+ $this->unsubstPaths( $filesLockSh );
+ $this->unsubstPaths( $filesLockEx );
+ }
+ }
+
+ // Try to lock those files for the scope of this function...
+ $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
+ $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
+ if ( !$status->isOK() ) {
+ return $status; // abort
+ }
+
+ // Clear any cache entries (after locks acquired)
+ foreach ( $this->fileBackends as $backend ) {
+ $backend->clearCache();
+ }
+ // Actually attempt the operation batch...
+ $status->merge( FileOp::attemptBatch( $performOps, $opts ) );
+
+ return $status;
+ }
+
+ /**
+ * Substitute the backend name in storage path parameters
+ * for a set of operations with a that of a given backend.
+ *
+ * @param $ops Array List of file operation arrays
+ * @param $backend FileBackend
+ * @return Array
+ */
+ protected function substOpPaths( array $ops, FileBackend $backend ) {
+ $newOps = array(); // operations
+ foreach ( $ops as $op ) {
+ $newOp = $op; // operation
+ foreach ( array( 'src', 'srcs', 'dst' ) as $par ) {
+ if ( isset( $newOp[$par] ) ) {
+ $newOp[$par] = preg_replace(
+ '!^mwstore://' . preg_quote( $this->name ) . '/!',
+ 'mwstore://' . $backend->getName() . '/',
+ $newOp[$par] // string or array
+ );
+ }
+ }
+ $newOps[] = $newOp;
+ }
+ return $newOps;
+ }
+
+ /**
+ * Replace the backend part of storage paths with this backend's name
+ *
+ * @param &$paths Array
+ * @return void
+ */
+ protected function unsubstPaths( array &$paths ) {
+ foreach ( $paths as &$path ) {
+ $path = preg_replace( '!^mwstore://([^/]+)!', "mwstore://{$this->name}", $path );
+ }
+ }
+
+ function prepare( array $params ) {
+ $status = Status::newGood();
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->prepare( $realParams ) );
+ }
+ return $status;
+ }
+
+ function secure( array $params ) {
+ $status = Status::newGood();
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->secure( $realParams ) );
+ }
+ return $status;
+ }
+
+ function clean( array $params ) {
+ $status = Status::newGood();
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->clean( $realParams ) );
+ }
+ return $status;
+ }
+
+ function fileExists( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ if ( $backend->fileExists( $realParams ) ) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ function getFileTimestamp( array $params ) {
+ // Skip non-master for consistent timestamps
+ $realParams = $this->substOpPaths( $params, $backend );
+ return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
+ }
+
+ function getFileSha1Base36( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $hash = $backend->getFileSha1Base36( $realParams );
+ if ( $hash !== false ) {
+ return $hash;
+ }
+ }
+ return false;
+ }
+
+ function getFileProps( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $props = $backend->getFileProps( $realParams );
+ if ( $props !== null ) {
+ return $props;
+ }
+ }
+ return null;
+ }
+
+ function streamFile( array $params ) {
+ $status = Status::newGood();
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $subStatus = $backend->streamFile( $realParams );
+ $status->merge( $subStatus );
+ if ( $subStatus->isOK() ) {
+ // Pass isOK() despite fatals from other backends
+ $status->setResult( true );
+ return $status;
+ } else { // failure
+ if ( headers_sent() ) {
+ return $status; // died mid-stream...so this is already fubar
+ } elseif ( strval( ob_get_contents() ) !== '' ) {
+ ob_clean(); // output was buffered but not sent; clear it
+ }
+ }
+ }
+ return $status;
+ }
+
+ function getLocalReference( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $fsFile = $backend->getLocalReference( $realParams );
+ if ( $fsFile ) {
+ return $fsFile;
+ }
+ }
+ return null;
+ }
+
+ function getLocalCopy( array $params ) {
+ # Hit all backends in case of failed operations (out of sync)
+ foreach ( $this->backends as $backend ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $tmpFile = $backend->getLocalCopy( $realParams );
+ if ( $tmpFile ) {
+ return $tmpFile;
+ }
+ }
+ return null;
+ }
+
+ function getFileList( array $params ) {
+ foreach ( $this->backends as $index => $backend ) {
+ # Get results from the first backend
+ $realParams = $this->substOpPaths( $params, $backend );
+ return $backend->getFileList( $realParams );
+ }
+ return array(); // sanity
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * Helper class for representing operations with transaction support.
+ * FileBackend::doOperations() will require these classes for supported operations.
+ *
+ * Use of large fields should be avoided as we want to be able to support
+ * potentially many FileOp classes in large arrays in memory.
+ *
+ * @ingroup FileBackend
+ * @since 1.19
+ */
+abstract class FileOp {
+ /** $var Array */
+ protected $params = array();
+ /** $var FileBackendBase */
+ protected $backend;
+ /** @var TempFSFile|null */
+ protected $tmpSourceFile, $tmpDestFile;
+
+ protected $state = self::STATE_NEW; // integer
+ protected $failed = false; // boolean
+ protected $useBackups = true; // boolean
+ protected $destSameAsSource = false; // boolean
+ protected $destAlreadyExists = false; // boolean
+
+ /* Object life-cycle */
+ const STATE_NEW = 1;
+ const STATE_CHECKED = 2;
+ const STATE_ATTEMPTED = 3;
+ const STATE_DONE = 4;
+
+ /**
+ * Build a new file operation transaction
+ *
+ * @params $backend FileBackend
+ * @params $params Array
+ */
+ final public function __construct( FileBackendBase $backend, array $params ) {
+ $this->backend = $backend;
+ foreach ( $this->allowedParams() as $name ) {
+ if ( isset( $params[$name] ) ) {
+ $this->params[$name] = $params[$name];
+ }
+ }
+ $this->params = $params;
+ }
+
+ /**
+ * Disable file backups for this operation
+ *
+ * @return void
+ */
+ final protected function disableBackups() {
+ $this->useBackups = false;
+ }
+
+ /**
+ * Attempt a series of file operations.
+ * Callers are responsible for handling file locking.
+ *
+ * @param $performOps Array List of FileOp operations
+ * @param $opts Array Batch operation options
+ * @return Status
+ */
+ final public static function attemptBatch( array $performOps, array $opts ) {
+ $status = Status::newGood();
+
+ $ignoreErrors = isset( $opts['ignoreErrors'] ) && $opts['ignoreErrors'];
+ $predicates = FileOp::newPredicates(); // account for previous op in prechecks
+ // Do pre-checks for each operation; abort on failure...
+ foreach ( $performOps as $index => $fileOp ) {
+ $status->merge( $fileOp->precheck( $predicates ) );
+ if ( !$status->isOK() ) { // operation failed?
+ if ( $ignoreErrors ) {
+ ++$status->failCount;
+ $status->success[$index] = false;
+ } else {
+ return $status;
+ }
+ }
+ }
+
+ // Attempt each operation; abort on failure...
+ foreach ( $performOps as $index => $fileOp ) {
+ if ( $fileOp->failed() ) {
+ continue; // nothing to do
+ } elseif ( $ignoreErrors ) {
+ $fileOp->disableBackups(); // no chance of revert() calls
+ }
+ $status->merge( $fileOp->attempt() );
+ if ( !$status->isOK() ) { // operation failed?
+ if ( $ignoreErrors ) {
+ ++$status->failCount;
+ $status->success[$index] = false;
+ } else {
+ // Revert everything done so far and abort.
+ // Do this by reverting each previous operation in reverse order.
+ $pos = $index - 1; // last one failed; no need to revert()
+ while ( $pos >= 0 ) {
+ if ( !$performOps[$pos]->failed() ) {
+ $status->merge( $performOps[$pos]->revert() );
+ }
+ $pos--;
+ }
+ return $status;
+ }
+ }
+ }
+
+ // Finish each operation...
+ foreach ( $performOps as $index => $fileOp ) {
+ if ( $fileOp->failed() ) {
+ continue; // nothing to do
+ }
+ $subStatus = $fileOp->finish();
+ if ( $subStatus->isOK() ) {
+ ++$status->successCount;
+ $status->success[$index] = true;
+ } else {
+ ++$status->failCount;
+ $status->success[$index] = false;
+ }
+ $status->merge( $subStatus );
+ }
+
+ // Make sure status is OK, despite any finish() fatals
+ $status->setResult( true, $status->value );
+
+ return $status;
+ }
+
+ /**
+ * Get the value of the parameter with the given name.
+ * Returns null if the parameter is not set.
+ *
+ * @param $name string
+ * @return mixed
+ */
+ final public function getParam( $name ) {
+ return isset( $this->params[$name] ) ? $this->params[$name] : null;
+ }
+
+ /**
+ * Check if this operation failed precheck() or attempt()
+ * @return type
+ */
+ final public function failed() {
+ return $this->failed;
+ }
+
+ /**
+ * Get a new empty predicates array for precheck()
+ *
+ * @return Array
+ */
+ final public static function newPredicates() {
+ return array( 'exists' => array() );
+ }
+
+ /**
+ * Check preconditions of the operation without writing anything
+ *
+ * @param $predicates Array
+ * @return Status
+ */
+ final public function precheck( array &$predicates ) {
+ if ( $this->state !== self::STATE_NEW ) {
+ return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
+ }
+ $this->state = self::STATE_CHECKED;
+ $status = $this->doPrecheck( $predicates );
+ if ( !$status->isOK() ) {
+ $this->failed = true;
+ }
+ return $status;
+ }
+
+ /**
+ * Attempt the operation, backing up files as needed; this must be reversible
+ *
+ * @return Status
+ */
+ final public function attempt() {
+ if ( $this->state !== self::STATE_CHECKED ) {
+ return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
+ } elseif ( $this->failed ) { // failed precheck
+ return Status::newFatal( 'fileop-fail-attempt-precheck' );
+ }
+ $this->state = self::STATE_ATTEMPTED;
+ $status = $this->doAttempt();
+ if ( !$status->isOK() ) {
+ $this->failed = true;
+ $this->logFailure( 'attempt' );
+ }
+ return $status;
+ }
+
+ /**
+ * Revert the operation; affected files are restored
+ *
+ * @return Status
+ */
+ final public function revert() {
+ if ( $this->state !== self::STATE_ATTEMPTED ) {
+ return Status::newFatal( 'fileop-fail-state', self::STATE_ATTEMPTED, $this->state );
+ }
+ $this->state = self::STATE_DONE;
+ if ( $this->failed ) {
+ $status = Status::newGood(); // nothing to revert
+ } else {
+ $status = $this->doRevert();
+ if ( !$status->isOK() ) {
+ $this->logFailure( 'revert' );
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Finish the operation; this may be irreversible
+ *
+ * @return Status
+ */
+ final public function finish() {
+ if ( $this->state !== self::STATE_ATTEMPTED ) {
+ return Status::newFatal( 'fileop-fail-state', self::STATE_ATTEMPTED, $this->state );
+ }
+ $this->state = self::STATE_DONE;
+ if ( $this->failed ) {
+ $status = Status::newGood(); // nothing to finish
+ } else {
+ $status = $this->doFinish();
+ }
+ return $status;
+ }
+
+ /**
+ * Get a list of storage paths read from for this operation
+ *
+ * @return Array
+ */
+ public function storagePathsRead() {
+ return array();
+ }
+
+ /**
+ * Get a list of storage paths written to for this operation
+ *
+ * @return Array
+ */
+ public function storagePathsChanged() {
+ return array();
+ }
+
+ /**
+ * @return Array List of allowed parameters
+ */
+ protected function allowedParams() {
+ return array();
+ }
+
+ /**
+ * @return Status
+ */
+ protected function doPrecheck( array &$predicates ) {
+ return Status::newGood();
+ }
+
+ /**
+ * @return Status
+ */
+ abstract protected function doAttempt();
+
+ /**
+ * @return Status
+ */
+ abstract protected function doRevert();
+
+ /**
+ * @return Status
+ */
+ protected function doFinish() {
+ return Status::newGood();
+ }
+
+ /**
+ * Check if the destination file exists and update the
+ * destAlreadyExists member variable. A bad status will
+ * be returned if there is no chance it can be overwritten.
+ *
+ * @param $predicates Array
+ * @return Status
+ */
+ protected function precheckDestExistence( array $predicates ) {
+ $status = Status::newGood();
+ if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
+ $this->destAlreadyExists = true;
+ if ( !$this->getParam( 'overwriteDest' ) && !$this->getParam( 'overwriteSame' ) ) {
+ $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
+ return $status;
+ }
+ } else {
+ $this->destAlreadyExists = false;
+ }
+ return $status;
+ }
+
+ /**
+ * Backup any file at the source to a temporary file
+ *
+ * @return Status
+ */
+ protected function backupSource() {
+ $status = Status::newGood();
+ if ( $this->useBackups ) {
+ // Check if a file already exists at the source...
+ $params = array( 'src' => $this->params['src'] );
+ if ( $this->backend->fileExists( $params ) ) {
+ // Create a temporary backup copy...
+ $this->tmpSourcePath = $this->backend->getLocalCopy( $params );
+ if ( $this->tmpSourcePath === null ) {
+ $status->fatal( 'backend-fail-backup', $this->params['src'] );
+ return $status;
+ }
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Backup the file at the destination to a temporary file.
+ * Don't bother backing it up unless we might overwrite the file.
+ * This assumes that the destination is in the backend and that
+ * the source is either in the backend or on the file system.
+ * This also handles the 'overwriteSame' check logic and updates
+ * the destSameAsSource member variable.
+ *
+ * @return Status
+ */
+ protected function checkAndBackupDest() {
+ $status = Status::newGood();
+ $this->destSameAsSource = false;
+
+ if ( $this->getParam( 'overwriteDest' ) ) {
+ if ( $this->useBackups ) {
+ // Create a temporary backup copy...
+ $params = array( 'src' => $this->params['dst'] );
+ $this->tmpDestFile = $this->backend->getLocalCopy( $params );
+ if ( !$this->tmpDestFile ) {
+ $status->fatal( 'backend-fail-backup', $this->params['dst'] );
+ return $status;
+ }
+ }
+ } elseif ( $this->getParam( 'overwriteSame' ) ) {
+ $shash = $this->getSourceSha1Base36();
+ // If there is a single source, then we can do some checks already.
+ // For things like concatenate(), we would need to build a temp file
+ // first and thus don't support 'overwriteSame' ($shash is null).
+ if ( $shash !== null ) {
+ $dhash = $this->getFileSha1Base36( $this->params['dst'] );
+ if ( !strlen( $shash ) || !strlen( $dhash ) ) {
+ $status->fatal( 'backend-fail-hashes' );
+ } elseif ( $shash !== $dhash ) {
+ // Give an error if the files are not identical
+ $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
+ } else {
+ $this->destSameAsSource = true;
+ }
+ return $status; // do nothing; either OK or bad status
+ }
+ } else {
+ $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
+ return $status;
+ }
+
+ return $status;
+ }
+
+ /**
+ * checkAndBackupDest() helper function to get the source file Sha1.
+ * Returns false on failure and null if there is no single source.
+ *
+ * @return string|false|null
+ */
+ protected function getSourceSha1Base36() {
+ return null; // N/A
+ }
+
+ /**
+ * checkAndBackupDest() helper function to get the Sha1 of a file.
+ *
+ * @return string|false False on failure
+ */
+ protected function getFileSha1Base36( $path ) {
+ // Source file is in backend
+ if ( FileBackend::isStoragePath( $path ) ) {
+ // For some backends (e.g. Swift, Azure) we can get
+ // standard hashes to use for this types of comparisons.
+ $hash = $this->backend->getFileSha1Base36( array( 'src' => $path ) );
+ // Source file is on file system
+ } else {
+ wfSuppressWarnings();
+ $hash = sha1_file( $path );
+ wfRestoreWarnings();
+ if ( $hash !== false ) {
+ $hash = wfBaseConvert( $hash, 16, 36, 31 );
+ }
+ }
+ return $hash;
+ }
+
+ /**
+ * Restore any temporary source backup file
+ *
+ * @return Status
+ */
+ protected function restoreSource() {
+ $status = Status::newGood();
+ // Restore any file that was at the destination
+ if ( $this->tmpSourcePath !== null ) {
+ $params = array(
+ 'src' => $this->tmpSourcePath,
+ 'dst' => $this->params['src'],
+ 'overwriteDest' => true
+ );
+ $status = $this->backend->store( $params );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Restore any temporary destination backup file
+ *
+ * @return Status
+ */
+ protected function restoreDest() {
+ $status = Status::newGood();
+ // Restore any file that was at the destination
+ if ( $this->tmpDestFile ) {
+ $params = array(
+ 'src' => $this->tmpDestFile->getPath(),
+ 'dst' => $this->params['dst'],
+ 'overwriteDest' => true
+ );
+ $status = $this->backend->store( $params );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Check if a file will exist in storage when this operation is attempted
+ *
+ * @param $source string Storage path
+ * @param $predicates Array
+ * @return bool
+ */
+ final protected function fileExists( $source, array $predicates ) {
+ if ( isset( $predicates['exists'][$source] ) ) {
+ return $predicates['exists'][$source]; // previous op assures this
+ } else {
+ return $this->backend->fileExists( array( 'src' => $source ) );
+ }
+ }
+
+ /**
+ * Log a file operation failure and preserve any temp files
+ *
+ * @param $fileOp FileOp
+ * @return void
+ */
+ final protected function logFailure( $action ) {
+ $params = $this->params;
+ $params['failedAction'] = $action;
+ // Preserve backup files just in case (for recovery)
+ if ( $this->tmpSourceFile ) {
+ $this->tmpSourceFile->preserve(); // don't purge
+ $params['srcBackupPath'] = $this->tmpSourceFile->getPath();
+ }
+ if ( $this->tmpDestFile ) {
+ $this->tmpDestFile->preserve(); // don't purge
+ $params['dstBackupPath'] = $this->tmpDestFile->getPath();
+ }
+ try {
+ wfDebugLog( 'FileOperation',
+ get_class( $this ) . ' failed:' . serialize( $params ) );
+ } catch ( Exception $e ) {
+ // bad config? debug log error?
+ }
+ }
+}
+
+/**
+ * Store a file into the backend from a file on the file system.
+ * Parameters similar to FileBackend::store(), which include:
+ * src : source path on file system
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ * overwriteSame : override any existing file at destination
+ */
+class StoreFileOp extends FileOp {
+ protected function allowedParams() {
+ return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if destination file exists
+ $status->merge( $this->precheckDestExistence( $predicates ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Check if the source file exists on the file system
+ if ( !is_file( $this->params['src'] ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['dst']] = true;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ // Create a destination backup copy as needed
+ if ( $this->destAlreadyExists ) {
+ $status->merge( $this->checkAndBackupDest() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ // Store the file at the destination
+ if ( !$this->destSameAsSource ) {
+ $status->merge( $this->backend->store( $this->params ) );
+ }
+ return $status;
+ }
+
+ protected function doRevert() {
+ $status = Status::newGood();
+ if ( !$this->destSameAsSource ) {
+ // Restore any file that was at the destination,
+ // overwritting what was put there in attempt()
+ $status->merge( $this->restoreDest() );
+ }
+ return $status;
+ }
+
+ protected function getSourceSha1Base36() {
+ return $this->getFileSha1Base36( $this->params['src'] );
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['dst'] );
+ }
+}
+
+/**
+ * Create a file in the backend with the given content.
+ * Parameters similar to FileBackend::create(), which include:
+ * content : a string of raw file contents
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ * overwriteSame : override any existing file at destination
+ */
+class CreateFileOp extends FileOp {
+ protected function allowedParams() {
+ return array( 'content', 'dst', 'overwriteDest', 'overwriteSame' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if destination file exists
+ $status->merge( $this->precheckDestExistence( $predicates ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['dst']] = true;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ // Create a destination backup copy as needed
+ if ( $this->destAlreadyExists ) {
+ $status->merge( $this->checkAndBackupDest() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ // Create the file at the destination
+ if ( !$this->destSameAsSource ) {
+ $status->merge( $this->backend->create( $this->params ) );
+ }
+ return $status;
+ }
+
+ protected function doRevert() {
+ $status = Status::newGood();
+ if ( !$this->destSameAsSource ) {
+ // Restore any file that was at the destination,
+ // overwritting what was put there in attempt()
+ $status->merge( $this->restoreDest() );
+ }
+ return $status;
+ }
+
+ protected function getSourceSha1Base36() {
+ return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['dst'] );
+ }
+}
+
+/**
+ * Copy a file from one storage path to another in the backend.
+ * Parameters similar to FileBackend::copy(), which include:
+ * src : source storage path
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ * overwriteSame : override any existing file at destination
+ */
+class CopyFileOp extends FileOp {
+ protected function allowedParams() {
+ return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if destination file exists
+ $status->merge( $this->precheckDestExistence( $predicates ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Check if the source file exists
+ if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['dst']] = true;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ // Create a destination backup copy as needed
+ if ( $this->destAlreadyExists ) {
+ $status->merge( $this->checkAndBackupDest() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ // Copy the file into the destination
+ if ( !$this->destSameAsSource ) {
+ $status->merge( $this->backend->copy( $this->params ) );
+ }
+ return $status;
+ }
+
+ protected function doRevert() {
+ $status = Status::newGood();
+ if ( !$this->destSameAsSource ) {
+ // Restore any file that was at the destination,
+ // overwritting what was put there in attempt()
+ $status->merge( $this->restoreDest() );
+ }
+ return $status;
+ }
+
+ protected function getSourceSha1Base36() {
+ return $this->getFileSha1Base36( $this->params['src'] );
+ }
+
+ public function storagePathsRead() {
+ return array( $this->params['src'] );
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['dst'] );
+ }
+}
+
+/**
+ * Move a file from one storage path to another in the backend.
+ * Parameters similar to FileBackend::move(), which include:
+ * src : source storage path
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ * overwriteSame : override any existing file at destination
+ */
+class MoveFileOp extends FileOp {
+ protected function allowedParams() {
+ return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if destination file exists
+ $status->merge( $this->precheckDestExistence( $predicates ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Check if the source file exists
+ if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['src']] = false;
+ $predicates['exists'][$this->params['dst']] = true;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ // Create a destination backup copy as needed
+ if ( $this->destAlreadyExists ) {
+ $status->merge( $this->checkAndBackupDest() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ if ( !$this->destSameAsSource ) {
+ // Move the file into the destination
+ $status->merge( $this->backend->move( $this->params ) );
+ } else {
+ // Create a source backup copy as needed
+ $status->merge( $this->backupSource() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Just delete source as the destination needs no changes
+ $params = array( 'src' => $this->params['src'] );
+ $status->merge( $this->backend->delete( $params ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ return $status;
+ }
+
+ protected function doRevert() {
+ $status = Status::newGood();
+ if ( !$this->destSameAsSource ) {
+ // Move the file back to the source
+ $params = array(
+ 'src' => $this->params['dst'],
+ 'dst' => $this->params['src']
+ );
+ $status->merge( $this->backend->move( $params ) );
+ if ( !$status->isOK() ) {
+ return $status; // also can't restore any dest file
+ }
+ // Restore any file that was at the destination
+ $status->merge( $this->restoreDest() );
+ } else {
+ // Restore any source file
+ return $this->restoreSource();
+ }
+
+ return $status;
+ }
+
+ protected function getSourceSha1Base36() {
+ return $this->getFileSha1Base36( $this->params['src'] );
+ }
+
+ public function storagePathsRead() {
+ return array( $this->params['src'] );
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['dst'] );
+ }
+}
+
+/**
+ * Combines files from severals storage paths into a new file in the backend.
+ * Parameters similar to FileBackend::concatenate(), which include:
+ * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
+ * dst : destination storage path
+ * overwriteDest : do nothing and pass if an identical file exists at destination
+ */
+class ConcatenateFileOp extends FileOp {
+ protected function allowedParams() {
+ return array( 'srcs', 'dst', 'overwriteDest' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if destination file exists
+ $status->merge( $this->precheckDestExistence( $predicates ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Check that source files exists
+ foreach ( $this->params['srcs'] as $source ) {
+ if ( !$this->fileExists( $source, $predicates ) ) {
+ $status->fatal( 'backend-fail-notexists', $source );
+ return $status;
+ }
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['dst']] = true;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ // Create a destination backup copy as needed
+ if ( $this->destAlreadyExists ) {
+ $status->merge( $this->checkAndBackupDest() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ // Concatenate the file at the destination
+ $status->merge( $this->backend->concatenate( $this->params ) );
+ return $status;
+ }
+
+ protected function doRevert() {
+ // Restore any file that was at the destination,
+ // overwritting what was put there in attempt()
+ return $this->restoreDest();
+ }
+
+ protected function getSourceSha1Base36() {
+ return null; // defer this until we finish building the new file
+ }
+
+ public function storagePathsRead() {
+ return $this->params['srcs'];
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['dst'] );
+ }
+}
+
+/**
+ * Delete a file at the storage path.
+ * Parameters similar to FileBackend::delete(), which include:
+ * src : source storage path
+ * ignoreMissingSource : don't return an error if the file does not exist
+ */
+class DeleteFileOp extends FileOp {
+ protected $needsDelete = true;
+
+ protected function allowedParams() {
+ return array( 'src', 'ignoreMissingSource' );
+ }
+
+ protected function doPrecheck( array &$predicates ) {
+ $status = Status::newGood();
+ // Check if the source file exists
+ if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
+ if ( !$this->getParam( 'ignoreMissingSource' ) ) {
+ $status->fatal( 'backend-fail-notexists', $this->params['src'] );
+ return $status;
+ }
+ $this->needsDelete = false;
+ }
+ // Update file existence predicates
+ $predicates['exists'][$this->params['src']] = false;
+ return $status;
+ }
+
+ protected function doAttempt() {
+ $status = Status::newGood();
+ if ( $this->needsDelete ) {
+ // Create a source backup copy as needed
+ $status->merge( $this->backupSource() );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ // Delete the source file
+ $status->merge( $this->backend->delete( $this->params ) );
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ }
+ return $status;
+ }
+
+ protected function doRevert() {
+ // Restore any source file that we deleted
+ return $this->restoreSource();
+ }
+
+ public function storagePathsChanged() {
+ return array( $this->params['src'] );
+ }
+}
+
+/**
+ * Placeholder operation that has no params and does nothing
+ */
+class NullFileOp extends FileOp {
+ protected function doAttempt() {
+ return Status::newGood();
+ }
+
+ protected function doRevert() {
+ return Status::newGood();
+ }
+}
--- /dev/null
+<?php
+
+/**
+ * Version of LockManager based on using DB table locks.
+ * This is meant for multi-wiki systems that may share files.
+ * All locks are blocking, so it might be useful to set a small
+ * lock-wait timeout via server config to curtail deadlocks.
+ *
+ * All lock requests for a resource, identified by a hash string, will map
+ * to one bucket. Each bucket maps to one or several peer DBs, each on their
+ * own server, all having the filelocks.sql tables (with row-level locking).
+ * A majority of peer DBs must agree for a lock to be acquired.
+ *
+ * Caching is used to avoid hitting servers that are down.
+ *
+ * @ingroup LockManager
+ */
+class DBLockManager extends LockManager {
+ /** @var Array Map of DB names to server config */
+ protected $dbServers; // (DB name => server config array)
+ /** @var Array Map of bucket indexes to peer DB lists */
+ protected $dbsByBucket; // (bucket index => (ldb1, ldb2, ...))
+ /** @var BagOStuff */
+ protected $statusCache;
+
+ protected $lockExpiry; // integer number of seconds
+ protected $safeDelay; // integer number of seconds
+
+ protected $session = 0; // random integer
+ /** @var Array Map of (locked key => lock type => count) */
+ protected $locksHeld = array();
+ /** @var Array Map Database connections (DB name => Database) */
+ protected $conns = array();
+
+ /**
+ * Construct a new instance from configuration.
+ * $config paramaters include:
+ * 'dbServers' : Associative array of DB names to server configuration.
+ * Configuration is an associative array that includes:
+ * 'host' - DB server name
+ * 'dbname' - DB name
+ * 'type' - DB type (mysql,postgres,...)
+ * 'user' - DB user
+ * 'password' - DB user password
+ * 'tablePrefix' - DB table prefix
+ * 'flags' - DB flags (see DatabaseBase)
+ * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0,
+ * each having an odd-numbered list of DB names (peers) as values.
+ * Any DB named 'localDBMaster' will automatically use the DB master
+ * settings for this wiki (without the need for a dbServers entry).
+ * 'lockExpiry' : Lock timeout (seconds) for dropped connections. [optional]
+ * This tells the DB server how long to wait before assuming
+ * connection failure and releasing all the locks for a session.
+ *
+ * @param Array $config
+ */
+ public function __construct( array $config ) {
+ $this->dbServers = $config['dbServers'];
+ // Sanitize dbsByBucket config to prevent PHP errors
+ $this->dbsByBucket = array_filter( $config['dbsByBucket'], 'is_array' );
+ $this->dbsByBucket = array_values( $this->dbsByBucket ); // consecutive
+
+ if ( isset( $config['lockExpiry'] ) ) {
+ $this->lockExpiry = $config['lockExpiry'];
+ } else {
+ $met = ini_get( 'max_execution_time' );
+ $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0
+ }
+ $this->safeDelay = ( $this->lockExpiry <= 0 )
+ ? 60 // pick a safe-ish number to match DB timeout default
+ : $this->lockExpiry; // cover worst case
+
+ foreach ( $this->dbsByBucket as $bucket ) {
+ if ( count( $bucket ) > 1 ) {
+ // Tracks peers that couldn't be queried recently to avoid lengthy
+ // connection timeouts. This is useless if each bucket has one peer.
+ $this->statusCache = wfGetMainCache();
+ break;
+ }
+ }
+
+ $this->session = '';
+ for ( $i = 0; $i < 5; $i++ ) {
+ $this->session .= mt_rand( 0, 2147483647 );
+ }
+ $this->session = wfBaseConvert( sha1( $this->session ), 16, 36, 31 );
+ }
+
+ protected function doLock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ $keysToLock = array();
+ // Get locks that need to be acquired (buckets => locks)...
+ foreach ( $keys as $key ) {
+ if ( isset( $this->locksHeld[$key][$type] ) ) {
+ ++$this->locksHeld[$key][$type];
+ } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
+ $this->locksHeld[$key][$type] = 1;
+ } else {
+ $bucket = $this->getBucketFromKey( $key );
+ $keysToLock[$bucket][] = $key;
+ }
+ }
+
+ $lockedKeys = array(); // files locked in this attempt
+ // Attempt to acquire these locks...
+ foreach ( $keysToLock as $bucket => $keys ) {
+ // Try to acquire the locks for this bucket
+ $res = $this->doLockingQueryAll( $bucket, $keys, $type );
+ if ( $res === 'cantacquire' ) {
+ // Resources already locked by another process.
+ // Abort and unlock everything we just locked.
+ $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) );
+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
+ return $status;
+ } elseif ( $res !== true ) {
+ // Couldn't contact any DBs for this bucket.
+ // Abort and unlock everything we just locked.
+ $status->fatal( 'lockmanager-fail-db-bucket', $bucket );
+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
+ return $status;
+ }
+ // Record these locks as active
+ foreach ( $keys as $key ) {
+ $this->locksHeld[$key][$type] = 1; // locked
+ }
+ // Keep track of what locks were made in this attempt
+ $lockedKeys = array_merge( $lockedKeys, $keys );
+ }
+
+ return $status;
+ }
+
+ protected function doUnlock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ foreach ( $keys as $key ) {
+ if ( !isset( $this->locksHeld[$key] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } elseif ( !isset( $this->locksHeld[$key][$type] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } else {
+ --$this->locksHeld[$key][$type];
+ if ( $this->locksHeld[$key][$type] <= 0 ) {
+ unset( $this->locksHeld[$key][$type] );
+ }
+ if ( !count( $this->locksHeld[$key] ) ) {
+ unset( $this->locksHeld[$key] ); // no SH or EX locks left for key
+ }
+ }
+ }
+
+ // Reference count the locks held and COMMIT when zero
+ if ( !count( $this->locksHeld ) ) {
+ $status->merge( $this->finishLockTransactions() );
+ }
+
+ return $status;
+ }
+
+ /**
+ * Get a connection to a lock DB and acquire locks on $keys.
+ * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118.
+ *
+ * @param $lockDb string
+ * @param $keys Array
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return bool Resources able to be locked
+ * @throws DBError
+ */
+ protected function doLockingQuery( $lockDb, array $keys, $type ) {
+ if ( $type == self::LOCK_EX ) { // writer locks
+ $db = $this->getConnection( $lockDb );
+ if ( !$db ) {
+ return false; // bad config
+ }
+ $data = array();
+ foreach ( $keys as $key ) {
+ $data[] = array( 'fle_key' => $key );
+ }
+ # Wait on any existing writers and block new ones if we get in
+ $db->insert( 'filelocks_exclusive', $data, __METHOD__ );
+ }
+ return true;
+ }
+
+ /**
+ * Attempt to acquire locks with the peers for a bucket.
+ * This should avoid throwing any exceptions.
+ *
+ * @param $bucket integer
+ * @param $keys Array List of resource keys to lock
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return bool|string One of (true, 'cantacquire', 'dberrors')
+ */
+ protected function doLockingQueryAll( $bucket, array $keys, $type ) {
+ $yesVotes = 0; // locks made on trustable DBs
+ $votesLeft = count( $this->dbsByBucket[$bucket] ); // remaining DBs
+ $quorum = floor( $votesLeft/2 + 1 ); // simple majority
+ // Get votes for each DB, in order, until we have enough...
+ foreach ( $this->dbsByBucket[$bucket] as $index => $lockDb ) {
+ // Check that DB is not *known* to be down
+ if ( $this->cacheCheckFailures( $lockDb ) ) {
+ try {
+ // Attempt to acquire the lock on this DB
+ if ( !$this->doLockingQuery( $lockDb, $keys, $type ) ) {
+ return 'cantacquire'; // vetoed; resource locked
+ }
+ ++$yesVotes; // success for this peer
+ if ( $yesVotes >= $quorum ) {
+ return true; // lock obtained
+ }
+ } catch ( DBConnectionError $e ) {
+ $this->cacheRecordFailure( $lockDb );
+ } catch ( DBError $e ) {
+ if ( $this->lastErrorIndicatesLocked( $lockDb ) ) {
+ return 'cantacquire'; // vetoed; resource locked
+ }
+ }
+ }
+ $votesLeft--;
+ $votesNeeded = $quorum - $yesVotes;
+ if ( $votesNeeded > $votesLeft ) {
+ // In "trust cache" mode we don't have to meet the quorum
+ break; // short-circuit
+ }
+ }
+ // At this point, we must not have meet the quorum
+ return 'dberrors'; // not enough votes to ensure correctness
+ }
+
+ /**
+ * Get (or reuse) a connection to a lock DB
+ *
+ * @param $lockDb string
+ * @return Database
+ * @throws DBError
+ */
+ protected function getConnection( $lockDb ) {
+ if ( !isset( $this->conns[$lockDb] ) ) {
+ $db = null;
+ if ( $lockDb === 'localDBMaster' ) {
+ $lb = wfGetLBFactory()->newMainLB();
+ $db = $lb->getConnection( DB_MASTER );
+ } elseif ( isset( $this->dbServers[$lockDb] ) ) {
+ $config = $this->dbServers[$lockDb];
+ $db = DatabaseBase::factory( $config['type'], $config );
+ }
+ if ( !$db ) {
+ return null; // config error?
+ }
+ $this->conns[$lockDb] = $db;
+ $this->conns[$lockDb]->clearFlag( DBO_TRX );
+ # If the connection drops, try to avoid letting the DB rollback
+ # and release the locks before the file operations are finished.
+ # This won't handle the case of DB server restarts however.
+ $options = array();
+ if ( $this->lockExpiry > 0 ) {
+ $options['connTimeout'] = $this->lockExpiry;
+ }
+ $this->conns[$lockDb]->setSessionOptions( $options );
+ $this->initConnection( $lockDb, $this->conns[$lockDb] );
+ }
+ if ( !$this->conns[$lockDb]->trxLevel() ) {
+ $this->conns[$lockDb]->begin(); // start transaction
+ }
+ return $this->conns[$lockDb];
+ }
+
+ /**
+ * Do additional initialization for new lock DB connection
+ *
+ * @param $lockDb string
+ * @param $db DatabaseBase
+ * @return void
+ * @throws DBError
+ */
+ protected function initConnection( $lockDb, DatabaseBase $db ) {}
+
+ /**
+ * Commit all changes to lock-active databases.
+ * This should avoid throwing any exceptions.
+ *
+ * @return Status
+ */
+ protected function finishLockTransactions() {
+ $status = Status::newGood();
+ foreach ( $this->conns as $lockDb => $db ) {
+ if ( $db->trxLevel() ) { // in transaction
+ try {
+ $db->rollback(); // finish transaction and kill any rows
+ } catch ( DBError $e ) {
+ $status->fatal( 'lockmanager-fail-db-release', $lockDb );
+ }
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Check if the last DB error for $lockDb indicates
+ * that a requested resource was locked by another process.
+ * This should avoid throwing any exceptions.
+ *
+ * @param $lockDb string
+ * @return bool
+ */
+ protected function lastErrorIndicatesLocked( $lockDb ) {
+ if ( isset( $this->conns[$lockDb] ) ) { // sanity
+ $db = $this->conns[$lockDb];
+ return ( $db->wasDeadlock() || $db->wasLockTimeout() );
+ }
+ return false;
+ }
+
+ /**
+ * Checks if the DB has not recently had connection/query errors.
+ * This just avoids wasting time on doomed connection attempts.
+ *
+ * @param $lockDb string
+ * @return bool
+ */
+ protected function cacheCheckFailures( $lockDb ) {
+ if ( $this->statusCache && $this->safeDelay > 0 ) {
+ $key = $this->getMissKey( $lockDb );
+ $misses = $this->statusCache->get( $key );
+ return !$misses;
+ }
+ return true;
+ }
+
+ /**
+ * Log a lock request failure to the cache
+ *
+ * @param $lockDb string
+ * @return bool Success
+ */
+ protected function cacheRecordFailure( $lockDb ) {
+ if ( $this->statusCache && $this->safeDelay > 0 ) {
+ $key = $this->getMissKey( $lockDb );
+ $misses = $this->statusCache->get( $key );
+ if ( $misses ) {
+ return $this->statusCache->incr( $key );
+ } else {
+ return $this->statusCache->add( $key, 1, $this->safeDelay );
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Get a cache key for recent query misses for a DB
+ *
+ * @param $lockDb string
+ * @return string
+ */
+ protected function getMissKey( $lockDb ) {
+ return 'lockmanager:querymisses:' . str_replace( ' ', '_', $lockDb );
+ }
+
+ /**
+ * Get the bucket for lock key.
+ * This should avoid throwing any exceptions.
+ *
+ * @param $key string (31 char hex key)
+ * @return integer
+ */
+ protected function getBucketFromKey( $key ) {
+ $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits)
+ return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->dbsByBucket );
+ }
+
+ /**
+ * Make sure remaining locks get cleared for sanity
+ */
+ function __destruct() {
+ foreach ( $this->conns as $lockDb => $db ) {
+ if ( $db->trxLevel() ) { // in transaction
+ try {
+ $db->rollback(); // finish transaction and kill any rows
+ } catch ( DBError $e ) {
+ // oh well
+ }
+ }
+ $db->close();
+ }
+ }
+}
+
+/**
+ * MySQL version of DBLockManager that supports shared locks.
+ * All locks are non-blocking, which avoids deadlocks.
+ *
+ * @ingroup LockManager
+ */
+class MySqlLockManager extends DBLockManager {
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_SH,
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ protected function initConnection( $lockDb, DatabaseBase $db ) {
+ # Let this transaction see lock rows from other transactions
+ $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" );
+ }
+
+ protected function doLockingQuery( $lockDb, array $keys, $type ) {
+ $db = $this->getConnection( $lockDb );
+ if ( !$db ) {
+ return false;
+ }
+ $data = array();
+ foreach ( $keys as $key ) {
+ $data[] = array( 'fls_key' => $key, 'fls_session' => $this->session );
+ }
+ # Block new writers...
+ $db->insert( 'filelocks_shared', $data, __METHOD__, array( 'IGNORE' ) );
+ # Actually do the locking queries...
+ if ( $type == self::LOCK_SH ) { // reader locks
+ # Bail if there are any existing writers...
+ $blocked = $db->selectField( 'filelocks_exclusive', '1',
+ array( 'fle_key' => $keys ),
+ __METHOD__
+ );
+ # Prospective writers that haven't yet updated filelocks_exclusive
+ # will recheck filelocks_shared after doing so and bail due to our entry.
+ } else { // writer locks
+ $encSession = $db->addQuotes( $this->session );
+ # Bail if there are any existing writers...
+ # The may detect readers, but the safe check for them is below.
+ # Note: if two writers come at the same time, both bail :)
+ $blocked = $db->selectField( 'filelocks_shared', '1',
+ array( 'fls_key' => $keys, "fls_session != $encSession" ),
+ __METHOD__
+ );
+ if ( !$blocked ) {
+ $data = array();
+ foreach ( $keys as $key ) {
+ $data[] = array( 'fle_key' => $key );
+ }
+ # Block new readers/writers...
+ $db->insert( 'filelocks_exclusive', $data, __METHOD__ );
+ # Bail if there are any existing readers...
+ $blocked = $db->selectField( 'filelocks_shared', '1',
+ array( 'fls_key' => $keys, "fls_session != $encSession" ),
+ __METHOD__
+ );
+ }
+ }
+ return !$blocked;
+ }
+}
--- /dev/null
+<?php
+
+/**
+ * Simple version of LockManager based on using FS lock files.
+ * All locks are non-blocking, which avoids deadlocks.
+ *
+ * This should work fine for small sites running off one server.
+ * Do not use this with 'lockDir' set to an NFS mount unless the
+ * NFS client is at least version 2.6.12. Otherwise, the BSD flock()
+ * locks will be ignored; see http://nfs.sourceforge.net/#section_d.
+ *
+ * @ingroup LockManager
+ */
+class FSLockManager extends LockManager {
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_SH,
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ protected $lockDir; // global dir for all servers
+
+ /** @var Array Map of (locked key => lock type => count) */
+ protected $locksHeld = array();
+ /** @var Array Map of (locked key => lock type => lock file handle) */
+ protected $handles = array();
+
+ function __construct( array $config ) {
+ $this->lockDir = $config['lockDirectory'];
+ }
+
+ protected function doLock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ $lockedKeys = array(); // files locked in this attempt
+ foreach ( $keys as $key ) {
+ $subStatus = $this->doSingleLock( $key, $type );
+ $status->merge( $subStatus );
+ if ( $status->isOK() ) {
+ // Don't append to $lockedKeys if $key is already locked.
+ // We do NOT want to unlock the key if we have to rollback.
+ if ( $subStatus->isGood() ) { // no warnings/fatals?
+ $lockedKeys[] = $key;
+ }
+ } else {
+ // Abort and unlock everything
+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
+ return $status;
+ }
+ }
+
+ return $status;
+ }
+
+ protected function doUnlock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ foreach ( $keys as $key ) {
+ $status->merge( $this->doSingleUnlock( $key, $type ) );
+ }
+
+ return $status;
+ }
+
+ /**
+ * Lock a single resource key
+ *
+ * @param $key string
+ * @param $type integer
+ * @return Status
+ */
+ protected function doSingleLock( $key, $type ) {
+ $status = Status::newGood();
+
+ if ( isset( $this->locksHeld[$key][$type] ) ) {
+ ++$this->locksHeld[$key][$type];
+ } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
+ $this->locksHeld[$key][$type] = 1;
+ } else {
+ wfSuppressWarnings();
+ $handle = fopen( $this->getLockPath( $key ), 'a+' );
+ wfRestoreWarnings();
+ if ( !$handle ) { // lock dir missing?
+ wfMkdirParents( $this->lockDir );
+ wfSuppressWarnings();
+ $handle = fopen( $this->getLockPath( $key ), 'a+' ); // try again
+ wfRestoreWarnings();
+ }
+ if ( $handle ) {
+ // Either a shared or exclusive lock
+ $lock = ( $type == self::LOCK_SH ) ? LOCK_SH : LOCK_EX;
+ if ( flock( $handle, $lock | LOCK_NB ) ) {
+ // Record this lock as active
+ $this->locksHeld[$key][$type] = 1;
+ $this->handles[$key][$type] = $handle;
+ } else {
+ fclose( $handle );
+ $status->fatal( 'lockmanager-fail-acquirelock', $key );
+ }
+ } else {
+ $status->fatal( 'lockmanager-fail-openlock', $key );
+ }
+ }
+
+ return $status;
+ }
+
+ /**
+ * Unlock a single resource key
+ *
+ * @param $key string
+ * @param $type integer
+ * @return Status
+ */
+ protected function doSingleUnlock( $key, $type ) {
+ $status = Status::newGood();
+
+ if ( !isset( $this->locksHeld[$key] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } elseif ( !isset( $this->locksHeld[$key][$type] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } else {
+ $handlesToClose = array();
+ --$this->locksHeld[$key][$type];
+ if ( $this->locksHeld[$key][$type] <= 0 ) {
+ unset( $this->locksHeld[$key][$type] );
+ // If a LOCK_SH comes in while we have a LOCK_EX, we don't
+ // actually add a handler, so check for handler existence.
+ if ( isset( $this->handles[$key][$type] ) ) {
+ // Mark this handle to be unlocked and closed
+ $handlesToClose[] = $this->handles[$key][$type];
+ unset( $this->handles[$key][$type] );
+ }
+ }
+ // Unlock handles to release locks and delete
+ // any lock files that end up with no locks on them...
+ if ( wfIsWindows() ) {
+ // Windows: for any process, including this one,
+ // calling unlink() on a locked file will fail
+ $status->merge( $this->closeLockHandles( $key, $handlesToClose ) );
+ $status->merge( $this->pruneKeyLockFiles( $key ) );
+ } else {
+ // Unix: unlink() can be used on files currently open by this
+ // process and we must do so in order to avoid race conditions
+ $status->merge( $this->pruneKeyLockFiles( $key ) );
+ $status->merge( $this->closeLockHandles( $key, $handlesToClose ) );
+ }
+ }
+
+ return $status;
+ }
+
+ private function closeLockHandles( $key, array $handlesToClose ) {
+ $status = Status::newGood();
+ foreach ( $handlesToClose as $handle ) {
+ wfSuppressWarnings();
+ if ( !flock( $handle, LOCK_UN ) ) {
+ $status->fatal( 'lockmanager-fail-releaselock', $key );
+ }
+ if ( !fclose( $handle ) ) {
+ $status->warning( 'lockmanager-fail-closelock', $key );
+ }
+ wfRestoreWarnings();
+ }
+ return $status;
+ }
+
+ private function pruneKeyLockFiles( $key ) {
+ $status = Status::newGood();
+ if ( !count( $this->locksHeld[$key] ) ) {
+ wfSuppressWarnings();
+ # No locks are held for the lock file anymore
+ if ( !unlink( $this->getLockPath( $key ) ) ) {
+ $status->warning( 'lockmanager-fail-deletelock', $key );
+ }
+ wfRestoreWarnings();
+ unset( $this->locksHeld[$key] );
+ unset( $this->handles[$key] );
+ }
+ return $status;
+ }
+
+ /**
+ * Get the path to the lock file for a key
+ * @param $key string
+ * @return string
+ */
+ protected function getLockPath( $key ) {
+ return "{$this->lockDir}/{$key}.lock";
+ }
+
+ function __destruct() {
+ // Make sure remaining locks get cleared for sanity
+ foreach ( $this->locksHeld as $key => $locks ) {
+ $this->doSingleUnlock( $key, 0 );
+ }
+ }
+}
--- /dev/null
+<?php
+
+/**
+ * Version of LockManager based on using lock daemon servers.
+ * This is meant for multi-wiki systems that may share files.
+ * All locks are non-blocking, which avoids deadlocks.
+ *
+ * All lock requests for a resource, identified by a hash string, will map
+ * to one bucket. Each bucket maps to one or several peer servers, each
+ * running LockServerDaemon.php, listening on a designated TCP port.
+ * A majority of peers must agree for a lock to be acquired.
+ *
+ * @ingroup LockManager
+ */
+class LSLockManager extends LockManager {
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_SH,
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ /** @var Array Map of server names to server config */
+ protected $lockServers; // (server name => server config array)
+ /** @var Array Map of bucket indexes to peer server lists */
+ protected $srvsByBucket; // (bucket index => (lsrv1, lsrv2, ...))
+
+ /** @var Array Map of (locked key => lock type => count) */
+ protected $locksHeld = array();
+ /** @var Array Map Server connections (server name => resource) */
+ protected $conns = array();
+
+ protected $connTimeout; // float number of seconds
+ protected $session = ''; // random SHA-1 string
+
+ /**
+ * Construct a new instance from configuration.
+ * $config paramaters include:
+ * 'lockServers' : Associative array of server names to configuration.
+ * Configuration is an associative array that includes:
+ * 'host' - IP address/hostname
+ * 'port' - TCP port
+ * 'authKey' - Secret string the lock server uses
+ * 'srvsByBucket' : Array of 1-16 consecutive integer keys, starting from 0,
+ * each having an odd-numbered list of server names (peers) as values.
+ * 'connTimeout' : Lock server connection attempt timeout. [optional]
+ *
+ * @param Array $config
+ */
+ public function __construct( array $config ) {
+ $this->lockServers = $config['lockServers'];
+ // Sanitize srvsByBucket config to prevent PHP errors
+ $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' );
+ $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive
+
+ if ( isset( $config['connTimeout'] ) ) {
+ $this->connTimeout = $config['connTimeout'];
+ } else {
+ $this->connTimeout = 3; // use some sane amount
+ }
+
+ $this->session = '';
+ for ( $i = 0; $i < 5; $i++ ) {
+ $this->session .= mt_rand( 0, 2147483647 );
+ }
+ $this->session = wfBaseConvert( sha1( $this->session ), 16, 36, 31 );
+ }
+
+ protected function doLock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ $keysToLock = array();
+ // Get locks that need to be acquired (buckets => locks)...
+ foreach ( $keys as $key ) {
+ if ( isset( $this->locksHeld[$key][$type] ) ) {
+ ++$this->locksHeld[$key][$type];
+ } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
+ $this->locksHeld[$key][$type] = 1;
+ } else {
+ $bucket = $this->getBucketFromKey( $key );
+ $keysToLock[$bucket][] = $key;
+ }
+ }
+
+ $lockedKeys = array(); // files locked in this attempt
+ // Attempt to acquire these locks...
+ foreach ( $keysToLock as $bucket => $keys ) {
+ // Try to acquire the locks for this bucket
+ $res = $this->doLockingRequestAll( $bucket, $keys, $type );
+ if ( $res === 'cantacquire' ) {
+ // Resources already locked by another process.
+ // Abort and unlock everything we just locked.
+ $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) );
+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
+ return $status;
+ } elseif ( $res !== true ) {
+ // Couldn't contact any servers for this bucket.
+ // Abort and unlock everything we just locked.
+ $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) );
+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
+ return $status;
+ }
+ // Record these locks as active
+ foreach ( $keys as $key ) {
+ $this->locksHeld[$key][$type] = 1; // locked
+ }
+ // Keep track of what locks were made in this attempt
+ $lockedKeys = array_merge( $lockedKeys, $keys );
+ }
+
+ return $status;
+ }
+
+ protected function doUnlock( array $keys, $type ) {
+ $status = Status::newGood();
+
+ foreach ( $keys as $key ) {
+ if ( !isset( $this->locksHeld[$key] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } elseif ( !isset( $this->locksHeld[$key][$type] ) ) {
+ $status->warning( 'lockmanager-notlocked', $key );
+ } else {
+ --$this->locksHeld[$key][$type];
+ if ( $this->locksHeld[$key][$type] <= 0 ) {
+ unset( $this->locksHeld[$key][$type] );
+ }
+ if ( !count( $this->locksHeld[$key] ) ) {
+ unset( $this->locksHeld[$key] ); // no SH or EX locks left for key
+ }
+ }
+ }
+
+ // Reference count the locks held and release locks when zero
+ if ( !count( $this->locksHeld ) ) {
+ $status->merge( $this->releaseLocks() );
+ }
+
+ return $status;
+ }
+
+ /**
+ * Get a connection to a lock server and acquire locks on $keys
+ *
+ * @param $lockSrv string
+ * @param $keys Array
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return bool Resources able to be locked
+ */
+ protected function doLockingRequest( $lockSrv, array $keys, $type ) {
+ if ( $type == self::LOCK_SH ) { // reader locks
+ $type = 'SH';
+ } elseif ( $type == self::LOCK_EX ) { // writer locks
+ $type = 'EX';
+ } else {
+ return true; // ok...
+ }
+
+ // Send out the command and get the response...
+ $response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys );
+
+ return ( $response === 'ACQUIRED' );
+ }
+
+ /**
+ * Send a command and get back the response
+ *
+ * @param $lockSrv string
+ * @param $action string
+ * @param $type string
+ * @param $values Array
+ * @return string|false
+ */
+ protected function sendCommand( $lockSrv, $action, $type, $values ) {
+ $conn = $this->getConnection( $lockSrv );
+ if ( !$conn ) {
+ return false; // no connection
+ }
+ $authKey = $this->lockServers[$lockSrv]['authKey'];
+ // Build of the command as a flat string...
+ $values = implode( '|', $values );
+ $key = sha1( $this->session . $action . $type . $values . $authKey );
+ // Send out the command...
+ if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) {
+ return false;
+ }
+ // Get the response...
+ $response = fgets( $conn );
+ if ( $response === false ) {
+ return false;
+ }
+ return trim( $response );
+ }
+
+ /**
+ * Attempt to acquire locks with the peers for a bucket
+ *
+ * @param $bucket integer
+ * @param $keys Array List of resource keys to lock
+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
+ * @return bool|string One of (true, 'cantacquire', 'srverrors')
+ */
+ protected function doLockingRequestAll( $bucket, array $keys, $type ) {
+ $yesVotes = 0; // locks made on trustable servers
+ $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers
+ $quorum = floor( $votesLeft/2 + 1 ); // simple majority
+ // Get votes for each peer, in order, until we have enough...
+ foreach ( $this->srvsByBucket[$bucket] as $index => $lockSrv ) {
+ // Attempt to acquire the lock on this peer
+ if ( !$this->doLockingRequest( $lockSrv, $keys, $type ) ) {
+ return 'cantacquire'; // vetoed; resource locked
+ }
+ ++$yesVotes; // success for this peer
+ if ( $yesVotes >= $quorum ) {
+ return true; // lock obtained
+ }
+ $votesLeft--;
+ $votesNeeded = $quorum - $yesVotes;
+ if ( $votesNeeded > $votesLeft ) {
+ // In "trust cache" mode we don't have to meet the quorum
+ break; // short-circuit
+ }
+ }
+ // At this point, we must not have meet the quorum
+ return 'srverrors'; // not enough votes to ensure correctness
+ }
+
+ /**
+ * Get (or reuse) a connection to a lock server
+ *
+ * @param $lockSrv string
+ * @return resource
+ */
+ protected function getConnection( $lockSrv ) {
+ if ( !isset( $this->conns[$lockSrv] ) ) {
+ $cfg = $this->lockServers[$lockSrv];
+ wfSuppressWarnings();
+ $errno = $errstr = '';
+ $conn = fsockopen( $cfg['host'], $cfg['port'], $errno, $errstr, $this->connTimeout );
+ wfRestoreWarnings();
+ if ( $conn === false ) {
+ return null;
+ }
+ $sec = floor( $this->connTimeout );
+ $usec = floor( ( $this->connTimeout - floor( $this->connTimeout ) ) * 1e6 );
+ stream_set_timeout( $conn, $sec, $usec );
+ $this->conns[$lockSrv] = $conn;
+ }
+ return $this->conns[$lockSrv];
+ }
+
+ /**
+ * Release all locks that this session is holding
+ *
+ * @return Status
+ */
+ protected function releaseLocks() {
+ $status = Status::newGood();
+ foreach ( $this->conns as $lockSrv => $conn ) {
+ $response = $this->sendCommand( $lockSrv, 'RELEASE_ALL', '', array() );
+ if ( $response !== 'RELEASED_ALL' ) {
+ $status->fatal( 'lockmanager-fail-svr-release', $lockSrv );
+ }
+ }
+ return $status;
+ }
+
+ /**
+ * Get the bucket for lock key
+ *
+ * @param $key string (31 char hex key)
+ * @return integer
+ */
+ protected function getBucketFromKey( $key ) {
+ $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits)
+ return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->srvsByBucket );
+ }
+
+ /**
+ * Make sure remaining locks get cleared for sanity
+ */
+ function __destruct() {
+ $this->releaseLocks();
+ foreach ( $this->conns as $lockSrv => $conn ) {
+ fclose( $conn );
+ }
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup LockManager
+ * @author Aaron Schulz
+ */
+
+/**
+ * Class for handling resource locking.
+ * Locks on resource keys can either be shared or exclusive.
+ *
+ * Implementations must keep track of what is locked by this proccess
+ * in-memory and support nested locking calls (using reference counting).
+ * At least LOCK_UW and LOCK_EX must be implemented. LOCK_SH can be a no-op.
+ * Locks should either be non-blocking or have low wait timeouts.
+ *
+ * Subclasses should avoid throwing exceptions at all costs.
+ *
+ * @ingroup LockManager
+ * @since 1.19
+ */
+abstract class LockManager {
+ /* Lock types; stronger locks have higher values */
+ const LOCK_SH = 1; // shared lock (for reads)
+ const LOCK_UW = 2; // shared lock (for reads used to write elsewhere)
+ const LOCK_EX = 3; // exclusive lock (for writes)
+
+ /** @var Array Mapping of lock types to the type actually used */
+ protected $lockTypeMap = array(
+ self::LOCK_SH => self::LOCK_SH,
+ self::LOCK_UW => self::LOCK_EX, // subclasses may use self::LOCK_SH
+ self::LOCK_EX => self::LOCK_EX
+ );
+
+ /**
+ * Construct a new instance from configuration
+ *
+ * @param $config Array
+ */
+ public function __construct( array $config ) {}
+
+ /**
+ * Lock the resources at the given abstract paths
+ *
+ * @param $paths Array List of resource names
+ * @param $type integer LockManager::LOCK_* constant
+ * @return Status
+ */
+ final public function lock( array $paths, $type = self::LOCK_EX ) {
+ $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
+ return $this->doLock( $keys, $this->lockTypeMap[$type] );
+ }
+
+ /**
+ * Unlock the resources at the given abstract paths
+ *
+ * @param $paths Array List of storage paths
+ * @param $type integer LockManager::LOCK_* constant
+ * @return Status
+ */
+ final public function unlock( array $paths, $type = self::LOCK_EX ) {
+ $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) );
+ return $this->doUnlock( $keys, $this->lockTypeMap[$type] );
+ }
+
+ /**
+ * Get the base 36 SHA-1 of a string, padded to 31 digits
+ *
+ * @param $path string
+ * @return string
+ */
+ final protected static function sha1Base36( $path ) {
+ return wfBaseConvert( sha1( $path ), 16, 36, 31 );
+ }
+
+ /**
+ * Lock resources with the given keys and lock type
+ *
+ * @param $key Array List of keys to lock (40 char hex hashes)
+ * @param $type integer LockManager::LOCK_* constant
+ * @return string
+ */
+ abstract protected function doLock( array $keys, $type );
+
+ /**
+ * Unlock resources with the given keys and lock type
+ *
+ * @param $key Array List of keys to unlock (40 char hex hashes)
+ * @param $type integer LockManager::LOCK_* constant
+ * @return string
+ */
+ abstract protected function doUnlock( array $keys, $type );
+}
+
+/**
+ * LockManager helper class to handle scoped locks, which
+ * release when an object is destroyed or goes out of scope.
+ *
+ * @ingroup LockManager
+ * @since 1.19
+ */
+class ScopedLock {
+ /** @var LockManager */
+ protected $manager;
+ /** @var Status */
+ protected $status;
+ /** @var Array List of resource paths*/
+ protected $paths;
+
+ protected $type; // integer lock type
+
+ /**
+ * @param $manager LockManager
+ * @param $paths Array List of storage paths
+ * @param $type integer LockManager::LOCK_* constant
+ * @param $status Status
+ */
+ protected function __construct(
+ LockManager $manager, array $paths, $type, Status $status
+ ) {
+ $this->manager = $manager;
+ $this->paths = $paths;
+ $this->status = $status;
+ $this->type = $type;
+ }
+
+ protected function __clone() {}
+
+ /**
+ * Get a ScopedLock object representing a lock on resource paths.
+ * Any locks are released once this object goes out of scope.
+ * The status object is updated with any errors or warnings.
+ *
+ * @param $manager LockManager
+ * @param $paths Array List of storage paths
+ * @param $type integer LockManager::LOCK_* constant
+ * @param $status Status
+ * @return ScopedLock|null Returns null on failure
+ */
+ public static function factory(
+ LockManager $manager, array $paths, $type, Status $status
+ ) {
+ $lockStatus = $manager->lock( $paths, $type );
+ $status->merge( $lockStatus );
+ if ( $lockStatus->isOK() ) {
+ return new self( $manager, $paths, $type, $status );
+ }
+ return null;
+ }
+
+ function __destruct() {
+ $wasOk = $this->status->isOK();
+ $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) );
+ if ( $wasOk ) {
+ // Make sure status is OK, despite any unlockFiles() fatals
+ $this->status->setResult( true, $this->status->value );
+ }
+ }
+}
+
+/**
+ * Simple version of LockManager that does nothing
+ */
+class NullLockManager extends LockManager {
+ protected function doLock( array $keys, $type ) {
+ return Status::newGood();
+ }
+
+ protected function doUnlock( array $keys, $type ) {
+ return Status::newGood();
+ }
+}
--- /dev/null
+<?php
+/**
+ * Class to handle file lock manager registration
+ *
+ * @ingroup LockManager
+ */
+class LockManagerGroup {
+ protected static $instance = null;
+
+ /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */
+ protected $managers = array();
+
+ protected function __construct() {}
+ protected function __clone() {}
+
+ public static function singleton() {
+ if ( self::$instance == null ) {
+ self::$instance = new self();
+ }
+ return self::$instance;
+ }
+
+ /**
+ * Register an array of file lock manager configurations
+ *
+ * @param $configs Array
+ * @return void
+ * @throws MWException
+ */
+ public function register( array $configs ) {
+ foreach ( $configs as $config ) {
+ if ( !isset( $config['name'] ) ) {
+ throw new MWException( "Cannot register a lock manager with no name." );
+ }
+ $name = $config['name'];
+ if ( !isset( $config['class'] ) ) {
+ throw new MWException( "Cannot register lock manager `{$name}` with no class." );
+ }
+ $class = $config['class'];
+ unset( $config['class'] ); // lock manager won't need this
+ $this->managers[$name] = array(
+ 'class' => $class,
+ 'config' => $config,
+ 'instance' => null
+ );
+ }
+ }
+
+ /**
+ * Get the lock manager object with a given name
+ *
+ * @param $name string
+ * @return LockManager
+ * @throws MWException
+ */
+ public function get( $name ) {
+ if ( !isset( $this->managers[$name] ) ) {
+ throw new MWException( "No lock manager defined with the name `$name`." );
+ }
+ // Lazy-load the actual lock manager instance
+ if ( !isset( $this->managers[$name]['instance'] ) ) {
+ $class = $this->managers[$name]['class'];
+ $config = $this->managers[$name]['config'];
+ $this->managers[$name]['instance'] = new $class( $config );
+ }
+ return $this->managers[$name]['instance'];
+ }
+}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ */
+
+/**
+ * Class representing a non-directory file on the file system
+ *
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ */
+class FSFile {
+ protected $path; // path to file
+
+ /**
+ * Sets up the file object
+ *
+ * @param String $path Path to temporary file on local disk
+ */
+ public function __construct( $path ) {
+ if ( FileBackend::isStoragePath( $path ) ) {
+ throw new MWException( __METHOD__ . " given storage path `$path`." );
+ }
+ $this->path = $path;
+ }
+
+ /**
+ * Returns the file system path
+ *
+ * @return String
+ */
+ public function getPath() {
+ return $this->path;
+ }
+
+ /**
+ * Checks if the file exists
+ *
+ * @return bool
+ */
+ public function exists() {
+ return is_file( $this->path );
+ }
+
+ /**
+ * Get the file size in bytes
+ *
+ * @return int|false
+ */
+ public function getSize() {
+ return filesize( $this->path );
+ }
+
+ /**
+ * Get the file's last-modified timestamp
+ *
+ * @return string|false TS_MW timestamp or false on failure
+ */
+ public function getTimestamp() {
+ wfSuppressWarnings();
+ $timestamp = filemtime( $this->path );
+ wfRestoreWarnings();
+ if ( $timestamp !== false ) {
+ $timestamp = wfTimestamp( TS_MW, $timestamp );
+ }
+ return $timestamp;
+ }
+
+ /**
+ * Get an associative array containing information about
+ * a file with the given storage path.
+ *
+ * @param $ext Mixed: the file extension, or true to extract it from the filename.
+ * Set it to false to ignore the extension.
+ *
+ * @return array
+ */
+ public function getProps( $ext = true ) {
+ wfProfileIn( __METHOD__ );
+ wfDebug( __METHOD__.": Getting file info for $this->path\n" );
+
+ $info = self::placeholderProps();
+ $info['fileExists'] = $this->exists();
+
+ if ( $info['fileExists'] ) {
+ $magic = MimeMagic::singleton();
+
+ # get the file extension
+ if ( $ext === true ) {
+ $i = strrpos( $this->path, '.' );
+ $ext = strtolower( $i ? substr( $this->path, $i + 1 ) : '' );
+ }
+
+ # mime type according to file contents
+ $info['file-mime'] = $magic->guessMimeType( $this->path, false );
+ # logical mime type
+ $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext );
+
+ list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] );
+ $info['media_type'] = $magic->getMediaType( $this->path, $info['mime'] );
+
+ # Get size in bytes
+ $info['size'] = $this->getSize();
+
+ # Height, width and metadata
+ $handler = MediaHandler::getHandler( $info['mime'] );
+ if ( $handler ) {
+ $tempImage = (object)array();
+ $info['metadata'] = $handler->getMetadata( $tempImage, $this->path );
+ $gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] );
+ if ( is_array( $gis ) ) {
+ $info = $this->extractImageSizeInfo( $gis ) + $info;
+ }
+ }
+ $info['sha1'] = $this->getSha1Base36();
+
+ wfDebug(__METHOD__.": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n");
+ } else {
+ wfDebug(__METHOD__.": $this->path NOT FOUND!\n");
+ }
+
+ wfProfileOut( __METHOD__ );
+ return $info;
+ }
+
+ /**
+ * Placeholder file properties to use for files that don't exist
+ *
+ * @return Array
+ */
+ public static function placeholderProps() {
+ $info = array();
+ $info['fileExists'] = false;
+ $info['mime'] = null;
+ $info['media_type'] = MEDIATYPE_UNKNOWN;
+ $info['metadata'] = '';
+ $info['sha1'] = '';
+ $info['width'] = 0;
+ $info['height'] = 0;
+ $info['bits'] = 0;
+ return $info;
+ }
+
+ protected function extractImageSizeInfo( array $gis ) {
+ $info = array();
+ # NOTE: $gis[2] contains a code for the image type. This is no longer used.
+ $info['width'] = $gis[0];
+ $info['height'] = $gis[1];
+ if ( isset( $gis['bits'] ) ) {
+ $info['bits'] = $gis['bits'];
+ } else {
+ $info['bits'] = 0;
+ }
+ return $info;
+ }
+
+ /**
+ * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case
+ * encoding, zero padded to 31 digits.
+ *
+ * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36
+ * fairly neatly.
+ *
+ * @return false|string False on failure
+ */
+ public function getSha1Base36() {
+ wfProfileIn( __METHOD__ );
+
+ wfSuppressWarnings();
+ $hash = sha1_file( $this->path );
+ wfRestoreWarnings();
+ if ( $hash !== false ) {
+ $hash = wfBaseConvert( $hash, 16, 36, 31 );
+ }
+
+ wfProfileOut( __METHOD__ );
+ return $hash;
+ }
+
+ /**
+ * Get an associative array containing information about a file in the local filesystem.
+ *
+ * @param $path String: absolute local filesystem path
+ * @param $ext Mixed: the file extension, or true to extract it from the filename.
+ * Set it to false to ignore the extension.
+ *
+ * @return array
+ */
+ static function getPropsFromPath( $path, $ext = true ) {
+ $fsFile = new self( $path );
+ return $fsFile->getProps( $ext );
+ }
+
+ /**
+ * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case
+ * encoding, zero padded to 31 digits.
+ *
+ * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36
+ * fairly neatly.
+ *
+ * @param $path string
+ *
+ * @return false|string False on failure
+ */
+ static function getSha1Base36FromPath( $path ) {
+ $fsFile = new self( $path );
+ return $fsFile->getSha1Base36();
+ }
+}
var $lastError, $redirected, $redirectedTitle;
+ /**
+ * @var FSFile|false
+ */
+ protected $fsFile;
+
/**
* @var MediaHandler
*/
protected $handler;
+ /**
+ * @var string
+ */
protected $url, $extension, $name, $path, $hashPath, $pageCount, $transformScript;
/**
/**
* Given a string or Title object return either a
* valid Title object with namespace NS_FILE or null
+ *
* @param $title Title|string
* @param $exception string|false Use 'exception' to throw an error on bad titles
* @return Title|null
static function checkExtensionCompatibility( File $old, $new ) {
$oldMime = $old->getMimeType();
$n = strrpos( $new, '.' );
- $newExt = self::normalizeExtension(
- $n ? substr( $new, $n + 1 ) : '' );
+ $newExt = self::normalizeExtension( $n ? substr( $new, $n + 1 ) : '' );
$mimeMagic = MimeMagic::singleton();
return $mimeMagic->isMatchingExtension( $newExt, $oldMime );
}
/**
* Return the associated title object
+ *
* @return Title|false
*/
- public function getTitle() { return $this->title; }
+ public function getTitle() {
+ return $this->title;
+ }
/**
* Return the title used to find this file
}
/**
- * Return the full filesystem path to the file. Note that this does
+ * Return the storage path to the file. Note that this does
* not mean that a file actually exists under that location.
*
* This path depends on whether directory hashing is active or not,
return $this->path;
}
+ /**
+ * Get an FS copy or original of this file and return the path.
+ * Returns false on failure. Callers must not alter the file.
+ * Temporary files are cleared automatically.
+ *
+ * @return string|false
+ */
+ public function getLocalRefPath() {
+ $this->assertRepoDefined();
+ if ( !isset( $this->fsFile ) ) {
+ $this->fsFile = $this->repo->getLocalReference( $this->getPath() );
+ if ( !$this->fsFile ) {
+ $this->fsFile = false; // null => false; cache negative hits
+ }
+ }
+ return ( $this->fsFile )
+ ? $this->fsFile->getPath()
+ : false;
+ }
+
/**
* Return the width of the image. Returns false if the width is unknown
* or undefined.
public function convertMetadataVersion($metadata, $version) {
$handler = $this->getHandler();
if ( !is_array( $metadata ) ) {
- //just to make the return type consistant
+ // Just to make the return type consistent
$metadata = unserialize( $metadata );
}
if ( $handler ) {
* Overridden by LocalFile,
* STUB
*/
- function getMediaType() { return MEDIATYPE_UNKNOWN; }
+ function getMediaType() {
+ return MEDIATYPE_UNKNOWN;
+ }
/**
* Checks if the output of transform() for this file is likely
* @return bool
*/
protected function _getIsSafeFile() {
+ global $wgTrustedMediaFormats;
+
if ( $this->allowInlineDisplay() ) {
return true;
}
return true;
}
- global $wgTrustedMediaFormats;
-
$type = $this->getMediaType();
$mime = $this->getMimeType();
#wfDebug("LocalFile::isSafeFile: type= $type, mime= $mime\n");
* @return bool
*/
function isTrustedFile() {
- #this could be implemented to check a flag in the databas,
+ #this could be implemented to check a flag in the database,
#look for signatures, etc
return false;
}
* @return boolean Whether file exists in the repository.
*/
public function exists() {
- return $this->getPath() && file_exists( $this->path );
+ return $this->getPath() && $this->repo->fileExists( $this->path );
}
/**
return null;
}
$extension = $this->getExtension();
- list( $thumbExt, $thumbMime ) = $this->handler->getThumbType( $extension, $this->getMimeType(), $params );
+ list( $thumbExt, $thumbMime ) = $this->handler->getThumbType(
+ $extension, $this->getMimeType(), $params );
$thumbName = $this->handler->makeParamString( $params ) . '-' . $name;
if ( $thumbExt != $extension ) {
$thumbName .= ".$thumbExt";
$params['height'] = $height;
}
$thumb = $this->transform( $params );
- if( is_null( $thumb ) || $thumb->isError() ) return '';
+ if ( is_null( $thumb ) || $thumb->isError() ) {
+ return '';
+ }
return $thumb->getUrl();
}
* Typical keys are width, height and page.
* @param $flags Integer: a bitfield, may contain self::RENDER_NOW to force rendering
*
- * @return MediaTransformOutput | false
+ * @return MediaTransformOutput|null
*/
protected function maybeDoTransform( $thumbName, $thumbUrl, $params, $flags = 0 ) {
global $wgIgnoreImageErrors, $wgThumbnailEpoch;
- $thumbPath = $this->getThumbPath( $thumbName );
- if ( $this->repo && $this->repo->canTransformVia404() && !($flags & self::RENDER_NOW ) ) {
+ $thumbPath = $this->getThumbPath( $thumbName ); // final thumb path
+
+ if ( $this->repo && $this->repo->canTransformVia404() && !( $flags & self::RENDER_NOW ) ) {
wfDebug( __METHOD__ . " transformation deferred." );
- return $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params );
+ return $this->handler->getTransform( $this, false, $thumbUrl, $params );
}
wfDebug( __METHOD__.": Doing stat for $thumbPath\n" );
$this->migrateThumbFile( $thumbName );
- if ( file_exists( $thumbPath ) && !($flags & self::RENDER_FORCE) ) {
- $thumbTime = filemtime( $thumbPath );
- if ( $thumbTime !== FALSE &&
- gmdate( 'YmdHis', $thumbTime ) >= $wgThumbnailEpoch ) {
-
- return $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params );
+ if ( $this->repo->fileExists( $thumbPath ) && !( $flags & self::RENDER_FORCE ) ) {
+ $timestamp = $this->repo->getFileTimestamp( $thumbPath );
+ if ( $timestamp !== false && $timestamp >= $wgThumbnailEpoch ) {
+ return $this->handler->getTransform( $this, false, $thumbUrl, $params );
}
- } elseif( $flags & self::RENDER_FORCE ) {
+ } elseif ( $flags & self::RENDER_FORCE ) {
wfDebug( __METHOD__ . " forcing rendering per flag File::RENDER_FORCE\n" );
}
- $thumb = $this->handler->doTransform( $this, $thumbPath, $thumbUrl, $params );
+
+ // Create a temp FS file with the same extension
+ $tmpFile = TempFSFile::factory( 'transform_', $this->getExtension() );
+ if ( !$tmpFile ) {
+ return new MediaTransformError( 'thumbnail_error',
+ $params['width'], 0, wfMsg( 'thumbnail-temp-create' ) );
+ }
+ $tmpThumbPath = $tmpFile->getPath(); // path of 0-byte temp file
+
+ // Actually render the thumbnail
+ $thumb = $this->handler->doTransform( $this, $tmpThumbPath, $thumbUrl, $params );
+ $tmpFile->bind( $thumb ); // keep alive with $thumb
// Ignore errors if requested
if ( !$thumb ) {
$thumb = null;
} elseif ( $thumb->isError() ) {
$this->lastError = $thumb->toText();
- if ( $wgIgnoreImageErrors && !($flags & self::RENDER_NOW) ) {
- $thumb = $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params );
+ if ( $wgIgnoreImageErrors && !( $flags & self::RENDER_NOW ) ) {
+ $thumb = $this->handler->getTransform( $this, $tmpThumbPath, $thumbUrl, $params );
+ }
+ } elseif ( $thumb->hasFile() && !$thumb->fileIsSource() ) {
+ // @TODO: use a FileRepo store function
+ $op = array( 'op' => 'store',
+ 'src' => $tmpThumbPath, 'dst' => $thumbPath, 'overwriteDest' => true );
+ // Copy any thumbnail from the FS into storage at $dstpath
+ $opts = array( 'ignoreErrors' => true, 'nonLocking' => true ); // performance
+ if ( !$this->getRepo()->getBackend()->doOperation( $op, $opts )->isOK() ) {
+ return new MediaTransformError( 'thumbnail_error',
+ $params['width'], 0, wfMsg( 'thumbnail-dest-create' ) );
}
}
return $thumb;
}
-
/**
* Transform a media file
*
// Purge. Useful in the event of Core -> Squid connection failure or squid
// purge collisions from elsewhere during failure. Don't keep triggering for
// "thumbs" which have the main image URL though (bug 13776)
- if ( $wgUseSquid && ( !$thumb || $thumb->isError() || $thumb->getUrl() != $this->getURL()) ) {
- SquidUpdate::purge( array( $thumbUrl ) );
+ if ( $wgUseSquid ) {
+ if ( !$thumb || $thumb->isError() || $thumb->getUrl() != $this->getURL() ) {
+ SquidUpdate::purge( array( $thumbUrl ) );
+ }
}
} while (false);
/**
* Get a MediaHandler instance for this file
+ *
* @return MediaHandler
*/
function getHandler() {
/**
* Get a ThumbnailImage representing a file type icon
+ *
* @return ThumbnailImage
*/
function iconThumb() {
global $wgStylePath, $wgStyleDirectory;
$try = array( 'fileicon-' . $this->getExtension() . '.png', 'fileicon.png' );
- foreach( $try as $icon ) {
+ foreach ( $try as $icon ) {
$path = '/common/images/icons/' . $icon;
$filepath = $wgStyleDirectory . $path;
- if( file_exists( $filepath ) ) {
+ if ( file_exists( $filepath ) ) { // always FS
return new ThumbnailImage( $this, $wgStylePath . $path, 120, 120 );
}
}
*/
function getThumbUrl( $suffix = false ) {
$this->assertRepoDefined();
- $path = $this->repo->getZoneUrl('thumb') . '/' . $this->getUrlRel();
+ $path = $this->repo->getZoneUrl( 'thumb' ) . '/' . $this->getUrlRel();
if ( $suffix !== false ) {
$path .= '/' . rawurlencode( $suffix );
}
}
/**
- * Get the virtual URL for an archived file's thumbs, or a specific thumb.
+ * Get the public zone virtual URL for a current version source file
*
* @param $suffix bool|string if not false, the name of a thumbnail file
*
* @return string
*/
- function getArchiveVirtualUrl( $suffix = false ) {
+ function getVirtualUrl( $suffix = false ) {
$this->assertRepoDefined();
- $path = $this->repo->getVirtualUrl() . '/public/archive/' . $this->getHashPath();
- if ( $suffix === false ) {
- $path = substr( $path, 0, -1 );
- } else {
- $path .= rawurlencode( $suffix );
+ $path = $this->repo->getVirtualUrl() . '/public/' . $this->getUrlRel();
+ if ( $suffix !== false ) {
+ $path .= '/' . rawurlencode( $suffix );
}
return $path;
}
/**
- * Get the virtual URL for a thumbnail file or directory
+ * Get the public zone virtual URL for an archived version source file
*
* @param $suffix bool|string if not false, the name of a thumbnail file
*
* @return string
*/
- function getThumbVirtualUrl( $suffix = false ) {
+ function getArchiveVirtualUrl( $suffix = false ) {
$this->assertRepoDefined();
- $path = $this->repo->getVirtualUrl() . '/thumb/' . $this->getUrlRel();
- if ( $suffix !== false ) {
- $path .= '/' . rawurlencode( $suffix );
+ $path = $this->repo->getVirtualUrl() . '/public/archive/' . $this->getHashPath();
+ if ( $suffix === false ) {
+ $path = substr( $path, 0, -1 );
+ } else {
+ $path .= rawurlencode( $suffix );
}
return $path;
}
/**
- * Get the virtual URL for the file itself
+ * Get the virtual URL for a thumbnail file or directory
*
* @param $suffix bool|string if not false, the name of a thumbnail file
*
* @return string
*/
- function getVirtualUrl( $suffix = false ) {
+ function getThumbVirtualUrl( $suffix = false ) {
$this->assertRepoDefined();
- $path = $this->repo->getVirtualUrl() . '/public/' . $this->getUrlRel();
+ $path = $this->repo->getVirtualUrl() . '/thumb/' . $this->getUrlRel();
if ( $suffix !== false ) {
$path .= '/' . rawurlencode( $suffix );
}
}
/**
- * Get the 14-character timestamp of the file upload, or false if
- * it doesn't exist
+ * Get the 14-character timestamp of the file upload
*
- * @return string
+ * @return string|false TS_MW timestamp or false on failure
*/
function getTimestamp() {
- $path = $this->getPath();
- if ( !file_exists( $path ) ) {
- return false;
- }
- return wfTimestamp( TS_MW, filemtime( $path ) );
+ $this->assertRepoDefined();
+ return $this->repo->getFileTimestamp( $this->getPath() );
}
/**
* @return string
*/
function getSha1() {
- return self::sha1Base36( $this->getPath() );
+ $this->assertRepoDefined();
+ return $this->repo->getFileSha1( $this->getPath() );
}
/**
* @return array
*/
static function getPropsFromPath( $path, $ext = true ) {
- wfProfileIn( __METHOD__ );
wfDebug( __METHOD__.": Getting file info for $path\n" );
- $info = array(
- 'fileExists' => file_exists( $path ) && !is_dir( $path )
- );
- $gis = false;
- if ( $info['fileExists'] ) {
- $magic = MimeMagic::singleton();
-
- if ( $ext === true ) {
- $i = strrpos( $path, '.' );
- $ext = strtolower( $i ? substr( $path, $i + 1 ) : '' );
- }
-
- # mime type according to file contents
- $info['file-mime'] = $magic->guessMimeType( $path, false );
- # logical mime type
- $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext );
-
- list( $info['major_mime'], $info['minor_mime'] ) = self::splitMime( $info['mime'] );
- $info['media_type'] = $magic->getMediaType( $path, $info['mime'] );
+ wfDeprecated( __METHOD__, '1.19' );
- # Get size in bytes
- $info['size'] = filesize( $path );
-
- # Height, width and metadata
- $handler = MediaHandler::getHandler( $info['mime'] );
- if ( $handler ) {
- $tempImage = (object)array();
- $info['metadata'] = $handler->getMetadata( $tempImage, $path );
- $gis = $handler->getImageSize( $tempImage, $path, $info['metadata'] );
- } else {
- $gis = false;
- $info['metadata'] = '';
- }
- $info['sha1'] = self::sha1Base36( $path );
-
- wfDebug(__METHOD__.": $path loaded, {$info['size']} bytes, {$info['mime']}.\n");
- } else {
- $info['mime'] = null;
- $info['media_type'] = MEDIATYPE_UNKNOWN;
- $info['metadata'] = '';
- $info['sha1'] = '';
- wfDebug(__METHOD__.": $path NOT FOUND!\n");
- }
- if( $gis ) {
- # NOTE: $gis[2] contains a code for the image type. This is no longer used.
- $info['width'] = $gis[0];
- $info['height'] = $gis[1];
- if ( isset( $gis['bits'] ) ) {
- $info['bits'] = $gis['bits'];
- } else {
- $info['bits'] = 0;
- }
- } else {
- $info['width'] = 0;
- $info['height'] = 0;
- $info['bits'] = 0;
- }
- wfProfileOut( __METHOD__ );
- return $info;
+ $fsFile = new FSFile( $path );
+ return $fsFile->getProps();
}
/**
* @return false|string False on failure
*/
static function sha1Base36( $path ) {
- wfSuppressWarnings();
- $hash = sha1_file( $path );
- wfRestoreWarnings();
- if ( $hash === false ) {
- return false;
- } else {
- return wfBaseConvert( $hash, 16, 36, 31 );
- }
+ wfDeprecated( __METHOD__, '1.19' );
+
+ $fsFile = new FSFile( $path );
+ return $fsFile->getSha1Base36();
}
/**
* @ingroup FileRepo
*/
class ForeignAPIFile extends File {
-
private $mExists;
protected $repoClass = 'ForeignApiRepo';
}
function getSha1() {
- return isset( $this->mInfo['sha1'] ) ?
- wfBaseConvert( strval( $this->mInfo['sha1'] ), 16, 36, 31 ) :
- null;
+ return isset( $this->mInfo['sha1'] )
+ ? wfBaseConvert( strval( $this->mInfo['sha1'] ), 16, 36, 31 )
+ : null;
}
function getTimestamp() {
return wfTimestamp( TS_MW,
- isset( $this->mInfo['timestamp'] ) ?
- strval( $this->mInfo['timestamp'] ) :
- null
+ isset( $this->mInfo['timestamp'] )
+ ? strval( $this->mInfo['timestamp'] )
+ : null
);
}
}
function getThumbnails() {
- $files = array();
$dir = $this->getThumbPath( $this->getName() );
- if ( is_dir( $dir ) ) {
- $handle = opendir( $dir );
- if ( $handle ) {
- while ( false !== ( $file = readdir($handle) ) ) {
- if ( $file[0] != '.' ) {
- $files[] = $file;
- }
- }
- closedir( $handle );
- }
+ $iter = $this->repo->getBackend()->getFileList( array( 'dir' => $dir ) );
+
+ $files = array();
+ foreach ( $iter as $file ) {
+ $files[] = $file;
}
+
return $files;
}
function purgeDescriptionPage() {
global $wgMemc, $wgContLang;
+
$url = $this->repo->getDescriptionRenderUrl( $this->getName(), $wgContLang->getCode() );
$key = $this->repo->getLocalCacheKey( 'RemoteFileDescription', 'url', md5($url) );
+
$wgMemc->delete( $key );
}
function purgeThumbnails( $options = array() ) {
global $wgMemc;
+
$key = $this->repo->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $this->getName() );
$wgMemc->delete( $key );
+
+ $backend = $this->repo->getBackend();
$files = $this->getThumbnails();
// Give media handler a chance to filter the purge list
$handler = $this->getHandler();
$dir = $this->getThumbPath( $this->getName() );
foreach ( $files as $file ) {
- unlink( $dir . $file );
- }
- if ( is_dir( $dir ) ) {
- rmdir( $dir ); // Might have already gone away, spews errors if we don't.
+ $op = array( 'op' => 'delete', 'src' => "{$dir}{$file}" );
+ $backend->doOperation( $op );
}
+ $backend->clean( array( 'dir' => $dir ) );
}
}
* Load metadata from the file itself
*/
function loadFromFile() {
- $this->setProps( self::getPropsFromPath( $this->getPath() ) );
+ $props = $this->repo->getFileProps( $this->getVirtualUrl() );
+ $this->setProps( $props );
}
function getCacheFields( $prefix = 'img_' ) {
*/
function migrateThumbFile( $thumbName ) {
$thumbDir = $this->getThumbPath();
- $thumbPath = "$thumbDir/$thumbName";
+ /* Old code for bug 2532
+ $thumbPath = "$thumbDir/$thumbName";
if ( is_dir( $thumbPath ) ) {
// Directory where file should be
// This happened occasionally due to broken migration code in 1.5
// Doesn't exist anymore
clearstatcache();
}
+ */
- if ( is_file( $thumbDir ) ) {
+ if ( $this->repo->fileExists( $thumbDir, FileRepo::FILES_ONLY ) ) {
// File where directory should be
- unlink( $thumbDir );
- // Doesn't exist anymore
- clearstatcache();
+ $op = array( 'op' => 'delete', 'src' => $thumbDir );
+ $this->repo->getBackend()->doOperation( $op );
}
}
} else {
$dir = $this->getThumbPath();
}
- $files = array();
- $files[] = $dir;
-
- if ( is_dir( $dir ) ) {
- $handle = opendir( $dir );
-
- if ( $handle ) {
- while ( false !== ( $file = readdir( $handle ) ) ) {
- if ( $file { 0 } != '.' ) {
- $files[] = $file;
- }
- }
- closedir( $handle );
- }
+ $backend = $this->repo->getBackend();
+ $files = array( $dir );
+ $iterator = $backend->getFileList( array( 'dir' => $dir ) );
+ foreach ( $iterator as $file ) {
+ $files[] = $file;
}
return $files;
}
}
-
/**
* Delete cached transformed files for the current version only.
*/
// Delete thumbnails
$files = $this->getThumbnails();
-
+
// Give media handler a chance to filter the purge list
if ( !empty( $options['forRefresh'] ) ) {
$handler = $this->getHandler();
$handler->filterThumbnailPurgeList( $files, $options );
}
}
-
+
$dir = array_shift( $files );
$this->purgeThumbList( $dir, $files );
* @param $dir string base dir of the files.
* @param $files array of strings: relative filenames (to $dir)
*/
- protected function purgeThumbList($dir, $files) {
+ protected function purgeThumbList( $dir, $files ) {
$fileListDebug = strtr(
var_export( $files, true ),
array("\n"=>'')
);
wfDebug( __METHOD__ . ": $fileListDebug\n" );
+ $backend = $this->repo->getBackend();
foreach ( $files as $file ) {
# Check that the base file name is part of the thumb name
# This is a basic sanity check to avoid erasing unrelated directories
if ( strpos( $file, $this->getName() ) !== false ) {
- wfSuppressWarnings();
- unlink( "$dir/$file" );
- wfRestoreWarnings();
+ $op = array( 'op' => 'delete', 'src' => "{$dir}/{$file}" );
+ $backend->doOperation( $op );
}
}
+ # Clear out directory if empty
+ $backend->clean( array( 'dir' => $dir ) );
}
/** purgeDescription inherited */
/**
* Upload a file and record it in the DB
- * @param $srcPath String: source path or virtual URL
+ * @param $srcPath String: source storage path or virtual URL
* @param $comment String: upload description
* @param $pageText String: text to use for the new description page,
* if a new description page is created
);
if ( $dbw->affectedRows() == 0 ) {
+ if ( $oldver == '' ) {
+ throw new MWException( "Empty oi_archive_name. Database and storage out of sync?" );
+ }
$reupload = true;
-
# Collision, this is an update of a file
# Insert previous contents into oldimage
$dbw->insertSelect( 'oldimage', 'image',
$this->load();
// Initialise now if necessary
if ( $this->sha1 == '' && $this->fileExists ) {
- $this->sha1 = File::sha1Base36( $this->getPath() );
+ $tmpPath = $this->getLocalRefPath();
+ $this->sha1 = FSFile::sha1Base36( $tmpPath );
if ( !wfReadOnly() && strval( $this->sha1 ) != '' ) {
$dbw = $this->repo->getMasterDB();
$dbw->update( 'image',
$dbw->begin();
$dstPath = $this->repo->getZonePath( 'public' ) . '/' . $this->getRel();
- $props = self::getPropsFromPath( $dstPath );
+ $props = $this->repo->getFileProps( $dstPath );
if ( !$props['fileExists'] ) {
return false;
}
--- /dev/null
+<?php
+/**
+ * @file
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ */
+
+/**
+ * This class is used to hold the location and do limited manipulation
+ * of files stored temporarily (usually this will be $wgTmpDirectory)
+ *
+ * @ingroup FileRepo
+ * @ingroup FileBackend
+ */
+class TempFSFile extends FSFile {
+ protected $canDelete = true; // garbage collect the temp file
+
+ /** @var Array of active temp files to purge on shutdown */
+ protected static $instances = array();
+
+ /**
+ * Make a new temporary file on the file system.
+ * Temporary files may be purged when the file object falls out of scope.
+ *
+ * @param $prefix string
+ * @param $extension string
+ * @return TempFSFile|null
+ */
+ public static function factory( $prefix, $extension = '' ) {
+ $base = wfTempDir() . '/' . $prefix . dechex( mt_rand( 0, 99999999 ) );
+ $ext = ( $extension != '' ) ? ".{$extension}" : "";
+ for ( $attempt = 1; true; $attempt++ ) {
+ $path = "{$base}-{$attempt}{$ext}";
+ wfSuppressWarnings();
+ $newFileHandle = fopen( $path, 'x' );
+ wfRestoreWarnings();
+ if ( $newFileHandle ) {
+ fclose( $newFileHandle );
+ break; // got it
+ }
+ if ( $attempt >= 15 ) {
+ return null; // give up
+ }
+ }
+ $tmpFile = new self( $path );
+ if ( php_sapi_name() != 'cli' ) {
+ self::$instances[] = $tmpFile; // defer purge till shutdown
+ }
+ return $tmpFile;
+ }
+
+ /**
+ * Purge this file off the file system
+ *
+ * @return bool Success
+ */
+ public function purge() {
+ $this->canDelete = false; // done
+ wfSuppressWarnings();
+ $ok = unlink( $this->path );
+ wfRestoreWarnings();
+ return $ok;
+ }
+
+ /**
+ * Clean up the temporary file only after an object goes out of scope
+ *
+ * @param $object Object
+ * @return void
+ */
+ public function bind( $object ) {
+ if ( is_object( $object ) ) {
+ $object->tempFSFileReferences[] = $this;
+ }
+ }
+
+ /**
+ * Set flag to not clean up after the temporary file
+ *
+ * @return void
+ */
+ public function preserve() {
+ $this->canDelete = false;
+ }
+
+ /**
+ * Cleans up after the temporary file by deleting it
+ */
+ function __destruct() {
+ if ( $this->canDelete ) {
+ wfSuppressWarnings();
+ unlink( $this->path );
+ wfRestoreWarnings();
+ }
+ }
+}
var $handler;
/**
- * @param $path
- * @param $mime
+ * @param $path string Storage path
+ * @param $mime string
* @return UnregisteredLocalFile
*/
static function newFromPath( $path, $mime ) {
if ( $path ) {
$this->path = $path;
} else {
+ $this->assertRepoDefined();
$this->path = $repo->getRootDirectory() . '/' .
$repo->getHashPath( $this->name ) . $this->name;
}
function getMimeType() {
if ( !isset( $this->mime ) ) {
$magic = MimeMagic::singleton();
- $this->mime = $magic->guessMimeType( $this->getPath() );
+ $this->mime = $magic->guessMimeType( $this->getLocalRefPath() );
}
return $this->mime;
}
if ( !$this->getHandler() ) {
return false;
}
- return $this->handler->getImageSize( $this, $this->getPath() );
+ return $this->handler->getImageSize( $this, $this->getLocalRefPath() );
}
function getMetadata() {
if ( !$this->getHandler() ) {
$this->metadata = false;
} else {
- $this->metadata = $this->handler->getMetadata( $this, $this->getPath() );
+ $this->metadata = $this->handler->getMetadata( $this, $this->getLocalRefPath() );
}
}
return $this->metadata;
}
function getSize() {
- if ( file_exists( $this->path ) ) {
- return filesize( $this->path );
- } else {
- return false;
+ $this->assertRepoDefined();
+ $props = $this->repo->getFileProps( $this->path );
+ if ( isset( $props['size'] ) ) {
+ return $props['size'];
}
+ return false; // doesn't exist
}
}
'srcWidth' => $image->getWidth(),
'srcHeight' => $image->getHeight(),
'mimeType' => $image->getMimeType(),
- 'srcPath' => $image->getPath(),
+ 'srcPath' => $image->getLocalRefPath(),
'dstPath' => $dstPath,
'dstUrl' => $dstUrl,
);
return new TransformParameterError( $params );
}
return new ThumbnailImage( $image, $image->getURL(), $params['width'],
- $params['height'], $image->getPath() );
+ $params['height'], $image->getLocalRefPath() );
}
}
}
$width = $params['width'];
$height = $params['height'];
- $srcPath = $image->getPath();
+ $srcPath = $image->getLocalRefPath();
$page = $params['page'];
if ( $page > $this->pageCount( $image ) ) {
return new MediaTransformError( 'thumbnail_error', $width, $height, wfMsg( 'djvu_page_error' ) );
global $wgEnableAutoRotation;
$gis = parent::getImageSize( $image, $path );
- // Don't just call $image->getMetadata(); File::getPropsFromPath() calls us with a bogus object.
+ // Don't just call $image->getMetadata(); FSFile::getPropsFromPath() calls us with a bogus object.
// This may mean we read EXIF data twice on initial upload.
if ( $wgEnableAutoRotation ) {
$meta = $this->getMetadata( $image, $path );
* Get handler-specific metadata which will be saved in the img_metadata field.
*
* @param $image File: the image object, or false if there isn't one.
- * Warning, File::getPropsFromPath might pass an (object)array() instead (!)
+ * Warning, FSFile::getPropsFromPath might pass an (object)array() instead (!)
* @param $path String: the filename
* @return String
*/
* @param $dstUrl String: Destination URL to use in output HTML
* @param $params Array: Arbitrary set of parameters validated by $this->validateParam()
*/
- function getTransform( $image, $dstPath, $dstUrl, $params ) {
+ final function getTransform( $image, $dstPath, $dstUrl, $params ) {
return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER );
}
* @param $image File
*/
function getPageDimensions( $image, $page ) {
- $gis = $this->getImageSize( $image, $image->getPath() );
+ $gis = $this->getImageSize( $image, $image->getLocalRefPath() );
return array(
'width' => $gis[0],
'height' => $gis[1]
return true;
}
- /**
- * Get a transform output object without actually doing the transform
- */
- function getTransform( $image, $dstPath, $dstUrl, $params ) {
- return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER );
- }
-
/**
* Validate thumbnail parameters and fill in the correct height
*
/**
* Get the width of the output box
*/
- function getWidth() {
+ public function getWidth() {
return $this->width;
}
/**
* Get the height of the output box
*/
- function getHeight() {
+ public function getHeight() {
return $this->height;
}
/**
* @return string The thumbnail URL
*/
- function getUrl() {
+ public function getUrl() {
return $this->url;
}
- /**
- * @return String: destination file path (local filesystem)
- */
- function getPath() {
- return $this->path;
- }
-
/**
* Fetch HTML for this transform output
*
*
* @return string
*/
- abstract function toHtml( $options = array() );
+ abstract public function toHtml( $options = array() );
/**
* This will be overridden to return true in error classes
*/
- function isError() {
+ public function isError() {
return false;
}
+ /**
+ * Check if an output thumbnail file was actually made.
+ * This will return false if there was an error, the
+ * thumnail is to be handled client-side only, or if
+ * transformation was deferred via TRANSFORM_LATER.
+ *
+ * @return Bool
+ */
+ public function hasFile() {
+ // If TRANSFORM_LATER, $this->path will be false
+ return ( !$this->isError() && $this->path );
+ }
+
+ /**
+ * Check if the output thumbnail file is the same as the source.
+ * This can occur if the requested width was bigger than the source.
+ *
+ * @return Bool
+ */
+ public function fileIsSource() {
+ return ( !$this->isError() && $this->path === $this->file->getLocalRefPath() );
+ }
+
+ /**
+ * Get the path of a file system copy of the thumbnail
+ *
+ * @return string|false Returns false if there isn't one
+ */
+ public function getLocalCopyPath() {
+ return $this->path;
+ }
+
+ /**
+ * Stream the file if there were no errors
+ *
+ * @param $headers Array Additional HTTP headers to send on success
+ * @return Bool success
+ */
+ public function streamFile( $headers = array() ) {
+ return $this->path && StreamFile::stream( $this->path, $headers );
+ }
+
/**
* Wrap some XHTML text in an anchor tag with the given attributes
*
* @param $params array
* @return array
*/
- function getDescLinkAttribs( $title = null, $params = '' ) {
+ public function getDescLinkAttribs( $title = null, $params = '' ) {
$query = $this->page ? ( 'page=' . urlencode( $this->page ) ) : '';
if( $params ) {
$query .= $query ? '&'.$params : $params;
$clientHeight = $params['height'];
$physicalWidth = $params['physicalWidth'];
$physicalHeight = $params['physicalHeight'];
- $srcPath = $image->getPath();
+ $srcPath = $image->getLocalRefPath();
if ( $flags & self::TRANSFORM_LATER ) {
return new ThumbnailImage( $image, $dstUrl, $clientWidth, $clientHeight, $dstPath );
$key = $oimage->getStorageKey();
$path = $repo->getZonePath( 'deleted' ) . '/' . $repo->getDeletedHashPath( $key ) . $key;
- StreamFile::stream( $path );
+ $repo->streamFile( $path );
}
/**
$repo = RepoGroup::singleton()->getLocalRepo();
$path = $repo->getZonePath( 'deleted' ) . '/' . $repo->getDeletedHashPath( $key ) . $key;
- StreamFile::stream( $path );
+ $repo->streamFile( $path );
}
private function showHistory() {
*/
public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) {
$this->mDesiredDestName = $name;
+ if ( FileBackend::isStoragePath( $tempPath ) ) {
+ throw new MWException( __METHOD__ . " given storage path `$tempPath`." );
+ }
$this->mTempPath = $tempPath;
$this->mFileSize = $fileSize;
$this->mRemoveTempFile = $removeTempFile;
return $this->mFileSize;
}
- /**
- * Append a file to the Repo file
- *
- * @deprecated since 1.19
- *
- * @param $srcPath String: path to source file
- * @param $toAppendPath String: path to the Repo file that will be appended to.
- * @return Status Status
- */
- protected function appendToUploadFile( $srcPath, $toAppendPath ) {
- wfDeprecated( __METHOD__, '1.19' );
-
- $repo = RepoGroup::singleton()->getLocalRepo();
- $status = $repo->append( $srcPath, $toAppendPath );
- return $status;
- }
-
- /**
- * Finish appending to the Repo file
- *
- * @deprecated since 1.19
- *
- * @param $toAppendPath String: path to the Repo file that will be appended to.
- * @return Status Status
- */
- protected function appendFinish( $toAppendPath ) {
- wfDeprecated( __METHOD__, '1.19' );
-
- $repo = RepoGroup::singleton()->getLocalRepo();
- $status = $repo->appendFinish( $toAppendPath );
- return $status;
- }
-
-
/**
* @param $srcPath String: the source path
* @return the real path if it was a virtual URL
function getRealPath( $srcPath ) {
$repo = RepoGroup::singleton()->getLocalRepo();
if ( $repo->isVirtualUrl( $srcPath ) ) {
- return $repo->resolveVirtualUrl( $srcPath );
+ // @TODO: just make uploads work with storage paths
+ // UploadFromStash loads files via virtuals URLs
+ $tmpFile = $repo->getLocalCopy( $srcPath );
+ $tmpFile->bind( $this ); // keep alive with $thumb
+ return $tmpFile->getPath();
}
return $srcPath;
}
# we need to populate mFinalExtension
$this->getTitle();
- $this->mFileProps = File::getPropsFromPath( $this->mTempPath, $this->mFinalExtension );
+ $this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension );
# check mime type, if desired
$mime = $this->mFileProps[ 'file-mime' ];
}
// Check dupes against existing files
- $hash = File::sha1Base36( $this->mTempPath );
+ $hash = FSFile::getSha1Base36FromPath( $this->mTempPath );
$dupes = RepoGroup::singleton()->findBySha1( $hash );
$title = $this->getTitle();
// Remove all matches against self
$fileList[] = $this->getVirtualChunkLocation( $i );
}
- // Concatinate into the mVirtualTempPath location;
- $status = $this->repo->concatenate( $fileList, $this->mVirtualTempPath, FileRepo::DELETE_SOURCE );
+ // Concatenate into the mVirtualTempPath location;
+ $status = $this->repo->concatenate( $fileList, $this->mVirtualTempPath, FileRepo::DELETE_SOURCE );
if( !$status->isOk() ){
return $status;
}
$this->mTempPath = $this->getRealPath( $this->mVirtualTempPath );
return $status;
}
+
+ /**
+ * Perform the upload, then remove the temp copy afterward
+ * @param $comment string
+ * @param $pageText string
+ * @param $watch bool
+ * @param $user User
+ * @return Status
+ */
+ public function performUpload( $comment, $pageText, $watch, $user ) {
+ $rv = parent::performUpload( $comment, $pageText, $watch, $user );
+ $this->repo->freeTemp( $this->mVirtualTempPath );
+ return $rv;
+ }
+
/**
* Returns the virtual chunk location:
* @param unknown_type $index
*
* @param $repo FileRepo
*/
- public function __construct( $repo, $user = null ) {
+ public function __construct( FileRepo $repo, $user = null ) {
// this might change based on wiki's configuration.
$this->repo = $repo;
// fetch fileprops
$path = $this->fileMetadata[$key]['us_path'];
- if ( $this->repo->isVirtualUrl( $path ) ) {
- $path = $this->repo->resolveVirtualUrl( $path );
- }
- $this->fileProps[$key] = File::getPropsFromPath( $path );
+ $this->fileProps[$key] = $this->repo->getFileProps( $path );
}
if ( ! $this->files[$key]->exists() ) {
wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" );
throw new UploadStashBadPathException( "path doesn't exist" );
}
- $fileProps = File::getPropsFromPath( $path );
+ $fileProps = FSFile::getPropsFromPath( $path );
wfDebug( __METHOD__ . " stashing file at '$path'\n" );
// we will be initializing from some tmpnam files that don't have extensions.
$error = array( 'unknown', 'no error recorded' );
}
}
- throw new UploadStashFileException( "error storing file in '$path': " . implode( '; ', $error ) );
+ throw new UploadStashFileException( "Error storing file in '$path': " . implode( '; ', $error ) );
}
$stashPath = $storeStatus->value;
'us_user' => $this->userId,
'us_key' => $key,
'us_orig_path' => $path,
- 'us_path' => $stashPath,
+ 'us_path' => $stashPath, // virtual URL
'us_size' => $fileProps['size'],
'us_sha1' => $fileProps['sha1'],
'us_mime' => $fileProps['mime'],
'uploadstash-refresh' => 'Refresh the list of files',
'invalid-chunk-offset' => 'Invalid chunk offset',
+# file backend
+'backend-fail-stream' => 'Could not stream file $1.',
+'backend-fail-backup' => 'Could not backup file $1.',
+'backend-fail-notexists' => 'The file $1 does not exist.',
+'backend-fail-hashes' => 'Could not get file hashes for comparison.',
+'backend-fail-notsame' => 'A non-identical file already exists at $1.',
+'backend-fail-invalidpath' => '$1 is not a valid storage path.',
+'backend-fail-delete' => 'Could not delete file $1.',
+'backend-fail-alreadyexists' => 'The file $1 already exists.',
+'backend-fail-store' => 'Could not store file $1 at $2',
+'backend-fail-copy' => 'Could not copy file $1 to $2',
+'backend-fail-move' => 'Could not move file $1 to $2',
+'backend-fail-opentemp' => 'Could not open temporary file.',
+'backend-fail-writetemp' => 'Could not write to temporary file.',
+'backend-fail-closetemp' => 'Could not close temporary file.',
+'backend-fail-read' => 'Could not read file $1',
+'backend-fail-create' => 'Could not create file $1',
+
+# lock manager
+'lockmanager-notlocked' => 'Could not unlock key "$1"; it is not locked.',
+'lockmanager-fail-closelock' => 'Could not close lock file for key "$1".',
+'lockmanager-fail-deletelock' => 'Could not delete lock file for key "$1".',
+'lockmanager-fail-openlock' => 'Could not open lock file for key "$1".',
+'lockmanager-fail-acquirelock' => 'Could not acquire lock for key "$1".',
+'lockmanager-fail-releaselock' => 'Could not release lock for key "$1".',
+'lockmanager-fail-acquirelocks' => 'Could not acquire locks for keys "$1".',
+'lockmanager-fail-db-bucket' => 'Could not contact enough lock databases in bucket $1',
+'lockmanager-fail-db-release' => 'Could not release locks on database $1',
+'lockmanager-fail-svr-release' => 'Could not release locks on server $1',
+
# img_auth script messages
'img-auth-accessdenied' => 'Access denied',
'img-auth-nopathinfo' => 'Missing PATH_INFO.
'thumbnail_error' => 'Error creating thumbnail: $1',
'djvu_page_error' => 'DjVu page out of range',
'djvu_no_xml' => 'Unable to fetch XML for DjVu file',
+'thumbnail-temp-create' => 'Unable to create temporary thumbnail file',
+'thumbnail-dest-create' => 'Unable to save thumbnail to destination',
'thumbnail_invalid_params' => 'Invalid thumbnail parameters',
'thumbnail_dest_directory' => 'Unable to create destination directory',
'thumbnail_image-type' => 'Image type not supported',
'upload-http-error',
),
+ 'filebackend-errors' => array(
+ 'backend-fail-stream',
+ 'backend-fail-backup',
+ 'backend-fail-notexists',
+ 'backend-fail-hashes',
+ 'backend-fail-notsame',
+ 'backend-fail-invalidpath',
+ 'backend-fail-delete',
+ 'backend-fail-alreadyexists',
+ 'backend-fail-store',
+ 'backend-fail-copy',
+ 'backend-fail-move',
+ 'backend-fail-opentemp',
+ 'backend-fail-writetemp',
+ 'backend-fail-closetemp',
+ 'backend-fail-read',
+ 'backend-fail-create',
+ ),
+
+ 'lockmanger-errors' => array(
+ 'lockmanager-notlocked',
+ 'lockmanager-fail-closelock',
+ 'lockmanager-fail-deletelock',
+ 'lockmanager-fail-acquirelock',
+ 'lockmanager-fail-openlock',
+ 'lockmanager-fail-releaselock',
+ 'lockmanager-fail-acquirelocks',
+ 'lockmanager-fail-db-bucket',
+ 'lockmanager-fail-db-release',
+ 'lockmanager-fail-svr-release'
+ ),
+
'zip' => array(
'zip-file-open-error',
'zip-wrong-format',
--- /dev/null
+<?php
+
+if ( php_sapi_name() !== 'cli' ) {
+ die( "This is not a valid entry point.\n" );
+}
+error_reporting( E_ALL );
+
+// Run the server...
+set_time_limit( 0 );
+LockServerDaemon::init(
+ getopt( '', array(
+ 'address:', 'port:', 'authKey:',
+ 'connTimeout::', 'lockTimeout::', 'maxClients::', 'maxBacklog::', 'maxLocks::',
+ ) )
+)->main();
+
+/**
+ * Simple lock server daemon that accepts lock/unlock requests.
+ * This should not require MediaWiki setup or PHP files.
+ */
+class LockServerDaemon {
+ /** @var resource */
+ protected $sock; // socket to listen/accept on
+ /** @var Array */
+ protected $shLocks = array(); // (key => session => 1)
+ /** @var Array */
+ protected $exLocks = array(); // (key => session)
+ /** @var Array */
+ protected $sessions = array(); // (session => resource)
+ /** @var Array */
+ protected $deadSessions = array(); // (session => UNIX timestamp)
+
+ /** @var Array */
+ protected $sessionIndexSh = array(); // (session => key => 1)
+ /** @var Array */
+ protected $sessionIndexEx = array(); // (session => key => 1)
+
+ protected $address; // string (IP/hostname)
+ protected $port; // integer
+ protected $authKey; // string key
+ protected $connTimeout; // array ( 'sec' => integer, 'usec' => integer )
+ protected $lockTimeout; // integer number of seconds
+ protected $maxLocks; // integer
+ protected $maxClients; // integer
+ protected $maxBacklog; // integer
+
+ protected $startTime; // integer UNIX timestamp
+ protected $lockCount = 0; // integer
+ protected $ticks = 0; // integer counter
+
+ protected static $instance = null;
+
+ /**
+ * @params $config Array
+ * @return LockServerDaemon
+ */
+ public function init( array $config ) {
+ if ( self::$instance ) {
+ throw new Exception( 'LockServer already initialized.' );
+ }
+ self::$instance = new self( $config );
+ return self::$instance;
+ }
+
+ /**
+ * @params $config Array
+ */
+ protected function __construct( array $config ) {
+ $required = array( 'address', 'port', 'authKey' );
+ foreach ( $required as $par ) {
+ if ( !isset( $config[$par] ) ) {
+ throw new Exception( "Parameter '$par' must be specified." );
+ }
+ }
+
+ $this->address = $config['address'];
+ $this->port = $config['port'];
+ $this->authKey = $config['authKey'];
+
+ $connTimeout = isset( $config['connTimeout'] )
+ ? $config['connTimeout']
+ : 1.5;
+ $this->connTimeout = array(
+ 'sec' => floor( $connTimeout ),
+ 'usec' => floor( ( $connTimeout - floor( $connTimeout ) ) * 1e6 )
+ );
+ $this->lockTimeout = isset( $config['lockTimeout'] )
+ ? $config['lockTimeout']
+ : 60;
+ $this->maxLocks = isset( $config['maxLocks'] )
+ ? $config['maxLocks']
+ : 5000;
+ $this->maxClients = isset( $config['maxClients'] )
+ ? $config['maxClients']
+ : 100;
+ $this->maxBacklog = isset( $config['maxBacklog'] )
+ ? $config['maxBacklog']
+ : 10;
+ }
+
+ /**
+ * @return void
+ */
+ protected function setupSocket() {
+ if ( !function_exists( 'socket_create' ) ) {
+ throw new Exception( "PHP sockets extension missing from PHP CLI mode." );
+ }
+ $sock = socket_create( AF_INET, SOCK_STREAM, SOL_TCP );
+ if ( $sock === false ) {
+ throw new Exception( "socket_create(): " . socket_strerror( socket_last_error() ) );
+ }
+ socket_set_option( $sock, SOL_SOCKET, SO_REUSEADDR, 1 ); // bypass 2MLS
+ if ( socket_bind( $sock, $this->address, $this->port ) === false ) {
+ throw new Exception( "socket_bind(): " .
+ socket_strerror( socket_last_error( $sock ) ) );
+ } elseif ( socket_listen( $sock, $this->maxBacklog ) === false ) {
+ throw new Exception( "socket_listen(): " .
+ socket_strerror( socket_last_error( $sock ) ) );
+ }
+ $this->sock = $sock;
+
+ $this->startTime = time();
+ }
+
+ /**
+ * @return void
+ */
+ public function main() {
+ // Setup socket and start listing
+ $this->setupSocket();
+ // Create a list of all the clients that will be connected to us.
+ $clients = array( $this->sock ); // start off with listening socket
+ do {
+ // Create a copy, so $clients doesn't get modified by socket_select()
+ $read = $clients; // clients-with-data
+ // Get a list of all the clients that have data to be read from
+ $changed = socket_select( $read, $write = NULL, $except = NULL, NULL );
+ if ( $changed === false ) {
+ trigger_error( 'socket_listen(): ' . socket_strerror( socket_last_error() ) );
+ continue;
+ } elseif ( $changed < 1 ) {
+ continue; // wait
+ }
+ // Check if there is a client trying to connect...
+ if ( in_array( $this->sock, $read ) && count( $clients ) < $this->maxClients ) {
+ // Accept the new client...
+ $newsock = socket_accept( $this->sock );
+ socket_set_option( $newsock, SOL_SOCKET, SO_RCVTIMEO, $this->connTimeout );
+ socket_set_option( $newsock, SOL_SOCKET, SO_SNDTIMEO, $this->connTimeout );
+ $clients[] = $newsock;
+ // Remove the listening socket from the clients-with-data array...
+ $key = array_search( $this->sock, $read );
+ unset( $read[$key] );
+ }
+ // Loop through all the clients that have data to read...
+ foreach ( $read as $read_sock ) {
+ // Read until newline or 65535 bytes are recieved.
+ // socket_read show errors when the client is disconnected.
+ $data = @socket_read( $read_sock, 65535, PHP_NORMAL_READ );
+ // Check if the client is disconnected
+ if ( $data === false ) {
+ // Remove client from $clients list
+ $key = array_search( $read_sock, $clients );
+ unset( $clients[$key] );
+ // Remove socket's session from tracking (if it exists)
+ $session = array_search( $read_sock, $this->sessions );
+ if ( $session !== false ) {
+ unset( $this->sessions[$session] );
+ // Record recently killed sessions that still have locks
+ if ( isset( $this->sessionIndexSh[$session] )
+ || isset( $this->sessionIndexEx[$session] ) )
+ {
+ $this->deadSessions[$session] = time();
+ }
+ }
+ } else {
+ // Perform the requested command...
+ $response = $this->doCommand( trim( $data ), $read_sock );
+ // Send the response to the client...
+ if ( socket_write( $read_sock, "$response\n" ) === false ) {
+ trigger_error( 'socket_write(): ' .
+ socket_strerror( socket_last_error( $read_sock ) ) );
+ }
+ }
+ }
+ // Prune dead locks every 10 socket events...
+ if ( ++$this->ticks >= 9 ) {
+ $this->ticks = 0;
+ $this->purgeExpiredLocks();
+ }
+ } while ( true );
+ }
+
+ /**
+ * @param $data string
+ * @param $sourceSock resource
+ * @return string
+ */
+ protected function doCommand( $data, $sourceSock ) {
+ $cmdArr = $this->getCommand( $data );
+ if ( is_string( $cmdArr ) ) {
+ return $cmdArr; // error
+ }
+ list( $function, $session, $type, $resources ) = $cmdArr;
+ // On first command, track the session => sock correspondence
+ if ( !isset( $this->sessions[$session] ) ) {
+ $this->sessions[$session] = $sourceSock;
+ }
+ if ( $function === 'ACQUIRE' ) {
+ return $this->lock( $session, $type, $resources );
+ } elseif ( $function === 'RELEASE' ) {
+ return $this->unlock( $session, $type, $resources );
+ } elseif ( $function === 'RELEASE_ALL' ) {
+ return $this->release( $session );
+ } elseif ( $function === 'STAT' ) {
+ return $this->stat();
+ }
+ return 'INTERNAL_ERROR';
+ }
+
+ /**
+ * @param $data string
+ * @return Array
+ */
+ protected function getCommand( $data ) {
+ $m = explode( ':', $data ); // <session, key, command, type, values>
+ if ( count( $m ) == 5 ) {
+ list( $session, $key, $command, $type, $values ) = $m;
+ if ( sha1( $session . $command . $type . $values . $this->authKey ) !== $key ) {
+ return 'BAD_KEY';
+ } elseif ( strlen( $session ) !== 31 ) {
+ return 'BAD_SESSION';
+ }
+ $values = explode( '|', $values );
+ if ( $command === 'ACQUIRE' ) {
+ $needsLockArgs = true;
+ } elseif ( $command === 'RELEASE' ) {
+ $needsLockArgs = true;
+ } elseif ( $command === 'RELEASE_ALL' ) {
+ $needsLockArgs = false;
+ } elseif ( $command === 'STAT' ) {
+ $needsLockArgs = false;
+ } else {
+ return 'BAD_COMMAND';
+ }
+ if ( $needsLockArgs ) {
+ if ( $type !== 'SH' && $type !== 'EX' ) {
+ return 'BAD_TYPE';
+ }
+ foreach ( $values as $value ) {
+ if ( strlen( $value ) !== 31 ) {
+ return 'BAD_FORMAT';
+ }
+ }
+ }
+ return array( $command, $session, $type, $values );
+ }
+ return 'BAD_FORMAT';
+ }
+
+ /**
+ * @param $session string
+ * @param $type string
+ * @param $keys Array
+ * @return string
+ */
+ protected function lock( $session, $type, $keys ) {
+ if ( $this->lockCount >= $this->maxLocks ) {
+ return 'TOO_MANY_LOCKS';
+ }
+ if ( $type === 'SH' ) {
+ // Check if any keys are already write-locked...
+ foreach ( $keys as $key ) {
+ if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] !== $session ) {
+ return 'CANT_ACQUIRE';
+ }
+ }
+ // Acquire the read-locks...
+ foreach ( $keys as $key ) {
+ $this->set_sh_lock( $key, $session );
+ }
+ return 'ACQUIRED';
+ } elseif ( $type === 'EX' ) {
+ // Check if any keys are already read-locked or write-locked...
+ foreach ( $keys as $key ) {
+ if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] !== $session ) {
+ return 'CANT_ACQUIRE';
+ }
+ if ( isset( $this->shLocks[$key] ) ) {
+ foreach ( $this->shLocks[$key] as $otherSession => $x ) {
+ if ( $otherSession !== $session ) {
+ return 'CANT_ACQUIRE';
+ }
+ }
+ }
+ }
+ // Acquire the write-locks...
+ foreach ( $keys as $key ) {
+ $this->set_ex_lock( $key, $session );
+ }
+ return 'ACQUIRED';
+ }
+ return 'INTERNAL_ERROR';
+ }
+
+ /**
+ * @param $session string
+ * @param $type string
+ * @param $keys Array
+ * @return string
+ */
+ protected function unlock( $session, $type, $keys ) {
+ if ( $type === 'SH' ) {
+ foreach ( $keys as $key ) {
+ $this->unset_sh_lock( $key, $session );
+ }
+ return 'RELEASED';
+ } elseif ( $type === 'EX' ) {
+ foreach ( $keys as $key ) {
+ $this->unset_ex_lock( $key, $session );
+ }
+ return 'RELEASED';
+ }
+ return 'INTERNAL_ERROR';
+ }
+
+ /**
+ * @param $session string
+ * @return string
+ */
+ protected function release( $session ) {
+ if ( isset( $this->sessionIndexSh[$session] ) ) {
+ foreach ( $this->sessionIndexSh[$session] as $key => $x ) {
+ $this->unset_sh_lock( $key, $session );
+ }
+ }
+ if ( isset( $this->sessionIndexEx[$session] ) ) {
+ foreach ( $this->sessionIndexEx[$session] as $key => $x ) {
+ $this->unset_ex_lock( $key, $session );
+ }
+ }
+ return 'RELEASED_ALL';
+ }
+
+ /**
+ * @return string
+ */
+ protected function stat() {
+ return ( time() - $this->startTime ) . ':' . memory_get_usage();
+ }
+
+ /**
+ * Clear locks for sessions that have been dead for a while
+ *
+ * @return void
+ */
+ protected function purgeExpiredLocks() {
+ $now = time();
+ foreach ( $this->deadSessions as $session => $timestamp ) {
+ if ( ( $now - $timestamp ) > $this->lockTimeout ) {
+ $this->release( $session );
+ unset( $this->deadSessions[$session] );
+ }
+ }
+ }
+
+ /**
+ * @param $key string
+ * @param $session string
+ * @return void
+ */
+ protected function set_sh_lock( $key, $session ) {
+ if ( !isset( $this->shLocks[$key][$session] ) ) {
+ $this->shLocks[$key][$session] = 1;
+ $this->sessionIndexSh[$session][$key] = 1;
+ ++$this->lockCount; // we are adding a lock
+ }
+ }
+
+ /**
+ * @param $key string
+ * @param $session string
+ * @return void
+ */
+ protected function set_ex_lock( $key, $session ) {
+ if ( !isset( $this->exLocks[$key][$session] ) ) {
+ $this->exLocks[$key] = $session;
+ $this->sessionIndexEx[$session][$key] = 1;
+ ++$this->lockCount; // we are adding a lock
+ }
+ }
+
+ /**
+ * @param $key string
+ * @param $session string
+ * @return void
+ */
+ protected function unset_sh_lock( $key, $session ) {
+ if ( isset( $this->shLocks[$key][$session] ) ) {
+ unset( $this->shLocks[$key][$session] );
+ if ( !count( $this->shLocks[$key] ) ) {
+ unset( $this->shLocks[$key] );
+ }
+ unset( $this->sessionIndexSh[$session][$key] );
+ if ( !count( $this->sessionIndexSh[$session] ) ) {
+ unset( $this->sessionIndexSh[$session] );
+ }
+ --$this->lockCount;
+ }
+ }
+
+ /**
+ * @param $key string
+ * @param $session string
+ * @return void
+ */
+ protected function unset_ex_lock( $key, $session ) {
+ if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] === $session ) {
+ unset( $this->exLocks[$key] );
+ unset( $this->sessionIndexEx[$session][$key] );
+ if ( !count( $this->sessionIndexEx[$session] ) ) {
+ unset( $this->sessionIndexEx[$session] );
+ }
+ --$this->lockCount;
+ }
+ }
+}
--- /dev/null
+-- Table to handle resource locking (EX) with row-level locking
+CREATE TABLE /*_*/filelocks_exclusive (
+ fle_key binary(31) NOT NULL PRIMARY KEY
+) ENGINE=InnoDB, CHECKSUM=0;
+
+-- Table to handle resource locking (SH) with row-level locking
+CREATE TABLE /*_*/filelocks_shared (
+ fls_key binary(31) NOT NULL,
+ fls_session binary(31) NOT NULL,
+ PRIMARY KEY (fls_key,fls_session)
+) ENGINE=InnoDB, CHECKSUM=0;
$wgStylePath = '/skins';
$wgExtensionAssetsPath = '/extensions';
$wgThumbnailScriptPath = false;
+ $backend = new FSFileBackend( array(
+ 'name' => 'local-backend',
+ 'lockManager' => 'fsLockManager',
+ 'containerPaths' => array(
+ 'images-public' => wfTempDir() . '/test-repo/public',
+ 'images-thumb' => wfTempDir() . '/test-repo/thumb',
+ 'images-temp' => wfTempDir() . '/test-repo/temp',
+ 'images-deleted' => wfTempDir() . '/test-repo/delete',
+ )
+ ) );
$wgLocalFileRepo = array(
- 'class' => 'LocalRepo',
- 'name' => 'local',
- 'directory' => wfTempDir() . '/test-repo',
- 'url' => 'http://example.com/images',
- 'deletedDir' => wfTempDir() . '/test-repo/delete',
- 'hashLevels' => 2,
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'url' => 'http://example.com/images',
+ 'hashLevels' => 2,
'transformVia404' => false,
+ 'backend' => $backend
);
$wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface';
$wgNamespaceAliases['Image'] = NS_FILE;
global $wgCapitalLinks;
$wgCapitalLinks = true;
+
+ $backend = new FSFileBackend( array(
+ 'name' => 'local-backend',
+ 'lockManager' => 'fsLockManager',
+ 'containerPaths' => array(
+ 'cont1' => "/testdir/local-backend/tempimages/cont1",
+ 'cont2' => "/testdir/local-backend/tempimages/cont2"
+ )
+ ) );
$info = array(
- 'name' => 'test',
- 'directory' => '/testdir',
- 'url' => '/testurl',
- 'hashLevels' => 2,
+ 'name' => 'test',
+ 'directory' => '/testdir',
+ 'url' => '/testurl',
+ 'hashLevels' => 2,
'transformVia404' => false,
+ 'backend' => $backend
);
$this->repo_hl0 = new LocalRepo( array( 'hashLevels' => 0 ) + $info );
$this->repo_hl2 = new LocalRepo( array( 'hashLevels' => 2 ) + $info );
}
function testGetArchivePath() {
- $this->assertEquals( '/testdir/archive', $this->file_hl0->getArchivePath() );
- $this->assertEquals( '/testdir/archive/a/a2', $this->file_hl2->getArchivePath() );
- $this->assertEquals( '/testdir/archive/!', $this->file_hl0->getArchivePath( '!' ) );
- $this->assertEquals( '/testdir/archive/a/a2/!', $this->file_hl2->getArchivePath( '!' ) );
+ $this->assertEquals( 'mwstore://local-backend/images-public/archive', $this->file_hl0->getArchivePath() );
+ $this->assertEquals( 'mwstore://local-backend/images-public/archive/a/a2', $this->file_hl2->getArchivePath() );
+ $this->assertEquals( 'mwstore://local-backend/images-public/archive/!', $this->file_hl0->getArchivePath( '!' ) );
+ $this->assertEquals( 'mwstore://local-backend/images-public/archive/a/a2/!', $this->file_hl2->getArchivePath( '!' ) );
}
function testGetThumbPath() {
- $this->assertEquals( '/testdir/thumb/Test!', $this->file_hl0->getThumbPath() );
- $this->assertEquals( '/testdir/thumb/a/a2/Test!', $this->file_hl2->getThumbPath() );
- $this->assertEquals( '/testdir/thumb/Test!/x', $this->file_hl0->getThumbPath( 'x' ) );
- $this->assertEquals( '/testdir/thumb/a/a2/Test!/x', $this->file_hl2->getThumbPath( 'x' ) );
+ $this->assertEquals( 'mwstore://local-backend/images-thumb/Test!', $this->file_hl0->getThumbPath() );
+ $this->assertEquals( 'mwstore://local-backend/images-thumb/a/a2/Test!', $this->file_hl2->getThumbPath() );
+ $this->assertEquals( 'mwstore://local-backend/images-thumb/Test!/x', $this->file_hl0->getThumbPath( 'x' ) );
+ $this->assertEquals( 'mwstore://local-backend/images-thumb/a/a2/Test!/x', $this->file_hl2->getThumbPath( 'x' ) );
}
function testGetArchiveUrl() {
* @param $filePath String: path to file on the filesystem
*/
public function deleteFileByContent( $filePath ) {
- $hash = File::sha1Base36( $filePath );
+ $hash = FSFile::getSha1Base36FromPath( $filePath );
$dupes = RepoGroup::singleton()->findBySha1( $hash );
$success = true;
foreach ( $dupes as $dupe ) {
}
$this->assertTrue( isset( $result['upload'] ) );
$this->assertEquals( 'Success', $result['upload']['result'] );
- $this->assertFalse( $exception );
+ $this->assertFalse( $exception, "No UsageException exception." );
// clean up
$this->deleteFileByFilename( $fileName );
--- /dev/null
+<?php
+
+// @TODO: fix empty dir leakage
+class FileBackendTest extends MediaWikiTestCase {
+ private $backend, $multiBackend;
+ private $filesToPrune, $pathsToPrune;
+
+ function setUp() {
+ parent::setUp();
+ $this->backend = new FSFileBackend( array(
+ 'name' => 'localtesting',
+ 'lockManager' => 'fsLockManager',
+ 'containerPaths' => array(
+ 'cont1' => wfTempDir() . '/localtesting/cont1',
+ 'cont2' => wfTempDir() . '/localtesting/cont2' )
+ ) );
+ $this->multiBackend = new FileBackendMultiWrite( array(
+ 'name' => 'localtestingmulti',
+ 'lockManager' => 'fsLockManager',
+ 'backends' => array(
+ array(
+ 'name' => 'localmutlitesting1',
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array(
+ 'cont1' => wfTempDir() . '/localtestingmulti1/cont1',
+ 'cont2' => wfTempDir() . '/localtestingmulti1/cont2' ),
+ 'isMultiMaster' => false
+ ),
+ array(
+ 'name' => 'localmutlitesting2',
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array(
+ 'cont1' => wfTempDir() . '/localtestingmulti2/cont1',
+ 'cont2' => wfTempDir() . '/localtestingmulti2/cont2' ),
+ 'isMultiMaster' => true
+ )
+ )
+ ) );
+ $this->filesToPrune = $this->pathsToPrune = array();
+ }
+
+ private function singleBasePath() {
+ return 'mwstore://localtesting';
+ }
+
+ /**
+ * @dataProvider provider_testStore
+ */
+ public function testStore( $op, $source, $dest ) {
+ $this->filesToPrune[] = $source;
+ $this->pathsToPrune[] = $dest;
+
+ file_put_contents( $source, "Unit test file" );
+ $status = $this->backend->doOperation( $op );
+
+ $this->assertEquals( true, $status->isOK(),
+ "Store from $source to $dest succeeded." );
+ $this->assertEquals( true, $status->isGood(),
+ "Store from $source to $dest succeeded without warnings." );
+ $this->assertEquals( true, file_exists( $source ),
+ "Source file $source still exists." );
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Destination file $dest exists." );
+
+ $props1 = FSFile::getPropsFromPath( $source );
+ $props2 = $this->backend->getFileProps( array( 'src' => $dest ) );
+ $this->assertEquals( $props1, $props2,
+ "Source and destination have the same props." );
+ }
+
+ public function provider_testStore() {
+ $cases = array();
+
+ $tmpName = TempFSFile::factory( "unittests_", 'txt' )->getPath();
+ $toPath = $this->singleBasePath() . '/cont1/fun/obj1.txt';
+ $op = array( 'op' => 'store', 'src' => $tmpName, 'dst' => $toPath );
+ $cases[] = array(
+ $op, // operation
+ $tmpName, // source
+ $toPath, // dest
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $tmpName, // source
+ $toPath, // dest
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testCopy
+ */
+ public function testCopy( $op, $source, $dest ) {
+ $this->pathsToPrune[] = $source;
+ $this->pathsToPrune[] = $dest;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." );
+
+ $status = $this->backend->doOperation( $op );
+ $this->assertEquals( true, $status->isOK(),
+ "Copy from $source to $dest succeeded." );
+ $this->assertEquals( true, $status->isGood(),
+ "Copy from $source to $dest succeeded without warnings." );
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $source ) ),
+ "Source file $source still exists." );
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Destination file $dest exists after copy." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source ) );
+ $props2 = $this->backend->getFileProps( array( 'src' => $dest ) );
+ $this->assertEquals( $props1, $props2,
+ "Source and destination have the same props." );
+ }
+
+ public function provider_testCopy() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/file.txt';
+ $dest = $this->singleBasePath() . '/cont2/fileMoved.txt';
+
+ $op = array( 'op' => 'copy', 'src' => $source, 'dst' => $dest );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testMove
+ */
+ public function testMove( $op, $source, $dest ) {
+ $this->pathsToPrune[] = $source;
+ $this->pathsToPrune[] = $dest;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." );
+
+ $status = $this->backend->doOperation( $op );
+ $this->assertEquals( true, $status->isOK(),
+ "Move from $source to $dest succeeded." );
+ $this->assertEquals( true, $status->isGood(),
+ "Move from $source to $dest succeeded without warnings." );
+ $this->assertEquals( false, $this->backend->fileExists( array( 'src' => $source ) ),
+ "Source file $source does not still exists." );
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Destination file $dest exists after move." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source ) );
+ $props2 = $this->backend->getFileProps( array( 'src' => $dest ) );
+ $this->assertEquals( false, $props1['fileExists'],
+ "Source file does not exist accourding to props." );
+ $this->assertEquals( true, $props2['fileExists'],
+ "Destination file exists accourding to props." );
+ }
+
+ public function provider_testMove() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/file.txt';
+ $dest = $this->singleBasePath() . '/cont2/fileMoved.txt';
+
+ $op = array( 'op' => 'move', 'src' => $source, 'dst' => $dest );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ $dest, // dest
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testDelete
+ */
+ public function testDelete( $op, $source, $withSource, $okStatus ) {
+ $this->pathsToPrune[] = $source;
+
+ if ( $withSource ) {
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." );
+ }
+
+ $status = $this->backend->doOperation( $op );
+ if ( $okStatus ) {
+ $this->assertEquals( true, $status->isOK(), "Deletion of file at $source succeeded." );
+ } else {
+ $this->assertEquals( false, $status->isOK(), "Deletion of file at $source failed." );
+ }
+
+ $this->assertEquals( false, $this->backend->fileExists( array( 'src' => $source ) ),
+ "Source file $source does not exist after move." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $source ) );
+ $this->assertEquals( false, $props1['fileExists'],
+ "Source file $source does not exist according to props." );
+ }
+
+ public function provider_testDelete() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont1/myfacefile.txt';
+
+ $op = array( 'op' => 'delete', 'src' => $source );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // with source
+ true // succeeds
+ );
+
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // without source
+ false // fails
+ );
+
+ $op['ignoreMissingSource'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // without source
+ true // succeeds
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testCreate
+ */
+ public function testCreate( $op, $dest, $alreadyExists, $okStatus, $newSize ) {
+ $this->pathsToPrune[] = $dest;
+
+ $oldText = 'blah...blah...waahwaah';
+ if ( $alreadyExists ) {
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => $oldText, 'dst' => $dest ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." );
+ }
+
+ $status = $this->backend->doOperation( $op );
+ if ( $okStatus ) {
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." );
+ } else {
+ $this->assertEquals( false, $status->isOK(), "Creation of file at $dest failed." );
+ }
+
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Dest file $dest exists after creation." );
+
+ $props1 = $this->backend->getFileProps( array( 'src' => $dest ) );
+ $this->assertEquals( true, $props1['fileExists'],
+ "Dest file $dest exists according to props." );
+ if ( $okStatus ) { // file content is what we saved
+ $this->assertEquals( $newSize, $props1['size'],
+ "Dest file $dest has expected size according to props." );
+ } else { // file content is some other previous text
+ $this->assertEquals( strlen( $oldText ), $props1['size'],
+ "Dest file $dest has different size that given text according to props." );
+ }
+ }
+
+ /**
+ * @dataProvider provider_testCreate
+ */
+ public function provider_testCreate() {
+ $cases = array();
+
+ $source = $this->singleBasePath() . '/cont2/myspacefile.txt';
+
+ $dummyText = 'hey hey';
+ $op = array( 'op' => 'create', 'content' => $dummyText, 'dst' => $source );
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ false, // no dest already exists
+ true, // succeeds
+ strlen( $dummyText )
+ );
+
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // dest already exists
+ false, // fails
+ strlen( $dummyText )
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $source, // source
+ true, // dest already exists
+ true, // succeeds
+ strlen( $dummyText )
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testConcatenate
+ */
+ public function testConcatenate( $op, $srcs, $srcsContent, $alreadyExists, $okStatus ) {
+ $this->pathsToPrune = array_merge( $this->pathsToPrune, $srcs );
+ $this->pathsToPrune[] = $op['dst'];
+
+ $expContent = '';
+ // Create sources
+ $ops = array();
+ foreach ( $srcs as $i => $source ) {
+ $ops[] = array(
+ 'op' => 'create', // operation
+ 'dst' => $source, // source
+ 'content' => $srcsContent[$i]
+ );
+ $expContent .= $srcsContent[$i];
+ }
+ $status = $this->backend->doOperations( $ops );
+
+ $this->assertEquals( true, $status->isOK(), "Creation of source files succeeded." );
+
+ $dest = $op['dst'];
+ if ( $alreadyExists ) {
+ $oldText = 'blah...blah...waahwaah';
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => $oldText, 'dst' => $dest ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." );
+ }
+
+ // Combine them
+ $status = $this->backend->doOperation( $op );
+ if ( $okStatus ) {
+ $this->assertEquals( true, $status->isOK(), "Creation of concat file at $dest succeeded." );
+ } else {
+ $this->assertEquals( false, $status->isOK(), "Creation of concat file at $dest failed." );
+ }
+
+ if ( $okStatus ) {
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Dest concat file $dest exists after creation." );
+ } else {
+ $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ),
+ "Dest concat file $dest exists after failed creation." );
+ }
+
+ $tmpFile = $this->backend->getLocalCopy( array( 'src' => $dest ) );
+ $this->assertNotNull( $tmpFile, "Creation of local copy of $dest succeeded." );
+
+ $contents = file_get_contents( $tmpFile->getPath() );
+ $this->assertNotEquals( false, $contents, "Local copy of $dest exists." );
+
+ if ( $okStatus ) {
+ $this->assertEquals( $expContent, $contents, "Concat file at $dest has correct contents." );
+ } else {
+ $this->assertNotEquals( $expContent, $contents, "Concat file at $dest has correct contents." );
+ }
+ }
+
+ function provider_testConcatenate() {
+ $cases = array();
+
+ $dest = $this->singleBasePath() . '/cont1/full_file.txt';
+ $srcs = array(
+ $this->singleBasePath() . '/cont1/file1.txt',
+ $this->singleBasePath() . '/cont1/file2.txt',
+ $this->singleBasePath() . '/cont1/file3.txt',
+ $this->singleBasePath() . '/cont1/file4.txt',
+ $this->singleBasePath() . '/cont1/file5.txt',
+ $this->singleBasePath() . '/cont1/file6.txt',
+ $this->singleBasePath() . '/cont1/file7.txt',
+ $this->singleBasePath() . '/cont1/file8.txt',
+ $this->singleBasePath() . '/cont1/file9.txt',
+ $this->singleBasePath() . '/cont1/file10.txt'
+ );
+ $content = array(
+ 'egfage',
+ 'ageageag',
+ 'rhokohlr',
+ 'shgmslkg',
+ 'kenga',
+ 'owagmal',
+ 'kgmae',
+ 'g eak;g',
+ 'lkaem;a',
+ 'legma'
+ );
+ $op = array( 'op' => 'concatenate', 'srcs' => $srcs, 'dst' => $dest );
+
+ $cases[] = array(
+ $op, // operation
+ $srcs, // sources
+ $content, // content for each source
+ false, // no dest already exists
+ true, // succeeds
+ );
+
+ $cases[] = array(
+ $op, // operation
+ $srcs, // sources
+ $content, // content for each source
+ true, // no dest already exists
+ false, // succeeds
+ );
+
+ $op['overwriteDest'] = true;
+ $cases[] = array(
+ $op, // operation
+ $srcs, // sources
+ $content, // content for each source
+ true, // no dest already exists
+ true, // succeeds
+ );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testGetLocalCopy
+ */
+ public function testGetLocalCopy( $src, $content ) {
+ $this->pathsToPrune[] = $src;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => $content, 'dst' => $src ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $src succeeded." );
+
+ $tmpFile = $this->backend->getLocalCopy( array( 'src' => $src ) );
+ $this->assertNotNull( $tmpFile, "Creation of local copy of $src succeeded." );
+
+ $contents = file_get_contents( $tmpFile->getPath() );
+ $this->assertNotEquals( false, $contents, "Local copy of $src exists." );
+ }
+
+ function provider_testGetLocalCopy() {
+ $cases = array();
+
+ $base = $this->singleBasePath();
+ $cases[] = array( "$base/cont1/a/z/some_file.txt", "some file contents" );
+ $cases[] = array( "$base/cont1/a/some-other_file.txt", "more file contents" );
+
+ return $cases;
+ }
+
+ /**
+ * @dataProvider provider_testGetReference
+ */
+ public function testGetLocalReference( $src, $content ) {
+ $this->pathsToPrune[] = $src;
+
+ $status = $this->backend->doOperation(
+ array( 'op' => 'create', 'content' => $content, 'dst' => $src ) );
+ $this->assertEquals( true, $status->isOK(), "Creation of file at $src succeeded." );
+
+ $tmpFile = $this->backend->getLocalReference( array( 'src' => $src ) );
+ $this->assertNotNull( $tmpFile, "Creation of local copy of $src succeeded." );
+
+ $contents = file_get_contents( $tmpFile->getPath() );
+ $this->assertNotEquals( false, $contents, "Local copy of $src exists." );
+ }
+
+ function provider_testGetReference() {
+ $cases = array();
+
+ $base = $this->singleBasePath();
+ $cases[] = array( "$base/cont1/a/z/some_file.txt", "some file contents" );
+ $cases[] = array( "$base/cont1/a/some-other_file.txt", "more file contents" );
+
+ return $cases;
+ }
+
+ // @TODO: testPrepare
+
+ // @TODO: testSecure
+
+ // @TODO: testClean
+
+ // @TODO: testDoOperations
+
+ public function testGetFileList() {
+ $base = $this->singleBasePath();
+ $files = array(
+ "$base/cont1/test1.txt",
+ "$base/cont1/test2.txt",
+ "$base/cont1/test3.txt",
+ "$base/cont1/subdir1/test1.txt",
+ "$base/cont1/subdir1/test2.txt",
+ "$base/cont1/subdir2/test3.txt",
+ "$base/cont1/subdir2/test4.txt",
+ "$base/cont1/subdir2/subdir/test1.txt",
+ "$base/cont1/subdir2/subdir/test2.txt",
+ "$base/cont1/subdir2/subdir/test3.txt",
+ "$base/cont1/subdir2/subdir/test4.txt",
+ "$base/cont1/subdir2/subdir/test5.txt",
+ "$base/cont1/subdir2/subdir/sub/test0.txt",
+ "$base/cont1/subdir2/subdir/sub/120-px-file.txt",
+ );
+ $this->pathsToPrune = array_merge( $this->pathsToPrune, $files );
+
+ // Add the files
+ $ops = array();
+ foreach ( $files as $file ) {
+ $ops[] = array( 'op' => 'create', 'content' => 'xxy', 'dst' => $file );
+ }
+ $status = $this->backend->doOperations( $ops );
+ $this->assertEquals( true, $status->isOK(), "Creation of files succeeded." );
+
+ // Expected listing
+ $expected = array(
+ "test1.txt",
+ "test2.txt",
+ "test3.txt",
+ "subdir1/test1.txt",
+ "subdir1/test2.txt",
+ "subdir2/test3.txt",
+ "subdir2/test1.txt",
+ "subdir2/subdir/test1.txt",
+ "subdir2/subdir/test2.txt",
+ "subdir2/subdir/test3.txt",
+ "subdir2/subdir/test4.txt",
+ "subdir2/subdir/test5.txt",
+ "subdir2/subdir/sub/test0.txt",
+ "subdir2/subdir/sub/120-px-file.txt",
+ );
+ $expected = sort( $expected );
+
+ // Actual listing (no trailing slash)
+ $list = array();
+ $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1" ) );
+ foreach ( $iter as $file ) {
+ $list[] = $file;
+ }
+
+ $this->assertEquals( $expected, sort( $list ), "Correct file listing." );
+
+ // Actual listing (with trailing slash)
+ $list = array();
+ $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1/" ) );
+ foreach ( $iter as $file ) {
+ $list[] = $file;
+ }
+
+ $this->assertEquals( $expected, sort( $list ), "Correct file listing." );
+
+ foreach ( $files as $file ) {
+ $this->backend->delete( array( 'src' => "$base/$files" ) );
+ }
+
+ $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1/not/exists" ) );
+ foreach ( $iter as $iter ) {} // no errors
+ }
+
+ function tearDown() {
+ parent::tearDown();
+ foreach ( $this->filesToPrune as $file ) {
+ @unlink( $file );
+ }
+ foreach ( $this->pathsToPrune as $file ) {
+ $this->backend->doOperation( array( 'op' => 'delete', 'src' => $file ) );
+ $this->multiBackend->doOperation( array( 'op' => 'delete', 'src' => $file ) );
+ }
+ $this->backend = $this->multiBackend = null;
+ $this->filesToPrune = $this->pathsToPrune = array();
+ }
+}
function setUp() {
parent::setUp();
- $this->filePath = dirname( __FILE__ ) . '/../../data/media/';
$this->handler = new BitmapHandler();
- $this->repo = new FSRepo(array(
- 'name' => 'temp',
- 'directory' => wfTempDir() . '/exif-test-' . time() . '-' . mt_rand(),
- 'url' => 'http://localhost/thumbtest'
- ));
+ $filePath = dirname( __FILE__ ) . '/../../data/media';
+ $tmpDir = wfTempDir() . '/exif-test-' . time() . '-' . mt_rand();
+ $this->backend = new FSFileBackend( array(
+ 'name' => 'localtesting',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array( 'images-thumb' => $tmpDir, 'data' => $filePath )
+ ) );
+ $this->repo = new FSRepo( array(
+ 'name' => 'temp',
+ 'url' => 'http://localhost/thumbtest',
+ 'backend' => $this->backend
+ ) );
if ( !wfDl( 'exif' ) ) {
$this->markTestSkipped( "This test needs the exif extension." );
}
if ( !BitmapHandler::canRotate() ) {
$this->markTestSkipped( "This test needs a rasterizer that can auto-rotate." );
}
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $name, $type );
+ $file = $this->dataFile( $name, $type );
$this->assertEquals( $info['width'], $file->getWidth(), "$name: width check" );
$this->assertEquals( $info['height'], $file->getHeight(), "$name: height check" );
}
throw new MWException('bogus test data format ' . $size);
}
- $file = $this->localFile( $name, $type );
- $thumb = $file->transform( $params, File::RENDER_NOW );
+ $file = $this->dataFile( $name, $type );
+ $thumb = $file->transform( $params, File::RENDER_NOW | File::RENDER_FORCE );
$this->assertEquals( $out[0], $thumb->getWidth(), "$name: thumb reported width check for $size" );
$this->assertEquals( $out[1], $thumb->getHeight(), "$name: thumb reported height check for $size" );
- $gis = getimagesize( $thumb->getPath() );
+ $gis = getimagesize( $thumb->getLocalCopyPath() );
if ($out[0] > $info['width']) {
// Physical image won't be scaled bigger than the original.
$this->assertEquals( $info['width'], $gis[0], "$name: thumb actual width check for $size");
}
}
- private function localFile( $name, $type ) {
- return new UnregisteredLocalFile( false, $this->repo, $this->filePath . $name, $type );
+ private function dataFile( $name, $type ) {
+ return new UnregisteredLocalFile( false, $this->repo,
+ "mwstore://localtesting/data/$name", $type );
}
function providerFiles() {
global $wgEnableAutoRotation;
$wgEnableAutoRotation = false;
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $name, $type );
+ $file = $this->dataFile( $name, $type );
$this->assertEquals( $info['width'], $file->getWidth(), "$name: width check" );
$this->assertEquals( $info['height'], $file->getHeight(), "$name: height check" );
throw new MWException('bogus test data format ' . $size);
}
- $file = $this->localFile( $name, $type );
- $thumb = $file->transform( $params, File::RENDER_NOW );
+ $file = $this->dataFile( $name, $type );
+ $thumb = $file->transform( $params, File::RENDER_NOW | File::RENDER_FORCE );
$this->assertEquals( $out[0], $thumb->getWidth(), "$name: thumb reported width check for $size" );
$this->assertEquals( $out[1], $thumb->getHeight(), "$name: thumb reported height check for $size" );
- $gis = getimagesize( $thumb->getPath() );
+ $gis = getimagesize( $thumb->getLocalCopyPath() );
if ($out[0] > $info['width']) {
// Physical image won't be scaled bigger than the original.
$this->assertEquals( $info['width'], $gis[0], "$name: thumb actual width check for $size");
array(
270,
array( self::TEST_HEIGHT, self::TEST_WIDTH )
- ),
+ ),
);
}
}
if ( !wfDl( 'exif' ) ) {
$this->markTestSkipped( "This test needs the exif extension." );
}
+ $filePath = dirname( __FILE__ ) . '/../../data/media';
+ $this->backend = new FSFileBackend( array(
+ 'name' => 'localtesting',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array( 'data' => $filePath )
+ ) );
+ $this->repo = new FSRepo( array(
+ 'name' => 'temp',
+ 'url' => 'http://localhost/thumbtest',
+ 'backend' => $this->backend
+ ) );
global $wgShowEXIF;
$this->show = $wgShowEXIF;
$wgShowEXIF = true;
}
public function testInvalidDate() {
- $file = UnregisteredLocalFile::newFromPath( dirname( __FILE__ ) .
- '/../../data/media/broken_exif_date.jpg', 'image/jpeg' );
+ $file = $this->dataFile( 'broken_exif_date.jpg', 'image/jpeg' );
// Throws an error if bug hit
$meta = $file->formatMetadata();
$meta['visible'][$dateIndex]['value'],
'File with invalid date metadata (bug 29471)' );
}
+
+ private function dataFile( $name, $type ) {
+ return new UnregisteredLocalFile( false, $this->repo,
+ "mwstore://localtesting/data/$name", $type );
+ }
}
class GIFHandlerTest extends MediaWikiTestCase {
public function setUp() {
- $this->filePath = dirname( __FILE__ ) . '/../../data/media/';
+ $this->filePath = dirname( __FILE__ ) . '/../../data/media';
+ $this->backend = new FSFileBackend( array(
+ 'name' => 'localtesting',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array( 'data' => $this->filePath )
+ ) );
+ $this->repo = new FSRepo( array(
+ 'name' => 'temp',
+ 'url' => 'http://localhost/thumbtest',
+ 'backend' => $this->backend
+ ) );
$this->handler = new GIFHandler();
}
public function testInvalidFile() {
- $res = $this->handler->getMetadata( null, $this->filePath . 'README' );
+ $res = $this->handler->getMetadata( null, $this->filePath . '/README' );
$this->assertEquals( GIFHandler::BROKEN_FILE, $res );
}
/**
* @dataProvider dataIsAnimated
*/
public function testIsAnimanted( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/gif' );
+ $file = $this->dataFile( $filename, 'image/gif' );
$actual = $this->handler->isAnimatedImage( $file );
$this->assertEquals( $expected, $actual );
}
* @dataProvider dataGetImageArea
*/
public function testGetImageArea( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/gif' );
+ $file = $this->dataFile( $filename, 'image/gif' );
$actual = $this->handler->getImageArea( $file, $file->getWidth(), $file->getHeight() );
$this->assertEquals( $expected, $actual );
}
* @dataProvider dataGetMetadata
*/
public function testGetMetadata( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/gif' );
- $actual = $this->handler->getMetadata( $file, $this->filePath . $filename );
+ $file = $this->dataFile( $filename, 'image/gif' );
+ $actual = $this->handler->getMetadata( $file, "$this->filePath/$filename" );
$this->assertEquals( unserialize( $expected ), unserialize( $actual ) );
}
+
public function dataGetMetadata() {
return array(
array( 'nonanimated.gif', 'a:4:{s:10:"frameCount";i:1;s:6:"looped";b:0;s:8:"duration";d:0.1000000000000000055511151231257827021181583404541015625;s:8:"metadata";a:2:{s:14:"GIFFileComment";a:1:{i:0;s:35:"GIF test file ⁕ Created with GIMP";}s:15:"_MW_GIF_VERSION";i:1;}}' ),
array( 'animated-xmp.gif', 'a:4:{s:10:"frameCount";i:4;s:6:"looped";b:1;s:8:"duration";d:2.399999999999999911182158029987476766109466552734375;s:8:"metadata";a:5:{s:6:"Artist";s:7:"Bawolff";s:16:"ImageDescription";a:2:{s:9:"x-default";s:18:"A file to test GIF";s:5:"_type";s:4:"lang";}s:15:"SublocationDest";s:13:"The interwebs";s:14:"GIFFileComment";a:1:{i:0;s:16:"GIƒ·test·file";}s:15:"_MW_GIF_VERSION";i:1;}}' ),
);
}
+
+ private function dataFile( $name, $type ) {
+ return new UnregisteredLocalFile( false, $this->repo,
+ "mwstore://localtesting/data/$name", $type );
+ }
}
class PNGHandlerTest extends MediaWikiTestCase {
public function setUp() {
- $this->filePath = dirname( __FILE__ ) . '/../../data/media/';
+ $this->filePath = dirname( __FILE__ ) . '/../../data/media';
+ $this->backend = new FSFileBackend( array(
+ 'name' => 'localtesting',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array( 'data' => $this->filePath )
+ ) );
+ $this->repo = new FSRepo( array(
+ 'name' => 'temp',
+ 'url' => 'http://localhost/thumbtest',
+ 'backend' => $this->backend
+ ) );
$this->handler = new PNGHandler();
}
public function testInvalidFile() {
- $res = $this->handler->getMetadata( null, $this->filePath . 'README' );
+ $res = $this->handler->getMetadata( null, $this->filePath . '/README' );
$this->assertEquals( PNGHandler::BROKEN_FILE, $res );
}
/**
* @dataProvider dataIsAnimated
*/
public function testIsAnimanted( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/png' );
+ $file = $this->dataFile( $filename, 'image/png' );
$actual = $this->handler->isAnimatedImage( $file );
$this->assertEquals( $expected, $actual );
}
* @dataProvider dataGetImageArea
*/
public function testGetImageArea( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/png' );
+ $file = $this->dataFile($filename, 'image/png' );
$actual = $this->handler->getImageArea( $file, $file->getWidth(), $file->getHeight() );
$this->assertEquals( $expected, $actual );
}
* @dataProvider dataGetMetadata
*/
public function testGetMetadata( $filename, $expected ) {
- $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename,
- 'image/png' );
- $actual = $this->handler->getMetadata( $file, $this->filePath . $filename );
+ $file = $this->dataFile( $filename, 'image/png' );
+ $actual = $this->handler->getMetadata( $file, "$this->filePath/$filename" );
// $this->assertEquals( unserialize( $expected ), unserialize( $actual ) );
$this->assertEquals( ( $expected ), ( $actual ) );
}
array( 'xmp.png', 'a:6:{s:10:"frameCount";i:0;s:9:"loopCount";i:1;s:8:"duration";d:0;s:8:"bitDepth";i:1;s:9:"colorType";s:14:"index-coloured";s:8:"metadata";a:2:{s:12:"SerialNumber";s:9:"123456789";s:15:"_MW_PNG_VERSION";i:1;}}' ),
);
}
+
+ private function dataFile( $name, $type ) {
+ return new UnregisteredLocalFile( false, $this->repo,
+ "mwstore://localtesting/data/$name", $type );
+ }
}
protected $file = false;
- /*function __construct($a = null,$b = array(),$c = null ) {
- parent::__construct($a,$b,$c);
- }*/
-
function setUp() {
global $wgContLang, $wgNamespaceProtection, $wgNamespaceAliases;
global $wgHooks, $IP;
$tmpGlobals['wgStylePath'] = '/skins';
$tmpGlobals['wgThumbnailScriptPath'] = false;
$tmpGlobals['wgLocalFileRepo'] = array(
- 'class' => 'LocalRepo',
- 'name' => 'local',
- 'directory' => wfTempDir() . '/test-repo',
- 'url' => 'http://example.com/images',
- 'deletedDir' => wfTempDir() . '/test-repo/delete',
- 'hashLevels' => 2,
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'url' => 'http://example.com/images',
+ 'hashLevels' => 2,
'transformVia404' => false,
+ 'backend' => 'local-backend'
);
-
+ $tmpGlobals['wgForeignFileRepos'] = array();
$tmpGlobals['wgEnableParserCache'] = false;
$tmpGlobals['wgHooks'] = $wgHooks;
$tmpGlobals['wgDeferredUpdateList'] = array();
$wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface';
$wgNamespaceAliases['Image'] = NS_FILE;
$wgNamespaceAliases['Image_talk'] = NS_FILE_TALK;
-
}
public function tearDown() {
$wgNamespaceProtection[NS_MEDIAWIKI] = $this->savedWeirdGlobals['mw_namespace_protection'];
$wgNamespaceAliases['Image'] = $this->savedWeirdGlobals['image_alias'];
$wgNamespaceAliases['Image_talk'] = $this->savedWeirdGlobals['image_talk_alias'];
+
+ // Restore backends
+ FileBackendGroup::destroySingleton();
+ FileBackendGroup::singleton()->register( $GLOBALS['wgFileBackends'] );
+ RepoGroup::destroySingleton();
}
function addDBData() {
+ $this->tablesUsed[] = 'image';
# Hack: insert a few Wikipedia in-project interwiki prefixes,
# for testing inter-language links
$this->db->insert( 'interwiki', array(
'wgExtensionAssetsPath' => '/extensions',
'wgActionPaths' => array(),
'wgLocalFileRepo' => array(
- 'class' => 'LocalRepo',
- 'name' => 'local',
- 'directory' => $this->uploadDir,
- 'url' => 'http://example.com/images',
- 'hashLevels' => 2,
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'url' => 'http://example.com/images',
+ 'hashLevels' => 2,
'transformVia404' => false,
+ 'backend' => 'local-backend'
),
'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ),
'wgStylePath' => '/skins',
$GLOBALS['wgOut'] = $context->getOutput();
$GLOBALS['wgUser'] = $context->getUser();
+ FileBackendGroup::destroySingleton(); // reset
+ $backend = array(
+ 'name' => 'local-backend',
+ 'class' => 'FSFileBackend',
+ 'lockManager' => 'nullLockManager',
+ 'containerPaths' => array(
+ 'images-public' => $this->uploadDir,
+ 'images-thumb' => $this->uploadDir . '/thumb' )
+ );
+ FileBackendGroup::singleton()->register( array( $backend ) );
+
global $wgHooks;
$wgHooks['ParserTestParser'][] = 'ParserTestParserHook::setup';
$wgHooks['ParserGetVariableValueTs'][] = 'ParserTest::getFakeTimestamp';
MagicWord::clearCache();
+ RepoGroup::destroySingleton();
# Publish the articles after we have the final language set
$this->publishTestArticles();
* after each test runs.
*/
protected function teardownGlobals() {
- RepoGroup::destroySingleton();
- LinkCache::singleton()->clear();
-
foreach ( $this->savedGlobals as $var => $val ) {
$GLOBALS[$var] = $val;
}
+ RepoGroup::destroySingleton();
+ LinkCache::singleton()->clear();
+ FileBackendGroup::destroySingleton();
+
$this->teardownUploadDir( $this->uploadDir );
}
$wgStyleSheetPath = '/skins';
$wgStylePath = '/skins';
$wgThumbnailScriptPath = false;
+ $backend = new FSFileBackend( array(
+ 'name' => 'local-backend',
+ 'lockManager' => 'fsLockManager',
+ 'containerPaths' => array(
+ 'images-public' => wfTempDir() . '/test-repo/public',
+ 'images-thumb' => wfTempDir() . '/test-repo/thumb',
+ 'images-temp' => wfTempDir() . '/test-repo/temp',
+ 'images-deleted' => wfTempDir() . '/test-repo/delete',
+ )
+ ) );
$wgLocalFileRepo = array(
- 'class' => 'LocalRepo',
- 'name' => 'local',
- 'directory' => wfTempDir() . '/test-repo',
- 'url' => 'http://example.com/images',
- 'deletedDir' => wfTempDir() . '/test-repo/delete',
- 'hashLevels' => 2,
+ 'class' => 'LocalRepo',
+ 'name' => 'local',
+ 'url' => 'http://example.com/images',
+ 'hashLevels' => 2,
'transformVia404' => false,
+ 'backend' => $backend
);
$wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface';
$wgNamespaceAliases['Image'] = NS_FILE;
$headers[] = 'Vary: Cookie';
}
+ // Check the source file storage path
if ( !$img ) {
wfThumbError( 404, wfMsg( 'badtitletext' ) );
wfProfileOut( __METHOD__ );
// Calculate time
wfSuppressWarnings();
$imsUnix = strtotime( $imsString );
- $stat = stat( $sourcePath );
wfRestoreWarnings();
- if ( $stat['mtime'] <= $imsUnix ) {
+ $sourceTsUnix = wfTimestamp( TS_UNIX, $img->getTimestamp() );
+ if ( $sourceTsUnix <= $imsUnix ) {
header( 'HTTP/1.1 304 Not Modified' );
wfProfileOut( __METHOD__ );
return;
// Stream the file if it exists already...
try {
$thumbName = $img->thumbName( $params );
- if ( $thumbName !== false ) { // valid params?
+ if ( strlen( $thumbName ) ) { // valid params?
$thumbPath = $img->getThumbPath( $thumbName );
- if ( is_file( $thumbPath ) ) {
- StreamFile::stream( $thumbPath, $headers );
+ if ( $img->getRepo()->fileExists( $thumbPath ) ) {
+ $img->getRepo()->streamFile( $thumbPath, $headers );
wfProfileOut( __METHOD__ );
return;
}
// Thumbnail isn't already there, so create the new thumbnail...
try {
$thumb = $img->transform( $params, File::RENDER_NOW );
- } catch( Exception $ex ) {
+ } catch ( Exception $ex ) {
// Tried to select a page on a non-paged file?
$thumb = false;
}
$errorMsg = wfMsgHtml( 'thumbnail_error', 'File::transform() returned false' );
} elseif ( $thumb->isError() ) {
$errorMsg = $thumb->getHtmlMsg();
- } elseif ( !$thumb->getPath() ) {
+ } elseif ( !$thumb->hasFile() ) {
$errorMsg = wfMsgHtml( 'thumbnail_error', 'No path supplied in thumbnail object' );
- } elseif ( $thumb->getPath() == $img->getPath() ) {
- $errorMsg = wfMsgHtml( 'thumbnail_error', 'Image was not scaled, ' .
- 'is the requested width bigger than the source?' );
+ } elseif ( $thumb->fileIsSource() ) {
+ $errorMsg = wfMsgHtml( 'thumbnail_error',
+ 'Image was not scaled, is the requested width bigger than the source?' );
}
if ( $errorMsg !== false ) {
wfThumbError( 500, $errorMsg );
} else {
// Stream the file if there were no errors
- StreamFile::stream( $thumb->getPath(), $headers );
+ $thumb->streamFile( $headers );
}
wfProfileOut( __METHOD__ );
EOT;
}
-
dépôts / lhc / web / wiklou.git / commitdiff