/** @var Array Prioritized list of FileBackendStore objects */
protected $backends = array(); // array of (backend index => backends)
protected $masterIndex = -1; // integer; index of master backend
- protected $syncChecks = 0; // integer bitfield
+ protected $syncChecks = 0; // integer; bitfield
+ protected $autoResync = false; // boolean
+
+ /** @var Array */
+ protected $noPushDirConts = array();
+ protected $noPushQuickOps = false; // boolean
/* Possible internal backend consistency checks */
const CHECK_SIZE = 1;
* Locking, journaling, and read-only checks are handled by the proxy backend.
*
* Additional $config params include:
- * - backends : Array of backend config and multi-backend settings.
- * Each value is the config used in the constructor of a
- * FileBackendStore class, but with these additional settings:
- * - class : The name of the backend class
- * - isMultiMaster : This must be set for one backend.
- * - template: : If given a backend name, this will use
- * the config of that backend as a template.
- * Values specified here take precedence.
- * - syncChecks : Integer bitfield of internal backend sync checks to perform.
- * Possible bits include the FileBackendMultiWrite::CHECK_* constants.
- * There are constants for SIZE, TIME, and SHA1.
- * The checks are done before allowing any file operations.
+ * - backends : Array of backend config and multi-backend settings.
+ * Each value is the config used in the constructor of a
+ * FileBackendStore class, but with these additional settings:
+ * - class : The name of the backend class
+ * - isMultiMaster : This must be set for one backend.
+ * - template: : If given a backend name, this will use
+ * the config of that backend as a template.
+ * Values specified here take precedence.
+ * - syncChecks : Integer bitfield of internal backend sync checks to perform.
+ * Possible bits include the FileBackendMultiWrite::CHECK_* constants.
+ * There are constants for SIZE, TIME, and SHA1.
+ * The checks are done before allowing any file operations.
+ * - autoResync : Automatically resync the clone backends to the master backend
+ * when pre-operation sync checks fail. This should only be used
+ * if the master backend is stable and not missing any files.
+ * - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend.
+ * - noPushDirConts : (hack) Only apply directory functions to the master backend.
+ *
* @param $config Array
* @throws MWException
*/
public function __construct( array $config ) {
parent::__construct( $config );
- $namesUsed = array();
+ $this->syncChecks = isset( $config['syncChecks'] )
+ ? $config['syncChecks']
+ : self::CHECK_SIZE;
+ $this->autoResync = !empty( $config['autoResync'] );
+ $this->noPushQuickOps = isset( $config['noPushQuickOps'] )
+ ? $config['noPushQuickOps']
+ : false;
+ $this->noPushDirConts = isset( $config['noPushDirConts'] )
+ ? $config['noPushDirConts']
+ : array();
// Construct backends here rather than via registration
// to keep these backends hidden from outside the proxy.
+ $namesUsed = array();
foreach ( $config['backends'] as $index => $config ) {
if ( isset( $config['template'] ) ) {
// Config is just a modified version of a registered backend's.
if ( $this->masterIndex < 0 ) { // need backends and must have a master
throw new MWException( 'No master backend defined.' );
}
- $this->syncChecks = isset( $config['syncChecks'] )
- ? $config['syncChecks']
- : self::CHECK_SIZE;
}
/**
}
// Clear any cache entries (after locks acquired)
$this->clearCache();
- // Do a consistency check to see if the backends agree
- $status->merge( $this->consistencyCheck( $this->fileStoragePathsForOps( $ops ) ) );
+ $opts['preserveCache'] = true; // only locked files are cached
+ // Get the list of paths to read/write...
+ $relevantPaths = $this->fileStoragePathsForOps( $ops );
+ // Check if the paths are valid and accessible on all backends...
+ $status->merge( $this->accessibilityCheck( $relevantPaths ) );
if ( !$status->isOK() ) {
return $status; // abort
}
+ // Do a consistency check to see if the backends are consistent...
+ $syncStatus = $this->consistencyCheck( $relevantPaths );
+ if ( !$syncStatus->isOK() ) {
+ wfDebugLog( 'FileOperation', get_class( $this ) .
+ " failed sync check: " . FormatJson::encode( $relevantPaths ) );
+ // Try to resync the clone backends to the master on the spot...
+ if ( !$this->autoResync || !$this->resyncFiles( $relevantPaths )->isOK() ) {
+ $status->merge( $syncStatus );
+ return $status; // abort
+ }
+ }
// Actually attempt the operation batch on the master backend...
$masterStatus = $mbe->doOperations( $realOps, $opts );
$status->merge( $masterStatus );
- // Propagate the operations to the clone backends...
- foreach ( $this->backends as $index => $backend ) {
- if ( $index !== $this->masterIndex ) { // not done already
- $realOps = $this->substOpBatchPaths( $ops, $backend );
- $status->merge( $backend->doOperations( $realOps, $opts ) );
+ // Propagate the operations to the clone backends if there were no unexpected errors
+ // and if there were either no expected errors or if the 'force' option was used.
+ // However, if nothing succeeded at all, then don't replicate any of the operations.
+ // If $ops only had one operation, this might avoid backend sync inconsistencies.
+ if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) {
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $index !== $this->masterIndex ) { // not done already
+ $realOps = $this->substOpBatchPaths( $ops, $backend );
+ $status->merge( $backend->doOperations( $realOps, $opts ) );
+ }
}
}
// Make 'success', 'successCount', and 'failCount' fields reflect
/**
* Check that a set of files are consistent across all internal backends
*
- * @param $paths Array
+ * @param $paths Array List of storage paths
* @return Status
*/
public function consistencyCheck( array $paths ) {
}
$mBackend = $this->backends[$this->masterIndex];
- foreach ( array_unique( $paths ) as $path ) {
+ foreach ( $paths as $path ) {
$params = array( 'src' => $path, 'latest' => true );
$mParams = $this->substOpPaths( $params, $mBackend );
// Stat the file on the 'master' backend
$mStat = $mBackend->getFileStat( $mParams );
if ( $this->syncChecks & self::CHECK_SHA1 ) {
- $mSha1 = $mBackend->getFileSha1( $mParams );
+ $mSha1 = $mBackend->getFileSha1Base36( $mParams );
} else {
$mSha1 = false;
}
- $mUsable = $mBackend->isPathUsableInternal( $mParams['src'] );
- // Check of all clone backends agree with the master...
+ // Check if all clone backends agree with the master...
foreach ( $this->backends as $index => $cBackend ) {
if ( $index === $this->masterIndex ) {
continue; // master
}
}
if ( $this->syncChecks & self::CHECK_SHA1 ) {
- if ( $cBackend->getFileSha1( $cParams ) !== $mSha1 ) { // wrong SHA1
+ if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1
$status->fatal( 'backend-fail-synced', $path );
continue;
}
$status->fatal( 'backend-fail-synced', $path );
}
}
- if ( $mUsable !== $cBackend->isPathUsableInternal( $cParams['src'] ) ) {
- $status->fatal( 'backend-fail-synced', $path );
+ }
+ }
+
+ return $status;
+ }
+
+ /**
+ * Check that a set of file paths are usable across all internal backends
+ *
+ * @param $paths Array List of storage paths
+ * @return Status
+ */
+ public function accessibilityCheck( array $paths ) {
+ $status = Status::newGood();
+ if ( count( $this->backends ) <= 1 ) {
+ return $status; // skip checks
+ }
+
+ foreach ( $paths as $path ) {
+ foreach ( $this->backends as $backend ) {
+ $realPath = $this->substPaths( $path, $backend );
+ if ( !$backend->isPathUsableInternal( $realPath ) ) {
+ $status->fatal( 'backend-fail-usable', $path );
+ }
+ }
+ }
+
+ return $status;
+ }
+
+ /**
+ * Check that a set of files are consistent across all internal backends
+ * and re-synchronize those files againt the "multi master" if needed.
+ *
+ * @param $paths Array List of storage paths
+ * @return Status
+ */
+ public function resyncFiles( array $paths ) {
+ $status = Status::newGood();
+
+ $mBackend = $this->backends[$this->masterIndex];
+ foreach ( $paths as $path ) {
+ $mPath = $this->substPaths( $path, $mBackend );
+ $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) );
+ $mExist = $mBackend->fileExists( array( 'src' => $mPath ) );
+ // Check if the master backend is available...
+ if ( $mExist === null ) {
+ $status->fatal( 'backend-fail-internal', $this->name );
+ }
+ // Check of all clone backends agree with the master...
+ foreach ( $this->backends as $index => $cBackend ) {
+ if ( $index === $this->masterIndex ) {
+ continue; // master
+ }
+ $cPath = $this->substPaths( $path, $cBackend );
+ $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath ) );
+ if ( $mSha1 === $cSha1 ) {
+ // already synced; nothing to do
+ } elseif ( $mSha1 ) { // file is in master
+ $fsFile = $mBackend->getLocalReference( array( 'src' => $mPath ) );
+ $status->merge( $cBackend->quickStore(
+ array( 'src' => $fsFile->getPath(), 'dst' => $cPath )
+ ) );
+ } elseif ( $mExist === false ) { // file is not in master
+ $status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) );
}
}
}
$paths[] = $op['dst'];
}
}
- return array_unique( $paths );
+ return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) );
}
/**
$masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
$status->merge( $masterStatus );
// Propagate the operations to the clone backends...
- foreach ( $this->backends as $index => $backend ) {
- if ( $index !== $this->masterIndex ) { // not done already
- $realOps = $this->substOpBatchPaths( $ops, $backend );
- $status->merge( $backend->doQuickOperations( $realOps ) );
+ if ( !$this->noPushQuickOps ) {
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $index !== $this->masterIndex ) { // not done already
+ $realOps = $this->substOpBatchPaths( $ops, $backend );
+ $status->merge( $backend->doQuickOperations( $realOps ) );
+ }
}
}
// Make 'success', 'successCount', and 'failCount' fields reflect
return $status;
}
+ /**
+ * @param $path string Storage path
+ * @return bool Path container should have dir changes pushed to all backends
+ */
+ protected function replicateContainerDirChanges( $path ) {
+ list( $b, $shortCont, $r ) = self::splitStoragePath( $path );
+ return !in_array( $shortCont, $this->noPushDirConts );
+ }
+
/**
* @see FileBackend::doPrepare()
* @return Status
*/
protected function doPrepare( array $params ) {
$status = Status::newGood();
- foreach ( $this->backends as $backend ) {
- $realParams = $this->substOpPaths( $params, $backend );
- $status->merge( $backend->doPrepare( $realParams ) );
+ $replicate = $this->replicateContainerDirChanges( $params['dir'] );
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $replicate || $index == $this->masterIndex ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->doPrepare( $realParams ) );
+ }
}
return $status;
}
*/
protected function doSecure( array $params ) {
$status = Status::newGood();
- foreach ( $this->backends as $backend ) {
- $realParams = $this->substOpPaths( $params, $backend );
- $status->merge( $backend->doSecure( $realParams ) );
+ $replicate = $this->replicateContainerDirChanges( $params['dir'] );
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $replicate || $index == $this->masterIndex ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->doSecure( $realParams ) );
+ }
}
return $status;
}
*/
protected function doPublish( array $params ) {
$status = Status::newGood();
- foreach ( $this->backends as $backend ) {
- $realParams = $this->substOpPaths( $params, $backend );
- $status->merge( $backend->doPublish( $realParams ) );
+ $replicate = $this->replicateContainerDirChanges( $params['dir'] );
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $replicate || $index == $this->masterIndex ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->doPublish( $realParams ) );
+ }
}
return $status;
}
*/
protected function doClean( array $params ) {
$status = Status::newGood();
- foreach ( $this->backends as $backend ) {
- $realParams = $this->substOpPaths( $params, $backend );
- $status->merge( $backend->doClean( $realParams ) );
+ $replicate = $this->replicateContainerDirChanges( $params['dir'] );
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $replicate || $index == $this->masterIndex ) {
+ $realParams = $this->substOpPaths( $params, $backend );
+ $status->merge( $backend->doClean( $realParams ) );
+ }
}
return $status;
}
/**
* @see FileBackend::fileExists()
* @param $params array
+ * @return bool|null
*/
public function fileExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
}
/**
- * @see FileBackend::getFileContents()
+ * @see FileBackend::getFileContentsMulti()
* @param $params array
* @return bool|string
*/
- public function getFileContents( array $params ) {
+ public function getFileContentsMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getFileContents( $realParams );
+ $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams );
+
+ $contents = array(); // (path => FSFile) mapping using the proxy backend's name
+ foreach ( $contentsM as $path => $data ) {
+ $contents[$this->unsubstPaths( $path )] = $data;
+ }
+ return $contents;
}
/**
}
/**
- * @see FileBackend::getLocalReference()
+ * @see FileBackend::getLocalReferenceMulti()
* @param $params array
* @return FSFile|null
*/
- public function getLocalReference( array $params ) {
+ public function getLocalReferenceMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
+ $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams );
+
+ $fsFiles = array(); // (path => FSFile) mapping using the proxy backend's name
+ foreach ( $fsFilesM as $path => $fsFile ) {
+ $fsFiles[$this->unsubstPaths( $path )] = $fsFile;
+ }
+ return $fsFiles;
}
/**
- * @see FileBackend::getLocalCopy()
+ * @see FileBackend::getLocalCopyMulti()
* @param $params array
* @return null|TempFSFile
*/
- public function getLocalCopy( array $params ) {
+ public function getLocalCopyMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
- return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
+ $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams );
+
+ $tempFiles = array(); // (path => TempFSFile) mapping using the proxy backend's name
+ foreach ( $tempFilesM as $path => $tempFile ) {
+ $tempFiles[$this->unsubstPaths( $path )] = $tempFile;
+ }
+ return $tempFiles;
}
/**
dépôts / lhc / web / wiklou.git / blobdiff