protected function doConcatenate( array $params ) {
$status = Status::newGood();
$tmpPath = $params['dst']; // convenience
+ unset( $params['latest'] ); // sanity
// Check that the specified temp file is valid...
wfSuppressWarnings();
- $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
+ $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
wfRestoreWarnings();
if ( !$ok ) { // not present or not empty
$status->fatal( 'backend-fail-opentemp', $tmpPath );
return $status;
}
- // Build up the temp file using the source chunks (in order)...
+ // Get local FS versions of the chunks needed for the concatenation...
+ $fsFiles = $this->getLocalReferenceMulti( $params );
+ foreach ( $fsFiles as $path => &$fsFile ) {
+ if ( !$fsFile ) { // chunk failed to download?
+ $fsFile = $this->getLocalReference( array( 'src' => $path ) );
+ if ( !$fsFile ) { // retry failed?
+ $status->fatal( 'backend-fail-read', $path );
+ return $status;
+ }
+ }
+ }
+ unset( $fsFile ); // unset reference so we can reuse $fsFile
+
+ // Get a handle for the destination temp file
$tmpHandle = fopen( $tmpPath, 'ab' );
if ( $tmpHandle === false ) {
$status->fatal( 'backend-fail-opentemp', $tmpPath );
return $status;
}
- foreach ( $params['srcs'] as $virtualSource ) {
- // Get a local FS version of the chunk
- $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
- if ( !$tmpFile ) {
- $status->fatal( 'backend-fail-read', $virtualSource );
- return $status;
- }
+
+ // Build up the temp file using the source chunks (in order)...
+ foreach ( $fsFiles as $virtualSource => $fsFile ) {
// Get a handle to the local FS version
- $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
+ $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
if ( $sourceHandle === false ) {
fclose( $tmpHandle );
$status->fatal( 'backend-fail-read', $virtualSource );
abstract protected function doGetFileStat( array $params );
/**
- * @see FileBackend::getFileContents()
- * @return bool|string
+ * @see FileBackend::getFileContentsMulti()
+ * @return Array
*/
- public function getFileContents( array $params ) {
+ public function getFileContentsMulti( array $params ) {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
- $tmpFile = $this->getLocalReference( $params );
- if ( !$tmpFile ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
- return false;
- }
- wfSuppressWarnings();
- $data = file_get_contents( $tmpFile->getPath() );
- wfRestoreWarnings();
+
+ $params = $this->setConcurrencyFlags( $params );
+ $contents = $this->doGetFileContentsMulti( $params );
+
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
- return $data;
+ return $contents;
+ }
+
+ /**
+ * @see FileBackendStore::getFileContentsMulti()
+ * @return Array
+ */
+ protected function doGetFileContentsMulti( array $params ) {
+ $contents = array();
+ foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
+ wfSuppressWarnings();
+ $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
+ wfRestoreWarnings();
+ }
+ return $contents;
}
/**
}
/**
- * @see FileBackend::getLocalReference()
- * @return TempFSFile|null
+ * @see FileBackend::getLocalReferenceMulti()
+ * @return Array
*/
- public function getLocalReference( array $params ) {
- $path = self::normalizeStoragePath( $params['src'] );
- if ( $path === null ) {
- return null; // invalid storage path
- }
+ final public function getLocalReferenceMulti( array $params ) {
wfProfileIn( __METHOD__ );
wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $params = $this->setConcurrencyFlags( $params );
+
+ $fsFiles = array(); // (path => FSFile)
$latest = !empty( $params['latest'] ); // use latest data?
- if ( $this->expensiveCache->has( $path, 'localRef' ) ) {
- $val = $this->expensiveCache->get( $path, 'localRef' );
- // If we want the latest data, check that this cached
- // value was in fact fetched with the latest available data.
- if ( !$latest || $val['latest'] ) {
- wfProfileOut( __METHOD__ . '-' . $this->name );
- wfProfileOut( __METHOD__ );
- return $val['object'];
+ // Reuse any files already in process cache...
+ foreach ( $params['srcs'] as $src ) {
+ $path = self::normalizeStoragePath( $src );
+ if ( $path === null ) {
+ $fsFiles[$src] = null; // invalid storage path
+ } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
+ $val = $this->expensiveCache->get( $path, 'localRef' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $val['latest'] ) {
+ $fsFiles[$src] = $val['object'];
+ }
}
}
- $tmpFile = $this->getLocalCopy( $params );
- if ( $tmpFile ) { // don't cache negatives
- $this->expensiveCache->set( $path, 'localRef',
- array( 'object' => $tmpFile, 'latest' => $latest ) );
+ // Fetch local references of any remaning files...
+ $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
+ foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
+ $fsFiles[$path] = $fsFile;
+ if ( $fsFile ) { // update the process cache...
+ $this->expensiveCache->set( $path, 'localRef',
+ array( 'object' => $fsFile, 'latest' => $latest ) );
+ }
}
+
+ wfProfileOut( __METHOD__ . '-' . $this->name );
+ wfProfileOut( __METHOD__ );
+ return $fsFiles;
+ }
+
+ /**
+ * @see FileBackendStore::getLocalReferenceMulti()
+ * @return Array
+ */
+ protected function doGetLocalReferenceMulti( array $params ) {
+ return $this->doGetLocalCopyMulti( $params );
+ }
+
+ /**
+ * @see FileBackend::getLocalCopyMulti()
+ * @return Array
+ */
+ final public function getLocalCopyMulti( array $params ) {
+ wfProfileIn( __METHOD__ );
+ wfProfileIn( __METHOD__ . '-' . $this->name );
+
+ $params = $this->setConcurrencyFlags( $params );
+ $tmpFiles = $this->doGetLocalCopyMulti( $params );
+
wfProfileOut( __METHOD__ . '-' . $this->name );
wfProfileOut( __METHOD__ );
- return $tmpFile;
+ return $tmpFiles;
}
+ /**
+ * @see FileBackendStore::getLocalCopyMulti()
+ * @return Array
+ */
+ abstract protected function doGetLocalCopyMulti( array $params );
+
/**
* @see FileBackend::streamFile()
* @return Status
$status = $this->doStreamFile( $params );
wfProfileOut( __METHOD__ . '-send-' . $this->name );
wfProfileOut( __METHOD__ . '-send' );
+ if ( !$status->isOK() ) {
+ // Per bug 41113, nasty things can happen if bad cache entries get
+ // stuck in cache. It's also possible that this error can come up
+ // with simple race conditions. Clear out the stat cache to be safe.
+ $this->clearCache( array( $params['src'] ) );
+ $this->deleteFileCache( $params['src'] );
+ trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
+ }
} else {
$status->fatal( 'backend-fail-stream', $params['src'] );
}
* Get a list of storage paths to lock for a list of operations
* Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
* each corresponding to a list of storage paths to be locked.
+ * All returned paths are normalized.
*
* @param $performOps Array List of FileOp objects
* @return Array ('sh' => list of paths, 'ex' => list of paths)
/**
* @see FileBackendStore::executeOpHandlesInternal()
+ * @param array $fileOpHandles
+ * @throws MWException
* @return Array List of corresponding Status objects
*/
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
/**
* Do a batch lookup from cache for container stats for all containers
* used in a list of container names, storage paths, or FileOp objects.
+ * This loads the persistent cache values into the process cache.
*
* @param $items Array
* @return void
/**
* Get the cache key for a file path
*
- * @param $path string Storage path
+ * @param $path string Normalized storage path
* @return string
*/
private function fileCacheKey( $path ) {
* @param $val mixed Information to cache
*/
final protected function setFileCache( $path, $val ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ return; // invalid storage path
+ }
$this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 );
}
* @param $path string Storage path
*/
final protected function deleteFileCache( $path ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ return; // invalid storage path
+ }
if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
trigger_error( "Unable to delete stat cache for file $path." );
}
/**
* Do a batch lookup from cache for file stats for all paths
* used in a list of storage paths or FileOp objects.
+ * This loads the persistent cache values into the process cache.
*
* @param $items Array List of storage paths or FileOps
* @return void
$paths = array_merge( $paths, $item->storagePathsRead() );
$paths = array_merge( $paths, $item->storagePathsChanged() );
} elseif ( self::isStoragePath( $item ) ) {
- $paths[] = $item;
+ $paths[] = FileBackend::normalizeStoragePath( $item );
}
}
+ // Get rid of any paths that failed normalization...
+ $paths = array_filter( $paths, 'strlen' ); // remove nulls
// Get all the corresponding cache keys for paths...
foreach ( $paths as $path ) {
list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
dépôts / lhc / web / wiklou.git / blobdiff