/** @var array Map of container names to sharding config */
protected $shardViaHashLevels = array();
- /** @var callback Method to get the MIME type of files */
+ /** @var callable Method to get the MIME type of files */
protected $mimeCallback;
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
$status->merge( $this->doConcatenate( $params ) );
$sec = microtime( true ) - $start_time;
if ( !$status->isOK() ) {
- wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
- count( $params['srcs'] ) . " file(s) [$sec sec]" );
+ wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name}" .
+ " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
}
}
/**
* @see FileBackendStore::doPrepare()
- * @param $container
+ * @param string $container
* @param string $dir
* @param array $params
* @return Status
/**
* @see FileBackendStore::doSecure()
- * @param $container
+ * @param string $container
* @param string $dir
* @param array $params
* @return Status
/**
* @see FileBackendStore::doPublish()
- * @param $container
+ * @param string $container
* @param string $dir
* @param array $params
* @return Status
/**
* @see FileBackendStore::doClean()
- * @param $container
+ * @param string $container
* @param string $dir
* @param array $params
* @return Status
$stat = $this->doGetFileStat( $params );
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
if ( is_array( $stat ) ) { // file exists
- $stat['latest'] = $latest;
+ // Strongly consistent backends can automatically set "latest"
+ $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
$this->cheapCache->set( $path, 'stat', $stat );
$this->setFileCache( $path, $stat ); // update persistent cache
if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
wfProfileOut( __METHOD__ . '-miss' );
$this->cheapCache->set( $path, 'xattr', array( 'map' => $fields, 'latest' => $latest ) );
+
return $fields;
}
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
* @param array $params
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
*/
abstract public function getDirectoryListInternal( $container, $dir, array $params );
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
* @param array $params
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
*/
abstract public function getFileListInternal( $container, $dir, array $params );
$this->clearCache();
}
- // Load from the persistent file and container caches
- $this->primeFileCache( $performOps );
- $this->primeContainerCache( $performOps );
+ // Build the list of paths involved
+ $paths = array();
+ foreach ( $performOps as $op ) {
+ $paths = array_merge( $paths, $op->storagePathsRead() );
+ $paths = array_merge( $paths, $op->storagePathsChanged() );
+ }
+
+ // Enlarge the cache to fit the stat entries of these files
+ $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
- // Actually attempt the operation batch...
- $opts = $this->setConcurrencyFlags( $opts );
- $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
+ // Load from the persistent container caches
+ $this->primeContainerCache( $paths );
+ // Get the latest stat info for all the files (having locked them)
+ $ok = $this->preloadFileStat( array( 'srcs' => $paths, 'latest' => true ) );
+
+ if ( $ok ) {
+ // Actually attempt the operation batch...
+ $opts = $this->setConcurrencyFlags( $opts );
+ $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
+ } else {
+ // If we could not even stat some files, then bail out...
+ $subStatus = Status::newFatal( 'backend-fail-internal', $this->name );
+ foreach ( $ops as $i => $op ) { // mark each op as failed
+ $subStatus->success[$i] = false;
+ ++$subStatus->failCount;
+ }
+ wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name} " .
+ " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
+ }
// Merge errors into status fields
$status->merge( $subStatus );
$status->success = $subStatus->success; // not done in merge()
+ // Shrink the stat cache back to normal size
+ $this->cheapCache->resize( self::CACHE_CHEAP_SIZE );
+
return $status;
}
protected function doClearCache( array $paths = null ) {
}
+ final public function preloadFileStat( array $params ) {
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+ $success = true; // no network errors
+
+ $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
+ $stats = $this->doGetFileStatMulti( $params );
+ if ( $stats === null ) {
+ return true; // not supported
+ }
+
+ $latest = !empty( $params['latest'] ); // use latest data?
+ foreach ( $stats as $path => $stat ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ continue; // this shouldn't happen
+ }
+ if ( is_array( $stat ) ) { // file exists
+ // Strongly consistent backends can automatically set "latest"
+ $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
+ $this->cheapCache->set( $path, 'stat', $stat );
+ $this->setFileCache( $path, $stat ); // update persistent cache
+ if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
+ }
+ if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
+ $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => $stat['xattr'], 'latest' => $latest ) );
+ }
+ } elseif ( $stat === false ) { // file does not exist
+ $this->cheapCache->set( $path, 'stat',
+ $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => false, 'latest' => $latest ) );
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => false, 'latest' => $latest ) );
+ wfDebug( __METHOD__ . ": File $path does not exist.\n" );
+ } else { // an error occurred
+ $success = false;
+ wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
+ }
+ }
+
+ return $success;
+ }
+
+ /**
+ * Get file stat information (concurrently if possible) for several files
+ *
+ * @see FileBackend::getFileStat()
+ *
+ * @param array $params Parameters include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * @return array|null Map of storage paths to array|bool|null (returns null if not supported)
+ * @since 1.23
+ */
+ protected function doGetFileStatMulti( array $params ) {
+ return null; // not supported
+ }
+
/**
* Is this a key/value store where directories are just virtual?
* Virtual directories exists in so much as files exists that are
* @return string
*/
private function containerCacheKey( $container ) {
- return wfMemcKey( 'backend', $this->getName(), 'container', $container );
+ return "filebackend:{$this->name}:{$this->wikiId}:container:{$container}";
}
/**
/**
* Do a batch lookup from cache for container stats for all containers
- * used in a list of container names, storage paths, or FileOp objects.
+ * used in a list of container names or storage paths objects.
* This loads the persistent cache values into the process cache.
*
* @param array $items
$contNames = array(); // (cache key => resolved container name)
// Get all the paths/containers from the items...
foreach ( $items as $item ) {
- if ( $item instanceof FileOp ) {
- $paths = array_merge( $paths, $item->storagePathsRead() );
- $paths = array_merge( $paths, $item->storagePathsChanged() );
- } elseif ( self::isStoragePath( $item ) ) {
+ if ( self::isStoragePath( $item ) ) {
$paths[] = $item;
} elseif ( is_string( $item ) ) { // full container name
$contNames[$this->containerCacheKey( $item )] = $item;
* @return string
*/
private function fileCacheKey( $path ) {
- return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
+ return "filebackend:{$this->name}:{$this->wikiId}:file:" . sha1( $path );
}
/**
}
$age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
$ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
- $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
+ $key = $this->fileCacheKey( $path );
+ // Set the cache unless it is currently salted with the value "PURGED".
+ // Using add() handles this except it also is a no-op in that case where
+ // the current value is not "latest" but $val is, so use CAS in that case.
+ if ( !$this->memCache->add( $key, $val, $ttl ) && !empty( $val['latest'] ) ) {
+ $this->memCache->merge(
+ $key,
+ function( BagOStuff $cache, $key, $cValue ) use ( $val ) {
+ return ( is_array( $cValue ) && empty( $cValue['latest'] ) )
+ ? $val // update the stat cache with the lastest info
+ : false; // do nothing (cache is salted or some error happened)
+ },
+ $ttl,
+ 1
+ );
+ }
}
/**
* used in a list of storage paths or FileOp objects.
* This loads the persistent cache values into the process cache.
*
- * @param array $items List of storage paths or FileOps
+ * @param array $items List of storage paths
*/
final protected function primeFileCache( array $items ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$pathNames = array(); // (cache key => storage path)
// Get all the paths/containers from the items...
foreach ( $items as $item ) {
- if ( $item instanceof FileOp ) {
- $paths = array_merge( $paths, $item->storagePathsRead() );
- $paths = array_merge( $paths, $item->storagePathsChanged() );
- } elseif ( self::isStoragePath( $item ) ) {
+ if ( self::isStoragePath( $item ) ) {
$paths[] = FileBackend::normalizeStoragePath( $item );
}
}
*/
final protected static function normalizeXAttributes( array $xattr ) {
$newXAttr = array( 'headers' => array(), 'metadata' => array() );
+
foreach ( $xattr['headers'] as $name => $value ) {
$newXAttr['headers'][strtolower( $name )] = $value;
}
+
foreach ( $xattr['metadata'] as $name => $value ) {
$newXAttr['metadata'][strtolower( $name )] = $value;
}
+
return $newXAttr;
}
dépôts / lhc / web / wiklou.git / blobdiff