From 23e2fbcd2f9708f0231387191259e1d2266a1a2b Mon Sep 17 00:00:00 2001 From: Aaron Date: Mon, 23 Apr 2012 13:27:58 -0700 Subject: [PATCH] [FileBackend] Added container stat caching to reduce RTTs to high latency backends. Change-Id: I73575fd65ca06a238803b30f8de6873801a224b3 --- .../filerepo/backend/FileBackendStore.php | 101 +++++++++++++++++- .../filerepo/backend/SwiftFileBackend.php | 45 ++++++-- 2 files changed, 139 insertions(+), 7 deletions(-) diff --git a/includes/filerepo/backend/FileBackendStore.php b/includes/filerepo/backend/FileBackendStore.php index 55dedc1e05..f4dd4c03b5 100644 --- a/includes/filerepo/backend/FileBackendStore.php +++ b/includes/filerepo/backend/FileBackendStore.php @@ -19,6 +19,9 @@ * @since 1.19 */ abstract class FileBackendStore extends FileBackend { + /** @var BagOStuff */ + protected $memCache; + /** @var Array Map of paths to small (RAM/disk) cache items */ protected $cache = array(); // (storage path => key => value) protected $maxCacheSize = 100; // integer; max paths with entries @@ -31,6 +34,16 @@ abstract class FileBackendStore extends FileBackend { protected $maxFileSize = 4294967296; // integer bytes (4GiB) + /** + * @see FileBackend::__construct() + * + * @param $config Array + */ + public function __construct( array $config ) { + parent::__construct( $config ); + $this->memCache = new EmptyBagOStuff(); // disabled by default + } + /** * Get the maximum allowable file size given backend * medium restrictions and basic performance constraints. @@ -390,11 +403,13 @@ abstract class FileBackendStore extends FileBackend { if ( $shard !== null ) { // confined to a single container/shard $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); + $this->deleteContainerCache( $fullCont ); // purge cache } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); + $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache } } @@ -760,9 +775,12 @@ abstract class FileBackendStore extends FileBackend { } } - // Clear any cache entries (after locks acquired) + // Clear any file cache entries (after locks acquired) $this->clearCache(); + // Load from the persistent container cache + $this->primeContainerCache( $performOps ); + // Actually attempt the operation batch... $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal ); @@ -1041,6 +1059,87 @@ abstract class FileBackendStore extends FileBackend { protected function resolveContainerPath( $container, $relStoragePath ) { return $relStoragePath; } + + /** + * Get the cache key for a container + * + * @param $container Resolved container name + * @return string + */ + private function containerCacheKey( $container ) { + return wfMemcKey( 'backend', $this->getName(), 'container', $container ); + } + + /** + * Set the cached info for a container + * + * @param $container Resolved container name + * @param $val mixed Information to cache + * @return void + */ + final protected function setContainerCache( $container, $val ) { + $this->memCache->set( $this->containerCacheKey( $container ), $val, 7*86400 ); + } + + /** + * Delete the cached info for a container + * + * @param $container Resolved container name + * @return void + */ + final protected function deleteContainerCache( $container ) { + $this->memCache->delete( $this->containerCacheKey( $container ) ); + } + + /** + * Do a batch lookup from cache for container stats for all containers + * used in a list of container names, storage paths, or FileOp objects. + * + * @param $items Array List of storage paths or FileOps + * @return void + */ + final protected function primeContainerCache( array $items ) { + $paths = array(); // list of storage paths + $contNames = array(); // (cache key => resolved container name) + // Get all the paths/containers from the items... + foreach ( $items as $item ) { + if ( $item instanceof FileOp ) { + $paths = array_merge( $paths, $item->storagePathsRead() ); + $paths = array_merge( $paths, $item->storagePathsChanged() ); + } elseif ( self::isStoragePath( $item ) ) { + $paths[] = $item; + } elseif ( is_string( $item ) ) { // full container name + $contNames[$this->containerCacheKey( $item )] = $item; + } + } + // Get all the corresponding cache keys for paths... + foreach ( $paths as $path ) { + list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path ); + if ( $fullCont !== null ) { // valid path for this backend + $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; + } + } + + $contInfo = array(); // (resolved container name => cache value) + // Get all cache entries for these container cache keys... + $values = $this->memCache->getBatch( array_keys( $contNames ) ); + foreach ( $values as $cacheKey => $val ) { + $contInfo[$contNames[$cacheKey]] = $val; + } + + // Populate the container process cache for the backend... + $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); + } + + /** + * Fill the backend-specific process cache given an array of + * resolved container names and their corresponding cached info. + * Only containers that actually exist should appear in the map. + * + * @param $containerInfo Array Map of resolved container names to cached info + * @return void + */ + protected function doPrimeContainerCache( array $containerInfo ) {} } /** diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php index c7e40e8d08..18ce4518ab 100644 --- a/includes/filerepo/backend/SwiftFileBackend.php +++ b/includes/filerepo/backend/SwiftFileBackend.php @@ -64,6 +64,8 @@ class SwiftFileBackend extends FileBackendStore { $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; + // Cache container info to mask latency + $this->memCache = wfGetMainCache(); } /** @@ -750,23 +752,30 @@ class SwiftFileBackend extends FileBackendStore { * Use $reCache if the file count or byte count is needed. * * @param $container string Container name - * @param $reCache bool Refresh the process cache + * @param $bypassCache bool Bypass all caches and load from Swift * @return CF_Container + * @throws InvalidResponseException */ - protected function getContainer( $container, $reCache = false ) { + protected function getContainer( $container, $bypassCache = false ) { $conn = $this->getConnection(); // Swift proxy connection - if ( $reCache ) { - unset( $this->connContainers[$container] ); // purge cache + if ( $bypassCache ) { // purge cache + unset( $this->connContainers[$container] ); + } elseif ( !isset( $this->connContainers[$container] ) ) { + $this->primeContainerCache( array( $container ) ); // check persistent cache } if ( !isset( $this->connContainers[$container] ) ) { $contObj = $conn->get_container( $container ); // NoSuchContainerException not thrown: container must exist if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache? reset( $this->connContainers ); - $key = key( $this->connContainers ); - unset( $this->connContainers[$key] ); + unset( $this->connContainers[key( $this->connContainers )] ); } $this->connContainers[$container] = $contObj; // cache it + if ( !$bypassCache ) { + $this->setContainerCache( $container, // update persistent cache + array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count ) + ); + } } return $this->connContainers[$container]; } @@ -776,6 +785,7 @@ class SwiftFileBackend extends FileBackendStore { * * @param $container string Container name * @return CF_Container + * @throws InvalidResponseException */ protected function createContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@ -789,6 +799,7 @@ class SwiftFileBackend extends FileBackendStore { * * @param $container string Container name * @return void + * @throws InvalidResponseException */ protected function deleteContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@ -796,6 +807,28 @@ class SwiftFileBackend extends FileBackendStore { unset( $this->connContainers[$container] ); // purge cache } + /** + * @see FileBackendStore::doPrimeContainerCache() + * @return void + */ + protected function doPrimeContainerCache( array $containerInfo ) { + try { + $conn = $this->getConnection(); // Swift proxy connection + foreach ( $containerInfo as $container => $info ) { + $this->connContainers[$container] = new CF_Container( + $conn->cfs_auth, + $conn->cfs_http, + $container, + $info['count'], + $info['bytes'] + ); + } + } catch ( InvalidResponseException $e ) { + } catch ( Exception $e ) { // some other exception? + $this->logException( $e, __METHOD__, array() ); + } + } + /** * Log an unexpected exception for this backend * -- 2.20.1