Limit the number of cached languages in MessageCache via MapCacheLRU
authorAaron Schulz <aschulz@wikimedia.org>
Mon, 25 Jun 2018 09:45:55 +0000 (10:45 +0100)
committerAaron Schulz <aschulz@wikimedia.org>
Tue, 3 Jul 2018 11:03:14 +0000 (11:03 +0000)
Change-Id: I37a128cfc553e0edaab524098461776cec3fe08a

includes/cache/MessageCache.php

index 0854a43..93959db 100644 (file)
@@ -46,14 +46,11 @@ class MessageCache {
        const LOCK_TTL = 30;
 
        /**
-        * Process local cache of loaded messages that are defined in
-        * MediaWiki namespace. First array level is a language code,
-        * second level is message key and the values are either message
-        * content prefixed with space, or !NONEXISTENT for negative
-        * caching.
-        * @var array $mCache
+        * Process cache of loaded messages that are defined in MediaWiki namespace
+        *
+        * @var MapCacheLRU Map of (language code => key => " <MESSAGE>" or "!TOO BIG")
         */
-       protected $mCache;
+       protected $cache;
 
        /**
         * @var bool[] Map of (language code => boolean)
@@ -80,12 +77,6 @@ class MessageCache {
        /** @var Parser */
        protected $mParser;
 
-       /**
-        * Variable for tracking which variables are already loaded
-        * @var array $mLoadedLanguages
-        */
-       protected $mLoadedLanguages = [];
-
        /**
         * @var bool $mInParser
         */
@@ -174,6 +165,8 @@ class MessageCache {
                $this->clusterCache = $clusterCache;
                $this->srvCache = $serverCache;
 
+               $this->cache = new MapCacheLRU( 5 ); // limit size for sanity
+
                $this->mDisable = !$useDB;
                $this->mExpiry = $expiry;
        }
@@ -247,7 +240,7 @@ class MessageCache {
         *
         * @param string $code Language to which load messages
         * @param int $mode Use MessageCache::FOR_UPDATE to skip process cache [optional]
-        * @throws MWException
+        * @throws InvalidArgumentException
         * @return bool
         */
        protected function load( $code, $mode = null ) {
@@ -256,7 +249,7 @@ class MessageCache {
                }
 
                # Don't do double loading...
-               if ( isset( $this->mLoadedLanguages[$code] ) && $mode != self::FOR_UPDATE ) {
+               if ( $this->cache->has( $code ) && $mode != self::FOR_UPDATE ) {
                        return true;
                }
 
@@ -296,8 +289,8 @@ class MessageCache {
                        $staleCache = $cache;
                } else {
                        $where[] = 'got from local cache';
+                       $this->cache->set( $code, $cache );
                        $success = true;
-                       $this->mCache[$code] = $cache;
                }
 
                if ( !$success ) {
@@ -326,7 +319,7 @@ class MessageCache {
                                                $staleCache = $cache;
                                        } else {
                                                $where[] = 'got from global cache';
-                                               $this->mCache[$code] = $cache;
+                                               $this->cache->set( $code, $cache );
                                                $this->saveToCaches( $cache, 'local-only', $code );
                                                $success = true;
                                        }
@@ -347,7 +340,7 @@ class MessageCache {
                                } elseif ( $staleCache ) {
                                        # Use the stale cache while some other thread constructs the new one
                                        $where[] = 'using stale cache';
-                                       $this->mCache[$code] = $staleCache;
+                                       $this->cache->set( $code, $staleCache );
                                        $success = true;
                                        break;
                                } elseif ( $failedAttempts > 0 ) {
@@ -371,13 +364,14 @@ class MessageCache {
                if ( !$success ) {
                        $where[] = 'loading FAILED - cache is disabled';
                        $this->mDisable = true;
-                       $this->mCache = false;
+                       $this->cache->set( $code, null );
                        wfDebugLog( 'MessageCacheError', __METHOD__ . ": Failed to load $code\n" );
                        # This used to throw an exception, but that led to nasty side effects like
                        # the whole wiki being instantly down if the memcached server died
-               } else {
-                       # All good, just record the success
-                       $this->mLoadedLanguages[$code] = true;
+               }
+
+               if ( !$this->cache->has( $code ) ) { // sanity
+                       throw new LogicException( "Process cache for '$code' should be set by now." );
                }
 
                $info = implode( ', ', $where );
@@ -417,7 +411,7 @@ class MessageCache {
                }
 
                $cache = $this->loadFromDB( $code, $mode );
-               $this->mCache[$code] = $cache;
+               $this->cache->set( $code, $cache );
                $saveSuccess = $this->saveToCaches( $cache, 'all', $code );
 
                if ( !$saveSuccess ) {
@@ -473,10 +467,10 @@ class MessageCache {
 
                $mostused = [];
                if ( $wgAdaptiveMessageCache && $code !== $wgLanguageCode ) {
-                       if ( !isset( $this->mCache[$wgLanguageCode] ) ) {
+                       if ( !$this->cache->has( $wgLanguageCode ) ) {
                                $this->load( $wgLanguageCode );
                        }
-                       $mostused = array_keys( $this->mCache[$wgLanguageCode] );
+                       $mostused = array_keys( $this->cache->get( $wgLanguageCode ) );
                        foreach ( $mostused as $key => $value ) {
                                $mostused[$key] = "$value/$code";
                        }
@@ -578,10 +572,10 @@ class MessageCache {
                // (a) Update the process cache with the new message text
                if ( $text === false ) {
                        // Page deleted
-                       $this->mCache[$code][$title] = '!NONEXISTENT';
+                       $this->cache->setField( $code, $title, '!NONEXISTENT' );
                } else {
                        // Ignore $wgMaxMsgCacheEntrySize so the process cache is up to date
-                       $this->mCache[$code][$title] = ' ' . $text;
+                       $this->cache->setField( $code, $title, ' ' . $text );
                }
 
                // (b) Update the shared caches in a deferred update with a fresh DB snapshot
@@ -600,24 +594,27 @@ class MessageCache {
                                }
                                // Load the messages from the master DB to avoid race conditions
                                $cache = $this->loadFromDB( $code, self::FOR_UPDATE );
-                               $this->mCache[$code] = $cache;
-                               // Load the process cache values and set the per-title cache keys
+                               // Check if an individual cache key should exist and update cache accordingly
                                $page = WikiPage::factory( Title::makeTitle( NS_MEDIAWIKI, $title ) );
                                $page->loadPageData( $page::READ_LATEST );
                                $text = $this->getMessageTextFromContent( $page->getContent() );
-                               // Check if an individual cache key should exist and update cache accordingly
                                if ( is_string( $text ) && strlen( $text ) > $wgMaxMsgCacheEntrySize ) {
-                                       $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title );
-                                       $this->wanCache->set( $titleKey, ' ' . $text, $this->mExpiry );
+                                       $this->wanCache->set(
+                                               $this->bigMessageCacheKey( $cache['HASH'], $title ),
+                                               ' ' . $text,
+                                               $this->mExpiry
+                                       );
                                }
                                // Mark this cache as definitely being "latest" (non-volatile) so
-                               // load() calls do try to refresh the cache with replica DB data
-                               $this->mCache[$code]['LATEST'] = time();
+                               // load() calls do not try to refresh the cache with replica DB data
+                               $cache['LATEST'] = time();
+                               // Update the process cache
+                               $this->cache->set( $code, $cache );
                                // Pre-emptively update the local datacenter cache so things like edit filter and
                                // blacklist changes are reflected immediately; these often use MediaWiki: pages.
                                // The datacenter handling replace() calls should be the same one handling edits
                                // as they require HTTP POST.
-                               $this->saveToCaches( $this->mCache[$code], 'all', $code );
+                               $this->saveToCaches( $cache, 'all', $code );
                                // Release the lock now that the cache is saved
                                ScopedCallback::consume( $scopedLock );
 
@@ -971,8 +968,8 @@ class MessageCache {
        public function getMsgFromNamespace( $title, $code ) {
                $this->load( $code );
 
-               if ( isset( $this->mCache[$code][$title] ) ) {
-                       $entry = $this->mCache[$code][$title];
+               if ( $this->cache->hasField( $code, $title ) ) {
+                       $entry = $this->cache->getField( $code, $title );
                        if ( substr( $entry, 0, 1 ) === ' ' ) {
                                // The message exists and is not '!TOO BIG'
                                return (string)substr( $entry, 1 );
@@ -984,40 +981,40 @@ class MessageCache {
                        $message = false;
                        Hooks::run( 'MessagesPreLoad', [ $title, &$message, $code ] );
                        if ( $message !== false ) {
-                               $this->mCache[$code][$title] = ' ' . $message;
+                               $this->cache->setField( $code, $title, ' ' . $message );
                        } else {
-                               $this->mCache[$code][$title] = '!NONEXISTENT';
+                               $this->cache->setField( $code, $title, '!NONEXISTENT' );
                        }
 
                        return $message;
                }
 
                // Individual message cache key
-               $titleKey = $this->bigMessageCacheKey( $this->mCache[$code]['HASH'], $title );
+               $bigKey = $this->bigMessageCacheKey( $this->cache->getField( $code, 'HASH' ), $title );
 
                if ( $this->mCacheVolatile[$code] ) {
                        $entry = false;
                        // Make sure that individual keys respect the WAN cache holdoff period too
                        LoggerFactory::getInstance( 'MessageCache' )->debug(
                                __METHOD__ . ': loading volatile key \'{titleKey}\'',
-                               [ 'titleKey' => $titleKey, 'code' => $code ] );
+                               [ 'titleKey' => $bigKey, 'code' => $code ] );
                } else {
                        // Try the individual message cache
-                       $entry = $this->wanCache->get( $titleKey );
+                       $entry = $this->wanCache->get( $bigKey );
                }
 
                if ( $entry !== false ) {
                        if ( substr( $entry, 0, 1 ) === ' ' ) {
-                               $this->mCache[$code][$title] = $entry;
+                               $this->cache->setField( $code, $title, $entry );
                                // The message exists, so make sure a string is returned
                                return (string)substr( $entry, 1 );
                        } elseif ( $entry === '!NONEXISTENT' ) {
-                               $this->mCache[$code][$title] = '!NONEXISTENT';
+                               $this->cache->setField( $code, $title, '!NONEXISTENT' );
 
                                return false;
                        } else {
                                // Corrupt/obsolete entry, delete it
-                               $this->wanCache->delete( $titleKey );
+                               $this->wanCache->delete( $bigKey );
                        }
                }
 
@@ -1040,14 +1037,14 @@ class MessageCache {
                        if ( $content ) {
                                $message = $this->getMessageTextFromContent( $content );
                                if ( is_string( $message ) ) {
-                                       $this->mCache[$code][$title] = ' ' . $message;
-                                       $this->wanCache->set( $titleKey, ' ' . $message, $this->mExpiry, $cacheOpts );
+                                       $this->cache->setField( $code, $title, ' ' . $message );
+                                       $this->wanCache->set( $bigKey, ' ' . $message, $this->mExpiry, $cacheOpts );
                                }
                        } else {
                                // A possibly temporary loading failure
                                LoggerFactory::getInstance( 'MessageCache' )->warning(
                                        __METHOD__ . ': failed to load message page text for \'{titleKey}\'',
-                                       [ 'titleKey' => $titleKey, 'code' => $code ] );
+                                       [ 'titleKey' => $bigKey, 'code' => $code ] );
                                $message = null; // no negative caching
                        }
                } else {
@@ -1056,8 +1053,8 @@ class MessageCache {
 
                if ( $message === false ) {
                        // Negative caching in case a "too big" message is no longer available (deleted)
-                       $this->mCache[$code][$title] = '!NONEXISTENT';
-                       $this->wanCache->set( $titleKey, '!NONEXISTENT', $this->mExpiry, $cacheOpts );
+                       $this->cache->setField( $code, $title, '!NONEXISTENT' );
+                       $this->wanCache->set( $bigKey, '!NONEXISTENT', $this->mExpiry, $cacheOpts );
                }
 
                return $message;
@@ -1192,13 +1189,12 @@ class MessageCache {
         *
         * Mainly used after a mass rebuild
         */
-       function clear() {
+       public function clear() {
                $langs = Language::fetchLanguageNames( null, 'mw' );
                foreach ( array_keys( $langs ) as $code ) {
                        $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) );
                }
-
-               $this->mLoadedLanguages = [];
+               $this->cache->clear();
        }
 
        /**
@@ -1235,12 +1231,12 @@ class MessageCache {
                global $wgContLang;
 
                $this->load( $code );
-               if ( !isset( $this->mCache[$code] ) ) {
+               if ( !$this->cache->has( $code ) ) {
                        // Apparently load() failed
                        return null;
                }
                // Remove administrative keys
-               $cache = $this->mCache[$code];
+               $cache = $this->cache->get( $code );
                unset( $cache['VERSION'] );
                unset( $cache['EXPIRY'] );
                unset( $cache['EXCESSIVE'] );