X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Fcache%2FMessageCache.php;h=5ada42fb77bd90b41774277a88ffd737ab8ff752;hb=f03d31af05dbe6acaad00bc76ff4aa02aee2f77f;hp=bf2ed2e4301d0bf982afc4fd77ceb32ccf513ec7;hpb=c9d843cbcef24202bd793e5bc4a870c831f59bf6;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php index bf2ed2e430..5ada42fb77 100644 --- a/includes/cache/MessageCache.php +++ b/includes/cache/MessageCache.php @@ -52,6 +52,13 @@ class MessageCache { */ protected $cache; + /** + * Map of (lowercase message key => index) for all software defined messages + * + * @var array + */ + protected $overridable; + /** * @var bool[] Map of (language code => boolean) */ @@ -191,6 +198,7 @@ class MessageCache { // either. $po = ParserOptions::newFromAnon(); $po->setAllowUnsafeRawHtml( false ); + $po->setTidy( true ); return $po; } @@ -199,6 +207,8 @@ class MessageCache { // from malicious sources. As a precaution, disable // the parser tag when parsing messages. $this->mParserOptions->setAllowUnsafeRawHtml( false ); + // For the same reason, tidy the output! + $this->mParserOptions->setTidy( true ); } return $this->mParserOptions; @@ -258,6 +268,8 @@ class MessageCache { return true; } + $this->overridable = array_flip( Language::getMessageKeysFor( $code ) ); + # 8 lines of code just to say (once) that message cache is disabled if ( $this->mDisable ) { static $shownDisabled = false; @@ -464,13 +476,7 @@ class MessageCache { $cache = []; - # Common conditions - $conds = [ - 'page_is_redirect' => 0, - 'page_namespace' => NS_MEDIAWIKI, - ]; - - $mostused = []; + $mostused = []; // list of "/" if ( $wgAdaptiveMessageCache && $code !== $wgLanguageCode ) { if ( !$this->cache->has( $wgLanguageCode ) ) { $this->load( $wgLanguageCode ); @@ -481,6 +487,14 @@ class MessageCache { } } + // Get the list of software-defined messages in core/extensions + $overridable = array_flip( Language::getMessageKeysFor( $wgLanguageCode ) ); + + // Common conditions + $conds = [ + 'page_is_redirect' => 0, + 'page_namespace' => NS_MEDIAWIKI, + ]; if ( count( $mostused ) ) { $conds['page_title'] = $mostused; } elseif ( $code !== $wgLanguageCode ) { @@ -492,31 +506,28 @@ class MessageCache { $dbr->buildLike( $dbr->anyString(), '/', $dbr->anyString() ); } - # Conditions to fetch oversized pages to ignore them - $bigConds = $conds; - $bigConds[] = 'page_len > ' . intval( $wgMaxMsgCacheEntrySize ); - - # Load titles for all oversized pages in the MediaWiki namespace + // Set the stubs for oversized software-defined messages in the main cache map $res = $dbr->select( 'page', [ 'page_title', 'page_latest' ], - $bigConds, + array_merge( $conds, [ 'page_len > ' . intval( $wgMaxMsgCacheEntrySize ) ] ), __METHOD__ . "($code)-big" ); foreach ( $res as $row ) { - $cache[$row->page_title] = '!TOO BIG'; + $name = $this->contLang->lcfirst( $row->page_title ); + // Include entries/stubs for all keys in $mostused in adaptive mode + if ( $wgAdaptiveMessageCache || $this->isMainCacheable( $name, $overridable ) ) { + $cache[$row->page_title] = '!TOO BIG'; + } // At least include revision ID so page changes are reflected in the hash $cache['EXCESSIVE'][$row->page_title] = $row->page_latest; } - # Conditions to load the remaining pages with their contents - $smallConds = $conds; - $smallConds[] = 'page_len <= ' . intval( $wgMaxMsgCacheEntrySize ); - + // Set the text for small software-defined messages in the main cache map $res = $dbr->select( [ 'page', 'revision', 'text' ], - [ 'page_title', 'old_id', 'old_text', 'old_flags' ], - $smallConds, + [ 'page_title', 'page_latest', 'old_id', 'old_text', 'old_flags' ], + array_merge( $conds, [ 'page_len <= ' . intval( $wgMaxMsgCacheEntrySize ) ] ), __METHOD__ . "($code)-small", [], [ @@ -524,23 +535,30 @@ class MessageCache { 'text' => [ 'JOIN', 'rev_text_id=old_id' ], ] ); - foreach ( $res as $row ) { - $text = Revision::getRevisionText( $row ); - if ( $text === false ) { - // Failed to fetch data; possible ES errors? - // Store a marker to fetch on-demand as a workaround... - // TODO Use a differnt marker - $entry = '!TOO BIG'; - wfDebugLog( - 'MessageCache', - __METHOD__ - . ": failed to load message page text for {$row->page_title} ($code)" - ); + $name = $this->contLang->lcfirst( $row->page_title ); + // Include entries/stubs for all keys in $mostused in adaptive mode + if ( $wgAdaptiveMessageCache || $this->isMainCacheable( $name, $overridable ) ) { + $text = Revision::getRevisionText( $row ); + if ( $text === false ) { + // Failed to fetch data; possible ES errors? + // Store a marker to fetch on-demand as a workaround... + // TODO Use a differnt marker + $entry = '!TOO BIG'; + wfDebugLog( + 'MessageCache', + __METHOD__ + . ": failed to load message page text for {$row->page_title} ($code)" + ); + } else { + $entry = ' ' . $text; + } + $cache[$row->page_title] = $entry; } else { - $entry = ' ' . $text; + // T193271: cache object gets too big and slow to generate. + // At least include revision ID so page changes are reflected in the hash. + $cache['EXCESSIVE'][$row->page_title] = $row->page_latest; } - $cache[$row->page_title] = $entry; } $cache['VERSION'] = MSG_CACHE_VERSION; @@ -551,10 +569,22 @@ class MessageCache { # stored and fetched from memcache. $cache['HASH'] = md5( serialize( $cache ) ); $cache['EXPIRY'] = wfTimestamp( TS_MW, time() + $this->mExpiry ); + unset( $cache['EXCESSIVE'] ); // only needed for hash return $cache; } + /** + * @param string $name Message name with lowercase first letter + * @param array $overridable Map of (key => unused) for software-defined messages + * @return bool + */ + private function isMainCacheable( $name, array $overridable ) { + // Include common conversion table pages. This also avoids problems with + // Installer::parse() bailing out due to disallowed DB queries (T207979). + return ( isset( $overridable[$name] ) || strpos( $name, 'conversiontable/' ) === 0 ); + } + /** * Updates cache as necessary when message page is changed * @@ -582,65 +612,103 @@ class MessageCache { // Ignore $wgMaxMsgCacheEntrySize so the process cache is up to date $this->cache->setField( $code, $title, ' ' . $text ); } - $fname = __METHOD__; // (b) Update the shared caches in a deferred update with a fresh DB snapshot - DeferredUpdates::addCallableUpdate( - function () use ( $title, $msg, $code, $fname ) { - global $wgMaxMsgCacheEntrySize; - // Allow one caller at a time to avoid race conditions - $scopedLock = $this->getReentrantScopedLock( - $this->clusterCache->makeKey( 'messages', $code ) - ); - if ( !$scopedLock ) { - LoggerFactory::getInstance( 'MessageCache' )->error( - $fname . ': could not acquire lock to update {title} ({code})', - [ 'title' => $title, 'code' => $code ] ); - return; - } - // Reload messages from the database and pre-populate dc-local caches - // as optimisation. Use the master DB to avoid race conditions. - $cache = $this->loadFromDB( $code, self::FOR_UPDATE ); - // Check if an individual cache key should exist and update cache accordingly - $page = WikiPage::factory( Title::makeTitle( NS_MEDIAWIKI, $title ) ); - $page->loadPageData( $page::READ_LATEST ); - $text = $this->getMessageTextFromContent( $page->getContent() ); - if ( is_string( $text ) && strlen( $text ) > $wgMaxMsgCacheEntrySize ) { - // Match logic of loadCachedMessagePageEntry() - $this->wanCache->set( - $this->bigMessageCacheKey( $cache['HASH'], $title ), - ' ' . $text, - $this->mExpiry - ); - } - // Mark this cache as definitely being "latest" (non-volatile) so - // load() calls do not try to refresh the cache with replica DB data - $cache['LATEST'] = time(); - // Update the process cache - $this->cache->set( $code, $cache ); - // Pre-emptively update the local datacenter cache so things like edit filter and - // blacklist changes are reflected immediately; these often use MediaWiki: pages. - // The datacenter handling replace() calls should be the same one handling edits - // as they require HTTP POST. - $this->saveToCaches( $cache, 'all', $code ); - // Release the lock now that the cache is saved - ScopedCallback::consume( $scopedLock ); - - // Relay the purge. Touching this check key expires cache contents - // and local cache (APC) validation hash across all datacenters. - $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) ); - - // Purge the message in the message blob store - $resourceloader = RequestContext::getMain()->getOutput()->getResourceLoader(); - $blobStore = $resourceloader->getMessageBlobStore(); - $blobStore->updateMessage( $this->contLang->lcfirst( $msg ) ); - - Hooks::run( 'MessageCacheReplace', [ $title, $text ] ); - }, + DeferredUpdates::addUpdate( + new MessageCacheUpdate( $code, $title, $msg ), DeferredUpdates::PRESEND ); } + /** + * @param string $code + * @param array[] $replacements List of (title, message key) pairs + * @throws MWException + */ + public function refreshAndReplaceInternal( $code, array $replacements ) { + global $wgMaxMsgCacheEntrySize; + + // Allow one caller at a time to avoid race conditions + $scopedLock = $this->getReentrantScopedLock( + $this->clusterCache->makeKey( 'messages', $code ) + ); + if ( !$scopedLock ) { + foreach ( $replacements as list( $title ) ) { + LoggerFactory::getInstance( 'MessageCache' )->error( + __METHOD__ . ': could not acquire lock to update {title} ({code})', + [ 'title' => $title, 'code' => $code ] ); + } + + return; + } + + // Load the existing cache to update it in the local DC cache. + // The other DCs will see a hash mismatch. + if ( $this->load( $code, self::FOR_UPDATE ) ) { + $cache = $this->cache->get( $code ); + } else { + // Err? Fall back to loading from the database. + $cache = $this->loadFromDB( $code, self::FOR_UPDATE ); + } + // Check if individual cache keys should exist and update cache accordingly + $newTextByTitle = []; // map of (title => content) + $newBigTitles = []; // map of (title => latest revision ID), like EXCESSIVE in loadFromDB() + foreach ( $replacements as list( $title ) ) { + $page = WikiPage::factory( Title::makeTitle( NS_MEDIAWIKI, $title ) ); + $page->loadPageData( $page::READ_LATEST ); + $text = $this->getMessageTextFromContent( $page->getContent() ); + // Remember the text for the blob store update later on + $newTextByTitle[$title] = $text; + // Note that if $text is false, then $cache should have a !NONEXISTANT entry + if ( !is_string( $text ) ) { + $cache[$title] = '!NONEXISTENT'; + } elseif ( strlen( $text ) > $wgMaxMsgCacheEntrySize ) { + $cache[$title] = '!TOO BIG'; + $newBigTitles[$title] = $page->getLatest(); + } else { + $cache[$title] = ' ' . $text; + } + } + // Update HASH for the new key. Incorporates various administrative keys, + // including the old HASH (and thereby the EXCESSIVE value from loadFromDB() + // and previous replace() calls), but that doesn't really matter since we + // only ever compare it for equality with a copy saved by saveToCaches(). + $cache['HASH'] = md5( serialize( $cache + [ 'EXCESSIVE' => $newBigTitles ] ) ); + // Update the too-big WAN cache entries now that we have the new HASH + foreach ( $newBigTitles as $title => $id ) { + // Match logic of loadCachedMessagePageEntry() + $this->wanCache->set( + $this->bigMessageCacheKey( $cache['HASH'], $title ), + ' ' . $newTextByTitle[$title], + $this->mExpiry + ); + } + // Mark this cache as definitely being "latest" (non-volatile) so + // load() calls do not try to refresh the cache with replica DB data + $cache['LATEST'] = time(); + // Update the process cache + $this->cache->set( $code, $cache ); + // Pre-emptively update the local datacenter cache so things like edit filter and + // blacklist changes are reflected immediately; these often use MediaWiki: pages. + // The datacenter handling replace() calls should be the same one handling edits + // as they require HTTP POST. + $this->saveToCaches( $cache, 'all', $code ); + // Release the lock now that the cache is saved + ScopedCallback::consume( $scopedLock ); + + // Relay the purge. Touching this check key expires cache contents + // and local cache (APC) validation hash across all datacenters. + $this->wanCache->touchCheckKey( $this->getCheckKey( $code ) ); + + // Purge the messages in the message blob store and fire any hook handlers + $resourceloader = RequestContext::getMain()->getOutput()->getResourceLoader(); + $blobStore = $resourceloader->getMessageBlobStore(); + foreach ( $replacements as list( $title, $msg ) ) { + $blobStore->updateMessage( $this->contLang->lcfirst( $msg ) ); + Hooks::run( 'MessageCacheReplace', [ $title, $newTextByTitle[$title] ] ); + } + } + /** * Is the given cache array expired due to time passing or a version change? * @@ -799,9 +867,8 @@ class MessageCache { Hooks::run( 'MessageCache::get', [ &$lckey ] ); // Loop through each language in the fallback list until we find something useful - $lang = wfGetLangObj( $langcode ); $message = $this->getMessageFromFallbackChain( - $lang, + wfGetLangObj( $langcode ), $lckey, !$this->mDisable && $useDB ); @@ -893,7 +960,6 @@ class MessageCache { $this->getMessagePageName( $langcode, $uckey ), $langcode ); - if ( $message !== false ) { return $message; } @@ -968,44 +1034,59 @@ class MessageCache { $this->load( $code ); $entry = $this->cache->getField( $code, $title ); + if ( $entry !== null ) { + // Message page exists as an override of a software messages if ( substr( $entry, 0, 1 ) === ' ' ) { // The message exists and is not '!TOO BIG' return (string)substr( $entry, 1 ); } elseif ( $entry === '!NONEXISTENT' ) { + // The text might be '-' or missing due to some data loss return false; } - // Fall through and try invididual message cache below - } else { - // Message does not have a MediaWiki page definition - $message = false; - Hooks::run( 'MessagesPreLoad', [ $title, &$message, $code ] ); - if ( $message !== false ) { - $this->cache->setField( $code, $title, ' ' . $message ); - } else { - $this->cache->setField( $code, $title, '!NONEXISTENT' ); - } - - return $message; - } - - if ( $this->cacheVolatile[$code] ) { - $entry = false; - // Make sure that individual keys respect the WAN cache holdoff period too - LoggerFactory::getInstance( 'MessageCache' )->debug( - __METHOD__ . ': loading volatile key \'{titleKey}\'', - [ 'titleKey' => $title, 'code' => $code ] ); - } else { - // Try the individual message cache + // Load the message page, utilizing the individual message cache. + // If the page does not exist, there will be no hook handler fallbacks. $entry = $this->loadCachedMessagePageEntry( $title, $code, $this->cache->getField( $code, 'HASH' ) ); + } else { + // Message page either does not exist or does not override a software message + $name = $this->contLang->lcfirst( $title ); + if ( !$this->isMainCacheable( $name, $this->overridable ) ) { + // Message page does not override any software-defined message. A custom + // message might be defined to have content or settings specific to the wiki. + // Load the message page, utilizing the individual message cache as needed. + $entry = $this->loadCachedMessagePageEntry( + $title, + $code, + $this->cache->getField( $code, 'HASH' ) + ); + } + if ( $entry === null || substr( $entry, 0, 1 ) !== ' ' ) { + // Message does not have a MediaWiki page definition; try hook handlers + $message = false; + Hooks::run( 'MessagesPreLoad', [ $title, &$message, $code ] ); + if ( $message !== false ) { + $this->cache->setField( $code, $title, ' ' . $message ); + } else { + $this->cache->setField( $code, $title, '!NONEXISTENT' ); + } + + return $message; + } } if ( $entry !== false && substr( $entry, 0, 1 ) === ' ' ) { - $this->cache->setField( $code, $title, $entry ); + if ( $this->cacheVolatile[$code] ) { + // Make sure that individual keys respect the WAN cache holdoff period too + LoggerFactory::getInstance( 'MessageCache' )->debug( + __METHOD__ . ': loading volatile key \'{titleKey}\'', + [ 'titleKey' => $title, 'code' => $code ] ); + } else { + $this->cache->setField( $code, $title, $entry ); + } // The message exists, so make sure a string is returned return (string)substr( $entry, 1 ); }