From: jenkins-bot Date: Thu, 4 Sep 2014 17:15:42 +0000 (+0000) Subject: Merge "LocalisationCache: Process one fallback at a time" X-Git-Tag: 1.31.0-rc.0~14166 X-Git-Url: https://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=commitdiff_plain;h=2fe29749ac48de2fe9caacff82414e8914e8017b;hp=ebaed675110376aa71f0c3b98760a6bdbfcfefd3 Merge "LocalisationCache: Process one fallback at a time" --- diff --git a/docs/hooks.txt b/docs/hooks.txt index 92497b2d61..4f0882e968 100644 --- a/docs/hooks.txt +++ b/docs/hooks.txt @@ -1635,6 +1635,13 @@ $code: language code &$alldata: The localisation data from core and extensions &purgeBlobs: whether to purge/update the message blobs via MessageBlobStore::clear() +'LocalisationCacheRecacheFallback': Called for each language when merging +fallback data into the cache. +$cache: The LocalisationCache object +$code: language code +&$alldata: The localisation data from core and extensions. Note some keys may + be omitted if they won't be merged into the final result. + 'LocalisationChecksBlacklist': When fetching the blacklist of localisation checks. &$blacklist: array of checks to blacklist. See the bottom of diff --git a/includes/cache/LocalisationCache.php b/includes/cache/LocalisationCache.php index bdfe51010b..bdfc75a6ef 100644 --- a/includes/cache/LocalisationCache.php +++ b/includes/cache/LocalisationCache.php @@ -843,73 +843,109 @@ class LocalisationCache { if ( $coreData['fallbackSequence'][$len - 1] !== 'en' ) { $coreData['fallbackSequence'][] = 'en'; } + } - # Load the fallback localisation item by item and merge it - foreach ( $coreData['fallbackSequence'] as $fbCode ) { - # Load the secondary localisation from the source file to - # avoid infinite cycles on cyclic fallbacks - $fbData = $this->readSourceFilesAndRegisterDeps( $fbCode, $deps ); - if ( $fbData === false ) { - continue; - } + $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] ); - foreach ( self::$allKeys as $key ) { - if ( !isset( $fbData[$key] ) ) { - continue; - } + wfProfileIn( __METHOD__ . '-fallbacks' ); + + # Load non-JSON localisation data for extensions + $extensionData = array_combine( + $codeSequence, + array_fill( 0, count( $codeSequence ), $initialData ) ); + foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) { + if ( isset( $wgMessagesDirs[$extension] ) ) { + # This extension has JSON message data; skip the PHP shim + continue; + } + + $data = $this->readPHPFile( $fileName, 'extension' ); + $used = false; - if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { - $this->mergeItem( $key, $coreData[$key], $fbData[$key] ); + foreach ( $data as $key => $item ) { + foreach ( $codeSequence as $csCode ) { + if ( isset( $item[$csCode] ) ) { + $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] ); + $used = true; } } } - } - $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] ); + if ( $used ) { + $deps[] = new FileDependency( $fileName ); + } + } - # Load core messages and the extension localisations. - wfProfileIn( __METHOD__ . '-extensions' ); + # Load the localisation data for each fallback, then merge it into the full array $allData = $initialData; - foreach ( $wgMessagesDirs as $dirs ) { - foreach ( (array)$dirs as $dir ) { - foreach ( $codeSequence as $csCode ) { + foreach ( $codeSequence as $csCode ) { + $csData = $initialData; + + # Load core messages and the extension localisations. + foreach ( $wgMessagesDirs as $dirs ) { + foreach ( (array)$dirs as $dir ) { $fileName = "$dir/$csCode.json"; $data = $this->readJSONFile( $fileName ); foreach ( $data as $key => $item ) { - $this->mergeItem( $key, $allData[$key], $item ); + $this->mergeItem( $key, $csData[$key], $item ); } $deps[] = new FileDependency( $fileName ); } } - } - foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) { - if ( isset( $wgMessagesDirs[$extension] ) ) { - # Already loaded the JSON files for this extension; skip the PHP shim - continue; + # Merge non-JSON extension data + if ( isset( $extensionData[$csCode] ) ) { + foreach ( $extensionData[$csCode] as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } } - $data = $this->readPHPFile( $fileName, 'extension' ); - $used = false; - - foreach ( $data as $key => $item ) { - if ( $this->mergeExtensionItem( $codeSequence, $key, $allData[$key], $item ) ) { - $used = true; + if ( $csCode === $code ) { + # Merge core data into extension data + foreach ( $coreData as $key => $item ) { + $this->mergeItem( $key, $csData[$key], $item ); + } + } else { + # Load the secondary localisation from the source file to + # avoid infinite cycles on cyclic fallbacks + $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps ); + if ( $fbData !== false ) { + # Only merge the keys that make sense to merge + foreach ( self::$allKeys as $key ) { + if ( !isset( $fbData[$key] ) ) { + continue; + } + + if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $csData[$key], $fbData[$key] ); + } + } } } - if ( $used ) { - $deps[] = new FileDependency( $fileName ); + # Allow extensions an opportunity to adjust the data for this + # fallback + wfRunHooks( 'LocalisationCacheRecacheFallback', array( $this, $csCode, &$csData ) ); + + # Merge the data for this fallback into the final array + if ( $csCode === $code ) { + $allData = $csData; + } else { + foreach ( self::$allKeys as $key ) { + if ( !isset( $csData[$key] ) ) { + continue; + } + + if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) { + $this->mergeItem( $key, $allData[$key], $csData[$key] ); + } + } } } - # Merge core data into extension data - foreach ( $coreData as $key => $item ) { - $this->mergeItem( $key, $allData[$key], $item ); - } - wfProfileOut( __METHOD__ . '-extensions' ); + wfProfileOut( __METHOD__ . '-fallbacks' ); # Add cache dependencies for any referenced globals $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' ); diff --git a/tests/phpunit/data/localisationcache/en.json b/tests/phpunit/data/localisationcache/en.json new file mode 100644 index 0000000000..27600cdb2c --- /dev/null +++ b/tests/phpunit/data/localisationcache/en.json @@ -0,0 +1,5 @@ +{ + "present-uk": "en", + "present-ru": "en", + "present-en": "en" +} diff --git a/tests/phpunit/data/localisationcache/ru.json b/tests/phpunit/data/localisationcache/ru.json new file mode 100644 index 0000000000..79e1444bca --- /dev/null +++ b/tests/phpunit/data/localisationcache/ru.json @@ -0,0 +1,4 @@ +{ + "present-uk": "ru", + "present-ru": "ru" +} diff --git a/tests/phpunit/data/localisationcache/uk.json b/tests/phpunit/data/localisationcache/uk.json new file mode 100644 index 0000000000..f63ce5d3f5 --- /dev/null +++ b/tests/phpunit/data/localisationcache/uk.json @@ -0,0 +1,3 @@ +{ + "present-uk": "uk" +} diff --git a/tests/phpunit/includes/LocalisationCacheTest.php b/tests/phpunit/includes/LocalisationCacheTest.php deleted file mode 100644 index f98410a1b8..0000000000 --- a/tests/phpunit/includes/LocalisationCacheTest.php +++ /dev/null @@ -1,34 +0,0 @@ -assertEquals( - $cache->getItem( 'ar', 'pluralRules' ), - $cache->getItem( 'arz', 'pluralRules' ), - 'arz plural rules (undefined) fallback to ar (defined)' - ); - - $this->assertEquals( - $cache->getItem( 'ar', 'compiledPluralRules' ), - $cache->getItem( 'arz', 'compiledPluralRules' ), - 'arz compiled plural rules (undefined) fallback to ar (defined)' - ); - - $this->assertNotEquals( - $cache->getItem( 'ksh', 'pluralRules' ), - $cache->getItem( 'de', 'pluralRules' ), - 'ksh plural rules (defined) dont fallback to de (defined)' - ); - - $this->assertNotEquals( - $cache->getItem( 'ksh', 'compiledPluralRules' ), - $cache->getItem( 'de', 'compiledPluralRules' ), - 'ksh compiled plural rules (defined) dont fallback to de (defined)' - ); - } -} diff --git a/tests/phpunit/includes/cache/LocalisationCacheTest.php b/tests/phpunit/includes/cache/LocalisationCacheTest.php new file mode 100644 index 0000000000..fc06a50126 --- /dev/null +++ b/tests/phpunit/includes/cache/LocalisationCacheTest.php @@ -0,0 +1,91 @@ +setMwGlobals( array( + 'wgMessagesDirs' => array( "$IP/tests/phpunit/data/localisationcache" ), + 'wgExtensionMessagesFiles' => array(), + 'wgHooks' => array(), + ) ); + } + + public function testPuralRulesFallback() { + $cache = new LocalisationCache( array( 'store' => 'detect' ) ); + + $this->assertEquals( + $cache->getItem( 'ar', 'pluralRules' ), + $cache->getItem( 'arz', 'pluralRules' ), + 'arz plural rules (undefined) fallback to ar (defined)' + ); + + $this->assertEquals( + $cache->getItem( 'ar', 'compiledPluralRules' ), + $cache->getItem( 'arz', 'compiledPluralRules' ), + 'arz compiled plural rules (undefined) fallback to ar (defined)' + ); + + $this->assertNotEquals( + $cache->getItem( 'ksh', 'pluralRules' ), + $cache->getItem( 'de', 'pluralRules' ), + 'ksh plural rules (defined) dont fallback to de (defined)' + ); + + $this->assertNotEquals( + $cache->getItem( 'ksh', 'compiledPluralRules' ), + $cache->getItem( 'de', 'compiledPluralRules' ), + 'ksh compiled plural rules (defined) dont fallback to de (defined)' + ); + } + + public function testRecacheFallbacks() { + $lc = new LocalisationCache( array( 'store' => 'detect' ) ); + $lc->recache( 'uk' ); + $this->assertEquals( + array( + 'present-uk' => 'uk', + 'present-ru' => 'ru', + 'present-en' => 'en', + ), + $lc->getItem( 'uk', 'messages' ), + 'Fallbacks are only used to fill missing data' + ); + } + + public function testRecacheFallbacksWithHooks() { + global $wgHooks; + + // Use hook to provide updates for messages. This is what the + // LocalisationUpdate extension does. See bug 68781. + $wgHooks['LocalisationCacheRecacheFallback'][] = function ( + LocalisationCache $lc, + $code, + array &$cache + ) { + if ( $code === 'ru' ) { + $cache['messages']['present-uk'] = 'ru-override'; + $cache['messages']['present-ru'] = 'ru-override'; + $cache['messages']['present-en'] = 'ru-override'; + } + }; + + $lc = new LocalisationCache( array( 'store' => 'detect' ) ); + $lc->recache( 'uk' ); + $this->assertEquals( + array( + 'present-uk' => 'uk', + 'present-ru' => 'ru-override', + 'present-en' => 'ru-override', + ), + $lc->getItem( 'uk', 'messages' ), + 'Updates provided by hooks follow the normal fallback order.' + ); + } +}