includes/cache/localisation/LocalisationCache.php

   1 <?php
   2 /**
   3  * Cache of the contents of localisation files.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 use CLDRPluralRuleParser\Evaluator;
  24 use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
  25 use MediaWiki\Config\ServiceOptions;
  26 use MediaWiki\Languages\LanguageNameUtils;
  27 use Psr\Log\LoggerInterface;
  28
  29 /**
  30  * Class for caching the contents of localisation files, Messages*.php
  31  * and *.i18n.php.
  32  *
  33  * An instance of this class is available using MediaWikiServices.
  34  *
  35  * The values retrieved from here are merged, containing items from extension
  36  * files, core messages files and the language fallback sequence (e.g. zh-cn ->
  37  * zh-hans -> en ). Some common errors are corrected, for example namespace
  38  * names with spaces instead of underscores, but heavyweight processing, such
  39  * as grammatical transformation, is done by the caller.
  40  */
  41 class LocalisationCache {
  42         const VERSION = 4;
  43
  44         /** @var ServiceOptions */
  45         private $options;
  46
  47         /**
  48          * True if recaching should only be done on an explicit call to recache().
  49          * Setting this reduces the overhead of cache freshness checking, which
  50          * requires doing a stat() for every extension i18n file.
  51          */
  52         private $manualRecache = false;
  53
  54         /**
  55          * The cache data. 3-d array, where the first key is the language code,
  56          * the second key is the item key e.g. 'messages', and the third key is
  57          * an item specific subkey index. Some items are not arrays and so for those
  58          * items, there are no subkeys.
  59          */
  60         protected $data = [];
  61
  62         /**
  63          * The persistent store object. An instance of LCStore.
  64          *
  65          * @var LCStore
  66          */
  67         private $store;
  68
  69         /**
  70          * @var LoggerInterface
  71          */
  72         private $logger;
  73
  74         /** @var callable[] See comment for parameter in constructor */
  75         private $clearStoreCallbacks;
  76
  77         /** @var LanguageNameUtils */
  78         private $langNameUtils;
  79
  80         /**
  81          * A 2-d associative array, code/key, where presence indicates that the item
  82          * is loaded. Value arbitrary.
  83          *
  84          * For split items, if set, this indicates that all of the subitems have been
  85          * loaded.
  86          *
  87          */
  88         private $loadedItems = [];
  89
  90         /**
  91          * A 3-d associative array, code/key/subkey, where presence indicates that
  92          * the subitem is loaded. Only used for the split items, i.e. messages.
  93          */
  94         private $loadedSubitems = [];
  95
  96         /**
  97          * An array where presence of a key indicates that that language has been
  98          * initialised. Initialisation includes checking for cache expiry and doing
  99          * any necessary updates.
 100          */
 101         private $initialisedLangs = [];
 102
 103         /**
 104          * An array mapping non-existent pseudo-languages to fallback languages. This
 105          * is filled by initShallowFallback() when data is requested from a language
 106          * that lacks a Messages*.php file.
 107          */
 108         private $shallowFallbacks = [];
 109
 110         /**
 111          * An array where the keys are codes that have been recached by this instance.
 112          */
 113         private $recachedLangs = [];
 114
 115         /**
 116          * All item keys
 117          */
 118         public static $allKeys = [
 119                 'fallback', 'namespaceNames', 'bookstoreList',
 120                 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns',
 121                 'digitTransformTable', 'separatorTransformTable',
 122                 'minimumGroupingDigits', 'fallback8bitEncoding',
 123                 'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset',
 124                 'namespaceAliases', 'dateFormats', 'datePreferences',
 125                 'datePreferenceMigrationMap', 'defaultDateFormat',
 126                 'specialPageAliases', 'imageFiles', 'preloadedMessages',
 127                 'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules',
 128                 'pluralRuleTypes', 'compiledPluralRules',
 129         ];
 130
 131         /**
 132          * Keys for items which consist of associative arrays, which may be merged
 133          * by a fallback sequence.
 134          */
 135         public static $mergeableMapKeys = [ 'messages', 'namespaceNames',
 136                 'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages'
 137         ];
 138
 139         /**
 140          * Keys for items which are a numbered array.
 141          */
 142         public static $mergeableListKeys = [];
 143
 144         /**
 145          * Keys for items which contain an array of arrays of equivalent aliases
 146          * for each subitem. The aliases may be merged by a fallback sequence.
 147          */
 148         public static $mergeableAliasListKeys = [ 'specialPageAliases' ];
 149
 150         /**
 151          * Keys for items which contain an associative array, and may be merged if
 152          * the primary value contains the special array key "inherit". That array
 153          * key is removed after the first merge.
 154          */
 155         public static $optionalMergeKeys = [ 'bookstoreList' ];
 156
 157         /**
 158          * Keys for items that are formatted like $magicWords
 159          */
 160         public static $magicWordKeys = [ 'magicWords' ];
 161
 162         /**
 163          * Keys for items where the subitems are stored in the backend separately.
 164          */
 165         public static $splitKeys = [ 'messages' ];
 166
 167         /**
 168          * Keys which are loaded automatically by initLanguage()
 169          */
 170         public static $preloadedKeys = [ 'dateFormats', 'namespaceNames' ];
 171
 172         /**
 173          * Associative array of cached plural rules. The key is the language code,
 174          * the value is an array of plural rules for that language.
 175          */
 176         private $pluralRules = null;
 177
 178         /**
 179          * Associative array of cached plural rule types. The key is the language
 180          * code, the value is an array of plural rule types for that language. For
 181          * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many'].
 182          * The index for each rule type matches the index for the rule in
 183          * $pluralRules, thus allowing correlation between the two. The reason we
 184          * don't just use the type names as the keys in $pluralRules is because
 185          * Language::convertPlural applies the rules based on numeric order (or
 186          * explicit numeric parameter), not based on the name of the rule type. For
 187          * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than
 188          * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}.
 189          */
 190         private $pluralRuleTypes = null;
 191
 192         private $mergeableKeys = null;
 193
 194         /**
 195          * Return a suitable LCStore as specified by the given configuration.
 196          *
 197          * @since 1.34
 198          * @param array $conf In the format of $wgLocalisationCacheConf
 199          * @param string|false|null $fallbackCacheDir In case 'storeDirectory' isn't specified
 200          * @return LCStore
 201          */
 202         public static function getStoreFromConf( array $conf, $fallbackCacheDir ) : LCStore {
 203                 $storeArg = [];
 204                 $storeArg['directory'] =
 205                         $conf['storeDirectory'] ?: $fallbackCacheDir;
 206
 207                 if ( !empty( $conf['storeClass'] ) ) {
 208                         $storeClass = $conf['storeClass'];
 209                 } elseif ( $conf['store'] === 'files' || $conf['store'] === 'file' ||
 210                         ( $conf['store'] === 'detect' && $storeArg['directory'] )
 211                 ) {
 212                         $storeClass = LCStoreCDB::class;
 213                 } elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) {
 214                         $storeClass = LCStoreDB::class;
 215                         $storeArg['server'] = $conf['storeServer'] ?? [];
 216                 } elseif ( $conf['store'] === 'array' ) {
 217                         $storeClass = LCStoreStaticArray::class;
 218                 } else {
 219                         throw new MWException(
 220                                 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.'
 221                         );
 222                 }
 223
 224                 return new $storeClass( $storeArg );
 225         }
 226
 227         /**
 228          * @var array
 229          * @since 1.34
 230          */
 231         public const CONSTRUCTOR_OPTIONS = [
 232                 // True to treat all files as expired until they are regenerated by this object.
 233                 'forceRecache',
 234                 'manualRecache',
 235                 'ExtensionMessagesFiles',
 236                 'MessagesDirs',
 237         ];
 238
 239         /**
 240          * For constructor parameters, see the documentation in DefaultSettings.php
 241          * for $wgLocalisationCacheConf.
 242          *
 243          * Do not construct this directly. Use MediaWikiServices.
 244          *
 245          * @param ServiceOptions $options
 246          * @param LCStore $store What backend to use for storage
 247          * @param LoggerInterface $logger
 248          * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be
 249          *   used to clear other caches that depend on this one, such as ResourceLoader's
 250          *   MessageBlobStore.
 251          * @param LanguageNameUtils $langNameUtils
 252          * @throws MWException
 253          */
 254         function __construct(
 255                 ServiceOptions $options,
 256                 LCStore $store,
 257                 LoggerInterface $logger,
 258                 array $clearStoreCallbacks,
 259                 LanguageNameUtils $langNameUtils
 260         ) {
 261                 $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
 262
 263                 $this->options = $options;
 264                 $this->store = $store;
 265                 $this->logger = $logger;
 266                 $this->clearStoreCallbacks = $clearStoreCallbacks;
 267                 $this->langNameUtils = $langNameUtils;
 268
 269                 // Keep this separate from $this->options so it can be mutable
 270                 $this->manualRecache = $options->get( 'manualRecache' );
 271         }
 272
 273         /**
 274          * Returns true if the given key is mergeable, that is, if it is an associative
 275          * array which can be merged through a fallback sequence.
 276          * @param string $key
 277          * @return bool
 278          */
 279         public function isMergeableKey( $key ) {
 280                 if ( $this->mergeableKeys === null ) {
 281                         $this->mergeableKeys = array_flip( array_merge(
 282                                 self::$mergeableMapKeys,
 283                                 self::$mergeableListKeys,
 284                                 self::$mergeableAliasListKeys,
 285                                 self::$optionalMergeKeys,
 286                                 self::$magicWordKeys
 287                         ) );
 288                 }
 289
 290                 return isset( $this->mergeableKeys[$key] );
 291         }
 292
 293         /**
 294          * Get a cache item.
 295          *
 296          * Warning: this may be slow for split items (messages), since it will
 297          * need to fetch all of the subitems from the cache individually.
 298          * @param string $code
 299          * @param string $key
 300          * @return mixed
 301          */
 302         public function getItem( $code, $key ) {
 303                 if ( !isset( $this->loadedItems[$code][$key] ) ) {
 304                         $this->loadItem( $code, $key );
 305                 }
 306
 307                 if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
 308                         return $this->shallowFallbacks[$code];
 309                 }
 310
 311                 return $this->data[$code][$key];
 312         }
 313
 314         /**
 315          * Get a subitem, for instance a single message for a given language.
 316          * @param string $code
 317          * @param string $key
 318          * @param string $subkey
 319          * @return mixed|null
 320          */
 321         public function getSubitem( $code, $key, $subkey ) {
 322                 if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
 323                         !isset( $this->loadedItems[$code][$key] )
 324                 ) {
 325                         $this->loadSubitem( $code, $key, $subkey );
 326                 }
 327
 328                 return $this->data[$code][$key][$subkey] ?? null;
 329         }
 330
 331         /**
 332          * Get the list of subitem keys for a given item.
 333          *
 334          * This is faster than array_keys($lc->getItem(...)) for the items listed in
 335          * self::$splitKeys.
 336          *
 337          * Will return null if the item is not found, or false if the item is not an
 338          * array.
 339          * @param string $code
 340          * @param string $key
 341          * @return bool|null|string|string[]
 342          */
 343         public function getSubitemList( $code, $key ) {
 344                 if ( in_array( $key, self::$splitKeys ) ) {
 345                         return $this->getSubitem( $code, 'list', $key );
 346                 } else {
 347                         $item = $this->getItem( $code, $key );
 348                         if ( is_array( $item ) ) {
 349                                 return array_keys( $item );
 350                         } else {
 351                                 return false;
 352                         }
 353                 }
 354         }
 355
 356         /**
 357          * Load an item into the cache.
 358          * @param string $code
 359          * @param string $key
 360          */
 361         protected function loadItem( $code, $key ) {
 362                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 363                         $this->initLanguage( $code );
 364                 }
 365
 366                 // Check to see if initLanguage() loaded it for us
 367                 if ( isset( $this->loadedItems[$code][$key] ) ) {
 368                         return;
 369                 }
 370
 371                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 372                         $this->loadItem( $this->shallowFallbacks[$code], $key );
 373
 374                         return;
 375                 }
 376
 377                 if ( in_array( $key, self::$splitKeys ) ) {
 378                         $subkeyList = $this->getSubitem( $code, 'list', $key );
 379                         foreach ( $subkeyList as $subkey ) {
 380                                 if ( isset( $this->data[$code][$key][$subkey] ) ) {
 381                                         continue;
 382                                 }
 383                                 $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey );
 384                         }
 385                 } else {
 386                         $this->data[$code][$key] = $this->store->get( $code, $key );
 387                 }
 388
 389                 $this->loadedItems[$code][$key] = true;
 390         }
 391
 392         /**
 393          * Load a subitem into the cache
 394          * @param string $code
 395          * @param string $key
 396          * @param string $subkey
 397          */
 398         protected function loadSubitem( $code, $key, $subkey ) {
 399                 if ( !in_array( $key, self::$splitKeys ) ) {
 400                         $this->loadItem( $code, $key );
 401
 402                         return;
 403                 }
 404
 405                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 406                         $this->initLanguage( $code );
 407                 }
 408
 409                 // Check to see if initLanguage() loaded it for us
 410                 if ( isset( $this->loadedItems[$code][$key] ) ||
 411                         isset( $this->loadedSubitems[$code][$key][$subkey] )
 412                 ) {
 413                         return;
 414                 }
 415
 416                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 417                         $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
 418
 419                         return;
 420                 }
 421
 422                 $value = $this->store->get( $code, "$key:$subkey" );
 423                 $this->data[$code][$key][$subkey] = $value;
 424                 $this->loadedSubitems[$code][$key][$subkey] = true;
 425         }
 426
 427         /**
 428          * Returns true if the cache identified by $code is missing or expired.
 429          *
 430          * @param string $code
 431          *
 432          * @return bool
 433          */
 434         public function isExpired( $code ) {
 435                 if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
 436                         $this->logger->debug( __METHOD__ . "($code): forced reload" );
 437
 438                         return true;
 439                 }
 440
 441                 $deps = $this->store->get( $code, 'deps' );
 442                 $keys = $this->store->get( $code, 'list' );
 443                 $preload = $this->store->get( $code, 'preload' );
 444                 // Different keys may expire separately for some stores
 445                 if ( $deps === null || $keys === null || $preload === null ) {
 446                         $this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" );
 447
 448                         return true;
 449                 }
 450
 451                 foreach ( $deps as $dep ) {
 452                         // Because we're unserializing stuff from cache, we
 453                         // could receive objects of classes that don't exist
 454                         // anymore (e.g. uninstalled extensions)
 455                         // When this happens, always expire the cache
 456                         if ( !$dep instanceof CacheDependency || $dep->isExpired() ) {
 457                                 $this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " .
 458                                         get_class( $dep ) );
 459
 460                                 return true;
 461                         }
 462                 }
 463
 464                 return false;
 465         }
 466
 467         /**
 468          * Initialise a language in this object. Rebuild the cache if necessary.
 469          * @param string $code
 470          * @throws MWException
 471          */
 472         protected function initLanguage( $code ) {
 473                 if ( isset( $this->initialisedLangs[$code] ) ) {
 474                         return;
 475                 }
 476
 477                 $this->initialisedLangs[$code] = true;
 478
 479                 # If the code is of the wrong form for a Messages*.php file, do a shallow fallback
 480                 if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) {
 481                         $this->initShallowFallback( $code, 'en' );
 482
 483                         return;
 484                 }
 485
 486                 # Recache the data if necessary
 487                 if ( !$this->manualRecache && $this->isExpired( $code ) ) {
 488                         if ( $this->langNameUtils->isSupportedLanguage( $code ) ) {
 489                                 $this->recache( $code );
 490                         } elseif ( $code === 'en' ) {
 491                                 throw new MWException( 'MessagesEn.php is missing.' );
 492                         } else {
 493                                 $this->initShallowFallback( $code, 'en' );
 494                         }
 495
 496                         return;
 497                 }
 498
 499                 # Preload some stuff
 500                 $preload = $this->getItem( $code, 'preload' );
 501                 if ( $preload === null ) {
 502                         if ( $this->manualRecache ) {
 503                                 // No Messages*.php file. Do shallow fallback to en.
 504                                 if ( $code === 'en' ) {
 505                                         throw new MWException( 'No localisation cache found for English. ' .
 506                                                 'Please run maintenance/rebuildLocalisationCache.php.' );
 507                                 }
 508                                 $this->initShallowFallback( $code, 'en' );
 509
 510                                 return;
 511                         } else {
 512                                 throw new MWException( 'Invalid or missing localisation cache.' );
 513                         }
 514                 }
 515                 $this->data[$code] = $preload;
 516                 foreach ( $preload as $key => $item ) {
 517                         if ( in_array( $key, self::$splitKeys ) ) {
 518                                 foreach ( $item as $subkey => $subitem ) {
 519                                         $this->loadedSubitems[$code][$key][$subkey] = true;
 520                                 }
 521                         } else {
 522                                 $this->loadedItems[$code][$key] = true;
 523                         }
 524                 }
 525         }
 526
 527         /**
 528          * Create a fallback from one language to another, without creating a
 529          * complete persistent cache.
 530          * @param string $primaryCode
 531          * @param string $fallbackCode
 532          */
 533         public function initShallowFallback( $primaryCode, $fallbackCode ) {
 534                 $this->data[$primaryCode] =& $this->data[$fallbackCode];
 535                 $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
 536                 $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
 537                 $this->shallowFallbacks[$primaryCode] = $fallbackCode;
 538         }
 539
 540         /**
 541          * Read a PHP file containing localisation data.
 542          * @param string $_fileName
 543          * @param string $_fileType
 544          * @throws MWException
 545          * @return array
 546          */
 547         protected function readPHPFile( $_fileName, $_fileType ) {
 548                 include $_fileName;
 549
 550                 $data = [];
 551                 if ( $_fileType == 'core' || $_fileType == 'extension' ) {
 552                         foreach ( self::$allKeys as $key ) {
 553                                 // Not all keys are set in language files, so
 554                                 // check they exist first
 555                                 if ( isset( $$key ) ) {
 556                                         $data[$key] = $$key;
 557                                 }
 558                         }
 559                 } elseif ( $_fileType == 'aliases' ) {
 560                         if ( isset( $aliases ) ) {
 561                                 $data['aliases'] = $aliases;
 562                         }
 563                 } else {
 564                         throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" );
 565                 }
 566
 567                 return $data;
 568         }
 569
 570         /**
 571          * Read a JSON file containing localisation messages.
 572          * @param string $fileName Name of file to read
 573          * @throws MWException If there is a syntax error in the JSON file
 574          * @return array Array with a 'messages' key, or empty array if the file doesn't exist
 575          */
 576         public function readJSONFile( $fileName ) {
 577                 if ( !is_readable( $fileName ) ) {
 578                         return [];
 579                 }
 580
 581                 $json = file_get_contents( $fileName );
 582                 if ( $json === false ) {
 583                         return [];
 584                 }
 585
 586                 $data = FormatJson::decode( $json, true );
 587                 if ( $data === null ) {
 588                         throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" );
 589                 }
 590
 591                 // Remove keys starting with '@', they're reserved for metadata and non-message data
 592                 foreach ( $data as $key => $unused ) {
 593                         if ( $key === '' || $key[0] === '@' ) {
 594                                 unset( $data[$key] );
 595                         }
 596                 }
 597
 598                 // The JSON format only supports messages, none of the other variables, so wrap the data
 599                 return [ 'messages' => $data ];
 600         }
 601
 602         /**
 603          * Get the compiled plural rules for a given language from the XML files.
 604          * @since 1.20
 605          * @param string $code
 606          * @return array|null
 607          */
 608         public function getCompiledPluralRules( $code ) {
 609                 $rules = $this->getPluralRules( $code );
 610                 if ( $rules === null ) {
 611                         return null;
 612                 }
 613                 try {
 614                         $compiledRules = Evaluator::compile( $rules );
 615                 } catch ( CLDRPluralRuleError $e ) {
 616                         $this->logger->debug( $e->getMessage() );
 617
 618                         return [];
 619                 }
 620
 621                 return $compiledRules;
 622         }
 623
 624         /**
 625          * Get the plural rules for a given language from the XML files.
 626          * Cached.
 627          * @since 1.20
 628          * @param string $code
 629          * @return array|null
 630          */
 631         public function getPluralRules( $code ) {
 632                 if ( $this->pluralRules === null ) {
 633                         $this->loadPluralFiles();
 634                 }
 635                 return $this->pluralRules[$code] ?? null;
 636         }
 637
 638         /**
 639          * Get the plural rule types for a given language from the XML files.
 640          * Cached.
 641          * @since 1.22
 642          * @param string $code
 643          * @return array|null
 644          */
 645         public function getPluralRuleTypes( $code ) {
 646                 if ( $this->pluralRuleTypes === null ) {
 647                         $this->loadPluralFiles();
 648                 }
 649                 return $this->pluralRuleTypes[$code] ?? null;
 650         }
 651
 652         /**
 653          * Load the plural XML files.
 654          */
 655         protected function loadPluralFiles() {
 656                 global $IP;
 657                 $cldrPlural = "$IP/languages/data/plurals.xml";
 658                 $mwPlural = "$IP/languages/data/plurals-mediawiki.xml";
 659                 // Load CLDR plural rules
 660                 $this->loadPluralFile( $cldrPlural );
 661                 if ( file_exists( $mwPlural ) ) {
 662                         // Override or extend
 663                         $this->loadPluralFile( $mwPlural );
 664                 }
 665         }
 666
 667         /**
 668          * Load a plural XML file with the given filename, compile the relevant
 669          * rules, and save the compiled rules in a process-local cache.
 670          *
 671          * @param string $fileName
 672          * @throws MWException
 673          */
 674         protected function loadPluralFile( $fileName ) {
 675                 // Use file_get_contents instead of DOMDocument::load (T58439)
 676                 $xml = file_get_contents( $fileName );
 677                 if ( !$xml ) {
 678                         throw new MWException( "Unable to read plurals file $fileName" );
 679                 }
 680                 $doc = new DOMDocument;
 681                 $doc->loadXML( $xml );
 682                 $rulesets = $doc->getElementsByTagName( "pluralRules" );
 683                 foreach ( $rulesets as $ruleset ) {
 684                         $codes = $ruleset->getAttribute( 'locales' );
 685                         $rules = [];
 686                         $ruleTypes = [];
 687                         $ruleElements = $ruleset->getElementsByTagName( "pluralRule" );
 688                         foreach ( $ruleElements as $elt ) {
 689                                 $ruleType = $elt->getAttribute( 'count' );
 690                                 if ( $ruleType === 'other' ) {
 691                                         // Don't record "other" rules, which have an empty condition
 692                                         continue;
 693                                 }
 694                                 $rules[] = $elt->nodeValue;
 695                                 $ruleTypes[] = $ruleType;
 696                         }
 697                         foreach ( explode( ' ', $codes ) as $code ) {
 698                                 $this->pluralRules[$code] = $rules;
 699                                 $this->pluralRuleTypes[$code] = $ruleTypes;
 700                         }
 701                 }
 702         }
 703
 704         /**
 705          * Read the data from the source files for a given language, and register
 706          * the relevant dependencies in the $deps array. If the localisation
 707          * exists, the data array is returned, otherwise false is returned.
 708          *
 709          * @param string $code
 710          * @param array &$deps
 711          * @return array
 712          */
 713         protected function readSourceFilesAndRegisterDeps( $code, &$deps ) {
 714                 global $IP;
 715
 716                 // This reads in the PHP i18n file with non-messages l10n data
 717                 $fileName = $this->langNameUtils->getMessagesFileName( $code );
 718                 if ( !file_exists( $fileName ) ) {
 719                         $data = [];
 720                 } else {
 721                         $deps[] = new FileDependency( $fileName );
 722                         $data = $this->readPHPFile( $fileName, 'core' );
 723                 }
 724
 725                 # Load CLDR plural rules for JavaScript
 726                 $data['pluralRules'] = $this->getPluralRules( $code );
 727                 # And for PHP
 728                 $data['compiledPluralRules'] = $this->getCompiledPluralRules( $code );
 729                 # Load plural rule types
 730                 $data['pluralRuleTypes'] = $this->getPluralRuleTypes( $code );
 731
 732                 $deps['plurals'] = new FileDependency( "$IP/languages/data/plurals.xml" );
 733                 $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" );
 734
 735                 return $data;
 736         }
 737
 738         /**
 739          * Merge two localisation values, a primary and a fallback, overwriting the
 740          * primary value in place.
 741          * @param string $key
 742          * @param mixed &$value
 743          * @param mixed $fallbackValue
 744          */
 745         protected function mergeItem( $key, &$value, $fallbackValue ) {
 746                 if ( !is_null( $value ) ) {
 747                         if ( !is_null( $fallbackValue ) ) {
 748                                 if ( in_array( $key, self::$mergeableMapKeys ) ) {
 749                                         $value = $value + $fallbackValue;
 750                                 } elseif ( in_array( $key, self::$mergeableListKeys ) ) {
 751                                         // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
 752                                         $value = array_unique( array_merge( $fallbackValue, $value ) );
 753                                 } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) {
 754                                         $value = array_merge_recursive( $value, $fallbackValue );
 755                                 } elseif ( in_array( $key, self::$optionalMergeKeys ) ) {
 756                                         if ( !empty( $value['inherit'] ) ) {
 757                                                 $value = array_merge( $fallbackValue, $value );
 758                                         }
 759
 760                                         if ( isset( $value['inherit'] ) ) {
 761                                                 unset( $value['inherit'] );
 762                                         }
 763                                 } elseif ( in_array( $key, self::$magicWordKeys ) ) {
 764                                         $this->mergeMagicWords( $value, $fallbackValue );
 765                                 }
 766                         }
 767                 } else {
 768                         $value = $fallbackValue;
 769                 }
 770         }
 771
 772         /**
 773          * @param mixed &$value
 774          * @param mixed $fallbackValue
 775          */
 776         protected function mergeMagicWords( &$value, $fallbackValue ) {
 777                 foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
 778                         if ( !isset( $value[$magicName] ) ) {
 779                                 $value[$magicName] = $fallbackInfo;
 780                         } else {
 781                                 $oldSynonyms = array_slice( $fallbackInfo, 1 );
 782                                 $newSynonyms = array_slice( $value[$magicName], 1 );
 783                                 $synonyms = array_values( array_unique( array_merge(
 784                                         $newSynonyms, $oldSynonyms ) ) );
 785                                 $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms );
 786                         }
 787                 }
 788         }
 789
 790         /**
 791          * Given an array mapping language code to localisation value, such as is
 792          * found in extension *.i18n.php files, iterate through a fallback sequence
 793          * to merge the given data with an existing primary value.
 794          *
 795          * Returns true if any data from the extension array was used, false
 796          * otherwise.
 797          * @param array $codeSequence
 798          * @param string $key
 799          * @param mixed &$value
 800          * @param mixed $fallbackValue
 801          * @return bool
 802          */
 803         protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) {
 804                 $used = false;
 805                 foreach ( $codeSequence as $code ) {
 806                         if ( isset( $fallbackValue[$code] ) ) {
 807                                 $this->mergeItem( $key, $value, $fallbackValue[$code] );
 808                                 $used = true;
 809                         }
 810                 }
 811
 812                 return $used;
 813         }
 814
 815         /**
 816          * Gets the combined list of messages dirs from
 817          * core and extensions
 818          *
 819          * @since 1.25
 820          * @return array
 821          */
 822         public function getMessagesDirs() {
 823                 global $IP;
 824
 825                 return [
 826                         'core' => "$IP/languages/i18n",
 827                         'exif' => "$IP/languages/i18n/exif",
 828                         'api' => "$IP/includes/api/i18n",
 829                         'oojs-ui' => "$IP/resources/lib/ooui/i18n",
 830                 ] + $this->options->get( 'MessagesDirs' );
 831         }
 832
 833         /**
 834          * Load localisation data for a given language for both core and extensions
 835          * and save it to the persistent cache store and the process cache
 836          * @param string $code
 837          * @throws MWException
 838          */
 839         public function recache( $code ) {
 840                 if ( !$code ) {
 841                         throw new MWException( "Invalid language code requested" );
 842                 }
 843                 $this->recachedLangs[ $code ] = true;
 844
 845                 # Initial values
 846                 $initialData = array_fill_keys( self::$allKeys, null );
 847                 $coreData = $initialData;
 848                 $deps = [];
 849
 850                 # Load the primary localisation from the source file
 851                 $data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
 852                 $this->logger->debug( __METHOD__ . ": got localisation for $code from source" );
 853
 854                 # Merge primary localisation
 855                 foreach ( $data as $key => $value ) {
 856                         $this->mergeItem( $key, $coreData[ $key ], $value );
 857                 }
 858
 859                 # Fill in the fallback if it's not there already
 860                 if ( ( is_null( $coreData['fallback'] ) || $coreData['fallback'] === false ) && $code === 'en' ) {
 861                         $coreData['fallback'] = false;
 862                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = [];
 863                 } else {
 864                         if ( !is_null( $coreData['fallback'] ) ) {
 865                                 $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) );
 866                         } else {
 867                                 $coreData['fallbackSequence'] = [];
 868                         }
 869                         $len = count( $coreData['fallbackSequence'] );
 870
 871                         # Before we add the 'en' fallback for messages, keep a copy of
 872                         # the original fallback sequence
 873                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'];
 874
 875                         # Ensure that the sequence ends at 'en' for messages
 876                         if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
 877                                 $coreData['fallbackSequence'][] = 'en';
 878                         }
 879                 }
 880
 881                 $codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] );
 882                 $messageDirs = $this->getMessagesDirs();
 883
 884                 # Load non-JSON localisation data for extensions
 885                 $extensionData = array_fill_keys( $codeSequence, $initialData );
 886                 foreach ( $this->options->get( 'ExtensionMessagesFiles' ) as $extension => $fileName ) {
 887                         if ( isset( $messageDirs[$extension] ) ) {
 888                                 # This extension has JSON message data; skip the PHP shim
 889                                 continue;
 890                         }
 891
 892                         $data = $this->readPHPFile( $fileName, 'extension' );
 893                         $used = false;
 894
 895                         foreach ( $data as $key => $item ) {
 896                                 foreach ( $codeSequence as $csCode ) {
 897                                         if ( isset( $item[$csCode] ) ) {
 898                                                 $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
 899                                                 $used = true;
 900                                         }
 901                                 }
 902                         }
 903
 904                         if ( $used ) {
 905                                 $deps[] = new FileDependency( $fileName );
 906                         }
 907                 }
 908
 909                 # Load the localisation data for each fallback, then merge it into the full array
 910                 $allData = $initialData;
 911                 foreach ( $codeSequence as $csCode ) {
 912                         $csData = $initialData;
 913
 914                         # Load core messages and the extension localisations.
 915                         foreach ( $messageDirs as $dirs ) {
 916                                 foreach ( (array)$dirs as $dir ) {
 917                                         $fileName = "$dir/$csCode.json";
 918                                         $data = $this->readJSONFile( $fileName );
 919
 920                                         foreach ( $data as $key => $item ) {
 921                                                 $this->mergeItem( $key, $csData[$key], $item );
 922                                         }
 923
 924                                         $deps[] = new FileDependency( $fileName );
 925                                 }
 926                         }
 927
 928                         # Merge non-JSON extension data
 929                         if ( isset( $extensionData[$csCode] ) ) {
 930                                 foreach ( $extensionData[$csCode] as $key => $item ) {
 931                                         $this->mergeItem( $key, $csData[$key], $item );
 932                                 }
 933                         }
 934
 935                         if ( $csCode === $code ) {
 936                                 # Merge core data into extension data
 937                                 foreach ( $coreData as $key => $item ) {
 938                                         $this->mergeItem( $key, $csData[$key], $item );
 939                                 }
 940                         } else {
 941                                 # Load the secondary localisation from the source file to
 942                                 # avoid infinite cycles on cyclic fallbacks
 943                                 $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
 944                                 # Only merge the keys that make sense to merge
 945                                 foreach ( self::$allKeys as $key ) {
 946                                         if ( !isset( $fbData[ $key ] ) ) {
 947                                                 continue;
 948                                         }
 949
 950                                         if ( is_null( $coreData[ $key ] ) || $this->isMergeableKey( $key ) ) {
 951                                                 $this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] );
 952                                         }
 953                                 }
 954                         }
 955
 956                         # Allow extensions an opportunity to adjust the data for this
 957                         # fallback
 958                         Hooks::run( 'LocalisationCacheRecacheFallback', [ $this, $csCode, &$csData ] );
 959
 960                         # Merge the data for this fallback into the final array
 961                         if ( $csCode === $code ) {
 962                                 $allData = $csData;
 963                         } else {
 964                                 foreach ( self::$allKeys as $key ) {
 965                                         if ( !isset( $csData[$key] ) ) {
 966                                                 continue;
 967                                         }
 968
 969                                         if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) {
 970                                                 $this->mergeItem( $key, $allData[$key], $csData[$key] );
 971                                         }
 972                                 }
 973                         }
 974                 }
 975
 976                 # Add cache dependencies for any referenced globals
 977                 $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' );
 978                 // The 'MessagesDirs' config setting is used in LocalisationCache::getMessagesDirs().
 979                 // We use the key 'wgMessagesDirs' for historical reasons.
 980                 $deps['wgMessagesDirs'] = new MainConfigDependency( 'MessagesDirs' );
 981                 $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' );
 982
 983                 # Add dependencies to the cache entry
 984                 $allData['deps'] = $deps;
 985
 986                 # Replace spaces with underscores in namespace names
 987                 $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] );
 988
 989                 # And do the same for special page aliases. $page is an array.
 990                 foreach ( $allData['specialPageAliases'] as &$page ) {
 991                         $page = str_replace( ' ', '_', $page );
 992                 }
 993                 # Decouple the reference to prevent accidental damage
 994                 unset( $page );
 995
 996                 # If there were no plural rules, return an empty array
 997                 if ( $allData['pluralRules'] === null ) {
 998                         $allData['pluralRules'] = [];
 999                 }
1000                 if ( $allData['compiledPluralRules'] === null ) {
1001                         $allData['compiledPluralRules'] = [];
1002                 }
1003                 # If there were no plural rule types, return an empty array
1004                 if ( $allData['pluralRuleTypes'] === null ) {
1005                         $allData['pluralRuleTypes'] = [];
1006                 }
1007
1008                 # Set the list keys
1009                 $allData['list'] = [];
1010                 foreach ( self::$splitKeys as $key ) {
1011                         $allData['list'][$key] = array_keys( $allData[$key] );
1012                 }
1013                 # Run hooks
1014                 $unused = true; // Used to be $purgeBlobs, removed in 1.34
1015                 Hooks::run( 'LocalisationCacheRecache', [ $this, $code, &$allData, &$unused ] );
1016
1017                 if ( is_null( $allData['namespaceNames'] ) ) {
1018                         throw new MWException( __METHOD__ . ': Localisation data failed sanity check! ' .
1019                                 'Check that your languages/messages/MessagesEn.php file is intact.' );
1020                 }
1021
1022                 # Set the preload key
1023                 $allData['preload'] = $this->buildPreload( $allData );
1024
1025                 # Save to the process cache and register the items loaded
1026                 $this->data[$code] = $allData;
1027                 foreach ( $allData as $key => $item ) {
1028                         $this->loadedItems[$code][$key] = true;
1029                 }
1030
1031                 # Save to the persistent cache
1032                 $this->store->startWrite( $code );
1033                 foreach ( $allData as $key => $value ) {
1034                         if ( in_array( $key, self::$splitKeys ) ) {
1035                                 foreach ( $value as $subkey => $subvalue ) {
1036                                         $this->store->set( "$key:$subkey", $subvalue );
1037                                 }
1038                         } else {
1039                                 $this->store->set( $key, $value );
1040                         }
1041                 }
1042                 $this->store->finishWrite();
1043
1044                 # Clear out the MessageBlobStore
1045                 # HACK: If using a null (i.e. disabled) storage backend, we
1046                 # can't write to the MessageBlobStore either
1047                 if ( !$this->store instanceof LCStoreNull ) {
1048                         foreach ( $this->clearStoreCallbacks as $callback ) {
1049                                 $callback();
1050                         }
1051                 }
1052         }
1053
1054         /**
1055          * Build the preload item from the given pre-cache data.
1056          *
1057          * The preload item will be loaded automatically, improving performance
1058          * for the commonly-requested items it contains.
1059          * @param array $data
1060          * @return array
1061          */
1062         protected function buildPreload( $data ) {
1063                 $preload = [ 'messages' => [] ];
1064                 foreach ( self::$preloadedKeys as $key ) {
1065                         $preload[$key] = $data[$key];
1066                 }
1067
1068                 foreach ( $data['preloadedMessages'] as $subkey ) {
1069                         $subitem = $data['messages'][$subkey] ?? null;
1070                         $preload['messages'][$subkey] = $subitem;
1071                 }
1072
1073                 return $preload;
1074         }
1075
1076         /**
1077          * Unload the data for a given language from the object cache.
1078          * Reduces memory usage.
1079          * @param string $code
1080          */
1081         public function unload( $code ) {
1082                 unset( $this->data[$code] );
1083                 unset( $this->loadedItems[$code] );
1084                 unset( $this->loadedSubitems[$code] );
1085                 unset( $this->initialisedLangs[$code] );
1086                 unset( $this->shallowFallbacks[$code] );
1087
1088                 foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
1089                         if ( $fbCode === $code ) {
1090                                 $this->unload( $shallowCode );
1091                         }
1092                 }
1093         }
1094
1095         /**
1096          * Unload all data
1097          */
1098         public function unloadAll() {
1099                 foreach ( $this->initialisedLangs as $lang => $unused ) {
1100                         $this->unload( $lang );
1101                 }
1102         }
1103
1104         /**
1105          * Disable the storage backend
1106          */
1107         public function disableBackend() {
1108                 $this->store = new LCStoreNull;
1109                 $this->manualRecache = false;
1110         }
1111 }