includes/cache/localisation/LocalisationCache.php

   1 <?php
   2 /**
   3  * Cache of the contents of localisation files.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 use CLDRPluralRuleParser\Evaluator;
  24 use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
  25 use MediaWiki\Config\ServiceOptions;
  26 use Psr\Log\LoggerInterface;
  27
  28 /**
  29  * Class for caching the contents of localisation files, Messages*.php
  30  * and *.i18n.php.
  31  *
  32  * An instance of this class is available using MediaWikiServices.
  33  *
  34  * The values retrieved from here are merged, containing items from extension
  35  * files, core messages files and the language fallback sequence (e.g. zh-cn ->
  36  * zh-hans -> en ). Some common errors are corrected, for example namespace
  37  * names with spaces instead of underscores, but heavyweight processing, such
  38  * as grammatical transformation, is done by the caller.
  39  */
  40 class LocalisationCache {
  41         const VERSION = 4;
  42
  43         /** @var ServiceOptions */
  44         private $options;
  45
  46         /**
  47          * True if recaching should only be done on an explicit call to recache().
  48          * Setting this reduces the overhead of cache freshness checking, which
  49          * requires doing a stat() for every extension i18n file.
  50          */
  51         private $manualRecache = false;
  52
  53         /**
  54          * The cache data. 3-d array, where the first key is the language code,
  55          * the second key is the item key e.g. 'messages', and the third key is
  56          * an item specific subkey index. Some items are not arrays and so for those
  57          * items, there are no subkeys.
  58          */
  59         protected $data = [];
  60
  61         /**
  62          * The persistent store object. An instance of LCStore.
  63          *
  64          * @var LCStore
  65          */
  66         private $store;
  67
  68         /**
  69          * @var LoggerInterface
  70          */
  71         private $logger;
  72
  73         /** @var callable[] See comment for parameter in constructor */
  74         private $clearStoreCallbacks;
  75
  76         /**
  77          * A 2-d associative array, code/key, where presence indicates that the item
  78          * is loaded. Value arbitrary.
  79          *
  80          * For split items, if set, this indicates that all of the subitems have been
  81          * loaded.
  82          */
  83         private $loadedItems = [];
  84
  85         /**
  86          * A 3-d associative array, code/key/subkey, where presence indicates that
  87          * the subitem is loaded. Only used for the split items, i.e. messages.
  88          */
  89         private $loadedSubitems = [];
  90
  91         /**
  92          * An array where presence of a key indicates that that language has been
  93          * initialised. Initialisation includes checking for cache expiry and doing
  94          * any necessary updates.
  95          */
  96         private $initialisedLangs = [];
  97
  98         /**
  99          * An array mapping non-existent pseudo-languages to fallback languages. This
 100          * is filled by initShallowFallback() when data is requested from a language
 101          * that lacks a Messages*.php file.
 102          */
 103         private $shallowFallbacks = [];
 104
 105         /**
 106          * An array where the keys are codes that have been recached by this instance.
 107          */
 108         private $recachedLangs = [];
 109
 110         /**
 111          * All item keys
 112          */
 113         public static $allKeys = [
 114                 'fallback', 'namespaceNames', 'bookstoreList',
 115                 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns',
 116                 'digitTransformTable', 'separatorTransformTable',
 117                 'minimumGroupingDigits', 'fallback8bitEncoding',
 118                 'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset',
 119                 'namespaceAliases', 'dateFormats', 'datePreferences',
 120                 'datePreferenceMigrationMap', 'defaultDateFormat',
 121                 'specialPageAliases', 'imageFiles', 'preloadedMessages',
 122                 'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules',
 123                 'pluralRuleTypes', 'compiledPluralRules',
 124         ];
 125
 126         /**
 127          * Keys for items which consist of associative arrays, which may be merged
 128          * by a fallback sequence.
 129          */
 130         public static $mergeableMapKeys = [ 'messages', 'namespaceNames',
 131                 'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages'
 132         ];
 133
 134         /**
 135          * Keys for items which are a numbered array.
 136          */
 137         public static $mergeableListKeys = [];
 138
 139         /**
 140          * Keys for items which contain an array of arrays of equivalent aliases
 141          * for each subitem. The aliases may be merged by a fallback sequence.
 142          */
 143         public static $mergeableAliasListKeys = [ 'specialPageAliases' ];
 144
 145         /**
 146          * Keys for items which contain an associative array, and may be merged if
 147          * the primary value contains the special array key "inherit". That array
 148          * key is removed after the first merge.
 149          */
 150         public static $optionalMergeKeys = [ 'bookstoreList' ];
 151
 152         /**
 153          * Keys for items that are formatted like $magicWords
 154          */
 155         public static $magicWordKeys = [ 'magicWords' ];
 156
 157         /**
 158          * Keys for items where the subitems are stored in the backend separately.
 159          */
 160         public static $splitKeys = [ 'messages' ];
 161
 162         /**
 163          * Keys which are loaded automatically by initLanguage()
 164          */
 165         public static $preloadedKeys = [ 'dateFormats', 'namespaceNames' ];
 166
 167         /**
 168          * Associative array of cached plural rules. The key is the language code,
 169          * the value is an array of plural rules for that language.
 170          */
 171         private $pluralRules = null;
 172
 173         /**
 174          * Associative array of cached plural rule types. The key is the language
 175          * code, the value is an array of plural rule types for that language. For
 176          * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many'].
 177          * The index for each rule type matches the index for the rule in
 178          * $pluralRules, thus allowing correlation between the two. The reason we
 179          * don't just use the type names as the keys in $pluralRules is because
 180          * Language::convertPlural applies the rules based on numeric order (or
 181          * explicit numeric parameter), not based on the name of the rule type. For
 182          * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than
 183          * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}.
 184          */
 185         private $pluralRuleTypes = null;
 186
 187         private $mergeableKeys = null;
 188
 189         /**
 190          * @todo Make this a const when HHVM support is dropped (T192166)
 191          *
 192          * @var array
 193          * @since 1.34
 194          */
 195         public static $constructorOptions = [
 196                 // True to treat all files as expired until they are regenerated by this object.
 197                 'forceRecache',
 198                 'manualRecache',
 199                 'ExtensionMessagesFiles',
 200                 'MessagesDirs',
 201         ];
 202
 203         /**
 204          * For constructor parameters, see the documentation in DefaultSettings.php
 205          * for $wgLocalisationCacheConf.
 206          *
 207          * Do not construct this directly. Use MediaWikiServices.
 208          *
 209          * @param ServiceOptions $options
 210          * @param LCStore $store What backend to use for storage
 211          * @param LoggerInterface $logger
 212          * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be
 213          *   used to clear other caches that depend on this one, such as ResourceLoader's
 214          *   MessageBlobStore.
 215          * @throws MWException
 216          */
 217         function __construct(
 218                 ServiceOptions $options,
 219                 LCStore $store,
 220                 LoggerInterface $logger,
 221                 array $clearStoreCallbacks = []
 222         ) {
 223                 $options->assertRequiredOptions( self::$constructorOptions );
 224
 225                 $this->options = $options;
 226                 $this->store = $store;
 227                 $this->logger = $logger;
 228                 $this->clearStoreCallbacks = $clearStoreCallbacks;
 229
 230                 // Keep this separate from $this->options so it can be mutable
 231                 $this->manualRecache = $options->get( 'manualRecache' );
 232         }
 233
 234         /**
 235          * Returns true if the given key is mergeable, that is, if it is an associative
 236          * array which can be merged through a fallback sequence.
 237          * @param string $key
 238          * @return bool
 239          */
 240         public function isMergeableKey( $key ) {
 241                 if ( $this->mergeableKeys === null ) {
 242                         $this->mergeableKeys = array_flip( array_merge(
 243                                 self::$mergeableMapKeys,
 244                                 self::$mergeableListKeys,
 245                                 self::$mergeableAliasListKeys,
 246                                 self::$optionalMergeKeys,
 247                                 self::$magicWordKeys
 248                         ) );
 249                 }
 250
 251                 return isset( $this->mergeableKeys[$key] );
 252         }
 253
 254         /**
 255          * Get a cache item.
 256          *
 257          * Warning: this may be slow for split items (messages), since it will
 258          * need to fetch all of the subitems from the cache individually.
 259          * @param string $code
 260          * @param string $key
 261          * @return mixed
 262          */
 263         public function getItem( $code, $key ) {
 264                 if ( !isset( $this->loadedItems[$code][$key] ) ) {
 265                         $this->loadItem( $code, $key );
 266                 }
 267
 268                 if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
 269                         return $this->shallowFallbacks[$code];
 270                 }
 271
 272                 return $this->data[$code][$key];
 273         }
 274
 275         /**
 276          * Get a subitem, for instance a single message for a given language.
 277          * @param string $code
 278          * @param string $key
 279          * @param string $subkey
 280          * @return mixed|null
 281          */
 282         public function getSubitem( $code, $key, $subkey ) {
 283                 if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
 284                         !isset( $this->loadedItems[$code][$key] )
 285                 ) {
 286                         $this->loadSubitem( $code, $key, $subkey );
 287                 }
 288
 289                 return $this->data[$code][$key][$subkey] ?? null;
 290         }
 291
 292         /**
 293          * Get the list of subitem keys for a given item.
 294          *
 295          * This is faster than array_keys($lc->getItem(...)) for the items listed in
 296          * self::$splitKeys.
 297          *
 298          * Will return null if the item is not found, or false if the item is not an
 299          * array.
 300          * @param string $code
 301          * @param string $key
 302          * @return bool|null|string|string[]
 303          */
 304         public function getSubitemList( $code, $key ) {
 305                 if ( in_array( $key, self::$splitKeys ) ) {
 306                         return $this->getSubitem( $code, 'list', $key );
 307                 } else {
 308                         $item = $this->getItem( $code, $key );
 309                         if ( is_array( $item ) ) {
 310                                 return array_keys( $item );
 311                         } else {
 312                                 return false;
 313                         }
 314                 }
 315         }
 316
 317         /**
 318          * Load an item into the cache.
 319          * @param string $code
 320          * @param string $key
 321          */
 322         protected function loadItem( $code, $key ) {
 323                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 324                         $this->initLanguage( $code );
 325                 }
 326
 327                 // Check to see if initLanguage() loaded it for us
 328                 if ( isset( $this->loadedItems[$code][$key] ) ) {
 329                         return;
 330                 }
 331
 332                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 333                         $this->loadItem( $this->shallowFallbacks[$code], $key );
 334
 335                         return;
 336                 }
 337
 338                 if ( in_array( $key, self::$splitKeys ) ) {
 339                         $subkeyList = $this->getSubitem( $code, 'list', $key );
 340                         foreach ( $subkeyList as $subkey ) {
 341                                 if ( isset( $this->data[$code][$key][$subkey] ) ) {
 342                                         continue;
 343                                 }
 344                                 $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey );
 345                         }
 346                 } else {
 347                         $this->data[$code][$key] = $this->store->get( $code, $key );
 348                 }
 349
 350                 $this->loadedItems[$code][$key] = true;
 351         }
 352
 353         /**
 354          * Load a subitem into the cache
 355          * @param string $code
 356          * @param string $key
 357          * @param string $subkey
 358          */
 359         protected function loadSubitem( $code, $key, $subkey ) {
 360                 if ( !in_array( $key, self::$splitKeys ) ) {
 361                         $this->loadItem( $code, $key );
 362
 363                         return;
 364                 }
 365
 366                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 367                         $this->initLanguage( $code );
 368                 }
 369
 370                 // Check to see if initLanguage() loaded it for us
 371                 if ( isset( $this->loadedItems[$code][$key] ) ||
 372                         isset( $this->loadedSubitems[$code][$key][$subkey] )
 373                 ) {
 374                         return;
 375                 }
 376
 377                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 378                         $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
 379
 380                         return;
 381                 }
 382
 383                 $value = $this->store->get( $code, "$key:$subkey" );
 384                 $this->data[$code][$key][$subkey] = $value;
 385                 $this->loadedSubitems[$code][$key][$subkey] = true;
 386         }
 387
 388         /**
 389          * Returns true if the cache identified by $code is missing or expired.
 390          *
 391          * @param string $code
 392          *
 393          * @return bool
 394          */
 395         public function isExpired( $code ) {
 396                 if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
 397                         $this->logger->debug( __METHOD__ . "($code): forced reload" );
 398
 399                         return true;
 400                 }
 401
 402                 $deps = $this->store->get( $code, 'deps' );
 403                 $keys = $this->store->get( $code, 'list' );
 404                 $preload = $this->store->get( $code, 'preload' );
 405                 // Different keys may expire separately for some stores
 406                 if ( $deps === null || $keys === null || $preload === null ) {
 407                         $this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" );
 408
 409                         return true;
 410                 }
 411
 412                 foreach ( $deps as $dep ) {
 413                         // Because we're unserializing stuff from cache, we
 414                         // could receive objects of classes that don't exist
 415                         // anymore (e.g. uninstalled extensions)
 416                         // When this happens, always expire the cache
 417                         if ( !$dep instanceof CacheDependency || $dep->isExpired() ) {
 418                                 $this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " .
 419                                         get_class( $dep ) );
 420
 421                                 return true;
 422                         }
 423                 }
 424
 425                 return false;
 426         }
 427
 428         /**
 429          * Initialise a language in this object. Rebuild the cache if necessary.
 430          * @param string $code
 431          * @throws MWException
 432          */
 433         protected function initLanguage( $code ) {
 434                 if ( isset( $this->initialisedLangs[$code] ) ) {
 435                         return;
 436                 }
 437
 438                 $this->initialisedLangs[$code] = true;
 439
 440                 # If the code is of the wrong form for a Messages*.php file, do a shallow fallback
 441                 if ( !Language::isValidBuiltInCode( $code ) ) {
 442                         $this->initShallowFallback( $code, 'en' );
 443
 444                         return;
 445                 }
 446
 447                 # Recache the data if necessary
 448                 if ( !$this->manualRecache && $this->isExpired( $code ) ) {
 449                         if ( Language::isSupportedLanguage( $code ) ) {
 450                                 $this->recache( $code );
 451                         } elseif ( $code === 'en' ) {
 452                                 throw new MWException( 'MessagesEn.php is missing.' );
 453                         } else {
 454                                 $this->initShallowFallback( $code, 'en' );
 455                         }
 456
 457                         return;
 458                 }
 459
 460                 # Preload some stuff
 461                 $preload = $this->getItem( $code, 'preload' );
 462                 if ( $preload === null ) {
 463                         if ( $this->manualRecache ) {
 464                                 // No Messages*.php file. Do shallow fallback to en.
 465                                 if ( $code === 'en' ) {
 466                                         throw new MWException( 'No localisation cache found for English. ' .
 467                                                 'Please run maintenance/rebuildLocalisationCache.php.' );
 468                                 }
 469                                 $this->initShallowFallback( $code, 'en' );
 470
 471                                 return;
 472                         } else {
 473                                 throw new MWException( 'Invalid or missing localisation cache.' );
 474                         }
 475                 }
 476                 $this->data[$code] = $preload;
 477                 foreach ( $preload as $key => $item ) {
 478                         if ( in_array( $key, self::$splitKeys ) ) {
 479                                 foreach ( $item as $subkey => $subitem ) {
 480                                         $this->loadedSubitems[$code][$key][$subkey] = true;
 481                                 }
 482                         } else {
 483                                 $this->loadedItems[$code][$key] = true;
 484                         }
 485                 }
 486         }
 487
 488         /**
 489          * Create a fallback from one language to another, without creating a
 490          * complete persistent cache.
 491          * @param string $primaryCode
 492          * @param string $fallbackCode
 493          */
 494         public function initShallowFallback( $primaryCode, $fallbackCode ) {
 495                 $this->data[$primaryCode] =& $this->data[$fallbackCode];
 496                 $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
 497                 $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
 498                 $this->shallowFallbacks[$primaryCode] = $fallbackCode;
 499         }
 500
 501         /**
 502          * Read a PHP file containing localisation data.
 503          * @param string $_fileName
 504          * @param string $_fileType
 505          * @throws MWException
 506          * @return array
 507          */
 508         protected function readPHPFile( $_fileName, $_fileType ) {
 509                 // Disable APC caching
 510                 Wikimedia\suppressWarnings();
 511                 $_apcEnabled = ini_set( 'apc.cache_by_default', '0' );
 512                 Wikimedia\restoreWarnings();
 513
 514                 include $_fileName;
 515
 516                 Wikimedia\suppressWarnings();
 517                 ini_set( 'apc.cache_by_default', $_apcEnabled );
 518                 Wikimedia\restoreWarnings();
 519
 520                 $data = [];
 521                 if ( $_fileType == 'core' || $_fileType == 'extension' ) {
 522                         foreach ( self::$allKeys as $key ) {
 523                                 // Not all keys are set in language files, so
 524                                 // check they exist first
 525                                 if ( isset( $$key ) ) {
 526                                         $data[$key] = $$key;
 527                                 }
 528                         }
 529                 } elseif ( $_fileType == 'aliases' ) {
 530                         if ( isset( $aliases ) ) {
 531                                 $data['aliases'] = $aliases;
 532                         }
 533                 } else {
 534                         throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" );
 535                 }
 536
 537                 return $data;
 538         }
 539
 540         /**
 541          * Read a JSON file containing localisation messages.
 542          * @param string $fileName Name of file to read
 543          * @throws MWException If there is a syntax error in the JSON file
 544          * @return array Array with a 'messages' key, or empty array if the file doesn't exist
 545          */
 546         public function readJSONFile( $fileName ) {
 547                 if ( !is_readable( $fileName ) ) {
 548                         return [];
 549                 }
 550
 551                 $json = file_get_contents( $fileName );
 552                 if ( $json === false ) {
 553                         return [];
 554                 }
 555
 556                 $data = FormatJson::decode( $json, true );
 557                 if ( $data === null ) {
 558                         throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" );
 559                 }
 560
 561                 // Remove keys starting with '@', they're reserved for metadata and non-message data
 562                 foreach ( $data as $key => $unused ) {
 563                         if ( $key === '' || $key[0] === '@' ) {
 564                                 unset( $data[$key] );
 565                         }
 566                 }
 567
 568                 // The JSON format only supports messages, none of the other variables, so wrap the data
 569                 return [ 'messages' => $data ];
 570         }
 571
 572         /**
 573          * Get the compiled plural rules for a given language from the XML files.
 574          * @since 1.20
 575          * @param string $code
 576          * @return array|null
 577          */
 578         public function getCompiledPluralRules( $code ) {
 579                 $rules = $this->getPluralRules( $code );
 580                 if ( $rules === null ) {
 581                         return null;
 582                 }
 583                 try {
 584                         $compiledRules = Evaluator::compile( $rules );
 585                 } catch ( CLDRPluralRuleError $e ) {
 586                         $this->logger->debug( $e->getMessage() );
 587
 588                         return [];
 589                 }
 590
 591                 return $compiledRules;
 592         }
 593
 594         /**
 595          * Get the plural rules for a given language from the XML files.
 596          * Cached.
 597          * @since 1.20
 598          * @param string $code
 599          * @return array|null
 600          */
 601         public function getPluralRules( $code ) {
 602                 if ( $this->pluralRules === null ) {
 603                         $this->loadPluralFiles();
 604                 }
 605                 return $this->pluralRules[$code] ?? null;
 606         }
 607
 608         /**
 609          * Get the plural rule types for a given language from the XML files.
 610          * Cached.
 611          * @since 1.22
 612          * @param string $code
 613          * @return array|null
 614          */
 615         public function getPluralRuleTypes( $code ) {
 616                 if ( $this->pluralRuleTypes === null ) {
 617                         $this->loadPluralFiles();
 618                 }
 619                 return $this->pluralRuleTypes[$code] ?? null;
 620         }
 621
 622         /**
 623          * Load the plural XML files.
 624          */
 625         protected function loadPluralFiles() {
 626                 global $IP;
 627                 $cldrPlural = "$IP/languages/data/plurals.xml";
 628                 $mwPlural = "$IP/languages/data/plurals-mediawiki.xml";
 629                 // Load CLDR plural rules
 630                 $this->loadPluralFile( $cldrPlural );
 631                 if ( file_exists( $mwPlural ) ) {
 632                         // Override or extend
 633                         $this->loadPluralFile( $mwPlural );
 634                 }
 635         }
 636
 637         /**
 638          * Load a plural XML file with the given filename, compile the relevant
 639          * rules, and save the compiled rules in a process-local cache.
 640          *
 641          * @param string $fileName
 642          * @throws MWException
 643          */
 644         protected function loadPluralFile( $fileName ) {
 645                 // Use file_get_contents instead of DOMDocument::load (T58439)
 646                 $xml = file_get_contents( $fileName );
 647                 if ( !$xml ) {
 648                         throw new MWException( "Unable to read plurals file $fileName" );
 649                 }
 650                 $doc = new DOMDocument;
 651                 $doc->loadXML( $xml );
 652                 $rulesets = $doc->getElementsByTagName( "pluralRules" );
 653                 foreach ( $rulesets as $ruleset ) {
 654                         $codes = $ruleset->getAttribute( 'locales' );
 655                         $rules = [];
 656                         $ruleTypes = [];
 657                         $ruleElements = $ruleset->getElementsByTagName( "pluralRule" );
 658                         foreach ( $ruleElements as $elt ) {
 659                                 $ruleType = $elt->getAttribute( 'count' );
 660                                 if ( $ruleType === 'other' ) {
 661                                         // Don't record "other" rules, which have an empty condition
 662                                         continue;
 663                                 }
 664                                 $rules[] = $elt->nodeValue;
 665                                 $ruleTypes[] = $ruleType;
 666                         }
 667                         foreach ( explode( ' ', $codes ) as $code ) {
 668                                 $this->pluralRules[$code] = $rules;
 669                                 $this->pluralRuleTypes[$code] = $ruleTypes;
 670                         }
 671                 }
 672         }
 673
 674         /**
 675          * Read the data from the source files for a given language, and register
 676          * the relevant dependencies in the $deps array. If the localisation
 677          * exists, the data array is returned, otherwise false is returned.
 678          *
 679          * @param string $code
 680          * @param array &$deps
 681          * @return array
 682          */
 683         protected function readSourceFilesAndRegisterDeps( $code, &$deps ) {
 684                 global $IP;
 685
 686                 // This reads in the PHP i18n file with non-messages l10n data
 687                 $fileName = Language::getMessagesFileName( $code );
 688                 if ( !file_exists( $fileName ) ) {
 689                         $data = [];
 690                 } else {
 691                         $deps[] = new FileDependency( $fileName );
 692                         $data = $this->readPHPFile( $fileName, 'core' );
 693                 }
 694
 695                 # Load CLDR plural rules for JavaScript
 696                 $data['pluralRules'] = $this->getPluralRules( $code );
 697                 # And for PHP
 698                 $data['compiledPluralRules'] = $this->getCompiledPluralRules( $code );
 699                 # Load plural rule types
 700                 $data['pluralRuleTypes'] = $this->getPluralRuleTypes( $code );
 701
 702                 $deps['plurals'] = new FileDependency( "$IP/languages/data/plurals.xml" );
 703                 $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" );
 704
 705                 return $data;
 706         }
 707
 708         /**
 709          * Merge two localisation values, a primary and a fallback, overwriting the
 710          * primary value in place.
 711          * @param string $key
 712          * @param mixed &$value
 713          * @param mixed $fallbackValue
 714          */
 715         protected function mergeItem( $key, &$value, $fallbackValue ) {
 716                 if ( !is_null( $value ) ) {
 717                         if ( !is_null( $fallbackValue ) ) {
 718                                 if ( in_array( $key, self::$mergeableMapKeys ) ) {
 719                                         $value = $value + $fallbackValue;
 720                                 } elseif ( in_array( $key, self::$mergeableListKeys ) ) {
 721                                         $value = array_unique( array_merge( $fallbackValue, $value ) );
 722                                 } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) {
 723                                         $value = array_merge_recursive( $value, $fallbackValue );
 724                                 } elseif ( in_array( $key, self::$optionalMergeKeys ) ) {
 725                                         if ( !empty( $value['inherit'] ) ) {
 726                                                 $value = array_merge( $fallbackValue, $value );
 727                                         }
 728
 729                                         if ( isset( $value['inherit'] ) ) {
 730                                                 unset( $value['inherit'] );
 731                                         }
 732                                 } elseif ( in_array( $key, self::$magicWordKeys ) ) {
 733                                         $this->mergeMagicWords( $value, $fallbackValue );
 734                                 }
 735                         }
 736                 } else {
 737                         $value = $fallbackValue;
 738                 }
 739         }
 740
 741         /**
 742          * @param mixed &$value
 743          * @param mixed $fallbackValue
 744          */
 745         protected function mergeMagicWords( &$value, $fallbackValue ) {
 746                 foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
 747                         if ( !isset( $value[$magicName] ) ) {
 748                                 $value[$magicName] = $fallbackInfo;
 749                         } else {
 750                                 $oldSynonyms = array_slice( $fallbackInfo, 1 );
 751                                 $newSynonyms = array_slice( $value[$magicName], 1 );
 752                                 $synonyms = array_values( array_unique( array_merge(
 753                                         $newSynonyms, $oldSynonyms ) ) );
 754                                 $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms );
 755                         }
 756                 }
 757         }
 758
 759         /**
 760          * Given an array mapping language code to localisation value, such as is
 761          * found in extension *.i18n.php files, iterate through a fallback sequence
 762          * to merge the given data with an existing primary value.
 763          *
 764          * Returns true if any data from the extension array was used, false
 765          * otherwise.
 766          * @param array $codeSequence
 767          * @param string $key
 768          * @param mixed &$value
 769          * @param mixed $fallbackValue
 770          * @return bool
 771          */
 772         protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) {
 773                 $used = false;
 774                 foreach ( $codeSequence as $code ) {
 775                         if ( isset( $fallbackValue[$code] ) ) {
 776                                 $this->mergeItem( $key, $value, $fallbackValue[$code] );
 777                                 $used = true;
 778                         }
 779                 }
 780
 781                 return $used;
 782         }
 783
 784         /**
 785          * Gets the combined list of messages dirs from
 786          * core and extensions
 787          *
 788          * @since 1.25
 789          * @return array
 790          */
 791         public function getMessagesDirs() {
 792                 global $IP;
 793
 794                 return [
 795                         'core' => "$IP/languages/i18n",
 796                         'exif' => "$IP/languages/i18n/exif",
 797                         'api' => "$IP/includes/api/i18n",
 798                         'oojs-ui' => "$IP/resources/lib/ooui/i18n",
 799                 ] + $this->options->get( 'MessagesDirs' );
 800         }
 801
 802         /**
 803          * Load localisation data for a given language for both core and extensions
 804          * and save it to the persistent cache store and the process cache
 805          * @param string $code
 806          * @throws MWException
 807          */
 808         public function recache( $code ) {
 809                 if ( !$code ) {
 810                         throw new MWException( "Invalid language code requested" );
 811                 }
 812                 $this->recachedLangs[$code] = true;
 813
 814                 # Initial values
 815                 $initialData = array_fill_keys( self::$allKeys, null );
 816                 $coreData = $initialData;
 817                 $deps = [];
 818
 819                 # Load the primary localisation from the source file
 820                 $data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
 821                 if ( $data === false ) {
 822                         $this->logger->debug( __METHOD__ . ": no localisation file for $code, using fallback to en" );
 823                         $coreData['fallback'] = 'en';
 824                 } else {
 825                         $this->logger->debug( __METHOD__ . ": got localisation for $code from source" );
 826
 827                         # Merge primary localisation
 828                         foreach ( $data as $key => $value ) {
 829                                 $this->mergeItem( $key, $coreData[$key], $value );
 830                         }
 831                 }
 832
 833                 # Fill in the fallback if it's not there already
 834                 if ( ( is_null( $coreData['fallback'] ) || $coreData['fallback'] === false ) && $code === 'en' ) {
 835                         $coreData['fallback'] = false;
 836                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = [];
 837                 } else {
 838                         if ( !is_null( $coreData['fallback'] ) ) {
 839                                 $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) );
 840                         } else {
 841                                 $coreData['fallbackSequence'] = [];
 842                         }
 843                         $len = count( $coreData['fallbackSequence'] );
 844
 845                         # Before we add the 'en' fallback for messages, keep a copy of
 846                         # the original fallback sequence
 847                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'];
 848
 849                         # Ensure that the sequence ends at 'en' for messages
 850                         if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
 851                                 $coreData['fallbackSequence'][] = 'en';
 852                         }
 853                 }
 854
 855                 $codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] );
 856                 $messageDirs = $this->getMessagesDirs();
 857
 858                 # Load non-JSON localisation data for extensions
 859                 $extensionData = array_fill_keys( $codeSequence, $initialData );
 860                 foreach ( $this->options->get( 'ExtensionMessagesFiles' ) as $extension => $fileName ) {
 861                         if ( isset( $messageDirs[$extension] ) ) {
 862                                 # This extension has JSON message data; skip the PHP shim
 863                                 continue;
 864                         }
 865
 866                         $data = $this->readPHPFile( $fileName, 'extension' );
 867                         $used = false;
 868
 869                         foreach ( $data as $key => $item ) {
 870                                 foreach ( $codeSequence as $csCode ) {
 871                                         if ( isset( $item[$csCode] ) ) {
 872                                                 $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
 873                                                 $used = true;
 874                                         }
 875                                 }
 876                         }
 877
 878                         if ( $used ) {
 879                                 $deps[] = new FileDependency( $fileName );
 880                         }
 881                 }
 882
 883                 # Load the localisation data for each fallback, then merge it into the full array
 884                 $allData = $initialData;
 885                 foreach ( $codeSequence as $csCode ) {
 886                         $csData = $initialData;
 887
 888                         # Load core messages and the extension localisations.
 889                         foreach ( $messageDirs as $dirs ) {
 890                                 foreach ( (array)$dirs as $dir ) {
 891                                         $fileName = "$dir/$csCode.json";
 892                                         $data = $this->readJSONFile( $fileName );
 893
 894                                         foreach ( $data as $key => $item ) {
 895                                                 $this->mergeItem( $key, $csData[$key], $item );
 896                                         }
 897
 898                                         $deps[] = new FileDependency( $fileName );
 899                                 }
 900                         }
 901
 902                         # Merge non-JSON extension data
 903                         if ( isset( $extensionData[$csCode] ) ) {
 904                                 foreach ( $extensionData[$csCode] as $key => $item ) {
 905                                         $this->mergeItem( $key, $csData[$key], $item );
 906                                 }
 907                         }
 908
 909                         if ( $csCode === $code ) {
 910                                 # Merge core data into extension data
 911                                 foreach ( $coreData as $key => $item ) {
 912                                         $this->mergeItem( $key, $csData[$key], $item );
 913                                 }
 914                         } else {
 915                                 # Load the secondary localisation from the source file to
 916                                 # avoid infinite cycles on cyclic fallbacks
 917                                 $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
 918                                 if ( $fbData !== false ) {
 919                                         # Only merge the keys that make sense to merge
 920                                         foreach ( self::$allKeys as $key ) {
 921                                                 if ( !isset( $fbData[$key] ) ) {
 922                                                         continue;
 923                                                 }
 924
 925                                                 if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) {
 926                                                         $this->mergeItem( $key, $csData[$key], $fbData[$key] );
 927                                                 }
 928                                         }
 929                                 }
 930                         }
 931
 932                         # Allow extensions an opportunity to adjust the data for this
 933                         # fallback
 934                         Hooks::run( 'LocalisationCacheRecacheFallback', [ $this, $csCode, &$csData ] );
 935
 936                         # Merge the data for this fallback into the final array
 937                         if ( $csCode === $code ) {
 938                                 $allData = $csData;
 939                         } else {
 940                                 foreach ( self::$allKeys as $key ) {
 941                                         if ( !isset( $csData[$key] ) ) {
 942                                                 continue;
 943                                         }
 944
 945                                         if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) {
 946                                                 $this->mergeItem( $key, $allData[$key], $csData[$key] );
 947                                         }
 948                                 }
 949                         }
 950                 }
 951
 952                 # Add cache dependencies for any referenced globals
 953                 $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' );
 954                 // The 'MessagesDirs' config setting is used in LocalisationCache::getMessagesDirs().
 955                 // We use the key 'wgMessagesDirs' for historical reasons.
 956                 $deps['wgMessagesDirs'] = new MainConfigDependency( 'MessagesDirs' );
 957                 $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' );
 958
 959                 # Add dependencies to the cache entry
 960                 $allData['deps'] = $deps;
 961
 962                 # Replace spaces with underscores in namespace names
 963                 $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] );
 964
 965                 # And do the same for special page aliases. $page is an array.
 966                 foreach ( $allData['specialPageAliases'] as &$page ) {
 967                         $page = str_replace( ' ', '_', $page );
 968                 }
 969                 # Decouple the reference to prevent accidental damage
 970                 unset( $page );
 971
 972                 # If there were no plural rules, return an empty array
 973                 if ( $allData['pluralRules'] === null ) {
 974                         $allData['pluralRules'] = [];
 975                 }
 976                 if ( $allData['compiledPluralRules'] === null ) {
 977                         $allData['compiledPluralRules'] = [];
 978                 }
 979                 # If there were no plural rule types, return an empty array
 980                 if ( $allData['pluralRuleTypes'] === null ) {
 981                         $allData['pluralRuleTypes'] = [];
 982                 }
 983
 984                 # Set the list keys
 985                 $allData['list'] = [];
 986                 foreach ( self::$splitKeys as $key ) {
 987                         $allData['list'][$key] = array_keys( $allData[$key] );
 988                 }
 989                 # Run hooks
 990                 $unused = true; // Used to be $purgeBlobs, removed in 1.34
 991                 Hooks::run( 'LocalisationCacheRecache', [ $this, $code, &$allData, &$unused ] );
 992
 993                 if ( is_null( $allData['namespaceNames'] ) ) {
 994                         throw new MWException( __METHOD__ . ': Localisation data failed sanity check! ' .
 995                                 'Check that your languages/messages/MessagesEn.php file is intact.' );
 996                 }
 997
 998                 # Set the preload key
 999                 $allData['preload'] = $this->buildPreload( $allData );
1000
1001                 # Save to the process cache and register the items loaded
1002                 $this->data[$code] = $allData;
1003                 foreach ( $allData as $key => $item ) {
1004                         $this->loadedItems[$code][$key] = true;
1005                 }
1006
1007                 # Save to the persistent cache
1008                 $this->store->startWrite( $code );
1009                 foreach ( $allData as $key => $value ) {
1010                         if ( in_array( $key, self::$splitKeys ) ) {
1011                                 foreach ( $value as $subkey => $subvalue ) {
1012                                         $this->store->set( "$key:$subkey", $subvalue );
1013                                 }
1014                         } else {
1015                                 $this->store->set( $key, $value );
1016                         }
1017                 }
1018                 $this->store->finishWrite();
1019
1020                 # Clear out the MessageBlobStore
1021                 # HACK: If using a null (i.e. disabled) storage backend, we
1022                 # can't write to the MessageBlobStore either
1023                 if ( !$this->store instanceof LCStoreNull ) {
1024                         foreach ( $this->clearStoreCallbacks as $callback ) {
1025                                 $callback();
1026                         }
1027                 }
1028         }
1029
1030         /**
1031          * Build the preload item from the given pre-cache data.
1032          *
1033          * The preload item will be loaded automatically, improving performance
1034          * for the commonly-requested items it contains.
1035          * @param array $data
1036          * @return array
1037          */
1038         protected function buildPreload( $data ) {
1039                 $preload = [ 'messages' => [] ];
1040                 foreach ( self::$preloadedKeys as $key ) {
1041                         $preload[$key] = $data[$key];
1042                 }
1043
1044                 foreach ( $data['preloadedMessages'] as $subkey ) {
1045                         $subitem = $data['messages'][$subkey] ?? null;
1046                         $preload['messages'][$subkey] = $subitem;
1047                 }
1048
1049                 return $preload;
1050         }
1051
1052         /**
1053          * Unload the data for a given language from the object cache.
1054          * Reduces memory usage.
1055          * @param string $code
1056          */
1057         public function unload( $code ) {
1058                 unset( $this->data[$code] );
1059                 unset( $this->loadedItems[$code] );
1060                 unset( $this->loadedSubitems[$code] );
1061                 unset( $this->initialisedLangs[$code] );
1062                 unset( $this->shallowFallbacks[$code] );
1063
1064                 foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
1065                         if ( $fbCode === $code ) {
1066                                 $this->unload( $shallowCode );
1067                         }
1068                 }
1069         }
1070
1071         /**
1072          * Unload all data
1073          */
1074         public function unloadAll() {
1075                 foreach ( $this->initialisedLangs as $lang => $unused ) {
1076                         $this->unload( $lang );
1077                 }
1078         }
1079
1080         /**
1081          * Disable the storage backend
1082          */
1083         public function disableBackend() {
1084                 $this->store = new LCStoreNull;
1085                 $this->manualRecache = false;
1086         }
1087 }