includes/cache/localisation/LocalisationCache.php

   1 <?php
   2 /**
   3  * Cache of the contents of localisation files.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 use CLDRPluralRuleParser\Evaluator;
  24 use CLDRPluralRuleParser\Error as CLDRPluralRuleError;
  25 use MediaWiki\Config\ServiceOptions;
  26 use Psr\Log\LoggerInterface;
  27
  28 /**
  29  * Class for caching the contents of localisation files, Messages*.php
  30  * and *.i18n.php.
  31  *
  32  * An instance of this class is available using MediaWikiServices.
  33  *
  34  * The values retrieved from here are merged, containing items from extension
  35  * files, core messages files and the language fallback sequence (e.g. zh-cn ->
  36  * zh-hans -> en ). Some common errors are corrected, for example namespace
  37  * names with spaces instead of underscores, but heavyweight processing, such
  38  * as grammatical transformation, is done by the caller.
  39  */
  40 class LocalisationCache {
  41         const VERSION = 4;
  42
  43         /** @var ServiceOptions */
  44         private $options;
  45
  46         /**
  47          * True if recaching should only be done on an explicit call to recache().
  48          * Setting this reduces the overhead of cache freshness checking, which
  49          * requires doing a stat() for every extension i18n file.
  50          */
  51         private $manualRecache = false;
  52
  53         /**
  54          * The cache data. 3-d array, where the first key is the language code,
  55          * the second key is the item key e.g. 'messages', and the third key is
  56          * an item specific subkey index. Some items are not arrays and so for those
  57          * items, there are no subkeys.
  58          */
  59         protected $data = [];
  60
  61         /**
  62          * The persistent store object. An instance of LCStore.
  63          *
  64          * @var LCStore
  65          */
  66         private $store;
  67
  68         /**
  69          * @var LoggerInterface
  70          */
  71         private $logger;
  72
  73         /** @var callable[] See comment for parameter in constructor */
  74         private $clearStoreCallbacks;
  75
  76         /**
  77          * A 2-d associative array, code/key, where presence indicates that the item
  78          * is loaded. Value arbitrary.
  79          *
  80          * For split items, if set, this indicates that all of the subitems have been
  81          * loaded.
  82          *
  83          */
  84         private $loadedItems = [];
  85
  86         /**
  87          * A 3-d associative array, code/key/subkey, where presence indicates that
  88          * the subitem is loaded. Only used for the split items, i.e. messages.
  89          */
  90         private $loadedSubitems = [];
  91
  92         /**
  93          * An array where presence of a key indicates that that language has been
  94          * initialised. Initialisation includes checking for cache expiry and doing
  95          * any necessary updates.
  96          */
  97         private $initialisedLangs = [];
  98
  99         /**
 100          * An array mapping non-existent pseudo-languages to fallback languages. This
 101          * is filled by initShallowFallback() when data is requested from a language
 102          * that lacks a Messages*.php file.
 103          */
 104         private $shallowFallbacks = [];
 105
 106         /**
 107          * An array where the keys are codes that have been recached by this instance.
 108          */
 109         private $recachedLangs = [];
 110
 111         /**
 112          * All item keys
 113          */
 114         public static $allKeys = [
 115                 'fallback', 'namespaceNames', 'bookstoreList',
 116                 'magicWords', 'messages', 'rtl', 'capitalizeAllNouns',
 117                 'digitTransformTable', 'separatorTransformTable',
 118                 'minimumGroupingDigits', 'fallback8bitEncoding',
 119                 'linkPrefixExtension', 'linkTrail', 'linkPrefixCharset',
 120                 'namespaceAliases', 'dateFormats', 'datePreferences',
 121                 'datePreferenceMigrationMap', 'defaultDateFormat',
 122                 'specialPageAliases', 'imageFiles', 'preloadedMessages',
 123                 'namespaceGenderAliases', 'digitGroupingPattern', 'pluralRules',
 124                 'pluralRuleTypes', 'compiledPluralRules',
 125         ];
 126
 127         /**
 128          * Keys for items which consist of associative arrays, which may be merged
 129          * by a fallback sequence.
 130          */
 131         public static $mergeableMapKeys = [ 'messages', 'namespaceNames',
 132                 'namespaceAliases', 'dateFormats', 'imageFiles', 'preloadedMessages'
 133         ];
 134
 135         /**
 136          * Keys for items which are a numbered array.
 137          */
 138         public static $mergeableListKeys = [];
 139
 140         /**
 141          * Keys for items which contain an array of arrays of equivalent aliases
 142          * for each subitem. The aliases may be merged by a fallback sequence.
 143          */
 144         public static $mergeableAliasListKeys = [ 'specialPageAliases' ];
 145
 146         /**
 147          * Keys for items which contain an associative array, and may be merged if
 148          * the primary value contains the special array key "inherit". That array
 149          * key is removed after the first merge.
 150          */
 151         public static $optionalMergeKeys = [ 'bookstoreList' ];
 152
 153         /**
 154          * Keys for items that are formatted like $magicWords
 155          */
 156         public static $magicWordKeys = [ 'magicWords' ];
 157
 158         /**
 159          * Keys for items where the subitems are stored in the backend separately.
 160          */
 161         public static $splitKeys = [ 'messages' ];
 162
 163         /**
 164          * Keys which are loaded automatically by initLanguage()
 165          */
 166         public static $preloadedKeys = [ 'dateFormats', 'namespaceNames' ];
 167
 168         /**
 169          * Associative array of cached plural rules. The key is the language code,
 170          * the value is an array of plural rules for that language.
 171          */
 172         private $pluralRules = null;
 173
 174         /**
 175          * Associative array of cached plural rule types. The key is the language
 176          * code, the value is an array of plural rule types for that language. For
 177          * example, $pluralRuleTypes['ar'] = ['zero', 'one', 'two', 'few', 'many'].
 178          * The index for each rule type matches the index for the rule in
 179          * $pluralRules, thus allowing correlation between the two. The reason we
 180          * don't just use the type names as the keys in $pluralRules is because
 181          * Language::convertPlural applies the rules based on numeric order (or
 182          * explicit numeric parameter), not based on the name of the rule type. For
 183          * example, {{plural:count|wordform1|wordform2|wordform3}}, rather than
 184          * {{plural:count|one=wordform1|two=wordform2|many=wordform3}}.
 185          */
 186         private $pluralRuleTypes = null;
 187
 188         private $mergeableKeys = null;
 189
 190         /**
 191          * Return a suitable LCStore as specified by the given configuration.
 192          *
 193          * @since 1.34
 194          * @param array $conf In the format of $wgLocalisationCacheConf
 195          * @param string|false|null $fallbackCacheDir In case 'storeDirectory' isn't specified
 196          * @return LCStore
 197          */
 198         public static function getStoreFromConf( array $conf, $fallbackCacheDir ) : LCStore {
 199                 $storeArg = [];
 200                 $storeArg['directory'] =
 201                         $conf['storeDirectory'] ?: $fallbackCacheDir;
 202
 203                 if ( !empty( $conf['storeClass'] ) ) {
 204                         $storeClass = $conf['storeClass'];
 205                 } elseif ( $conf['store'] === 'files' || $conf['store'] === 'file' ||
 206                         ( $conf['store'] === 'detect' && $storeArg['directory'] )
 207                 ) {
 208                         $storeClass = LCStoreCDB::class;
 209                 } elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) {
 210                         $storeClass = LCStoreDB::class;
 211                         $storeArg['server'] = $conf['storeServer'] ?? [];
 212                 } elseif ( $conf['store'] === 'array' ) {
 213                         $storeClass = LCStoreStaticArray::class;
 214                 } else {
 215                         throw new MWException(
 216                                 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.'
 217                         );
 218                 }
 219
 220                 return new $storeClass( $storeArg );
 221         }
 222
 223         /**
 224          * @var array
 225          * @since 1.34
 226          */
 227         public const CONSTRUCTOR_OPTIONS = [
 228                 // True to treat all files as expired until they are regenerated by this object.
 229                 'forceRecache',
 230                 'manualRecache',
 231                 'ExtensionMessagesFiles',
 232                 'MessagesDirs',
 233         ];
 234
 235         /**
 236          * For constructor parameters, see the documentation in DefaultSettings.php
 237          * for $wgLocalisationCacheConf.
 238          *
 239          * Do not construct this directly. Use MediaWikiServices.
 240          *
 241          * @param ServiceOptions $options
 242          * @param LCStore $store What backend to use for storage
 243          * @param LoggerInterface $logger
 244          * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be
 245          *   used to clear other caches that depend on this one, such as ResourceLoader's
 246          *   MessageBlobStore.
 247          * @throws MWException
 248          */
 249         function __construct(
 250                 ServiceOptions $options,
 251                 LCStore $store,
 252                 LoggerInterface $logger,
 253                 array $clearStoreCallbacks = []
 254         ) {
 255                 $options->assertRequiredOptions( self::CONSTRUCTOR_OPTIONS );
 256
 257                 $this->options = $options;
 258                 $this->store = $store;
 259                 $this->logger = $logger;
 260                 $this->clearStoreCallbacks = $clearStoreCallbacks;
 261
 262                 // Keep this separate from $this->options so it can be mutable
 263                 $this->manualRecache = $options->get( 'manualRecache' );
 264         }
 265
 266         /**
 267          * Returns true if the given key is mergeable, that is, if it is an associative
 268          * array which can be merged through a fallback sequence.
 269          * @param string $key
 270          * @return bool
 271          */
 272         public function isMergeableKey( $key ) {
 273                 if ( $this->mergeableKeys === null ) {
 274                         $this->mergeableKeys = array_flip( array_merge(
 275                                 self::$mergeableMapKeys,
 276                                 self::$mergeableListKeys,
 277                                 self::$mergeableAliasListKeys,
 278                                 self::$optionalMergeKeys,
 279                                 self::$magicWordKeys
 280                         ) );
 281                 }
 282
 283                 return isset( $this->mergeableKeys[$key] );
 284         }
 285
 286         /**
 287          * Get a cache item.
 288          *
 289          * Warning: this may be slow for split items (messages), since it will
 290          * need to fetch all of the subitems from the cache individually.
 291          * @param string $code
 292          * @param string $key
 293          * @return mixed
 294          */
 295         public function getItem( $code, $key ) {
 296                 if ( !isset( $this->loadedItems[$code][$key] ) ) {
 297                         $this->loadItem( $code, $key );
 298                 }
 299
 300                 if ( $key === 'fallback' && isset( $this->shallowFallbacks[$code] ) ) {
 301                         return $this->shallowFallbacks[$code];
 302                 }
 303
 304                 return $this->data[$code][$key];
 305         }
 306
 307         /**
 308          * Get a subitem, for instance a single message for a given language.
 309          * @param string $code
 310          * @param string $key
 311          * @param string $subkey
 312          * @return mixed|null
 313          */
 314         public function getSubitem( $code, $key, $subkey ) {
 315                 if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
 316                         !isset( $this->loadedItems[$code][$key] )
 317                 ) {
 318                         $this->loadSubitem( $code, $key, $subkey );
 319                 }
 320
 321                 return $this->data[$code][$key][$subkey] ?? null;
 322         }
 323
 324         /**
 325          * Get the list of subitem keys for a given item.
 326          *
 327          * This is faster than array_keys($lc->getItem(...)) for the items listed in
 328          * self::$splitKeys.
 329          *
 330          * Will return null if the item is not found, or false if the item is not an
 331          * array.
 332          * @param string $code
 333          * @param string $key
 334          * @return bool|null|string|string[]
 335          */
 336         public function getSubitemList( $code, $key ) {
 337                 if ( in_array( $key, self::$splitKeys ) ) {
 338                         return $this->getSubitem( $code, 'list', $key );
 339                 } else {
 340                         $item = $this->getItem( $code, $key );
 341                         if ( is_array( $item ) ) {
 342                                 return array_keys( $item );
 343                         } else {
 344                                 return false;
 345                         }
 346                 }
 347         }
 348
 349         /**
 350          * Load an item into the cache.
 351          * @param string $code
 352          * @param string $key
 353          */
 354         protected function loadItem( $code, $key ) {
 355                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 356                         $this->initLanguage( $code );
 357                 }
 358
 359                 // Check to see if initLanguage() loaded it for us
 360                 if ( isset( $this->loadedItems[$code][$key] ) ) {
 361                         return;
 362                 }
 363
 364                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 365                         $this->loadItem( $this->shallowFallbacks[$code], $key );
 366
 367                         return;
 368                 }
 369
 370                 if ( in_array( $key, self::$splitKeys ) ) {
 371                         $subkeyList = $this->getSubitem( $code, 'list', $key );
 372                         foreach ( $subkeyList as $subkey ) {
 373                                 if ( isset( $this->data[$code][$key][$subkey] ) ) {
 374                                         continue;
 375                                 }
 376                                 $this->data[$code][$key][$subkey] = $this->getSubitem( $code, $key, $subkey );
 377                         }
 378                 } else {
 379                         $this->data[$code][$key] = $this->store->get( $code, $key );
 380                 }
 381
 382                 $this->loadedItems[$code][$key] = true;
 383         }
 384
 385         /**
 386          * Load a subitem into the cache
 387          * @param string $code
 388          * @param string $key
 389          * @param string $subkey
 390          */
 391         protected function loadSubitem( $code, $key, $subkey ) {
 392                 if ( !in_array( $key, self::$splitKeys ) ) {
 393                         $this->loadItem( $code, $key );
 394
 395                         return;
 396                 }
 397
 398                 if ( !isset( $this->initialisedLangs[$code] ) ) {
 399                         $this->initLanguage( $code );
 400                 }
 401
 402                 // Check to see if initLanguage() loaded it for us
 403                 if ( isset( $this->loadedItems[$code][$key] ) ||
 404                         isset( $this->loadedSubitems[$code][$key][$subkey] )
 405                 ) {
 406                         return;
 407                 }
 408
 409                 if ( isset( $this->shallowFallbacks[$code] ) ) {
 410                         $this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
 411
 412                         return;
 413                 }
 414
 415                 $value = $this->store->get( $code, "$key:$subkey" );
 416                 $this->data[$code][$key][$subkey] = $value;
 417                 $this->loadedSubitems[$code][$key][$subkey] = true;
 418         }
 419
 420         /**
 421          * Returns true if the cache identified by $code is missing or expired.
 422          *
 423          * @param string $code
 424          *
 425          * @return bool
 426          */
 427         public function isExpired( $code ) {
 428                 if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) {
 429                         $this->logger->debug( __METHOD__ . "($code): forced reload" );
 430
 431                         return true;
 432                 }
 433
 434                 $deps = $this->store->get( $code, 'deps' );
 435                 $keys = $this->store->get( $code, 'list' );
 436                 $preload = $this->store->get( $code, 'preload' );
 437                 // Different keys may expire separately for some stores
 438                 if ( $deps === null || $keys === null || $preload === null ) {
 439                         $this->logger->debug( __METHOD__ . "($code): cache missing, need to make one" );
 440
 441                         return true;
 442                 }
 443
 444                 foreach ( $deps as $dep ) {
 445                         // Because we're unserializing stuff from cache, we
 446                         // could receive objects of classes that don't exist
 447                         // anymore (e.g. uninstalled extensions)
 448                         // When this happens, always expire the cache
 449                         if ( !$dep instanceof CacheDependency || $dep->isExpired() ) {
 450                                 $this->logger->debug( __METHOD__ . "($code): cache for $code expired due to " .
 451                                         get_class( $dep ) );
 452
 453                                 return true;
 454                         }
 455                 }
 456
 457                 return false;
 458         }
 459
 460         /**
 461          * Initialise a language in this object. Rebuild the cache if necessary.
 462          * @param string $code
 463          * @throws MWException
 464          */
 465         protected function initLanguage( $code ) {
 466                 if ( isset( $this->initialisedLangs[$code] ) ) {
 467                         return;
 468                 }
 469
 470                 $this->initialisedLangs[$code] = true;
 471
 472                 # If the code is of the wrong form for a Messages*.php file, do a shallow fallback
 473                 if ( !Language::isValidBuiltInCode( $code ) ) {
 474                         $this->initShallowFallback( $code, 'en' );
 475
 476                         return;
 477                 }
 478
 479                 # Recache the data if necessary
 480                 if ( !$this->manualRecache && $this->isExpired( $code ) ) {
 481                         if ( Language::isSupportedLanguage( $code ) ) {
 482                                 $this->recache( $code );
 483                         } elseif ( $code === 'en' ) {
 484                                 throw new MWException( 'MessagesEn.php is missing.' );
 485                         } else {
 486                                 $this->initShallowFallback( $code, 'en' );
 487                         }
 488
 489                         return;
 490                 }
 491
 492                 # Preload some stuff
 493                 $preload = $this->getItem( $code, 'preload' );
 494                 if ( $preload === null ) {
 495                         if ( $this->manualRecache ) {
 496                                 // No Messages*.php file. Do shallow fallback to en.
 497                                 if ( $code === 'en' ) {
 498                                         throw new MWException( 'No localisation cache found for English. ' .
 499                                                 'Please run maintenance/rebuildLocalisationCache.php.' );
 500                                 }
 501                                 $this->initShallowFallback( $code, 'en' );
 502
 503                                 return;
 504                         } else {
 505                                 throw new MWException( 'Invalid or missing localisation cache.' );
 506                         }
 507                 }
 508                 $this->data[$code] = $preload;
 509                 foreach ( $preload as $key => $item ) {
 510                         if ( in_array( $key, self::$splitKeys ) ) {
 511                                 foreach ( $item as $subkey => $subitem ) {
 512                                         $this->loadedSubitems[$code][$key][$subkey] = true;
 513                                 }
 514                         } else {
 515                                 $this->loadedItems[$code][$key] = true;
 516                         }
 517                 }
 518         }
 519
 520         /**
 521          * Create a fallback from one language to another, without creating a
 522          * complete persistent cache.
 523          * @param string $primaryCode
 524          * @param string $fallbackCode
 525          */
 526         public function initShallowFallback( $primaryCode, $fallbackCode ) {
 527                 $this->data[$primaryCode] =& $this->data[$fallbackCode];
 528                 $this->loadedItems[$primaryCode] =& $this->loadedItems[$fallbackCode];
 529                 $this->loadedSubitems[$primaryCode] =& $this->loadedSubitems[$fallbackCode];
 530                 $this->shallowFallbacks[$primaryCode] = $fallbackCode;
 531         }
 532
 533         /**
 534          * Read a PHP file containing localisation data.
 535          * @param string $_fileName
 536          * @param string $_fileType
 537          * @throws MWException
 538          * @return array
 539          */
 540         protected function readPHPFile( $_fileName, $_fileType ) {
 541                 include $_fileName;
 542
 543                 $data = [];
 544                 if ( $_fileType == 'core' || $_fileType == 'extension' ) {
 545                         foreach ( self::$allKeys as $key ) {
 546                                 // Not all keys are set in language files, so
 547                                 // check they exist first
 548                                 if ( isset( $$key ) ) {
 549                                         $data[$key] = $$key;
 550                                 }
 551                         }
 552                 } elseif ( $_fileType == 'aliases' ) {
 553                         if ( isset( $aliases ) ) {
 554                                 $data['aliases'] = $aliases;
 555                         }
 556                 } else {
 557                         throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" );
 558                 }
 559
 560                 return $data;
 561         }
 562
 563         /**
 564          * Read a JSON file containing localisation messages.
 565          * @param string $fileName Name of file to read
 566          * @throws MWException If there is a syntax error in the JSON file
 567          * @return array Array with a 'messages' key, or empty array if the file doesn't exist
 568          */
 569         public function readJSONFile( $fileName ) {
 570                 if ( !is_readable( $fileName ) ) {
 571                         return [];
 572                 }
 573
 574                 $json = file_get_contents( $fileName );
 575                 if ( $json === false ) {
 576                         return [];
 577                 }
 578
 579                 $data = FormatJson::decode( $json, true );
 580                 if ( $data === null ) {
 581                         throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" );
 582                 }
 583
 584                 // Remove keys starting with '@', they're reserved for metadata and non-message data
 585                 foreach ( $data as $key => $unused ) {
 586                         if ( $key === '' || $key[0] === '@' ) {
 587                                 unset( $data[$key] );
 588                         }
 589                 }
 590
 591                 // The JSON format only supports messages, none of the other variables, so wrap the data
 592                 return [ 'messages' => $data ];
 593         }
 594
 595         /**
 596          * Get the compiled plural rules for a given language from the XML files.
 597          * @since 1.20
 598          * @param string $code
 599          * @return array|null
 600          */
 601         public function getCompiledPluralRules( $code ) {
 602                 $rules = $this->getPluralRules( $code );
 603                 if ( $rules === null ) {
 604                         return null;
 605                 }
 606                 try {
 607                         $compiledRules = Evaluator::compile( $rules );
 608                 } catch ( CLDRPluralRuleError $e ) {
 609                         $this->logger->debug( $e->getMessage() );
 610
 611                         return [];
 612                 }
 613
 614                 return $compiledRules;
 615         }
 616
 617         /**
 618          * Get the plural rules for a given language from the XML files.
 619          * Cached.
 620          * @since 1.20
 621          * @param string $code
 622          * @return array|null
 623          */
 624         public function getPluralRules( $code ) {
 625                 if ( $this->pluralRules === null ) {
 626                         $this->loadPluralFiles();
 627                 }
 628                 return $this->pluralRules[$code] ?? null;
 629         }
 630
 631         /**
 632          * Get the plural rule types for a given language from the XML files.
 633          * Cached.
 634          * @since 1.22
 635          * @param string $code
 636          * @return array|null
 637          */
 638         public function getPluralRuleTypes( $code ) {
 639                 if ( $this->pluralRuleTypes === null ) {
 640                         $this->loadPluralFiles();
 641                 }
 642                 return $this->pluralRuleTypes[$code] ?? null;
 643         }
 644
 645         /**
 646          * Load the plural XML files.
 647          */
 648         protected function loadPluralFiles() {
 649                 global $IP;
 650                 $cldrPlural = "$IP/languages/data/plurals.xml";
 651                 $mwPlural = "$IP/languages/data/plurals-mediawiki.xml";
 652                 // Load CLDR plural rules
 653                 $this->loadPluralFile( $cldrPlural );
 654                 if ( file_exists( $mwPlural ) ) {
 655                         // Override or extend
 656                         $this->loadPluralFile( $mwPlural );
 657                 }
 658         }
 659
 660         /**
 661          * Load a plural XML file with the given filename, compile the relevant
 662          * rules, and save the compiled rules in a process-local cache.
 663          *
 664          * @param string $fileName
 665          * @throws MWException
 666          */
 667         protected function loadPluralFile( $fileName ) {
 668                 // Use file_get_contents instead of DOMDocument::load (T58439)
 669                 $xml = file_get_contents( $fileName );
 670                 if ( !$xml ) {
 671                         throw new MWException( "Unable to read plurals file $fileName" );
 672                 }
 673                 $doc = new DOMDocument;
 674                 $doc->loadXML( $xml );
 675                 $rulesets = $doc->getElementsByTagName( "pluralRules" );
 676                 foreach ( $rulesets as $ruleset ) {
 677                         $codes = $ruleset->getAttribute( 'locales' );
 678                         $rules = [];
 679                         $ruleTypes = [];
 680                         $ruleElements = $ruleset->getElementsByTagName( "pluralRule" );
 681                         foreach ( $ruleElements as $elt ) {
 682                                 $ruleType = $elt->getAttribute( 'count' );
 683                                 if ( $ruleType === 'other' ) {
 684                                         // Don't record "other" rules, which have an empty condition
 685                                         continue;
 686                                 }
 687                                 $rules[] = $elt->nodeValue;
 688                                 $ruleTypes[] = $ruleType;
 689                         }
 690                         foreach ( explode( ' ', $codes ) as $code ) {
 691                                 $this->pluralRules[$code] = $rules;
 692                                 $this->pluralRuleTypes[$code] = $ruleTypes;
 693                         }
 694                 }
 695         }
 696
 697         /**
 698          * Read the data from the source files for a given language, and register
 699          * the relevant dependencies in the $deps array. If the localisation
 700          * exists, the data array is returned, otherwise false is returned.
 701          *
 702          * @param string $code
 703          * @param array &$deps
 704          * @return array
 705          */
 706         protected function readSourceFilesAndRegisterDeps( $code, &$deps ) {
 707                 global $IP;
 708
 709                 // This reads in the PHP i18n file with non-messages l10n data
 710                 $fileName = Language::getMessagesFileName( $code );
 711                 if ( !file_exists( $fileName ) ) {
 712                         $data = [];
 713                 } else {
 714                         $deps[] = new FileDependency( $fileName );
 715                         $data = $this->readPHPFile( $fileName, 'core' );
 716                 }
 717
 718                 # Load CLDR plural rules for JavaScript
 719                 $data['pluralRules'] = $this->getPluralRules( $code );
 720                 # And for PHP
 721                 $data['compiledPluralRules'] = $this->getCompiledPluralRules( $code );
 722                 # Load plural rule types
 723                 $data['pluralRuleTypes'] = $this->getPluralRuleTypes( $code );
 724
 725                 $deps['plurals'] = new FileDependency( "$IP/languages/data/plurals.xml" );
 726                 $deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" );
 727
 728                 return $data;
 729         }
 730
 731         /**
 732          * Merge two localisation values, a primary and a fallback, overwriting the
 733          * primary value in place.
 734          * @param string $key
 735          * @param mixed &$value
 736          * @param mixed $fallbackValue
 737          */
 738         protected function mergeItem( $key, &$value, $fallbackValue ) {
 739                 if ( !is_null( $value ) ) {
 740                         if ( !is_null( $fallbackValue ) ) {
 741                                 if ( in_array( $key, self::$mergeableMapKeys ) ) {
 742                                         $value = $value + $fallbackValue;
 743                                 } elseif ( in_array( $key, self::$mergeableListKeys ) ) {
 744                                         // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
 745                                         $value = array_unique( array_merge( $fallbackValue, $value ) );
 746                                 } elseif ( in_array( $key, self::$mergeableAliasListKeys ) ) {
 747                                         $value = array_merge_recursive( $value, $fallbackValue );
 748                                 } elseif ( in_array( $key, self::$optionalMergeKeys ) ) {
 749                                         if ( !empty( $value['inherit'] ) ) {
 750                                                 $value = array_merge( $fallbackValue, $value );
 751                                         }
 752
 753                                         if ( isset( $value['inherit'] ) ) {
 754                                                 unset( $value['inherit'] );
 755                                         }
 756                                 } elseif ( in_array( $key, self::$magicWordKeys ) ) {
 757                                         $this->mergeMagicWords( $value, $fallbackValue );
 758                                 }
 759                         }
 760                 } else {
 761                         $value = $fallbackValue;
 762                 }
 763         }
 764
 765         /**
 766          * @param mixed &$value
 767          * @param mixed $fallbackValue
 768          */
 769         protected function mergeMagicWords( &$value, $fallbackValue ) {
 770                 foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
 771                         if ( !isset( $value[$magicName] ) ) {
 772                                 $value[$magicName] = $fallbackInfo;
 773                         } else {
 774                                 $oldSynonyms = array_slice( $fallbackInfo, 1 );
 775                                 $newSynonyms = array_slice( $value[$magicName], 1 );
 776                                 $synonyms = array_values( array_unique( array_merge(
 777                                         $newSynonyms, $oldSynonyms ) ) );
 778                                 $value[$magicName] = array_merge( [ $fallbackInfo[0] ], $synonyms );
 779                         }
 780                 }
 781         }
 782
 783         /**
 784          * Given an array mapping language code to localisation value, such as is
 785          * found in extension *.i18n.php files, iterate through a fallback sequence
 786          * to merge the given data with an existing primary value.
 787          *
 788          * Returns true if any data from the extension array was used, false
 789          * otherwise.
 790          * @param array $codeSequence
 791          * @param string $key
 792          * @param mixed &$value
 793          * @param mixed $fallbackValue
 794          * @return bool
 795          */
 796         protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) {
 797                 $used = false;
 798                 foreach ( $codeSequence as $code ) {
 799                         if ( isset( $fallbackValue[$code] ) ) {
 800                                 $this->mergeItem( $key, $value, $fallbackValue[$code] );
 801                                 $used = true;
 802                         }
 803                 }
 804
 805                 return $used;
 806         }
 807
 808         /**
 809          * Gets the combined list of messages dirs from
 810          * core and extensions
 811          *
 812          * @since 1.25
 813          * @return array
 814          */
 815         public function getMessagesDirs() {
 816                 global $IP;
 817
 818                 return [
 819                         'core' => "$IP/languages/i18n",
 820                         'exif' => "$IP/languages/i18n/exif",
 821                         'api' => "$IP/includes/api/i18n",
 822                         'oojs-ui' => "$IP/resources/lib/ooui/i18n",
 823                 ] + $this->options->get( 'MessagesDirs' );
 824         }
 825
 826         /**
 827          * Load localisation data for a given language for both core and extensions
 828          * and save it to the persistent cache store and the process cache
 829          * @param string $code
 830          * @throws MWException
 831          */
 832         public function recache( $code ) {
 833                 if ( !$code ) {
 834                         throw new MWException( "Invalid language code requested" );
 835                 }
 836                 $this->recachedLangs[ $code ] = true;
 837
 838                 # Initial values
 839                 $initialData = array_fill_keys( self::$allKeys, null );
 840                 $coreData = $initialData;
 841                 $deps = [];
 842
 843                 # Load the primary localisation from the source file
 844                 $data = $this->readSourceFilesAndRegisterDeps( $code, $deps );
 845                 $this->logger->debug( __METHOD__ . ": got localisation for $code from source" );
 846
 847                 # Merge primary localisation
 848                 foreach ( $data as $key => $value ) {
 849                         $this->mergeItem( $key, $coreData[ $key ], $value );
 850                 }
 851
 852                 # Fill in the fallback if it's not there already
 853                 if ( ( is_null( $coreData['fallback'] ) || $coreData['fallback'] === false ) && $code === 'en' ) {
 854                         $coreData['fallback'] = false;
 855                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'] = [];
 856                 } else {
 857                         if ( !is_null( $coreData['fallback'] ) ) {
 858                                 $coreData['fallbackSequence'] = array_map( 'trim', explode( ',', $coreData['fallback'] ) );
 859                         } else {
 860                                 $coreData['fallbackSequence'] = [];
 861                         }
 862                         $len = count( $coreData['fallbackSequence'] );
 863
 864                         # Before we add the 'en' fallback for messages, keep a copy of
 865                         # the original fallback sequence
 866                         $coreData['originalFallbackSequence'] = $coreData['fallbackSequence'];
 867
 868                         # Ensure that the sequence ends at 'en' for messages
 869                         if ( !$len || $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
 870                                 $coreData['fallbackSequence'][] = 'en';
 871                         }
 872                 }
 873
 874                 $codeSequence = array_merge( [ $code ], $coreData['fallbackSequence'] );
 875                 $messageDirs = $this->getMessagesDirs();
 876
 877                 # Load non-JSON localisation data for extensions
 878                 $extensionData = array_fill_keys( $codeSequence, $initialData );
 879                 foreach ( $this->options->get( 'ExtensionMessagesFiles' ) as $extension => $fileName ) {
 880                         if ( isset( $messageDirs[$extension] ) ) {
 881                                 # This extension has JSON message data; skip the PHP shim
 882                                 continue;
 883                         }
 884
 885                         $data = $this->readPHPFile( $fileName, 'extension' );
 886                         $used = false;
 887
 888                         foreach ( $data as $key => $item ) {
 889                                 foreach ( $codeSequence as $csCode ) {
 890                                         if ( isset( $item[$csCode] ) ) {
 891                                                 $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
 892                                                 $used = true;
 893                                         }
 894                                 }
 895                         }
 896
 897                         if ( $used ) {
 898                                 $deps[] = new FileDependency( $fileName );
 899                         }
 900                 }
 901
 902                 # Load the localisation data for each fallback, then merge it into the full array
 903                 $allData = $initialData;
 904                 foreach ( $codeSequence as $csCode ) {
 905                         $csData = $initialData;
 906
 907                         # Load core messages and the extension localisations.
 908                         foreach ( $messageDirs as $dirs ) {
 909                                 foreach ( (array)$dirs as $dir ) {
 910                                         $fileName = "$dir/$csCode.json";
 911                                         $data = $this->readJSONFile( $fileName );
 912
 913                                         foreach ( $data as $key => $item ) {
 914                                                 $this->mergeItem( $key, $csData[$key], $item );
 915                                         }
 916
 917                                         $deps[] = new FileDependency( $fileName );
 918                                 }
 919                         }
 920
 921                         # Merge non-JSON extension data
 922                         if ( isset( $extensionData[$csCode] ) ) {
 923                                 foreach ( $extensionData[$csCode] as $key => $item ) {
 924                                         $this->mergeItem( $key, $csData[$key], $item );
 925                                 }
 926                         }
 927
 928                         if ( $csCode === $code ) {
 929                                 # Merge core data into extension data
 930                                 foreach ( $coreData as $key => $item ) {
 931                                         $this->mergeItem( $key, $csData[$key], $item );
 932                                 }
 933                         } else {
 934                                 # Load the secondary localisation from the source file to
 935                                 # avoid infinite cycles on cyclic fallbacks
 936                                 $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
 937                                 # Only merge the keys that make sense to merge
 938                                 foreach ( self::$allKeys as $key ) {
 939                                         if ( !isset( $fbData[ $key ] ) ) {
 940                                                 continue;
 941                                         }
 942
 943                                         if ( is_null( $coreData[ $key ] ) || $this->isMergeableKey( $key ) ) {
 944                                                 $this->mergeItem( $key, $csData[ $key ], $fbData[ $key ] );
 945                                         }
 946                                 }
 947                         }
 948
 949                         # Allow extensions an opportunity to adjust the data for this
 950                         # fallback
 951                         Hooks::run( 'LocalisationCacheRecacheFallback', [ $this, $csCode, &$csData ] );
 952
 953                         # Merge the data for this fallback into the final array
 954                         if ( $csCode === $code ) {
 955                                 $allData = $csData;
 956                         } else {
 957                                 foreach ( self::$allKeys as $key ) {
 958                                         if ( !isset( $csData[$key] ) ) {
 959                                                 continue;
 960                                         }
 961
 962                                         if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) {
 963                                                 $this->mergeItem( $key, $allData[$key], $csData[$key] );
 964                                         }
 965                                 }
 966                         }
 967                 }
 968
 969                 # Add cache dependencies for any referenced globals
 970                 $deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' );
 971                 // The 'MessagesDirs' config setting is used in LocalisationCache::getMessagesDirs().
 972                 // We use the key 'wgMessagesDirs' for historical reasons.
 973                 $deps['wgMessagesDirs'] = new MainConfigDependency( 'MessagesDirs' );
 974                 $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' );
 975
 976                 # Add dependencies to the cache entry
 977                 $allData['deps'] = $deps;
 978
 979                 # Replace spaces with underscores in namespace names
 980                 $allData['namespaceNames'] = str_replace( ' ', '_', $allData['namespaceNames'] );
 981
 982                 # And do the same for special page aliases. $page is an array.
 983                 foreach ( $allData['specialPageAliases'] as &$page ) {
 984                         $page = str_replace( ' ', '_', $page );
 985                 }
 986                 # Decouple the reference to prevent accidental damage
 987                 unset( $page );
 988
 989                 # If there were no plural rules, return an empty array
 990                 if ( $allData['pluralRules'] === null ) {
 991                         $allData['pluralRules'] = [];
 992                 }
 993                 if ( $allData['compiledPluralRules'] === null ) {
 994                         $allData['compiledPluralRules'] = [];
 995                 }
 996                 # If there were no plural rule types, return an empty array
 997                 if ( $allData['pluralRuleTypes'] === null ) {
 998                         $allData['pluralRuleTypes'] = [];
 999                 }
1000
1001                 # Set the list keys
1002                 $allData['list'] = [];
1003                 foreach ( self::$splitKeys as $key ) {
1004                         $allData['list'][$key] = array_keys( $allData[$key] );
1005                 }
1006                 # Run hooks
1007                 $unused = true; // Used to be $purgeBlobs, removed in 1.34
1008                 Hooks::run( 'LocalisationCacheRecache', [ $this, $code, &$allData, &$unused ] );
1009
1010                 if ( is_null( $allData['namespaceNames'] ) ) {
1011                         throw new MWException( __METHOD__ . ': Localisation data failed sanity check! ' .
1012                                 'Check that your languages/messages/MessagesEn.php file is intact.' );
1013                 }
1014
1015                 # Set the preload key
1016                 $allData['preload'] = $this->buildPreload( $allData );
1017
1018                 # Save to the process cache and register the items loaded
1019                 $this->data[$code] = $allData;
1020                 foreach ( $allData as $key => $item ) {
1021                         $this->loadedItems[$code][$key] = true;
1022                 }
1023
1024                 # Save to the persistent cache
1025                 $this->store->startWrite( $code );
1026                 foreach ( $allData as $key => $value ) {
1027                         if ( in_array( $key, self::$splitKeys ) ) {
1028                                 foreach ( $value as $subkey => $subvalue ) {
1029                                         $this->store->set( "$key:$subkey", $subvalue );
1030                                 }
1031                         } else {
1032                                 $this->store->set( $key, $value );
1033                         }
1034                 }
1035                 $this->store->finishWrite();
1036
1037                 # Clear out the MessageBlobStore
1038                 # HACK: If using a null (i.e. disabled) storage backend, we
1039                 # can't write to the MessageBlobStore either
1040                 if ( !$this->store instanceof LCStoreNull ) {
1041                         foreach ( $this->clearStoreCallbacks as $callback ) {
1042                                 $callback();
1043                         }
1044                 }
1045         }
1046
1047         /**
1048          * Build the preload item from the given pre-cache data.
1049          *
1050          * The preload item will be loaded automatically, improving performance
1051          * for the commonly-requested items it contains.
1052          * @param array $data
1053          * @return array
1054          */
1055         protected function buildPreload( $data ) {
1056                 $preload = [ 'messages' => [] ];
1057                 foreach ( self::$preloadedKeys as $key ) {
1058                         $preload[$key] = $data[$key];
1059                 }
1060
1061                 foreach ( $data['preloadedMessages'] as $subkey ) {
1062                         $subitem = $data['messages'][$subkey] ?? null;
1063                         $preload['messages'][$subkey] = $subitem;
1064                 }
1065
1066                 return $preload;
1067         }
1068
1069         /**
1070          * Unload the data for a given language from the object cache.
1071          * Reduces memory usage.
1072          * @param string $code
1073          */
1074         public function unload( $code ) {
1075                 unset( $this->data[$code] );
1076                 unset( $this->loadedItems[$code] );
1077                 unset( $this->loadedSubitems[$code] );
1078                 unset( $this->initialisedLangs[$code] );
1079                 unset( $this->shallowFallbacks[$code] );
1080
1081                 foreach ( $this->shallowFallbacks as $shallowCode => $fbCode ) {
1082                         if ( $fbCode === $code ) {
1083                                 $this->unload( $shallowCode );
1084                         }
1085                 }
1086         }
1087
1088         /**
1089          * Unload all data
1090          */
1091         public function unloadAll() {
1092                 foreach ( $this->initialisedLangs as $lang => $unused ) {
1093                         $this->unload( $lang );
1094                 }
1095         }
1096
1097         /**
1098          * Disable the storage backend
1099          */
1100         public function disableBackend() {
1101                 $this->store = new LCStoreNull;
1102                 $this->manualRecache = false;
1103         }
1104 }