From 308e6427aef169a575a339e6a8e0558d29403a1d Mon Sep 17 00:00:00 2001 From: Amir Sarabadani Date: Mon, 26 Aug 2019 18:28:26 +0200 Subject: [PATCH] Revert "Make LocalisationCache a service" This reverts commits: - 76a940350d36c323ebedb4ab45cc81ed1c6b6c92 - b78b8804d076618e967c7b31ec15a1bd9e35d1d0 - 2e52f48c2ed8dcf480843e2186f685a86810e2ac - e4468a1d6b6b9fdc5b64800febdc8748d21f213d Bug: T231200 Bug: T231198 Change-Id: I1a7e46a979ae5c9c8130dd3927f6663a216ba753 --- RELEASE-NOTES-1.34 | 8 - autoload.php | 1 - includes/DefaultSettings.php | 2 - includes/MediaWikiServices.php | 18 - includes/ServiceWiring.php | 39 -- .../cache/localisation/LocalisationCache.php | 158 +++-- includes/installer/Installer.php | 11 +- includes/language/LanguageCode.php | 1 + includes/language/LanguageNameUtils.php | 319 ---------- languages/Language.php | 311 ++++++---- languages/data/Names.php | 2 +- maintenance/rebuildLocalisationCache.php | 26 +- tests/common/TestsAutoLoader.php | 3 - .../phpunit/MediaWikiIntegrationTestCase.php | 1 - .../includes/api/ApiQuerySiteinfoTest.php | 1 - .../includes/cache/LocalisationCacheTest.php | 54 +- .../includes/logging/LogFormatterTest.php | 4 +- tests/phpunit/languages/LanguageTest.php | 205 ++++--- .../language/LanguageNameUtilsTest.php | 66 --- .../language/LanguageNameUtilsTestTrait.php | 555 ------------------ 20 files changed, 402 insertions(+), 1383 deletions(-) delete mode 100644 includes/language/LanguageNameUtils.php delete mode 100644 tests/phpunit/unit/includes/language/LanguageNameUtilsTest.php delete mode 100644 tests/phpunit/unit/includes/language/LanguageNameUtilsTestTrait.php diff --git a/RELEASE-NOTES-1.34 b/RELEASE-NOTES-1.34 index df25d30c59..620a6f51de 100644 --- a/RELEASE-NOTES-1.34 +++ b/RELEASE-NOTES-1.34 @@ -354,8 +354,6 @@ because of Phabricator reports. * The UserIsBlockedFrom hook is only called if a block is found first, and should only be used to unblock a blocked user. * … -* Language::$dataCache has been removed (without prior deprecation, for - practical reasons). Use MediaWikiServices instead to get a LocalisationCache. === Deprecations in 1.34 === * The MWNamespace class is deprecated. Use NamespaceInfo. @@ -465,12 +463,6 @@ because of Phabricator reports. * Constructing MovePage directly is deprecated. Use MovePageFactory. * TempFSFile::factory() has been deprecated. Use TempFSFileFactory instead. * wfIsBadImage() is deprecated. Use the BadFileLookup service instead. -* Language::getLocalisationCache() is deprecated. Use MediaWikiServices. -* The following Language methods are deprecated: isSupportedLanguage, - isValidCode, isValidBuiltInCode, isKnownLanguageTag, fetchLanguageNames, - fetchLanguageName, getFileName, getMessagesFileName, getJsonMessagesFileName. - Use the new LanguageNameUtils class instead. (Note that fetchLanguageName(s) - are called getLanguageName(s) in the new class.) === Other changes in 1.34 === * … diff --git a/autoload.php b/autoload.php index 0a1ec2fb34..60bc5b5532 100644 --- a/autoload.php +++ b/autoload.php @@ -892,7 +892,6 @@ $wgAutoloadLocalClasses = [ 'MediaWiki\\Languages\\Data\\CrhExceptions' => __DIR__ . '/languages/data/CrhExceptions.php', 'MediaWiki\\Languages\\Data\\Names' => __DIR__ . '/languages/data/Names.php', 'MediaWiki\\Languages\\Data\\ZhConversion' => __DIR__ . '/languages/data/ZhConversion.php', - 'MediaWiki\\Languages\\LanguageNameUtils' => __DIR__ . '/includes/language/LanguageNameUtils.php', 'MediaWiki\\Logger\\ConsoleLogger' => __DIR__ . '/includes/debug/logger/ConsoleLogger.php', 'MediaWiki\\Logger\\ConsoleSpi' => __DIR__ . '/includes/debug/logger/ConsoleSpi.php', 'MediaWiki\\Logger\\LegacyLogger' => __DIR__ . '/includes/debug/logger/LegacyLogger.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 8341dac19a..0b14e56ebb 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -2636,8 +2636,6 @@ $wgLocalisationCacheConf = [ 'store' => 'detect', 'storeClass' => false, 'storeDirectory' => false, - 'storeServer' => [], - 'forceRecache' => false, 'manualRecache' => false, ]; diff --git a/includes/MediaWikiServices.php b/includes/MediaWikiServices.php index e926c3289e..6013aafc9e 100644 --- a/includes/MediaWikiServices.php +++ b/includes/MediaWikiServices.php @@ -14,12 +14,10 @@ use GlobalVarConfig; use Hooks; use IBufferingStatsdDataFactory; use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface; -use LocalisationCache; use MediaWiki\Block\BlockManager; use MediaWiki\Block\BlockRestrictionStore; use MediaWiki\FileBackend\FSFile\TempFSFileFactory; use MediaWiki\Http\HttpRequestFactory; -use MediaWiki\Languages\LanguageNameUtils; use MediaWiki\Page\MovePageFactory; use MediaWiki\Permissions\PermissionManager; use MediaWiki\Preferences\PreferencesFactory; @@ -625,14 +623,6 @@ class MediaWikiServices extends ServiceContainer { return $this->getService( 'InterwikiLookup' ); } - /** - * @since 1.34 - * @return LanguageNameUtils - */ - public function getLanguageNameUtils() { - return $this->getService( 'LanguageNameUtils' ); - } - /** * @since 1.28 * @return LinkCache @@ -660,14 +650,6 @@ class MediaWikiServices extends ServiceContainer { return $this->getService( 'LinkRendererFactory' ); } - /** - * @since 1.34 - * @return LocalisationCache - */ - public function getLocalisationCache() : LocalisationCache { - return $this->getService( 'LocalisationCache' ); - } - /** * @since 1.28 * @return \BagOStuff diff --git a/includes/ServiceWiring.php b/includes/ServiceWiring.php index f331d579f0..b30726415e 100644 --- a/includes/ServiceWiring.php +++ b/includes/ServiceWiring.php @@ -48,7 +48,6 @@ use MediaWiki\FileBackend\FSFile\TempFSFileFactory; use MediaWiki\Http\HttpRequestFactory; use MediaWiki\Interwiki\ClassicInterwikiLookup; use MediaWiki\Interwiki\InterwikiLookup; -use MediaWiki\Languages\LanguageNameUtils; use MediaWiki\Linker\LinkRenderer; use MediaWiki\Linker\LinkRendererFactory; use MediaWiki\Logger\LoggerFactory; @@ -257,13 +256,6 @@ return [ ); }, - 'LanguageNameUtils' => function ( MediaWikiServices $services ) : LanguageNameUtils { - return new LanguageNameUtils( new ServiceOptions( - LanguageNameUtils::$constructorOptions, - $services->getMainConfig() - ) ); - }, - 'LinkCache' => function ( MediaWikiServices $services ) : LinkCache { return new LinkCache( $services->getTitleFormatter(), @@ -290,37 +282,6 @@ return [ ); }, - 'LocalisationCache' => function ( MediaWikiServices $services ) : LocalisationCache { - $conf = $services->getMainConfig()->get( 'LocalisationCacheConf' ); - - $logger = LoggerFactory::getInstance( 'localisation' ); - - $store = LocalisationCache::getStoreFromConf( - $conf, $services->getMainConfig()->get( 'CacheDirectory' ) ); - $logger->debug( 'LocalisationCache: using store ' . get_class( $store ) ); - - return new $conf['class']( - new ServiceOptions( - LocalisationCache::$constructorOptions, - // Two of the options are stored in $wgLocalisationCacheConf - $conf, - // In case someone set that config variable and didn't reset all keys, set defaults. - [ - 'forceRecache' => false, - 'manualRecache' => false, - ], - // Some other options come from config itself - $services->getMainConfig() - ), - $store, - $logger, - [ function () use ( $services ) { - $services->getResourceLoader()->getMessageBlobStore()->clear(); - } ], - $services->getLanguageNameUtils() - ); - }, - 'LocalServerObjectCache' => function ( MediaWikiServices $services ) : BagOStuff { $config = $services->getMainConfig(); $cacheId = \ObjectCache::detectLocalServerCache(); diff --git a/includes/cache/localisation/LocalisationCache.php b/includes/cache/localisation/LocalisationCache.php index ed9421e255..ffc7cd00d6 100644 --- a/includes/cache/localisation/LocalisationCache.php +++ b/includes/cache/localisation/LocalisationCache.php @@ -22,15 +22,14 @@ use CLDRPluralRuleParser\Evaluator; use CLDRPluralRuleParser\Error as CLDRPluralRuleError; -use MediaWiki\Config\ServiceOptions; -use MediaWiki\Languages\LanguageNameUtils; -use Psr\Log\LoggerInterface; +use MediaWiki\Logger\LoggerFactory; +use MediaWiki\MediaWikiServices; /** * Class for caching the contents of localisation files, Messages*.php * and *.i18n.php. * - * An instance of this class is available using MediaWikiServices. + * An instance of this class is available using Language::getLocalisationCache(). * * The values retrieved from here are merged, containing items from extension * files, core messages files and the language fallback sequence (e.g. zh-cn -> @@ -41,8 +40,8 @@ use Psr\Log\LoggerInterface; class LocalisationCache { const VERSION = 4; - /** @var ServiceOptions */ - private $options; + /** Configuration associative array */ + private $conf; /** * True if recaching should only be done on an explicit call to recache(). @@ -51,6 +50,11 @@ class LocalisationCache { */ private $manualRecache = false; + /** + * True to treat all files as expired until they are regenerated by this object. + */ + private $forceRecache = false; + /** * The cache data. 3-d array, where the first key is the language code, * the second key is the item key e.g. 'messages', and the third key is @@ -67,16 +71,10 @@ class LocalisationCache { private $store; /** - * @var LoggerInterface + * @var \Psr\Log\LoggerInterface */ private $logger; - /** @var callable[] See comment for parameter in constructor */ - private $clearStoreCallbacks; - - /** @var LanguageNameUtils */ - private $langNameUtils; - /** * A 2-d associative array, code/key, where presence indicates that the item * is loaded. Value arbitrary. @@ -191,83 +189,59 @@ class LocalisationCache { private $mergeableKeys = null; /** - * Return a suitable LCStore as specified by the given configuration. + * For constructor parameters, see the documentation in DefaultSettings.php + * for $wgLocalisationCacheConf. * - * @param array $conf In the format of $wgLocalisationCacheConf - * @param string|false|null $fallbackCacheDir In case 'storeDirectory' isn't specified - * @return LCStore + * @param array $conf + * @throws MWException */ - public static function getStoreFromConf( array $conf, $fallbackCacheDir ) : LCStore { + function __construct( $conf ) { + global $wgCacheDirectory; + + $this->conf = $conf; + $this->logger = LoggerFactory::getInstance( 'localisation' ); + + $directory = !empty( $conf['storeDirectory'] ) ? $conf['storeDirectory'] : $wgCacheDirectory; $storeArg = []; - $storeArg['directory'] = - $conf['storeDirectory'] ?: $fallbackCacheDir; + $storeArg['directory'] = $directory; if ( !empty( $conf['storeClass'] ) ) { $storeClass = $conf['storeClass']; - } elseif ( $conf['store'] === 'files' || $conf['store'] === 'file' || - ( $conf['store'] === 'detect' && $storeArg['directory'] ) - ) { - $storeClass = LCStoreCDB::class; - } elseif ( $conf['store'] === 'db' || $conf['store'] === 'detect' ) { - $storeClass = LCStoreDB::class; - $storeArg['server'] = $conf['storeServer'] ?? []; - } elseif ( $conf['store'] === 'array' ) { - $storeClass = LCStoreStaticArray::class; } else { - throw new MWException( - 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.' - ); + switch ( $conf['store'] ) { + case 'files': + case 'file': + $storeClass = LCStoreCDB::class; + break; + case 'db': + $storeClass = LCStoreDB::class; + $storeArg['server'] = $conf['storeServer'] ?? []; + break; + case 'array': + $storeClass = LCStoreStaticArray::class; + break; + case 'detect': + if ( $directory ) { + $storeClass = LCStoreCDB::class; + } else { + $storeClass = LCStoreDB::class; + $storeArg['server'] = $conf['storeServer'] ?? []; + } + break; + default: + throw new MWException( + 'Please set $wgLocalisationCacheConf[\'store\'] to something sensible.' + ); + } } + $this->logger->debug( static::class . ": using store $storeClass" ); - return new $storeClass( $storeArg ); - } - - /** - * @todo Make this a const when HHVM support is dropped (T192166) - * - * @var array - * @since 1.34 - */ - public static $constructorOptions = [ - // True to treat all files as expired until they are regenerated by this object. - 'forceRecache', - 'manualRecache', - 'ExtensionMessagesFiles', - 'MessagesDirs', - ]; - - /** - * For constructor parameters, see the documentation in DefaultSettings.php - * for $wgLocalisationCacheConf. - * - * Do not construct this directly. Use MediaWikiServices. - * - * @param ServiceOptions $options - * @param LCStore $store What backend to use for storage - * @param LoggerInterface $logger - * @param callable[] $clearStoreCallbacks To be called whenever the cache is cleared. Can be - * used to clear other caches that depend on this one, such as ResourceLoader's - * MessageBlobStore. - * @param LanguageNameUtils $langNameUtils - * @throws MWException - */ - function __construct( - ServiceOptions $options, - LCStore $store, - LoggerInterface $logger, - array $clearStoreCallbacks, - LanguageNameUtils $langNameUtils - ) { - $options->assertRequiredOptions( self::$constructorOptions ); - - $this->options = $options; - $this->store = $store; - $this->logger = $logger; - $this->clearStoreCallbacks = $clearStoreCallbacks; - $this->langNameUtils = $langNameUtils; - - // Keep this separate from $this->options so it can be mutable - $this->manualRecache = $options->get( 'manualRecache' ); + $this->store = new $storeClass( $storeArg ); + foreach ( [ 'manualRecache', 'forceRecache' ] as $var ) { + if ( isset( $conf[$var] ) ) { + $this->$var = $conf[$var]; + } + } } /** @@ -432,7 +406,7 @@ class LocalisationCache { * @return bool */ public function isExpired( $code ) { - if ( $this->options->get( 'forceRecache' ) && !isset( $this->recachedLangs[$code] ) ) { + if ( $this->forceRecache && !isset( $this->recachedLangs[$code] ) ) { $this->logger->debug( __METHOD__ . "($code): forced reload" ); return true; @@ -477,7 +451,7 @@ class LocalisationCache { $this->initialisedLangs[$code] = true; # If the code is of the wrong form for a Messages*.php file, do a shallow fallback - if ( !$this->langNameUtils->isValidBuiltInCode( $code ) ) { + if ( !Language::isValidBuiltInCode( $code ) ) { $this->initShallowFallback( $code, 'en' ); return; @@ -485,7 +459,7 @@ class LocalisationCache { # Recache the data if necessary if ( !$this->manualRecache && $this->isExpired( $code ) ) { - if ( $this->langNameUtils->isSupportedLanguage( $code ) ) { + if ( Language::isSupportedLanguage( $code ) ) { $this->recache( $code ); } elseif ( $code === 'en' ) { throw new MWException( 'MessagesEn.php is missing.' ); @@ -723,7 +697,7 @@ class LocalisationCache { global $IP; // This reads in the PHP i18n file with non-messages l10n data - $fileName = $this->langNameUtils->getMessagesFileName( $code ); + $fileName = Language::getMessagesFileName( $code ); if ( !file_exists( $fileName ) ) { $data = []; } else { @@ -830,12 +804,14 @@ class LocalisationCache { public function getMessagesDirs() { global $IP; + $config = MediaWikiServices::getInstance()->getMainConfig(); + $messagesDirs = $config->get( 'MessagesDirs' ); return [ 'core' => "$IP/languages/i18n", 'exif' => "$IP/languages/i18n/exif", 'api' => "$IP/includes/api/i18n", 'oojs-ui' => "$IP/resources/lib/ooui/i18n", - ] + $this->options->get( 'MessagesDirs' ); + ] + $messagesDirs; } /** @@ -845,6 +821,8 @@ class LocalisationCache { * @throws MWException */ public function recache( $code ) { + global $wgExtensionMessagesFiles; + if ( !$code ) { throw new MWException( "Invalid language code requested" ); } @@ -896,7 +874,7 @@ class LocalisationCache { # Load non-JSON localisation data for extensions $extensionData = array_fill_keys( $codeSequence, $initialData ); - foreach ( $this->options->get( 'ExtensionMessagesFiles' ) as $extension => $fileName ) { + foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) { if ( isset( $messageDirs[$extension] ) ) { # This extension has JSON message data; skip the PHP shim continue; @@ -1060,9 +1038,8 @@ class LocalisationCache { # HACK: If using a null (i.e. disabled) storage backend, we # can't write to the MessageBlobStore either if ( !$this->store instanceof LCStoreNull ) { - foreach ( $this->clearStoreCallbacks as $callback ) { - $callback(); - } + $blobStore = MediaWikiServices::getInstance()->getResourceLoader()->getMessageBlobStore(); + $blobStore->clear(); } } @@ -1123,4 +1100,5 @@ class LocalisationCache { $this->store = new LCStoreNull; $this->manualRecache = false; } + } diff --git a/includes/installer/Installer.php b/includes/installer/Installer.php index 414222bc45..de154561c4 100644 --- a/includes/installer/Installer.php +++ b/includes/installer/Installer.php @@ -412,17 +412,14 @@ abstract class Installer { // This will be overridden in the web installer with the user-specified language RequestContext::getMain()->setLanguage( 'en' ); + // Disable the i18n cache + // TODO: manage LocalisationCache singleton in MediaWikiServices + Language::getLocalisationCache()->disableBackend(); + // Disable all global services, since we don't have any configuration yet! MediaWikiServices::disableStorageBackend(); $mwServices = MediaWikiServices::getInstance(); - - // Disable i18n cache - $mwServices->getLocalisationCache()->disableBackend(); - - // Clear language cache so the old i18n cache doesn't sneak back in - Language::clearCaches(); - // Disable object cache (otherwise CACHE_ANYTHING will try CACHE_DB and // SqlBagOStuff will then throw since we just disabled wfGetDB) $wgObjectCaches = $mwServices->getMainConfig()->get( 'ObjectCaches' ); diff --git a/includes/language/LanguageCode.php b/includes/language/LanguageCode.php index 1d2f0b4e83..7d954d3803 100644 --- a/includes/language/LanguageCode.php +++ b/includes/language/LanguageCode.php @@ -21,6 +21,7 @@ /** * Methods for dealing with language codes. + * @todo Move some of the code-related static methods out of Language into this class * * @since 1.29 * @ingroup Language diff --git a/includes/language/LanguageNameUtils.php b/includes/language/LanguageNameUtils.php deleted file mode 100644 index 08d9ab3e0d..0000000000 --- a/includes/language/LanguageNameUtils.php +++ /dev/null @@ -1,319 +0,0 @@ -assertRequiredOptions( self::$constructorOptions ); - $this->options = $options; - } - - /** - * Checks whether any localisation is available for that language tag in MediaWiki - * (MessagesXx.php or xx.json exists). - * - * @param string $code Language tag (in lower case) - * @return bool Whether language is supported - */ - public function isSupportedLanguage( $code ) { - if ( !$this->isValidBuiltInCode( $code ) ) { - return false; - } - - if ( $code === 'qqq' ) { - // Special code for internal use, not supported even though there is a qqq.json - return false; - } - - return is_readable( $this->getMessagesFileName( $code ) ) || - is_readable( $this->getJsonMessagesFileName( $code ) ); - } - - /** - * Returns true if a language code string is of a valid form, whether or not it exists. This - * includes codes which are used solely for customisation via the MediaWiki namespace. - * - * @param string $code - * - * @return bool - */ - public function isValidCode( $code ) { - Assert::parameterType( 'string', $code, '$code' ); - if ( !isset( $this->validCodeCache[$code] ) ) { - // People think language codes are HTML-safe, so enforce it. Ideally we should only - // allow a-zA-Z0-9- but .+ and other chars are often used for {{int:}} hacks. See bugs - // T39564, T39587, T38938. - $this->validCodeCache[$code] = - // Protect against path traversal - strcspn( $code, ":/\\\000&<>'\"" ) === strlen( $code ) && - !preg_match( MediaWikiTitleCodec::getTitleInvalidRegex(), $code ); - } - return $this->validCodeCache[$code]; - } - - /** - * Returns true if a language code is of a valid form for the purposes of internal customisation - * of MediaWiki, via Messages*.php or *.json. - * - * @param string $code - * @return bool - */ - public function isValidBuiltInCode( $code ) { - Assert::parameterType( 'string', $code, '$code' ); - - return (bool)preg_match( '/^[a-z0-9-]{2,}$/', $code ); - } - - /** - * Returns true if a language code is an IETF tag known to MediaWiki. - * - * @param string $tag - * - * @return bool - */ - public function isKnownLanguageTag( $tag ) { - // Quick escape for invalid input to avoid exceptions down the line when code tries to - // process tags which are not valid at all. - if ( !$this->isValidBuiltInCode( $tag ) ) { - return false; - } - - if ( isset( Data\Names::$names[$tag] ) || $this->getLanguageName( $tag, $tag ) !== '' ) { - return true; - } - - return false; - } - - /** - * Get an array of language names, indexed by code. - * @param null|string $inLanguage Code of language in which to return the names - * Use self::AUTONYMS for autonyms (native names) - * @param string $include One of: - * self::ALL all available languages - * self::DEFINED only if the language is defined in MediaWiki or wgExtraLanguageNames - * (default) - * self::SUPPORTED only if the language is in self::DEFINED *and* has a message file - * @return array Language code => language name (sorted by key) - */ - public function getLanguageNames( $inLanguage = self::AUTONYMS, $include = self::DEFINED ) { - $cacheKey = $inLanguage === self::AUTONYMS ? 'null' : $inLanguage; - $cacheKey .= ":$include"; - if ( !$this->languageNameCache ) { - $this->languageNameCache = new HashBagOStuff( [ 'maxKeys' => 20 ] ); - } - - $ret = $this->languageNameCache->get( $cacheKey ); - if ( !$ret ) { - $ret = $this->getLanguageNamesUncached( $inLanguage, $include ); - $this->languageNameCache->set( $cacheKey, $ret ); - } - return $ret; - } - - /** - * Uncached helper for getLanguageNames - * @param null|string $inLanguage As getLanguageNames - * @param string $include As getLanguageNames - * @return array Language code => language name (sorted by key) - */ - private function getLanguageNamesUncached( $inLanguage, $include ) { - // If passed an invalid language code to use, fallback to en - if ( $inLanguage !== self::AUTONYMS && !$this->isValidCode( $inLanguage ) ) { - $inLanguage = 'en'; - } - - $names = []; - - if ( $inLanguage !== self::AUTONYMS ) { - # TODO: also include for self::AUTONYMS, when this code is more efficient - Hooks::run( 'LanguageGetTranslatedLanguageNames', [ &$names, $inLanguage ] ); - } - - $mwNames = $this->options->get( 'ExtraLanguageNames' ) + Data\Names::$names; - if ( $this->options->get( 'UsePigLatinVariant' ) ) { - // Pig Latin (for variant development) - $mwNames['en-x-piglatin'] = 'Igpay Atinlay'; - } - - foreach ( $mwNames as $mwCode => $mwName ) { - # - Prefer own MediaWiki native name when not using the hook - # - For other names just add if not added through the hook - if ( $mwCode === $inLanguage || !isset( $names[$mwCode] ) ) { - $names[$mwCode] = $mwName; - } - } - - if ( $include === self::ALL ) { - ksort( $names ); - return $names; - } - - $returnMw = []; - $coreCodes = array_keys( $mwNames ); - foreach ( $coreCodes as $coreCode ) { - $returnMw[$coreCode] = $names[$coreCode]; - } - - if ( $include === self::SUPPORTED ) { - $namesMwFile = []; - # We do this using a foreach over the codes instead of a directory loop so that messages - # files in extensions will work correctly. - foreach ( $returnMw as $code => $value ) { - if ( is_readable( $this->getMessagesFileName( $code ) ) || - is_readable( $this->getJsonMessagesFileName( $code ) ) - ) { - $namesMwFile[$code] = $names[$code]; - } - } - - ksort( $namesMwFile ); - return $namesMwFile; - } - - ksort( $returnMw ); - # self::DEFINED option; default if it's not one of the other two options - # (self::ALL/self::SUPPORTED) - return $returnMw; - } - - /** - * @param string $code The code of the language for which to get the name - * @param null|string $inLanguage Code of language in which to return the name (self::AUTONYMS - * for autonyms) - * @param string $include See getLanguageNames(), except this defaults to self::ALL instead of - * self::DEFINED - * @return string Language name or empty - * @since 1.20 - */ - public function getLanguageName( $code, $inLanguage = self::AUTONYMS, $include = self::ALL ) { - $code = strtolower( $code ); - $array = $this->getLanguageNames( $inLanguage, $include ); - return $array[$code] ?? ''; - } - - /** - * Get the name of a file for a certain language code - * @param string $prefix Prepend this to the filename - * @param string $code Language code - * @param string $suffix Append this to the filename - * @throws MWException - * @return string $prefix . $mangledCode . $suffix - */ - public function getFileName( $prefix, $code, $suffix = '.php' ) { - if ( !$this->isValidBuiltInCode( $code ) ) { - throw new MWException( "Invalid language code \"$code\"" ); - } - - return $prefix . str_replace( '-', '_', ucfirst( $code ) ) . $suffix; - } - - /** - * @param string $code - * @return string - */ - public function getMessagesFileName( $code ) { - global $IP; - $file = $this->getFileName( "$IP/languages/messages/Messages", $code, '.php' ); - Hooks::run( 'Language::getMessagesFileName', [ $code, &$file ] ); - return $file; - } - - /** - * @param string $code - * @return string - * @throws MWException - */ - public function getJsonMessagesFileName( $code ) { - global $IP; - - if ( !$this->isValidBuiltInCode( $code ) ) { - throw new MWException( "Invalid language code \"$code\"" ); - } - - return "$IP/languages/i18n/$code.json"; - } -} diff --git a/languages/Language.php b/languages/Language.php index 872614c448..bb256c9c99 100644 --- a/languages/Language.php +++ b/languages/Language.php @@ -27,8 +27,8 @@ */ use CLDRPluralRuleParser\Evaluator; -use MediaWiki\Languages\LanguageNameUtils; use MediaWiki\MediaWikiServices; +use Wikimedia\Assert\Assert; /** * Internationalisation code @@ -38,24 +38,21 @@ class Language { /** * Return autonyms in fetchLanguageName(s). * @since 1.32 - * @deprecated since 1.34, LanguageNameUtils::AUTONYMS */ - const AS_AUTONYMS = LanguageNameUtils::AUTONYMS; + const AS_AUTONYMS = null; /** * Return all known languages in fetchLanguageName(s). * @since 1.32 - * @deprecated since 1.34, use LanguageNameUtils::ALL */ - const ALL = LanguageNameUtils::ALL; + const ALL = 'all'; /** * Return in fetchLanguageName(s) only the languages for which we have at * least some localisation. * @since 1.32 - * @deprecated since 1.34, use LanguageNameUtils::SUPPORTED */ - const SUPPORTED = LanguageNameUtils::SUPPORTED; + const SUPPORTED = 'mwfile'; /** * @var LanguageConverter @@ -78,11 +75,10 @@ class Language { */ public $transformData = []; - /** @var LocalisationCache */ - private $localisationCache; - - /** @var LanguageNameUtils */ - private $langNameUtils; + /** + * @var LocalisationCache + */ + public static $dataCache; public static $mLangObjCache = []; @@ -98,7 +94,6 @@ class Language { */ const STRICT_FALLBACKS = 1; - // TODO Make these const once we drop HHVM support (T192166) public static $mWeekdayMsgs = [ 'sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday' @@ -183,6 +178,12 @@ class Language { */ private static $grammarTransformations; + /** + * Cache for language names + * @var HashBagOStuff|null + */ + private static $languageNameCache; + /** * Unicode directional formatting characters, for embedBidi() */ @@ -238,12 +239,11 @@ class Language { * @return Language */ protected static function newFromCode( $code, $fallback = false ) { - $langNameUtils = MediaWikiServices::getInstance()->getLanguageNameUtils(); - if ( !$langNameUtils->isValidCode( $code ) ) { + if ( !self::isValidCode( $code ) ) { throw new MWException( "Invalid language code \"$code\"" ); } - if ( !$langNameUtils->isValidBuiltInCode( $code ) ) { + if ( !self::isValidBuiltInCode( $code ) ) { // It's not possible to customise this code with class files, so // just return a Language object. This is to support uselang= hacks. $lang = new Language; @@ -262,7 +262,7 @@ class Language { // Keep trying the fallback list until we find an existing class $fallbacks = self::getFallbacksFor( $code ); foreach ( $fallbacks as $fallbackCode ) { - if ( !$langNameUtils->isValidBuiltInCode( $fallbackCode ) ) { + if ( !self::isValidBuiltInCode( $fallbackCode ) ) { throw new MWException( "Invalid fallback '$fallbackCode' in fallback sequence for '$code'" ); } @@ -283,30 +283,37 @@ class Language { * @since 1.32 */ public static function clearCaches() { - if ( !defined( 'MW_PHPUNIT_TEST' ) && !defined( 'MEDIAWIKI_INSTALL' ) ) { - throw new MWException( __METHOD__ . ' must not be used outside tests/installer' ); - } - if ( !defined( 'MEDIAWIKI_INSTALL' ) ) { - MediaWikiServices::getInstance()->resetServiceForTesting( 'LocalisationCache' ); - MediaWikiServices::getInstance()->resetServiceForTesting( 'LanguageNameUtils' ); + if ( !defined( 'MW_PHPUNIT_TEST' ) ) { + throw new MWException( __METHOD__ . ' must not be used outside tests' ); } + self::$dataCache = null; + // Reinitialize $dataCache, since it's expected to always be available + self::getLocalisationCache(); self::$mLangObjCache = []; self::$fallbackLanguageCache = []; self::$grammarTransformations = null; + self::$languageNameCache = null; } /** * Checks whether any localisation is available for that language tag * in MediaWiki (MessagesXx.php exists). * - * @deprecated since 1.34, use LanguageNameUtils * @param string $code Language tag (in lower case) * @return bool Whether language is supported * @since 1.21 */ public static function isSupportedLanguage( $code ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->isSupportedLanguage( $code ); + if ( !self::isValidBuiltInCode( $code ) ) { + return false; + } + + if ( $code === 'qqq' ) { + return false; + } + + return is_readable( self::getMessagesFileName( $code ) ) || + is_readable( self::getJsonMessagesFileName( $code ) ); } /** @@ -374,55 +381,77 @@ class Language { * not it exists. This includes codes which are used solely for * customisation via the MediaWiki namespace. * - * @deprecated since 1.34, use LanguageNameUtils - * * @param string $code * * @return bool */ public static function isValidCode( $code ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils()->isValidCode( $code ); + static $cache = []; + Assert::parameterType( 'string', $code, '$code' ); + if ( !isset( $cache[$code] ) ) { + // People think language codes are html safe, so enforce it. + // Ideally we should only allow a-zA-Z0-9- + // but, .+ and other chars are often used for {{int:}} hacks + // see bugs T39564, T39587, T38938 + $cache[$code] = + // Protect against path traversal + strcspn( $code, ":/\\\000&<>'\"" ) === strlen( $code ) + && !preg_match( MediaWikiTitleCodec::getTitleInvalidRegex(), $code ); + } + return $cache[$code]; } /** * Returns true if a language code is of a valid form for the purposes of * internal customisation of MediaWiki, via Messages*.php or *.json. * - * @deprecated since 1.34, use LanguageNameUtils - * * @param string $code * * @since 1.18 * @return bool */ public static function isValidBuiltInCode( $code ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->isValidBuiltInCode( $code ); + Assert::parameterType( 'string', $code, '$code' ); + + return (bool)preg_match( '/^[a-z0-9-]{2,}$/', $code ); } /** * Returns true if a language code is an IETF tag known to MediaWiki. * - * @deprecated since 1.34, use LanguageNameUtils - * * @param string $tag * * @since 1.21 * @return bool */ public static function isKnownLanguageTag( $tag ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->isKnownLanguageTag( $tag ); + // Quick escape for invalid input to avoid exceptions down the line + // when code tries to process tags which are not valid at all. + if ( !self::isValidBuiltInCode( $tag ) ) { + return false; + } + + if ( isset( MediaWiki\Languages\Data\Names::$names[$tag] ) + || self::fetchLanguageName( $tag, $tag ) !== '' + ) { + return true; + } + + return false; } /** * Get the LocalisationCache instance * - * @deprecated since 1.34, use MediaWikiServices * @return LocalisationCache */ public static function getLocalisationCache() { - return MediaWikiServices::getInstance()->getLocalisationCache(); + if ( is_null( self::$dataCache ) ) { + global $wgLocalisationCacheConf; + $class = $wgLocalisationCacheConf['class']; + self::$dataCache = new $class( $wgLocalisationCacheConf ); + } + return self::$dataCache; } function __construct() { @@ -433,9 +462,7 @@ class Language { } else { $this->mCode = str_replace( '_', '-', strtolower( substr( static::class, 8 ) ) ); } - $services = MediaWikiServices::getInstance(); - $this->localisationCache = $services->getLocalisationCache(); - $this->langNameUtils = $services->getLanguageNameUtils(); + self::getLocalisationCache(); } /** @@ -467,7 +494,7 @@ class Language { * @return array */ public function getBookstoreList() { - return $this->localisationCache->getItem( $this->mCode, 'bookstoreList' ); + return self::$dataCache->getItem( $this->mCode, 'bookstoreList' ); } /** @@ -484,7 +511,7 @@ class Language { getCanonicalNamespaces(); $this->namespaceNames = $wgExtraNamespaces + - $this->localisationCache->getItem( $this->mCode, 'namespaceNames' ); + self::$dataCache->getItem( $this->mCode, 'namespaceNames' ); $this->namespaceNames += $validNamespaces; $this->namespaceNames[NS_PROJECT] = $wgMetaNamespace; @@ -591,7 +618,7 @@ class Language { global $wgExtraGenderNamespaces; $ns = $wgExtraGenderNamespaces + - (array)$this->localisationCache->getItem( $this->mCode, 'namespaceGenderAliases' ); + (array)self::$dataCache->getItem( $this->mCode, 'namespaceGenderAliases' ); return $ns[$index][$gender] ?? $this->getNsText( $index ); } @@ -613,7 +640,7 @@ class Language { return false; } else { // Check what is in i18n files - $aliases = $this->localisationCache->getItem( $this->mCode, 'namespaceGenderAliases' ); + $aliases = self::$dataCache->getItem( $this->mCode, 'namespaceGenderAliases' ); return count( $aliases ) > 0; } } @@ -637,7 +664,7 @@ class Language { */ public function getNamespaceAliases() { if ( is_null( $this->namespaceAliases ) ) { - $aliases = $this->localisationCache->getItem( $this->mCode, 'namespaceAliases' ); + $aliases = self::$dataCache->getItem( $this->mCode, 'namespaceAliases' ); if ( !$aliases ) { $aliases = []; } else { @@ -651,8 +678,8 @@ class Language { } global $wgExtraGenderNamespaces; - $genders = $wgExtraGenderNamespaces + (array)$this->localisationCache - ->getItem( $this->mCode, 'namespaceGenderAliases' ); + $genders = $wgExtraGenderNamespaces + + (array)self::$dataCache->getItem( $this->mCode, 'namespaceGenderAliases' ); foreach ( $genders as $index => $forms ) { foreach ( $forms as $alias ) { $aliases[$alias] = $index; @@ -740,7 +767,7 @@ class Language { if ( $usemsg && wfMessage( $msg )->exists() ) { return $this->getMessageFromDB( $msg ); } - $name = $this->langNameUtils->getLanguageName( $code ); + $name = self::fetchLanguageName( $code ); if ( $name ) { return $name; # if it's defined as a language name, show that } else { @@ -753,21 +780,21 @@ class Language { * @return string[]|bool List of date format preference keys, or false if disabled. */ public function getDatePreferences() { - return $this->localisationCache->getItem( $this->mCode, 'datePreferences' ); + return self::$dataCache->getItem( $this->mCode, 'datePreferences' ); } /** * @return array */ function getDateFormats() { - return $this->localisationCache->getItem( $this->mCode, 'dateFormats' ); + return self::$dataCache->getItem( $this->mCode, 'dateFormats' ); } /** * @return array|string */ public function getDefaultDateFormat() { - $df = $this->localisationCache->getItem( $this->mCode, 'defaultDateFormat' ); + $df = self::$dataCache->getItem( $this->mCode, 'defaultDateFormat' ); if ( $df === 'dmy or mdy' ) { global $wgAmericanDates; return $wgAmericanDates ? 'mdy' : 'dmy'; @@ -780,7 +807,7 @@ class Language { * @return array */ public function getDatePreferenceMigrationMap() { - return $this->localisationCache->getItem( $this->mCode, 'datePreferenceMigrationMap' ); + return self::$dataCache->getItem( $this->mCode, 'datePreferenceMigrationMap' ); } /** @@ -801,8 +828,6 @@ class Language { /** * Get an array of language names, indexed by code. - * - * @deprecated since 1.34, use LanguageNameUtils::getLanguageNames * @param null|string $inLanguage Code of language in which to return the names * Use self::AS_AUTONYMS for autonyms (native names) * @param string $include One of: @@ -813,12 +838,95 @@ class Language { * @since 1.20 */ public static function fetchLanguageNames( $inLanguage = self::AS_AUTONYMS, $include = 'mw' ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->getLanguageNames( $inLanguage, $include ); + $cacheKey = $inLanguage === self::AS_AUTONYMS ? 'null' : $inLanguage; + $cacheKey .= ":$include"; + if ( self::$languageNameCache === null ) { + self::$languageNameCache = new HashBagOStuff( [ 'maxKeys' => 20 ] ); + } + + $ret = self::$languageNameCache->get( $cacheKey ); + if ( !$ret ) { + $ret = self::fetchLanguageNamesUncached( $inLanguage, $include ); + self::$languageNameCache->set( $cacheKey, $ret ); + } + return $ret; + } + + /** + * Uncached helper for fetchLanguageNames + * @param null|string $inLanguage Code of language in which to return the names + * Use self::AS_AUTONYMS for autonyms (native names) + * @param string $include One of: + * self::ALL all available languages + * 'mw' only if the language is defined in MediaWiki or wgExtraLanguageNames (default) + * self::SUPPORTED only if the language is in 'mw' *and* has a message file + * @return array Language code => language name (sorted by key) + */ + private static function fetchLanguageNamesUncached( + $inLanguage = self::AS_AUTONYMS, + $include = 'mw' + ) { + global $wgExtraLanguageNames, $wgUsePigLatinVariant; + + // If passed an invalid language code to use, fallback to en + if ( $inLanguage !== self::AS_AUTONYMS && !self::isValidCode( $inLanguage ) ) { + $inLanguage = 'en'; + } + + $names = []; + + if ( $inLanguage ) { + # TODO: also include when $inLanguage is null, when this code is more efficient + Hooks::run( 'LanguageGetTranslatedLanguageNames', [ &$names, $inLanguage ] ); + } + + $mwNames = $wgExtraLanguageNames + MediaWiki\Languages\Data\Names::$names; + if ( $wgUsePigLatinVariant ) { + // Pig Latin (for variant development) + $mwNames['en-x-piglatin'] = 'Igpay Atinlay'; + } + + foreach ( $mwNames as $mwCode => $mwName ) { + # - Prefer own MediaWiki native name when not using the hook + # - For other names just add if not added through the hook + if ( $mwCode === $inLanguage || !isset( $names[$mwCode] ) ) { + $names[$mwCode] = $mwName; + } + } + + if ( $include === self::ALL ) { + ksort( $names ); + return $names; + } + + $returnMw = []; + $coreCodes = array_keys( $mwNames ); + foreach ( $coreCodes as $coreCode ) { + $returnMw[$coreCode] = $names[$coreCode]; + } + + if ( $include === self::SUPPORTED ) { + $namesMwFile = []; + # We do this using a foreach over the codes instead of a directory + # loop so that messages files in extensions will work correctly. + foreach ( $returnMw as $code => $value ) { + if ( is_readable( self::getMessagesFileName( $code ) ) + || is_readable( self::getJsonMessagesFileName( $code ) ) + ) { + $namesMwFile[$code] = $names[$code]; + } + } + + ksort( $namesMwFile ); + return $namesMwFile; + } + + ksort( $returnMw ); + # 'mw' option; default if it's not one of the other two options (all/mwfile) + return $returnMw; } /** - * @deprecated since 1.34, use LanguageNameUtils::getLanguageName * @param string $code The code of the language for which to get the name * @param null|string $inLanguage Code of language in which to return the name * (SELF::AS_AUTONYMS for autonyms) @@ -831,8 +939,9 @@ class Language { $inLanguage = self::AS_AUTONYMS, $include = self::ALL ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->getLanguageName( $code, $inLanguage, $include ); + $code = strtolower( $code ); + $array = self::fetchLanguageNames( $inLanguage, $include ); + return !array_key_exists( $code, $array ) ? '' : $array[$code]; } /** @@ -2165,8 +2274,7 @@ class Language { } if ( !isset( $this->dateFormatStrings[$type][$pref] ) ) { - $df = - $this->localisationCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); + $df = self::$dataCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); if ( $type === 'pretty' && $df === null ) { $df = $this->getDateFormatString( 'date', $pref ); @@ -2174,8 +2282,7 @@ class Language { if ( !$wasDefault && $df === null ) { $pref = $this->getDefaultDateFormat(); - $df = $this->getLocalisationCache() - ->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); + $df = self::$dataCache->getSubitem( $this->mCode, 'dateFormats', "$pref $type" ); } $this->dateFormatStrings[$type][$pref] = $df; @@ -2539,14 +2646,14 @@ class Language { * @return string|null */ public function getMessage( $key ) { - return $this->localisationCache->getSubitem( $this->mCode, 'messages', $key ); + return self::$dataCache->getSubitem( $this->mCode, 'messages', $key ); } /** * @return array */ function getAllMessages() { - return $this->localisationCache->getItem( $this->mCode, 'messages' ); + return self::$dataCache->getItem( $this->mCode, 'messages' ); } /** @@ -2788,7 +2895,7 @@ class Language { * @return string */ function fallback8bitEncoding() { - return $this->localisationCache->getItem( $this->mCode, 'fallback8bitEncoding' ); + return self::$dataCache->getItem( $this->mCode, 'fallback8bitEncoding' ); } /** @@ -2978,7 +3085,7 @@ class Language { * @return bool */ function isRTL() { - return $this->localisationCache->getItem( $this->mCode, 'rtl' ); + return self::$dataCache->getItem( $this->mCode, 'rtl' ); } /** @@ -3054,7 +3161,7 @@ class Language { * @return array */ function capitalizeAllNouns() { - return $this->localisationCache->getItem( $this->mCode, 'capitalizeAllNouns' ); + return self::$dataCache->getItem( $this->mCode, 'capitalizeAllNouns' ); } /** @@ -3087,7 +3194,7 @@ class Language { * @return bool */ function linkPrefixExtension() { - return $this->localisationCache->getItem( $this->mCode, 'linkPrefixExtension' ); + return self::$dataCache->getItem( $this->mCode, 'linkPrefixExtension' ); } /** @@ -3095,7 +3202,7 @@ class Language { * @return array */ function getMagicWords() { - return $this->localisationCache->getItem( $this->mCode, 'magicWords' ); + return self::$dataCache->getItem( $this->mCode, 'magicWords' ); } /** @@ -3105,7 +3212,7 @@ class Language { */ function getMagic( $mw ) { $rawEntry = $this->mMagicExtensions[$mw->mId] ?? - $this->localisationCache->getSubitem( $this->mCode, 'magicWords', $mw->mId ); + self::$dataCache->getSubitem( $this->mCode, 'magicWords', $mw->mId ); if ( !is_array( $rawEntry ) ) { wfWarn( "\"$rawEntry\" is not a valid magic word for \"$mw->mId\"" ); @@ -3140,7 +3247,7 @@ class Language { if ( is_null( $this->mExtendedSpecialPageAliases ) ) { // Initialise array $this->mExtendedSpecialPageAliases = - $this->localisationCache->getItem( $this->mCode, 'specialPageAliases' ); + self::$dataCache->getItem( $this->mCode, 'specialPageAliases' ); } return $this->mExtendedSpecialPageAliases; @@ -3305,28 +3412,28 @@ class Language { * @return string */ function digitGroupingPattern() { - return $this->localisationCache->getItem( $this->mCode, 'digitGroupingPattern' ); + return self::$dataCache->getItem( $this->mCode, 'digitGroupingPattern' ); } /** * @return array */ function digitTransformTable() { - return $this->localisationCache->getItem( $this->mCode, 'digitTransformTable' ); + return self::$dataCache->getItem( $this->mCode, 'digitTransformTable' ); } /** * @return array */ function separatorTransformTable() { - return $this->localisationCache->getItem( $this->mCode, 'separatorTransformTable' ); + return self::$dataCache->getItem( $this->mCode, 'separatorTransformTable' ); } /** * @return int|null */ function minimumGroupingDigits() { - return $this->localisationCache->getItem( $this->mCode, 'minimumGroupingDigits' ); + return self::$dataCache->getItem( $this->mCode, 'minimumGroupingDigits' ); } /** @@ -4226,7 +4333,7 @@ class Language { * @return string */ public function linkTrail() { - return $this->localisationCache->getItem( $this->mCode, 'linkTrail' ); + return self::$dataCache->getItem( $this->mCode, 'linkTrail' ); } /** @@ -4236,7 +4343,7 @@ class Language { * @return string */ public function linkPrefixCharset() { - return $this->localisationCache->getItem( $this->mCode, 'linkPrefixCharset' ); + return self::$dataCache->getItem( $this->mCode, 'linkPrefixCharset' ); } /** @@ -4338,8 +4445,6 @@ class Language { /** * Get the name of a file for a certain language code - * - * @deprecated since 1.34, use LanguageNameUtils * @param string $prefix Prepend this to the filename * @param string $code Language code * @param string $suffix Append this to the filename @@ -4347,30 +4452,38 @@ class Language { * @return string $prefix . $mangledCode . $suffix */ public static function getFileName( $prefix, $code, $suffix = '.php' ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->getFileName( $prefix, $code, $suffix ); + if ( !self::isValidBuiltInCode( $code ) ) { + throw new MWException( "Invalid language code \"$code\"" ); + } + + return $prefix . str_replace( '-', '_', ucfirst( $code ) ) . $suffix; } /** - * @deprecated since 1.34, use LanguageNameUtils * @param string $code * @return string */ public static function getMessagesFileName( $code ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->getMessagesFileName( $code ); + global $IP; + $file = self::getFileName( "$IP/languages/messages/Messages", $code, '.php' ); + Hooks::run( 'Language::getMessagesFileName', [ $code, &$file ] ); + return $file; } /** - * @deprecated since 1.34, use LanguageNameUtils * @param string $code * @return string * @throws MWException * @since 1.23 */ public static function getJsonMessagesFileName( $code ) { - return MediaWikiServices::getInstance()->getLanguageNameUtils() - ->getJsonMessagesFileName( $code ); + global $IP; + + if ( !self::isValidBuiltInCode( $code ) ) { + throw new MWException( "Invalid language code \"$code\"" ); + } + + return "$IP/languages/i18n/$code.json"; } /** @@ -4820,13 +4933,11 @@ class Language { * @return array Associative array with plural form, and plural rule as key-value pairs */ public function getCompiledPluralRules() { - $pluralRules = - $this->localisationCache->getItem( strtolower( $this->mCode ), 'compiledPluralRules' ); + $pluralRules = self::$dataCache->getItem( strtolower( $this->mCode ), 'compiledPluralRules' ); $fallbacks = self::getFallbacksFor( $this->mCode ); if ( !$pluralRules ) { foreach ( $fallbacks as $fallbackCode ) { - $pluralRules = $this->localisationCache - ->getItem( strtolower( $fallbackCode ), 'compiledPluralRules' ); + $pluralRules = self::$dataCache->getItem( strtolower( $fallbackCode ), 'compiledPluralRules' ); if ( $pluralRules ) { break; } @@ -4841,13 +4952,11 @@ class Language { * @return array Associative array with plural form number and plural rule as key-value pairs */ public function getPluralRules() { - $pluralRules = - $this->localisationCache->getItem( strtolower( $this->mCode ), 'pluralRules' ); + $pluralRules = self::$dataCache->getItem( strtolower( $this->mCode ), 'pluralRules' ); $fallbacks = self::getFallbacksFor( $this->mCode ); if ( !$pluralRules ) { foreach ( $fallbacks as $fallbackCode ) { - $pluralRules = $this->localisationCache - ->getItem( strtolower( $fallbackCode ), 'pluralRules' ); + $pluralRules = self::$dataCache->getItem( strtolower( $fallbackCode ), 'pluralRules' ); if ( $pluralRules ) { break; } @@ -4862,13 +4971,11 @@ class Language { * @return array Associative array with plural form number and plural rule type as key-value pairs */ public function getPluralRuleTypes() { - $pluralRuleTypes = - $this->localisationCache->getItem( strtolower( $this->mCode ), 'pluralRuleTypes' ); + $pluralRuleTypes = self::$dataCache->getItem( strtolower( $this->mCode ), 'pluralRuleTypes' ); $fallbacks = self::getFallbacksFor( $this->mCode ); if ( !$pluralRuleTypes ) { foreach ( $fallbacks as $fallbackCode ) { - $pluralRuleTypes = $this->localisationCache - ->getItem( strtolower( $fallbackCode ), 'pluralRuleTypes' ); + $pluralRuleTypes = self::$dataCache->getItem( strtolower( $fallbackCode ), 'pluralRuleTypes' ); if ( $pluralRuleTypes ) { break; } diff --git a/languages/data/Names.php b/languages/data/Names.php index 1d80f6b257..00f35b2564 100644 --- a/languages/data/Names.php +++ b/languages/data/Names.php @@ -39,7 +39,7 @@ namespace MediaWiki\Languages\Data; * If you are adding support for such a language, add it also to * the relevant section in shared.css. * - * Do not use this class directly. Use LanguageNameUtils::getLanguageNames(), which + * Do not use this class directly. Use Language::fetchLanguageNames(), which * includes support for the CLDR extension. * * @ingroup Language diff --git a/maintenance/rebuildLocalisationCache.php b/maintenance/rebuildLocalisationCache.php index 8a519e7178..4213d5f85d 100644 --- a/maintenance/rebuildLocalisationCache.php +++ b/maintenance/rebuildLocalisationCache.php @@ -29,10 +29,6 @@ * @ingroup Maintenance */ -use MediaWiki\Config\ServiceOptions; -use MediaWiki\Logger\LoggerFactory; -use MediaWiki\MediaWikiServices; - require_once __DIR__ . '/Maintenance.php'; /** @@ -62,7 +58,7 @@ class RebuildLocalisationCache extends Maintenance { } public function execute() { - global $wgLocalisationCacheConf, $wgCacheDirectory; + global $wgLocalisationCacheConf; $force = $this->hasOption( 'force' ); $threads = $this->getOption( 'threads', 1 ); @@ -81,25 +77,13 @@ class RebuildLocalisationCache extends Maintenance { $conf = $wgLocalisationCacheConf; $conf['manualRecache'] = false; // Allow fallbacks to create CDB files - $conf['forceRecache'] = $force || !empty( $conf['forceRecache'] ); + if ( $force ) { + $conf['forceRecache'] = true; + } if ( $this->hasOption( 'outdir' ) ) { $conf['storeDirectory'] = $this->getOption( 'outdir' ); } - // XXX Copy-pasted from ServiceWiring.php. Do we need a factory for this one caller? - $lc = new LocalisationCacheBulkLoad( - new ServiceOptions( - LocalisationCache::$constructorOptions, - $conf, - MediaWikiServices::getInstance()->getMainConfig() - ), - LocalisationCache::getStoreFromConf( $conf, $wgCacheDirectory ), - LoggerFactory::getInstance( 'localisation' ), - [ function () { - MediaWikiServices::getInstance()->getResourceLoader() - ->getMessageBlobStore()->clear(); - } ], - MediaWikiServices::getInstance()->getLanguageNameUtils() - ); + $lc = new LocalisationCacheBulkLoad( $conf ); $allCodes = array_keys( Language::fetchLanguageNames( null, 'mwfile' ) ); if ( $this->hasOption( 'lang' ) ) { diff --git a/tests/common/TestsAutoLoader.php b/tests/common/TestsAutoLoader.php index 99b548ed7f..7c8df1a6f5 100644 --- a/tests/common/TestsAutoLoader.php +++ b/tests/common/TestsAutoLoader.php @@ -221,9 +221,6 @@ $wgAutoloadClasses += [ # tests/phpunit/unit/includes 'BadFileLookupTest' => "$testDir/phpunit/unit/includes/BadFileLookupTest.php", - # tests/phpunit/unit/includes/language - 'LanguageNameUtilsTestTrait' => "$testDir/phpunit/unit/includes/language/LanguageNameUtilsTestTrait.php", - # tests/phpunit/unit/includes/libs/filebackend/fsfile 'TempFSFileTestTrait' => "$testDir/phpunit/unit/includes/libs/filebackend/fsfile/TempFSFileTestTrait.php", diff --git a/tests/phpunit/MediaWikiIntegrationTestCase.php b/tests/phpunit/MediaWikiIntegrationTestCase.php index 5261b19644..33518ef929 100644 --- a/tests/phpunit/MediaWikiIntegrationTestCase.php +++ b/tests/phpunit/MediaWikiIntegrationTestCase.php @@ -404,7 +404,6 @@ abstract class MediaWikiIntegrationTestCase extends PHPUnit\Framework\TestCase { $wgRequest = new FauxRequest(); MediaWiki\Session\SessionManager::resetCache(); - Language::clearCaches(); } public function run( PHPUnit_Framework_TestResult $result = null ) { diff --git a/tests/phpunit/includes/api/ApiQuerySiteinfoTest.php b/tests/phpunit/includes/api/ApiQuerySiteinfoTest.php index 6308b82c58..282188d264 100644 --- a/tests/phpunit/includes/api/ApiQuerySiteinfoTest.php +++ b/tests/phpunit/includes/api/ApiQuerySiteinfoTest.php @@ -160,7 +160,6 @@ class ApiQuerySiteinfoTest extends ApiTestCase { 'wgExtraInterlanguageLinkPrefixes' => [ 'self' ], 'wgExtraLanguageNames' => [ 'self' => 'Recursion' ], ] ); - $this->resetServices(); MessageCache::singleton()->enable(); diff --git a/tests/phpunit/includes/cache/LocalisationCacheTest.php b/tests/phpunit/includes/cache/LocalisationCacheTest.php index 39526fb9c7..42957b6084 100644 --- a/tests/phpunit/includes/cache/LocalisationCacheTest.php +++ b/tests/phpunit/includes/cache/LocalisationCacheTest.php @@ -1,9 +1,4 @@ createMock( LanguageNameUtils::class ); - $mockLangNameUtils->method( 'isValidBuiltInCode' )->will( $this->returnCallback( - function ( $code ) { - // Copy-paste, but it's only one line - return (bool)preg_match( '/^[a-z0-9-]{2,}$/', $code ); - } - ) ); - $mockLangNameUtils->method( 'isSupportedLanguage' )->will( $this->returnCallback( - function ( $code ) { - return in_array( $code, [ - 'ar', - 'arz', - 'ba', - 'de', - 'en', - 'ksh', - 'ru', - ] ); - } - ) ); - $mockLangNameUtils->method( 'getMessagesFileName' )->will( $this->returnCallback( - function ( $code ) { - global $IP; - $code = str_replace( '-', '_', ucfirst( $code ) ); - return "$IP/languages/messages/Messages$code.php"; - } - ) ); - $mockLangNameUtils->expects( $this->never() )->method( $this->anythingBut( - 'isValidBuiltInCode', 'isSupportedLanguage', 'getMessagesFileName' - ) ); - - $lc = $this->getMockBuilder( LocalisationCache::class ) - ->setConstructorArgs( [ - new ServiceOptions( LocalisationCache::$constructorOptions, [ - 'forceRecache' => false, - 'manualRecache' => false, - 'ExtensionMessagesFiles' => [], - 'MessagesDirs' => [], - ] ), - new LCStoreDB( [] ), - new NullLogger, - [], - $mockLangNameUtils - ] ) + $lc = $this->getMockBuilder( \LocalisationCache::class ) + ->setConstructorArgs( [ [ 'store' => 'detect' ] ] ) ->setMethods( [ 'getMessagesDirs' ] ) ->getMock(); $lc->expects( $this->any() )->method( 'getMessagesDirs' ) @@ -79,7 +31,7 @@ class LocalisationCacheTest extends MediaWikiTestCase { return $lc; } - public function testPluralRulesFallback() { + public function testPuralRulesFallback() { $cache = $this->getMockLocalisationCache(); $this->assertEquals( diff --git a/tests/phpunit/includes/logging/LogFormatterTest.php b/tests/phpunit/includes/logging/LogFormatterTest.php index 839272f356..4bb9d5ab1a 100644 --- a/tests/phpunit/includes/logging/LogFormatterTest.php +++ b/tests/phpunit/includes/logging/LogFormatterTest.php @@ -39,13 +39,13 @@ class LogFormatterTest extends MediaWikiLangTestCase { global $wgExtensionMessagesFiles; self::$oldExtMsgFiles = $wgExtensionMessagesFiles; $wgExtensionMessagesFiles['LogTests'] = __DIR__ . '/LogTests.i18n.php'; - Language::clearCaches(); + Language::getLocalisationCache()->recache( 'en' ); } public static function tearDownAfterClass() { global $wgExtensionMessagesFiles; $wgExtensionMessagesFiles = self::$oldExtMsgFiles; - Language::clearCaches(); + Language::getLocalisationCache()->recache( 'en' ); parent::tearDownAfterClass(); } diff --git a/tests/phpunit/languages/LanguageTest.php b/tests/phpunit/languages/LanguageTest.php index 6f618a2cc9..2f6fa39b36 100644 --- a/tests/phpunit/languages/LanguageTest.php +++ b/tests/phpunit/languages/LanguageTest.php @@ -3,24 +3,6 @@ use Wikimedia\TestingAccessWrapper; class LanguageTest extends LanguageClassesTestCase { - use LanguageNameUtilsTestTrait; - - /** @var array Copy of $wgHooks from before we unset LanguageGetTranslatedLanguageNames */ - private $origHooks; - - public function setUp() { - global $wgHooks; - - parent::setUp(); - - // Don't allow installed hooks to run, except if a test restores them via origHooks (needed - // for testIsKnownLanguageTag_cldr) - $this->origHooks = $wgHooks; - $newHooks = $wgHooks; - unset( $newHooks['LanguageGetTranslatedLanguageNames'] ); - $this->setMwGlobals( 'wgHooks', $newHooks ); - } - /** * @covers Language::convertDoubleWidth * @covers Language::normalizeForSearch @@ -528,6 +510,84 @@ class LanguageTest extends LanguageClassesTestCase { ); } + /** + * Test Language::isValidBuiltInCode() + * @dataProvider provideLanguageCodes + * @covers Language::isValidBuiltInCode + */ + public function testBuiltInCodeValidation( $code, $expected, $message = '' ) { + $this->assertEquals( $expected, + (bool)Language::isValidBuiltInCode( $code ), + "validating code $code $message" + ); + } + + public static function provideLanguageCodes() { + return [ + [ 'fr', true, 'Two letters, minor case' ], + [ 'EN', false, 'Two letters, upper case' ], + [ 'tyv', true, 'Three letters' ], + [ 'be-tarask', true, 'With dash' ], + [ 'be-x-old', true, 'With extension (two dashes)' ], + [ 'be_tarask', false, 'Reject underscores' ], + ]; + } + + /** + * Test Language::isKnownLanguageTag() + * @dataProvider provideKnownLanguageTags + * @covers Language::isKnownLanguageTag + */ + public function testKnownLanguageTag( $code, $message = '' ) { + $this->assertTrue( + (bool)Language::isKnownLanguageTag( $code ), + "validating code $code - $message" + ); + } + + public static function provideKnownLanguageTags() { + return [ + [ 'fr', 'simple code' ], + [ 'bat-smg', 'an MW legacy tag' ], + [ 'sgs', 'an internal standard MW name, for which a legacy tag is used externally' ], + ]; + } + + /** + * @covers Language::isKnownLanguageTag + */ + public function testKnownCldrLanguageTag() { + if ( !class_exists( 'LanguageNames' ) ) { + $this->markTestSkipped( 'The LanguageNames class is not available. ' + . 'The CLDR extension is probably not installed.' ); + } + + $this->assertTrue( + (bool)Language::isKnownLanguageTag( 'pal' ), + 'validating code "pal" an ancient language, which probably will ' + . 'not appear in Names.php, but appears in CLDR in English' + ); + } + + /** + * Negative tests for Language::isKnownLanguageTag() + * @dataProvider provideUnKnownLanguageTags + * @covers Language::isKnownLanguageTag + */ + public function testUnknownLanguageTag( $code, $message = '' ) { + $this->assertFalse( + (bool)Language::isKnownLanguageTag( $code ), + "checking that code $code is invalid - $message" + ); + } + + public static function provideUnknownLanguageTags() { + return [ + [ 'mw', 'non-existent two-letter code' ], + [ 'foo"getItem( 'zh', 'mainpage' ); + $oldCacheObj = Language::$dataCache; + $this->assertNotCount( 0, + TestingAccessWrapper::newFromObject( Language::$dataCache )->loadedItems ); + // Populate $mLangObjCache $lang = Language::factory( 'en' ); $this->assertNotCount( 0, Language::$mLangObjCache ); @@ -1764,11 +1830,36 @@ class LanguageTest extends LanguageClassesTestCase { $lang->getGrammarTransformations(); $this->assertNotNull( $languageClass->grammarTransformations ); + // Populate $languageNameCache + Language::fetchLanguageNames(); + $this->assertNotNull( $languageClass->languageNameCache ); + Language::clearCaches(); + $this->assertNotSame( $oldCacheObj, Language::$dataCache ); + $this->assertCount( 0, + TestingAccessWrapper::newFromObject( Language::$dataCache )->loadedItems ); $this->assertCount( 0, Language::$mLangObjCache ); $this->assertCount( 0, $languageClass->fallbackLanguageCache ); $this->assertNull( $languageClass->grammarTransformations ); + $this->assertNull( $languageClass->languageNameCache ); + } + + /** + * @dataProvider provideIsSupportedLanguage + * @covers Language::isSupportedLanguage + */ + public function testIsSupportedLanguage( $code, $expected, $comment ) { + $this->assertEquals( $expected, Language::isSupportedLanguage( $code ), $comment ); + } + + public static function provideIsSupportedLanguage() { + return [ + [ 'en', true, 'is supported language' ], + [ 'fi', true, 'is supported language' ], + [ 'bunny', false, 'is not supported language' ], + [ 'FI', false, 'is not supported language, input should be in lower case' ], + ]; } /** @@ -1874,82 +1965,4 @@ class LanguageTest extends LanguageClassesTestCase { [ 'èl', 'Ll' , 'Non-ASCII is overridden', [ 'è' => 'L' ] ], ]; } - - // The following methods are for LanguageNameUtilsTestTrait - - private function isSupportedLanguage( $code ) { - return Language::isSupportedLanguage( $code ); - } - - private function isValidCode( $code ) { - return Language::isValidCode( $code ); - } - - private function isValidBuiltInCode( $code ) { - return Language::isValidBuiltInCode( $code ); - } - - private function isKnownLanguageTag( $code ) { - return Language::isKnownLanguageTag( $code ); - } - - /** - * Call getLanguageName() and getLanguageNames() using the Language static methods. - * - * @param array $options To set globals for testing Language - * @param string $expected - * @param string $code - * @param mixed ...$otherArgs Optionally, pass $inLanguage and/or $include. - */ - private function assertGetLanguageNames( array $options, $expected, $code, ...$otherArgs ) { - if ( $options ) { - foreach ( $options as $key => $val ) { - $this->setMwGlobals( "wg$key", $val ); - } - $this->resetServices(); - } - $this->assertSame( $expected, - Language::fetchLanguageNames( ...$otherArgs )[strtolower( $code )] ?? '' ); - $this->assertSame( $expected, Language::fetchLanguageName( $code, ...$otherArgs ) ); - } - - private function getLanguageNames( ...$args ) { - return Language::fetchLanguageNames( ...$args ); - } - - private function getLanguageName( ...$args ) { - return Language::fetchLanguageName( ...$args ); - } - - private static function getFileName( ...$args ) { - return Language::getFileName( ...$args ); - } - - private static function getMessagesFileName( $code ) { - return Language::getMessagesFileName( $code ); - } - - private static function getJsonMessagesFileName( $code ) { - return Language::getJsonMessagesFileName( $code ); - } - - /** - * @todo This really belongs in the cldr extension's tests. - * - * @covers MediaWiki\Languages\LanguageNameUtils::isKnownLanguageTag - * @covers Language::isKnownLanguageTag - */ - public function testIsKnownLanguageTag_cldr() { - if ( !class_exists( 'LanguageNames' ) ) { - $this->markTestSkipped( 'The LanguageNames class is not available. ' - . 'The CLDR extension is probably not installed.' ); - } - - // We need to restore the extension's hook that we removed. - $this->setMwGlobals( 'wgHooks', $this->origHooks ); - - // "pal" is an ancient language, which probably will not appear in Names.php, but appears in - // CLDR in English - $this->assertTrue( Language::isKnownLanguageTag( 'pal' ) ); - } } diff --git a/tests/phpunit/unit/includes/language/LanguageNameUtilsTest.php b/tests/phpunit/unit/includes/language/LanguageNameUtilsTest.php deleted file mode 100644 index 6fbd4a2863..0000000000 --- a/tests/phpunit/unit/includes/language/LanguageNameUtilsTest.php +++ /dev/null @@ -1,66 +0,0 @@ - [], - 'LanguageCode' => 'en', - 'UsePigLatinVariant' => false, - ] - ) ); - } - - use LanguageNameUtilsTestTrait; - - private function isSupportedLanguage( $code ) { - return $this->newObj()->isSupportedLanguage( $code ); - } - - private function isValidCode( $code ) { - return $this->newObj()->isValidCode( $code ); - } - - private function isValidBuiltInCode( $code ) { - return $this->newObj()->isValidBuiltInCode( $code ); - } - - private function isKnownLanguageTag( $code ) { - return $this->newObj()->isKnownLanguageTag( $code ); - } - - private function assertGetLanguageNames( array $options, $expected, $code, ...$otherArgs ) { - $this->assertSame( $expected, $this->newObj( $options ) - ->getLanguageNames( ...$otherArgs )[strtolower( $code )] ?? '' ); - $this->assertSame( $expected, - $this->newObj( $options )->getLanguageName( $code, ...$otherArgs ) ); - } - - private function getLanguageNames( ...$args ) { - return $this->newObj()->getLanguageNames( ...$args ); - } - - private function getLanguageName( ...$args ) { - return $this->newObj()->getLanguageName( ...$args ); - } - - private static function getFileName( ...$args ) { - return self::newObj()->getFileName( ...$args ); - } - - private static function getMessagesFileName( $code ) { - return self::newObj()->getMessagesFileName( $code ); - } - - private static function getJsonMessagesFileName( $code ) { - return self::newObj()->getJsonMessagesFileName( $code ); - } -} diff --git a/tests/phpunit/unit/includes/language/LanguageNameUtilsTestTrait.php b/tests/phpunit/unit/includes/language/LanguageNameUtilsTestTrait.php deleted file mode 100644 index bd777e9a58..0000000000 --- a/tests/phpunit/unit/includes/language/LanguageNameUtilsTestTrait.php +++ /dev/null @@ -1,555 +0,0 @@ -assertSame( $expected, $this->isSupportedLanguage( $code ) ); - } - - public static function provideIsSupportedLanguage() { - return [ - 'en' => [ 'en', true ], - 'fi' => [ 'fi', true ], - 'bunny' => [ 'bunny', false ], - 'qqq' => [ 'qqq', false ], - 'uppercase is not considered supported' => [ 'FI', false ], - ]; - } - - abstract protected function isValidCode( $code ); - - /** - * We don't test that the result is cached, because that should only be noticeable if the - * configuration changes in between calls, and 1) that should never happen in normal operation, - * 2) if you do it you deserve whatever you get, and 3) once the static Language method is - * dropped and the invalid title regex is moved to something injected instead of a static call, - * the cache will be undetectable. - * - * @todo Should we test changes to $wgLegalTitleChars here? Does anybody actually change that? - * Is it possible to change it usefully without breaking everything? - * - * @dataProvider provideIsValidCode - * @covers MediaWiki\Languages\LanguageNameUtils::isValidCode - * @covers Language::isValidCode - * - * @param string $code - * @param bool $expected - */ - public function testIsValidCode( $code, $expected ) { - $this->assertSame( $expected, $this->isValidCode( $code ) ); - } - - public static function provideIsValidCode() { - $ret = [ - 'en' => [ 'en', true ], - 'en-GB' => [ 'en-GB', true ], - 'Funny chars' => [ "%!$()*,-.;=?@^_`~\x80\xA2\xFF+", true ], - 'Percent escape not allowed' => [ 'a%aF', false ], - 'Percent with only one following char is okay' => [ '%a', true ], - 'Percent with non-hex following chars is okay' => [ '%AG', true ], - 'Named char reference "a"' => [ 'a&a', false ], - 'Named char reference "A"' => [ 'a&A', false ], - 'Named char reference "0"' => [ 'a&0', false ], - 'Named char reference non-ASCII' => [ "a&\x92", false ], - 'Numeric char reference' => [ "a�", false ], - 'Hex char reference 0' => [ "a�", false ], - 'Hex char reference A' => [ "a ", false ], - 'Lone ampersand is valid for title but not lang code' => [ '&', false ], - 'Ampersand followed by just # is valid for title but not lang code' => [ '&#', false ], - 'Ampersand followed by # and non-x/digit is valid for title but not lang code' => - [ '&#a', false ], - ]; - $disallowedChars = ":/\\\000&<>'\""; - foreach ( str_split( $disallowedChars ) as $char ) { - $ret["Disallowed character $char"] = [ "a{$char}a", false ]; - } - return $ret; - } - - abstract protected function isValidBuiltInCode( $code ); - - /** - * @dataProvider provideIsValidBuiltInCode - * @covers MediaWiki\Languages\LanguageNameUtils::isValidBuiltInCode - * @covers Language::isValidBuiltInCode - * - * @param string $code - * @param bool $expected - */ - public function testIsValidBuiltInCode( $code, $expected ) { - $this->assertSame( $expected, $this->isValidBuiltInCode( $code ) ); - } - - public static function provideIsValidBuiltInCode() { - return [ - 'Two letters, lowercase' => [ 'fr', true ], - 'Two letters, uppercase' => [ 'EN', false ], - 'Three letters' => [ 'tyv', true ], - 'With dash' => [ 'be-tarask', true ], - 'With extension (two dashes)' => [ 'be-x-old', true ], - 'Reject underscores' => [ 'be_tarask', false ], - 'One letter' => [ 'a', false ], - 'Only digits' => [ '00', true ], - 'Only dashes' => [ '--', true ], - 'Unreasonably long' => [ str_repeat( 'x', 100 ), true ], - 'qqq' => [ 'qqq', true ], - ]; - } - - abstract protected function isKnownLanguageTag( $code ); - - /** - * @dataProvider provideIsKnownLanguageTag - * @covers MediaWiki\Languages\LanguageNameUtils::isKnownLanguageTag - * @covers Language::isKnownLanguageTag - * - * @param string $code - * @param bool $expected - */ - public function testIsKnownLanguageTag( $code, $expected ) { - $this->assertSame( $expected, $this->isKnownLanguageTag( $code ) ); - } - - public static function provideIsKnownLanguageTag() { - $invalidBuiltInCodes = array_filter( static::provideIsValidBuiltInCode(), - function ( $arr ) { - // If isValidBuiltInCode() returns false, we want to also, but if it returns true, - // we could still return false from isKnownLanguageTag(), so skip those. - return !$arr[1]; - } - ); - return array_merge( $invalidBuiltInCodes, [ - 'Simple code' => [ 'fr', true ], - 'An MW legacy tag' => [ 'bat-smg', true ], - 'An internal standard MW name, for which a legacy tag is used externally' => - [ 'sgs', true ], - 'Non-existent two-letter code' => [ 'mw', false ], - 'Very invalid language code' => [ 'foo"assertGetLanguageNames( [], $expected, $code, ...$otherArgs ); - } - - public static function provideGetLanguageNames() { - // @todo There are probably lots of interesting tests to add here. - return [ - 'Simple code' => [ 'Deutsch', 'de' ], - 'Simple code in a different language (doesn\'t work without hook)' => - [ 'Deutsch', 'de', 'fr' ], - 'Invalid code' => [ '', '&' ], - 'Pig Latin not enabled' => [ '', 'en-x-piglatin', AUTONYMS, ALL ], - 'qqq doesn\'t have a name' => [ '', 'qqq', AUTONYMS, ALL ], - 'An MW legacy tag is recognized' => [ 'žemaitėška', 'bat-smg' ], - // @todo Is the next test's result desired? - 'An MW legacy tag is not supported' => [ '', 'bat-smg', AUTONYMS, SUPPORTED ], - 'An internal standard name, for which a legacy tag is used externally, is supported' => - [ 'žemaitėška', 'sgs', AUTONYMS, SUPPORTED ], - ]; - } - - /** - * @dataProvider provideGetLanguageNames_withHook - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageName - * @covers Language::fetchLanguageNames - * @covers Language::fetchLanguageName - * - * @param string $expected Expected return value of getLanguageName() - * @param string $code - * @param mixed ...$otherArgs Optionally, pass $inLanguage and/or $include. - */ - public function testGetLanguageNames_withHook( $expected, $code, ...$otherArgs ) { - $this->setTemporaryHook( 'LanguageGetTranslatedLanguageNames', - function ( &$names, $inLanguage ) { - switch ( $inLanguage ) { - case 'de': - $names = [ - 'de' => 'Deutsch', - 'en' => 'Englisch', - 'fr' => 'Französisch', - ]; - break; - - case 'en': - $names = [ - 'de' => 'German', - 'en' => 'English', - 'fr' => 'French', - 'sqsqsqsq' => '!!?!', - 'bat-smg' => 'Samogitian', - ]; - break; - - case 'fr': - $names = [ - 'de' => 'allemand', - 'en' => 'anglais', - // Deliberate mistake (no cedilla) - 'fr' => 'francais', - ]; - break; - } - } - ); - - // Really we could dispense with assertGetLanguageNames() and just call - // testGetLanguageNames() here, but it looks weird to call a test method from another test - // method. - $this->assertGetLanguageNames( [], $expected, $code, ...$otherArgs ); - } - - public static function provideGetLanguageNames_withHook() { - return [ - 'Simple code in a different language' => [ 'allemand', 'de', 'fr' ], - 'Invalid inLanguage defaults to English' => [ 'German', 'de', '&' ], - 'If inLanguage not provided, default to autonym' => [ 'Deutsch', 'de' ], - 'Hooks ignored for explicitly-requested autonym' => [ 'français', 'fr', 'fr' ], - 'Hooks don\'t make a language supported' => [ '', 'bat-smg', 'en', SUPPORTED ], - 'Hooks don\'t make a language defined' => [ '', 'sqsqsqsq', 'en', DEFINED ], - 'Hooks do make a language name returned with ALL' => [ '!!?!', 'sqsqsqsq', 'en', ALL ], - ]; - } - - /** - * @dataProvider provideGetLanguageNames_ExtraLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageName - * @covers Language::fetchLanguageNames - * @covers Language::fetchLanguageName - * - * @param string $expected Expected return value of getLanguageName() - * @param string $code - * @param mixed ...$otherArgs Optionally, pass $inLanguage and/or $include. - */ - public function testGetLanguageNames_ExtraLanguageNames( $expected, $code, ...$otherArgs ) { - $this->setTemporaryHook( 'LanguageGetTranslatedLanguageNames', - function ( &$names ) { - $names['de'] = 'die deutsche Sprache'; - } - ); - $this->assertGetLanguageNames( - [ 'ExtraLanguageNames' => [ 'de' => 'deutsche Sprache', 'sqsqsqsq' => '!!?!' ] ], - $expected, $code, ...$otherArgs - ); - } - - public static function provideGetLanguageNames_ExtraLanguageNames() { - return [ - 'Simple extra language name' => [ '!!?!', 'sqsqsqsq' ], - 'Extra language is defined' => [ '!!?!', 'sqsqsqsq', AUTONYMS, DEFINED ], - 'Extra language is not supported' => [ '', 'sqsqsqsq', AUTONYMS, SUPPORTED ], - 'Extra language overrides default' => [ 'deutsche Sprache', 'de' ], - 'Extra language overrides hook for explicitly requested autonym' => - [ 'deutsche Sprache', 'de', 'de' ], - 'Hook overrides extra language for non-autonym' => - [ 'die deutsche Sprache', 'de', 'fr' ], - ]; - } - - /** - * Test that getLanguageNames() defaults to DEFINED, and getLanguageName() defaults to ALL. - * - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageName - * @covers Language::fetchLanguageNames - * @covers Language::fetchLanguageName - */ - public function testGetLanguageNames_parameterDefault() { - $this->setTemporaryHook( 'LanguageGetTranslatedLanguageNames', - function ( &$names ) { - $names = [ 'sqsqsqsq' => '!!?!' ]; - } - ); - - // We use 'en' here because the hook is not run if we're requesting autonyms, although in - // this case (language that isn't defined by MediaWiki itself) that behavior seems wrong. - $this->assertArrayNotHasKey( 'sqsqsqsq', $this->getLanguageNames(), 'en' ); - - $this->assertSame( '!!?!', $this->getLanguageName( 'sqsqsqsq', 'en' ) ); - } - - /** - * @dataProvider provideGetLanguageNames_sorted - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers Language::fetchLanguageNames - * - * @param mixed ...$args To pass to method - */ - public function testGetLanguageNames_sorted( ...$args ) { - $names = $this->getLanguageNames( ...$args ); - $sortedNames = $names; - ksort( $sortedNames ); - $this->assertSame( $sortedNames, $names ); - } - - public static function provideGetLanguageNames_sorted() { - return [ - [], - [ AUTONYMS ], - [ AUTONYMS, 'mw' ], - [ AUTONYMS, ALL ], - [ AUTONYMS, SUPPORTED ], - [ 'he', 'mw' ], - [ 'he', ALL ], - [ 'he', SUPPORTED ], - ]; - } - - /** - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers Language::fetchLanguageNames - */ - public function testGetLanguageNames_hookNotCalledForAutonyms() { - $count = 0; - $this->setTemporaryHook( 'LanguageGetTranslatedLanguageNames', - function () use ( &$count ) { - $count++; - } - ); - - $this->getLanguageNames(); - $this->assertSame( 0, $count, 'Hook must not be called for autonyms' ); - - // We test elsewhere that the hook works, but the following verifies that our test is - // working and $count isn't being incremented above only because we're checking autonyms. - $this->getLanguageNames( 'fr' ); - $this->assertSame( 1, $count, 'Hook must be called for non-autonyms' ); - } - - /** - * @dataProvider provideGetLanguageNames_pigLatin - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageName - * @covers Language::fetchLanguageNames - * @covers Language::fetchLanguageName - * - * @param string $expected - * @param mixed ...$otherArgs Optionally, pass $inLanguage and/or $include. - */ - public function testGetLanguageNames_pigLatin( $expected, ...$otherArgs ) { - $this->setTemporaryHook( 'LanguageGetTranslatedLanguageNames', - function ( &$names, $inLanguage ) { - switch ( $inLanguage ) { - case 'fr': - $names = [ 'en-x-piglatin' => 'latin de cochons' ]; - break; - - case 'en-x-piglatin': - // Deliberately lowercase - $names = [ 'en-x-piglatin' => 'igpay atinlay' ]; - break; - } - } - ); - - $this->assertGetLanguageNames( - [ 'UsePigLatinVariant' => true ], $expected, 'en-x-piglatin', ...$otherArgs ); - } - - public static function provideGetLanguageNames_pigLatin() { - return [ - 'Simple test' => [ 'Igpay Atinlay' ], - 'Not supported' => [ '', AUTONYMS, SUPPORTED ], - 'Foreign language' => [ 'latin de cochons', 'fr' ], - 'Hook doesn\'t override explicit autonym' => - [ 'Igpay Atinlay', 'en-x-piglatin', 'en-x-piglatin' ], - ]; - } - - /** - * Just for the sake of completeness, test that ExtraLanguageNames will not override the name - * for pig Latin. Nobody actually cares about this and if anything current behavior is probably - * wrong, but once we're testing the whole file we may as well be comprehensive. - * - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNames - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageNamesUncached - * @covers MediaWiki\Languages\LanguageNameUtils::getLanguageName - * @covers Language::fetchLanguageNames - * @covers Language::fetchLanguageName - */ - public function testGetLanguageNames_pigLatinAndExtraLanguageNames() { - $this->assertGetLanguageNames( - [ - 'UsePigLatinVariant' => true, - 'ExtraLanguageNames' => [ 'en-x-piglatin' => 'igpay atinlay' ] - ], - 'Igpay Atinlay', - 'en-x-piglatin' - ); - } - - abstract protected static function getFileName( ...$args ); - - /** - * @dataProvider provideGetFileName - * @covers MediaWiki\Languages\LanguageNameUtils::getFileName - * @covers Language::getFileName - * - * @param string $expected - * @param mixed ...$args To pass to method - */ - public function testGetFileName( $expected, ...$args ) { - $this->assertSame( $expected, $this->getFileName( ...$args ) ); - } - - public static function provideGetFileName() { - return [ - 'Simple case' => [ 'MessagesXx.php', 'Messages', 'xx' ], - 'With extension' => [ 'MessagesXx.ext', 'Messages', 'xx', '.ext' ], - 'Replacing dashes' => [ '!__?', '!', '--', '?' ], - 'Empty prefix and extension' => [ 'Xx', '', 'xx', '' ], - 'Uppercase only first letter' => [ 'Messages_a.php', 'Messages', '-a' ], - ]; - } - - abstract protected function getMessagesFileName( $code ); - - /** - * @dataProvider provideGetMessagesFileName - * @covers MediaWiki\Languages\LanguageNameUtils::getMessagesFileName - * @covers Language::getMessagesFileName - * - * @param string $code - * @param string $expected - */ - public function testGetMessagesFileName( $code, $expected ) { - $this->assertSame( $expected, $this->getMessagesFileName( $code ) ); - } - - public static function provideGetMessagesFileName() { - global $IP; - return [ - 'Simple case' => [ 'en', "$IP/languages/messages/MessagesEn.php" ], - 'Replacing dashes' => [ '--', "$IP/languages/messages/Messages__.php" ], - 'Uppercase only first letter' => [ '-a', "$IP/languages/messages/Messages_a.php" ], - ]; - } - - /** - * @covers MediaWiki\Languages\LanguageNameUtils::getMessagesFileName - * @covers Language::getMessagesFileName - */ - public function testGetMessagesFileName_withHook() { - $called = 0; - - $this->setTemporaryHook( 'Language::getMessagesFileName', - function ( $code, &$file ) use ( &$called ) { - global $IP; - - $called++; - - $this->assertSame( 'ab-cd', $code ); - $this->assertSame( "$IP/languages/messages/MessagesAb_cd.php", $file ); - $file = 'bye-bye'; - } - ); - - $this->assertSame( 'bye-bye', $this->getMessagesFileName( 'ab-cd' ) ); - $this->assertSame( 1, $called ); - } - - abstract protected function getJsonMessagesFileName( $code ); - - /** - * @covers MediaWiki\Languages\LanguageNameUtils::getJsonMessagesFileName - * @covers Language::getJsonMessagesFileName - */ - public function testGetJsonMessagesFileName() { - global $IP; - - // Not so much to test here, one test seems to be enough - $expected = "$IP/languages/i18n/en--123.json"; - $this->assertSame( $expected, $this->getJsonMessagesFileName( 'en--123' ) ); - } - - /** - * getFileName, getMessagesFileName, and getJsonMessagesFileName all throw if they get an - * invalid code. To save boilerplate, test them all in one method. - * - * @dataProvider provideExceptionFromInvalidCode - * @covers MediaWiki\Languages\LanguageNameUtils::getFileName - * @covers MediaWiki\Languages\LanguageNameUtils::getMessagesFileName - * @covers MediaWiki\Languages\LanguageNameUtils::getJsonMessagesFileName - * @covers Language::getFileName - * @covers Language::getMessagesFileName - * @covers Language::getJsonMessagesFileName - * - * @param callable $callback Will throw when passed $code - * @param string $code - */ - public function testExceptionFromInvalidCode( $callback, $code ) { - $this->setExpectedException( MWException::class, "Invalid language code \"$code\"" ); - - $callback( $code ); - } - - public static function provideExceptionFromInvalidCode() { - $ret = []; - foreach ( static::provideIsValidBuiltInCode() as $desc => list( $code, $valid ) ) { - if ( $valid ) { - // Won't get an exception from this one - continue; - } - - // For getFileName, we define an anonymous function because of the extra first param - $ret["getFileName: $desc"] = [ - function ( $code ) { - return static::getFileName( 'Messages', $code ); - }, - $code - ]; - - $ret["getMessagesFileName: $desc"] = - [ [ static::class, 'getMessagesFileName' ], $code ]; - - $ret["getJsonMessagesFileName: $desc"] = - [ [ static::class, 'getJsonMessagesFileName' ], $code ]; - } - return $ret; - } -} -- 2.20.1