Add action=query&meta=languageinfo API module
authorLucas Werkmeister <lucas.werkmeister@wikimedia.de>
Thu, 16 May 2019 09:42:05 +0000 (11:42 +0200)
committerLucas Werkmeister <lucas.werkmeister@wikimedia.de>
Mon, 3 Jun 2019 10:46:03 +0000 (12:46 +0200)
This API module can be used to get information about all the languages
supported by this MediaWiki installation. Since parts of this
information, such as the fallback chain, are expensive to retrieve if
the localization cache is not populated, we apply continuation if the
request is taking too long (suggested by Anomie in T217239#4994301); we
don’t expect this to happen in Wikimedia production, though.

Bug: T74153
Bug: T220415
Change-Id: Ic66991cd85ed4439a47bfb1412dbe24c23bd9819

autoload.php
includes/api/ApiQuery.php
includes/api/ApiQueryLanguageinfo.php [new file with mode: 0644]
includes/api/i18n/en.json
includes/api/i18n/qqq.json
tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php [new file with mode: 0644]

index 275d20e..0d317f8 100644 (file)
@@ -113,6 +113,7 @@ $wgAutoloadLocalClasses = [
        'ApiQueryInfo' => __DIR__ . '/includes/api/ApiQueryInfo.php',
        'ApiQueryLangBacklinks' => __DIR__ . '/includes/api/ApiQueryLangBacklinks.php',
        'ApiQueryLangLinks' => __DIR__ . '/includes/api/ApiQueryLangLinks.php',
+       'ApiQueryLanguageinfo' => __DIR__ . '/includes/api/ApiQueryLanguageinfo.php',
        'ApiQueryLinks' => __DIR__ . '/includes/api/ApiQueryLinks.php',
        'ApiQueryLogEvents' => __DIR__ . '/includes/api/ApiQueryLogEvents.php',
        'ApiQueryMyStashedFiles' => __DIR__ . '/includes/api/ApiQueryMyStashedFiles.php',
index ae6b1a1..0c89c1e 100644 (file)
@@ -115,6 +115,7 @@ class ApiQuery extends ApiBase {
                'userinfo' => ApiQueryUserInfo::class,
                'filerepoinfo' => ApiQueryFileRepoInfo::class,
                'tokens' => ApiQueryTokens::class,
+               'languageinfo' => ApiQueryLanguageinfo::class,
        ];
 
        /**
diff --git a/includes/api/ApiQueryLanguageinfo.php b/includes/api/ApiQueryLanguageinfo.php
new file mode 100644 (file)
index 0000000..72b59b0
--- /dev/null
@@ -0,0 +1,245 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+/**
+ * API module to enumerate language information.
+ *
+ * @ingroup API
+ */
+class ApiQueryLanguageinfo extends ApiQueryBase {
+
+       /**
+        * The maximum time for {@link execute()};
+        * if execution takes longer than this, apply continuation.
+        *
+        * If the localization cache is used, this time is not expected to ever be
+        * exceeded; on the other hand, if it is not used, a typical request will
+        * not yield more than a handful of languages before the time is exceeded
+        * and continuation is applied, if one of the expensive props is requested.
+        *
+        * @var float
+        */
+       const MAX_EXECUTE_SECONDS = 2.0;
+
+       /** @var callable|null */
+       private $microtimeFunction;
+
+       /**
+        * @param ApiQuery $queryModule
+        * @param string $moduleName
+        * @param callable|null $microtimeFunction Function to use instead of microtime(), for testing.
+        * Should accept no arguments and return float seconds. (null means real microtime().)
+        */
+       public function __construct(
+               ApiQuery $queryModule,
+               $moduleName,
+               $microtimeFunction = null
+       ) {
+               parent::__construct( $queryModule, $moduleName, 'li' );
+               $this->microtimeFunction = $microtimeFunction;
+       }
+
+       /** @return float */
+       private function microtime() {
+               if ( $this->microtimeFunction ) {
+                       return ( $this->microtimeFunction )();
+               } else {
+                       return microtime( true );
+               }
+       }
+
+       public function execute() {
+               $endTime = $this->microtime() + self::MAX_EXECUTE_SECONDS;
+
+               $props = array_flip( $this->getParameter( 'prop' ) );
+               $includeCode = isset( $props['code'] );
+               $includeBcp47 = isset( $props['bcp47'] );
+               $includeDir = isset( $props['dir'] );
+               $includeAutonym = isset( $props['autonym'] );
+               $includeName = isset( $props['name'] );
+               $includeFallbacks = isset( $props['fallbacks'] );
+               $includeVariants = isset( $props['variants'] );
+
+               $targetLanguageCode = $this->getLanguage()->getCode();
+               $include = 'all';
+
+               $availableLanguageCodes = array_keys( Language::fetchLanguageNames(
+                       // MediaWiki and extensions may return different sets of language codes
+                       // when asked for language names in different languages;
+                       // asking for English language names is most likely to give us the full set,
+                       // even though we may not need those at all
+                       'en',
+                       $include
+               ) );
+               $selectedLanguageCodes = $this->getParameter( 'code' );
+               if ( $selectedLanguageCodes === [ '*' ] ) {
+                       $languageCodes = $availableLanguageCodes;
+               } else {
+                       $languageCodes = array_values( array_intersect(
+                               $availableLanguageCodes,
+                               $selectedLanguageCodes
+                       ) );
+                       $unrecognizedCodes = array_values( array_diff(
+                               $selectedLanguageCodes,
+                               $availableLanguageCodes
+                       ) );
+                       if ( $unrecognizedCodes !== [] ) {
+                               $this->addWarning( [
+                                       'apiwarn-unrecognizedvalues',
+                                       $this->encodeParamName( 'code' ),
+                                       Message::listParam( $unrecognizedCodes, 'comma' ),
+                                       count( $unrecognizedCodes ),
+                               ] );
+                       }
+               }
+               // order of $languageCodes is guaranteed by Language::fetchLanguageNames()
+               // and preserved by array_values() + array_intersect()
+
+               $continue = $this->getParameter( 'continue' );
+               if ( $continue === null ) {
+                       $continue = reset( $languageCodes );
+               }
+
+               $result = $this->getResult();
+               $rootPath = [
+                       $this->getQuery()->getModuleName(),
+                       $this->getModuleName(),
+               ];
+               $result->addArrayType( $rootPath, 'assoc' );
+
+               foreach ( $languageCodes as $languageCode ) {
+                       if ( $languageCode < $continue ) {
+                               continue;
+                       }
+
+                       $now = $this->microtime();
+                       if ( $now >= $endTime ) {
+                               $this->setContinueEnumParameter( 'continue', $languageCode );
+                               break;
+                       }
+
+                       $info = [];
+                       ApiResult::setArrayType( $info, 'assoc' );
+
+                       if ( $includeCode ) {
+                               $info['code'] = $languageCode;
+                       }
+
+                       if ( $includeBcp47 ) {
+                               $bcp47 = LanguageCode::bcp47( $languageCode );
+                               $info['bcp47'] = $bcp47;
+                       }
+
+                       if ( $includeDir ) {
+                               $dir = Language::factory( $languageCode )->getDir();
+                               $info['dir'] = $dir;
+                       }
+
+                       if ( $includeAutonym ) {
+                               $autonym = Language::fetchLanguageName(
+                                       $languageCode,
+                                       Language::AS_AUTONYMS,
+                                       $include
+                               );
+                               $info['autonym'] = $autonym;
+                       }
+
+                       if ( $includeName ) {
+                               $name = Language::fetchLanguageName(
+                                       $languageCode,
+                                       $targetLanguageCode,
+                                       $include
+                               );
+                               $info['name'] = $name;
+                       }
+
+                       if ( $includeFallbacks ) {
+                               $fallbacks = Language::getFallbacksFor(
+                                       $languageCode,
+                                       // allow users to distinguish between implicit and explicit 'en' fallbacks
+                                       Language::STRICT_FALLBACKS
+                               );
+                               ApiResult::setIndexedTagName( $fallbacks, 'fb' );
+                               $info['fallbacks'] = $fallbacks;
+                       }
+
+                       if ( $includeVariants ) {
+                               $variants = Language::factory( $languageCode )->getVariants();
+                               ApiResult::setIndexedTagName( $variants, 'var' );
+                               $info['variants'] = $variants;
+                       }
+
+                       $fit = $result->addValue( $rootPath, $languageCode, $info );
+                       if ( !$fit ) {
+                               $this->setContinueEnumParameter( 'continue', $languageCode );
+                               break;
+                       }
+               }
+       }
+
+       public function getCacheMode( $params ) {
+               return 'public';
+       }
+
+       public function getAllowedParams() {
+               return [
+                       'prop' => [
+                               self::PARAM_DFLT => 'code',
+                               self::PARAM_ISMULTI => true,
+                               self::PARAM_TYPE => [
+                                       'code',
+                                       'bcp47',
+                                       'dir',
+                                       'autonym',
+                                       'name',
+                                       'fallbacks',
+                                       'variants',
+                               ],
+                               self::PARAM_HELP_MSG_PER_VALUE => [],
+                       ],
+                       'code' => [
+                               self::PARAM_DFLT => '*',
+                               self::PARAM_ISMULTI => true,
+                       ],
+                       'continue' => [
+                               self::PARAM_HELP_MSG => 'api-help-param-continue',
+                       ],
+               ];
+       }
+
+       protected function getExamplesMessages() {
+               $pathUrl = 'action=' . $this->getQuery()->getModuleName() .
+                       '&meta=' . $this->getModuleName();
+               $pathMsg = $this->getModulePath();
+               $prefix = $this->getModulePrefix();
+
+               return [
+                       "$pathUrl"
+                               => "apihelp-$pathMsg-example-simple",
+                       "$pathUrl&{$prefix}prop=autonym|name&lang=de"
+                               => "apihelp-$pathMsg-example-autonym-name-de",
+                       "$pathUrl&{$prefix}prop=fallbacks|variants&{$prefix}code=oc"
+                               => "apihelp-$pathMsg-example-fallbacks-variants-oc",
+                       "$pathUrl&{$prefix}prop=bcp47|dir"
+                               => "apihelp-$pathMsg-example-bcp47-dir",
+               ];
+       }
+
+}
index aded1f9..1280889 100644 (file)
        "apihelp-query+langlinks-param-inlanguagecode": "Language code for localised language names.",
        "apihelp-query+langlinks-example-simple": "Get interlanguage links from the page <kbd>Main Page</kbd>.",
 
+       "apihelp-query+languageinfo-summary": "Return information about available languages.",
+       "apihelp-query+languageinfo-extended-description": "[[mw:API:Query#Continuing queries|Continuation]] may be applied if retrieving the information takes too long for one request.",
+       "apihelp-query+languageinfo-param-prop": "Which information to get for each language.",
+       "apihelp-query+languageinfo-paramvalue-prop-code": "The language code. (This code is MediaWiki-specific, though there are overlaps with other standards.)",
+       "apihelp-query+languageinfo-paramvalue-prop-bcp47": "The BCP-47 language code.",
+       "apihelp-query+languageinfo-paramvalue-prop-dir": "The writing direction of the language (either <code>ltr</code> or <code>rtl</code>).",
+       "apihelp-query+languageinfo-paramvalue-prop-autonym": "The autonym of the language, that is, the name in that language.",
+       "apihelp-query+languageinfo-paramvalue-prop-name": "The name of the language in the language specified by the <var>lilang</var> parameter, with language fallbacks applied if necessary.",
+       "apihelp-query+languageinfo-paramvalue-prop-fallbacks": "The language codes of the fallback languages configured for this language. The implicit final fallback to 'en' is not included (but some languages may fall back to 'en' explicitly).",
+       "apihelp-query+languageinfo-paramvalue-prop-variants": "The language codes of the variants supported by this language.",
+       "apihelp-query+languageinfo-param-code": "Language codes of the languages that should be returned, or <code>*</code> for all languages.",
+       "apihelp-query+languageinfo-example-simple": "Get the language codes of all supported languages.",
+       "apihelp-query+languageinfo-example-autonym-name-de": "Get the autonyms and German names of all supported languages.",
+       "apihelp-query+languageinfo-example-fallbacks-variants-oc": "Get the fallback languages and variants of Occitan.",
+       "apihelp-query+languageinfo-example-bcp47-dir": "Get the BCP-47 language code and direction of all supported languages.",
+
        "apihelp-query+links-summary": "Returns all links from the given pages.",
        "apihelp-query+links-param-namespace": "Show links in these namespaces only.",
        "apihelp-query+links-param-limit": "How many links to return.",
index 06ac6a7..36c4953 100644 (file)
        "apihelp-query+langlinks-param-dir": "{{doc-apihelp-param|query+langlinks|dir}}",
        "apihelp-query+langlinks-param-inlanguagecode": "{{doc-apihelp-param|query+langlinks|inlanguagecode}}",
        "apihelp-query+langlinks-example-simple": "{{doc-apihelp-example|query+langlinks}}",
+       "apihelp-query+languageinfo-summary": "{{doc-apihelp-summary|query+languageinfo}}",
+       "apihelp-query+languageinfo-extended-description": "{{doc-apihelp-extended-description|query+languageinfo}}",
+       "apihelp-query+languageinfo-param-prop": "{{doc-apihelp-param|query+languageinfo|prop|paramvalues=1}}",
+       "apihelp-query+languageinfo-paramvalue-prop-code": "{{doc-apihelp-paramvalue|query+languageinfo|prop|code}}",
+       "apihelp-query+languageinfo-paramvalue-prop-bcp47": "{{doc-apihelp-paramvalue|query+languageinfo|prop|bcp47}}",
+       "apihelp-query+languageinfo-paramvalue-prop-dir": "{{doc-apihelp-paramvalue|query+languageinfo|prop|dir}}",
+       "apihelp-query+languageinfo-paramvalue-prop-autonym": "{{doc-apihelp-paramvalue|query+languageinfo|prop|autonym}}",
+       "apihelp-query+languageinfo-paramvalue-prop-name": "{{doc-apihelp-paramvalue|query+languageinfo|prop|name}}",
+       "apihelp-query+languageinfo-paramvalue-prop-fallbacks": "{{doc-apihelp-paramvalue|query+languageinfo|prop|fallbacks}}",
+       "apihelp-query+languageinfo-paramvalue-prop-variants": "{{doc-apihelp-paramvalue|query+languageinfo|prop|variants}}",
+       "apihelp-query+languageinfo-param-code": "{{doc-apihelp-param|query+languageinfo|code}}",
+       "apihelp-query+languageinfo-example-simple": "{{doc-apihelp-example|query+languageinfo}}",
+       "apihelp-query+languageinfo-example-autonym-name-de": "{{doc-apihelp-example|query+languageinfo}}",
+       "apihelp-query+languageinfo-example-fallbacks-variants-oc": "{{doc-apihelp-example|query+languageinfo}}",
+       "apihelp-query+languageinfo-example-bcp47-dir": "{{doc-apihelp-example|query+languageinfo}}",
        "apihelp-query+links-summary": "{{doc-apihelp-summary|query+links}}",
        "apihelp-query+links-param-namespace": "{{doc-apihelp-param|query+links|namespace}}",
        "apihelp-query+links-param-limit": "{{doc-apihelp-param|query+links|limit}}",
diff --git a/tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php b/tests/phpunit/includes/api/ApiQueryLanguageinfoTest.php
new file mode 100644 (file)
index 0000000..f20a061
--- /dev/null
@@ -0,0 +1,175 @@
+<?php
+
+/**
+ * @group API
+ * @group medium
+ *
+ * @covers ApiQueryLanguageinfo
+ */
+class ApiQueryLanguageinfoTest extends ApiTestCase {
+
+       protected function setUp() {
+               parent::setUp();
+               // register custom language names so this test is independent of CLDR
+               $this->setTemporaryHook(
+                       'LanguageGetTranslatedLanguageNames',
+                       function ( array &$names, $code ) {
+                               switch ( $code ) {
+                                       case 'en':
+                                               $names['sh'] = 'Serbo-Croatian';
+                                               $names['qtp'] = 'a custom language code MediaWiki knows nothing about';
+                                               break;
+                                       case 'pt':
+                                               $names['de'] = 'alemão';
+                                               break;
+                               }
+                       }
+               );
+       }
+
+       private function doQuery( array $params, $microtimeFunction = null ): array {
+               $params += [
+                       'action' => 'query',
+                       'meta' => 'languageinfo',
+                       'uselang' => 'en',
+               ];
+
+               if ( $microtimeFunction !== null ) {
+                       // hook into the module manager to override the factory function
+                       // so we can call the constructor with the custom $microtimeFunction
+                       $this->setTemporaryHook(
+                               'ApiQuery::moduleManager',
+                               function ( ApiModuleManager $moduleManager ) use ( $microtimeFunction ) {
+                                       $moduleManager->addModule(
+                                               'languageinfo',
+                                               'meta',
+                                               ApiQueryLanguageinfo::class,
+                                               function ( $parent, $name ) use ( $microtimeFunction ) {
+                                                       return new ApiQueryLanguageinfo(
+                                                               $parent,
+                                                               $name,
+                                                               $microtimeFunction
+                                                       );
+                                               }
+                                       );
+                               }
+                       );
+               }
+
+               $res = $this->doApiRequest( $params );
+
+               $this->assertArrayNotHasKey( 'warnings', $res[0] );
+
+               return [ $res[0]['query']['languageinfo'], $res[0]['continue'] ?? null ];
+       }
+
+       public function testAllPropsForSingleLanguage() {
+               list( $response, $continue ) = $this->doQuery( [
+                       'liprop' => 'code|bcp47|dir|autonym|name|fallbacks|variants',
+                       'licode' => 'sh',
+               ] );
+
+               $this->assertArrayEquals( [
+                       'sh' => [
+                               'code' => 'sh',
+                               'bcp47' => 'sh',
+                               'autonym' => 'srpskohrvatski / српскохрватски',
+                               'name' => 'Serbo-Croatian',
+                               'fallbacks' => [ 'bs', 'sr-el', 'hr' ],
+                               'dir' => 'ltr',
+                               'variants' => [ 'sh' ],
+                       ],
+               ], $response );
+       }
+
+       public function testAllPropsForSingleCustomLanguage() {
+               list( $response, $continue ) = $this->doQuery( [
+                       'liprop' => 'code|bcp47|dir|autonym|name|fallbacks|variants',
+                       'licode' => 'qtp', // reserved for local use by ISO 639; registered in setUp()
+               ] );
+
+               $this->assertArrayEquals( [
+                       'qtp' => [
+                               'code' => 'qtp',
+                               'bcp47' => 'qtp',
+                               'autonym' => '',
+                               'name' => 'a custom language code MediaWiki knows nothing about',
+                               'fallbacks' => [],
+                               'dir' => 'ltr',
+                               'variants' => [ 'qtp' ],
+                       ],
+               ], $response );
+       }
+
+       public function testNameInOtherLanguageForSingleLanguage() {
+               list( $response, $continue ) = $this->doQuery( [
+                       'liprop' => 'name',
+                       'licode' => 'de',
+                       'uselang' => 'pt',
+               ] );
+
+               $this->assertArrayEquals( [ 'de' => [ 'name' => 'alemão' ] ], $response );
+       }
+
+       public function testContinuationNecessary() {
+               $time = 0;
+               $microtimeFunction = function () use ( &$time ) {
+                       return $time += 0.75;
+               };
+
+               list( $response, $continue ) = $this->doQuery( [], $microtimeFunction );
+
+               $this->assertCount( 2, $response );
+               $this->assertArrayHasKey( 'licontinue', $continue );
+       }
+
+       public function testContinuationNotNecessary() {
+               $time = 0;
+               $microtimeFunction = function () use ( &$time ) {
+                       return $time += 1.5;
+               };
+
+               list( $response, $continue ) = $this->doQuery( [
+                       'licode' => 'de',
+               ], $microtimeFunction );
+
+               $this->assertNull( $continue );
+       }
+
+       public function testContinuationInAlphabeticalOrderNotParameterOrder() {
+               $time = 0;
+               $microtimeFunction = function () use ( &$time ) {
+                       return $time += 0.75;
+               };
+               $params = [ 'licode' => 'en|ru|zh|de|yue' ];
+
+               list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction );
+
+               $this->assertCount( 2, $response );
+               $this->assertArrayHasKey( 'licontinue', $continue );
+               $this->assertSame( [ 'de', 'en' ], array_keys( $response ) );
+
+               $time = 0;
+               $params = $continue + $params;
+               list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction );
+
+               $this->assertCount( 2, $response );
+               $this->assertArrayHasKey( 'licontinue', $continue );
+               $this->assertSame( [ 'ru', 'yue' ], array_keys( $response ) );
+
+               $time = 0;
+               $params = $continue + $params;
+               list( $response, $continue ) = $this->doQuery( $params, $microtimeFunction );
+
+               $this->assertCount( 1, $response );
+               $this->assertNull( $continue );
+               $this->assertSame( [ 'zh' ], array_keys( $response ) );
+       }
+
+       public function testResponseHasModulePathEvenIfEmpty() {
+               list( $response, $continue ) = $this->doQuery( [ 'licode' => '' ] );
+               $this->assertEmpty( $response );
+               // the real test is that $res[0]['query']['languageinfo'] in doQuery() didn’t fail
+       }
+
+}