Add populateInterwiki maintenance script

This has been in Wikibase since 2012, but is not at all
specific to Wikibase, and doesn't even require Wikibase
to be enabled.

Having this in core will make it available to more people
and without the hassle of having to clone Wikibase to
use the script.

Bug: T114577
Change-Id: Ib521a65e616bdc4b81206a084289cb4750f0d1f5

diff --git a/autoload.php b/autoload.php
index 0283d3c..0d127b7 100644 (file)
--- a/autoload.php
+++ b/autoload.php
@@ -1083,6 +1083,7 @@ $wgAutoloadLocalClasses = [
        'PopulateContentModel' => __DIR__ . '/maintenance/populateContentModel.php',
        'PopulateFilearchiveSha1' => __DIR__ . '/maintenance/populateFilearchiveSha1.php',
        'PopulateImageSha1' => __DIR__ . '/maintenance/populateImageSha1.php',
+       'PopulateInterwiki' => __DIR__ . '/maintenance/populateInterwiki.php',
        'PopulateLogSearch' => __DIR__ . '/maintenance/populateLogSearch.php',
        'PopulateLogUsertext' => __DIR__ . '/maintenance/populateLogUsertext.php',
        'PopulateParentId' => __DIR__ . '/maintenance/populateParentId.php',

diff --git a/maintenance/populateInterwiki.php b/maintenance/populateInterwiki.php
new file mode 100644 (file)
index 0000000..5d32b99
--- /dev/null
+++ b/maintenance/populateInterwiki.php
@@ -0,0 +1,171 @@
+<?php
+
+/**
+ * Maintenance script that populates the interwiki table with list of sites from
+ * a source wiki, such as English Wikipedia. (the default source)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ * @author Katie Filbert < aude.wiki@gmail.com >
+ */
+
+require_once __DIR__ . '/Maintenance.php';
+
+class PopulateInterwiki extends Maintenance {
+
+       /**
+        * @var string
+        */
+       private $source;
+
+       /**
+        * @var BagOStuff
+        */
+       private $cache;
+
+       public function __construct() {
+               parent::__construct();
+
+               $this->addDescription( <<<TEXT
+This script will populate the interwiki table, pulling in interwiki links that are used on Wikipedia
+or another MediaWiki wiki.
+
+When the script has finished, it will make a note of this in the database, and will not run again
+without the --force option.
+
+--source parameter is the url for the source wiki api, such as "https://en.wikipedia.org/w/api.php"
+(the default) from which the script fetches the interwiki data and uses here to populate
+the interwiki database table.
+TEXT
+               );
+
+               $this->addOption( 'source', 'Source wiki for interwiki table, such as '
+                       . 'https://en.wikipedia.org/w/api.php (the default)', false, true );
+               $this->addOption( 'force', 'Run regardless of whether the database says it has '
+                       . 'been run already.' );
+       }
+
+       public function execute() {
+               $force = $this->getOption( 'force', false );
+               $this->source = $this->getOption( 'source', 'https://en.wikipedia.org/w/api.php' );
+
+               $this->cache = wfGetMainCache();
+
+               $data = $this->fetchLinks();
+
+               if ( $data === false ) {
+                       $this->error( "Error during fetching data." );
+               } else {
+                       $this->doPopulate( $data, $force );
+               }
+       }
+
+       /**
+        * @return array[]|bool The 'interwikimap' sub-array or false on failure.
+        */
+       protected function fetchLinks() {
+               $url = wfArrayToCgi( [
+                       'action' => 'query',
+                       'meta' => 'siteinfo',
+                       'siprop' => 'interwikimap',
+                       'sifilteriw' => 'local',
+                       'format' => 'json'
+               ] );
+
+               if ( !empty( $this->source ) ) {
+                       $url = rtrim( $this->source, '?' ) . '?' . $url;
+               }
+
+               $json = Http::get( $url );
+               $data = json_decode( $json, true );
+
+               if ( is_array( $data ) ) {
+                       return $data['query']['interwikimap'];
+               } else {
+                       return false;
+               }
+       }
+
+       /**
+        * @param array[] $data
+        * @param bool $force
+        *
+        * @return bool
+        */
+       protected function doPopulate( array $data, $force ) {
+               $dbw = wfGetDB( DB_MASTER );
+
+               if ( !$force ) {
+                       $row = $dbw->selectRow(
+                               'updatelog',
+                               '1',
+                               [ 'ul_key' => 'populate interwiki' ],
+                               __METHOD__
+                       );
+
+                       if ( $row ) {
+                               $this->output( "Interwiki table already populated.  Use php " .
+                                       "maintenance/populateInterwiki.php\n--force from the command line " .
+                                       "to override.\n" );
+                               return true;
+                       }
+               }
+
+               foreach ( $data as $d ) {
+                       $prefix = $d['prefix'];
+
+                       $row = $dbw->selectRow(
+                               'interwiki',
+                               '1',
+                               [ 'iw_prefix' => $prefix ],
+                               __METHOD__
+                       );
+
+                       if ( ! $row ) {
+                               $dbw->insert(
+                                       'interwiki',
+                                       [
+                                               'iw_prefix' => $prefix,
+                                               'iw_url' => $d['url'],
+                                               'iw_local' => 1
+                                       ],
+                                       __METHOD__,
+                                       'IGNORE'
+                               );
+                       }
+
+                       $this->clearCacheEntry( $prefix );
+               }
+
+               $this->output( "Interwiki links are populated.\n" );
+
+               return true;
+       }
+
+       /**
+        * @param string $prefix
+        */
+       private function clearCacheEntry( $prefix ) {
+               $key = wfMemcKey( 'interwiki', $prefix );
+               $this->cache->delete( $key );
+       }
+
+}
+
+$maintClass = PopulateInterwiki::class;
+require_once RUN_MAINTENANCE_IF_MAIN;