maintenance/populateInterwiki.php

   1 <?php
   2
   3 /**
   4  * Maintenance script that populates the interwiki table with list of sites from
   5  * a source wiki, such as English Wikipedia. (the default source)
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup Maintenance
  24  * @author Katie Filbert < aude.wiki@gmail.com >
  25  */
  26
  27 require_once __DIR__ . '/Maintenance.php';
  28
  29 class PopulateInterwiki extends Maintenance {
  30
  31         /**
  32          * @var string
  33          */
  34         private $source;
  35
  36         /**
  37          * @var BagOStuff
  38          */
  39         private $cache;
  40
  41         public function __construct() {
  42                 parent::__construct();
  43
  44                 $this->addDescription( <<<TEXT
  45 This script will populate the interwiki table, pulling in interwiki links that are used on Wikipedia
  46 or another MediaWiki wiki.
  47
  48 When the script has finished, it will make a note of this in the database, and will not run again
  49 without the --force option.
  50
  51 --source parameter is the url for the source wiki api, such as "https://en.wikipedia.org/w/api.php"
  52 (the default) from which the script fetches the interwiki data and uses here to populate
  53 the interwiki database table.
  54 TEXT
  55                 );
  56
  57                 $this->addOption( 'source', 'Source wiki for interwiki table, such as '
  58                         . 'https://en.wikipedia.org/w/api.php (the default)', false, true );
  59                 $this->addOption( 'force', 'Run regardless of whether the database says it has '
  60                         . 'been run already.' );
  61         }
  62
  63         public function execute() {
  64                 $force = $this->getOption( 'force', false );
  65                 $this->source = $this->getOption( 'source', 'https://en.wikipedia.org/w/api.php' );
  66
  67                 $this->cache = wfGetMainCache();
  68
  69                 $data = $this->fetchLinks();
  70
  71                 if ( $data === false ) {
  72                         $this->error( "Error during fetching data." );
  73                 } else {
  74                         $this->doPopulate( $data, $force );
  75                 }
  76         }
  77
  78         /**
  79          * @return array[]|bool The 'interwikimap' sub-array or false on failure.
  80          */
  81         protected function fetchLinks() {
  82                 $url = wfArrayToCgi( [
  83                         'action' => 'query',
  84                         'meta' => 'siteinfo',
  85                         'siprop' => 'interwikimap',
  86                         'sifilteriw' => 'local',
  87                         'format' => 'json'
  88                 ] );
  89
  90                 if ( !empty( $this->source ) ) {
  91                         $url = rtrim( $this->source, '?' ) . '?' . $url;
  92                 }
  93
  94                 $json = Http::get( $url );
  95                 $data = json_decode( $json, true );
  96
  97                 if ( is_array( $data ) ) {
  98                         return $data['query']['interwikimap'];
  99                 } else {
 100                         return false;
 101                 }
 102         }
 103
 104         /**
 105          * @param array[] $data
 106          * @param bool $force
 107          *
 108          * @return bool
 109          */
 110         protected function doPopulate( array $data, $force ) {
 111                 $dbw = wfGetDB( DB_MASTER );
 112
 113                 if ( !$force ) {
 114                         $row = $dbw->selectRow(
 115                                 'updatelog',
 116                                 '1',
 117                                 [ 'ul_key' => 'populate interwiki' ],
 118                                 __METHOD__
 119                         );
 120
 121                         if ( $row ) {
 122                                 $this->output( "Interwiki table already populated.  Use php " .
 123                                         "maintenance/populateInterwiki.php\n--force from the command line " .
 124                                         "to override.\n" );
 125                                 return true;
 126                         }
 127                 }
 128
 129                 foreach ( $data as $d ) {
 130                         $prefix = $d['prefix'];
 131
 132                         $row = $dbw->selectRow(
 133                                 'interwiki',
 134                                 '1',
 135                                 [ 'iw_prefix' => $prefix ],
 136                                 __METHOD__
 137                         );
 138
 139                         if ( ! $row ) {
 140                                 $dbw->insert(
 141                                         'interwiki',
 142                                         [
 143                                                 'iw_prefix' => $prefix,
 144                                                 'iw_url' => $d['url'],
 145                                                 'iw_local' => 1
 146                                         ],
 147                                         __METHOD__,
 148                                         'IGNORE'
 149                                 );
 150                         }
 151
 152                         $this->clearCacheEntry( $prefix );
 153                 }
 154
 155                 $this->output( "Interwiki links are populated.\n" );
 156
 157                 return true;
 158         }
 159
 160         /**
 161          * @param string $prefix
 162          */
 163         private function clearCacheEntry( $prefix ) {
 164                 $key = wfMemcKey( 'interwiki', $prefix );
 165                 $this->cache->delete( $key );
 166         }
 167
 168 }
 169
 170 $maintClass = PopulateInterwiki::class;
 171 require_once RUN_MAINTENANCE_IF_MAIN;