maintenance/populateContentModel.php

   1 <?php
   2 /**
   3  * Populate the page_content_model and {rev,ar}_content_{model,format} fields.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 require_once __DIR__ . '/Maintenance.php';
  25
  26 use Wikimedia\Rdbms\IDatabase;
  27 use MediaWiki\MediaWikiServices;
  28
  29 /**
  30  * Usage:
  31  *  populateContentModel.php --ns=1 --table=page
  32  */
  33 class PopulateContentModel extends Maintenance {
  34         protected $wikiId;
  35         /** @var WANObjectCache */
  36         protected $wanCache;
  37
  38         public function __construct() {
  39                 parent::__construct();
  40                 $this->addDescription( 'Populate the various content_* fields' );
  41                 $this->addOption( 'ns', 'Namespace to run in, or "all" for all namespaces', true, true );
  42                 $this->addOption( 'table', 'Table to run in', true, true );
  43                 $this->setBatchSize( 100 );
  44         }
  45
  46         public function execute() {
  47                 $dbw = $this->getDB( DB_MASTER );
  48
  49                 $this->wikiId = $dbw->getDomainID();
  50                 $this->wanCache = MediaWikiServices::getInstance()->getMainWANObjectCache();
  51
  52                 $ns = $this->getOption( 'ns' );
  53                 if ( !ctype_digit( $ns ) && $ns !== 'all' ) {
  54                         $this->fatalError( 'Invalid namespace' );
  55                 }
  56                 $ns = $ns === 'all' ? 'all' : (int)$ns;
  57                 $table = $this->getOption( 'table' );
  58                 switch ( $table ) {
  59                         case 'revision':
  60                         case 'archive':
  61                                 $this->populateRevisionOrArchive( $dbw, $table, $ns );
  62                                 break;
  63                         case 'page':
  64                                 $this->populatePage( $dbw, $ns );
  65                                 break;
  66                         default:
  67                                 $this->fatalError( "Invalid table name: $table" );
  68                 }
  69         }
  70
  71         protected function clearCache( $page_id, $rev_id ) {
  72                 $contentModelKey = $this->wanCache->makeKey( 'page-content-model', $rev_id );
  73                 $revisionKey =
  74                         $this->wanCache->makeGlobalKey( 'revision', $this->wikiId, $page_id, $rev_id );
  75
  76                 // WikiPage content model cache
  77                 $this->wanCache->delete( $contentModelKey );
  78
  79                 // Revision object cache, which contains a content model
  80                 $this->wanCache->delete( $revisionKey );
  81         }
  82
  83         private function updatePageRows( IDatabase $dbw, $pageIds, $model ) {
  84                 $count = count( $pageIds );
  85                 $this->output( "Setting $count rows to $model..." );
  86                 $dbw->update(
  87                         'page',
  88                         [ 'page_content_model' => $model ],
  89                         [ 'page_id' => $pageIds ],
  90                         __METHOD__
  91                 );
  92                 wfWaitForSlaves();
  93                 $this->output( "done.\n" );
  94         }
  95
  96         protected function populatePage( IDatabase $dbw, $ns ) {
  97                 $toSave = [];
  98                 $lastId = 0;
  99                 $nsCondition = $ns === 'all' ? [] : [ 'page_namespace' => $ns ];
 100                 $batchSize = $this->getBatchSize();
 101                 do {
 102                         $rows = $dbw->select(
 103                                 'page',
 104                                 [ 'page_namespace', 'page_title', 'page_id' ],
 105                                 [
 106                                         'page_content_model' => null,
 107                                         'page_id > ' . $dbw->addQuotes( $lastId ),
 108                                 ] + $nsCondition,
 109                                 __METHOD__,
 110                                 [ 'LIMIT' => $batchSize, 'ORDER BY' => 'page_id ASC' ]
 111                         );
 112                         $this->output( "Fetched {$rows->numRows()} rows.\n" );
 113                         foreach ( $rows as $row ) {
 114                                 $title = Title::newFromRow( $row );
 115                                 $model = ContentHandler::getDefaultModelFor( $title );
 116                                 $toSave[$model][] = $row->page_id;
 117                                 if ( count( $toSave[$model] ) >= $batchSize ) {
 118                                         $this->updatePageRows( $dbw, $toSave[$model], $model );
 119                                         unset( $toSave[$model] );
 120                                 }
 121                                 $lastId = $row->page_id;
 122                         }
 123                 } while ( $rows->numRows() >= $batchSize );
 124                 foreach ( $toSave as $model => $pages ) {
 125                         $this->updatePageRows( $dbw, $pages, $model );
 126                 }
 127         }
 128
 129         private function updateRevisionOrArchiveRows( IDatabase $dbw, $ids, $model, $table ) {
 130                 $prefix = $table === 'archive' ? 'ar' : 'rev';
 131                 $model_column = "{$prefix}_content_model";
 132                 $format_column = "{$prefix}_content_format";
 133                 $key = "{$prefix}_id";
 134
 135                 $count = count( $ids );
 136                 $format = ContentHandler::getForModelID( $model )->getDefaultFormat();
 137                 $this->output( "Setting $count rows to $model / $format..." );
 138                 $dbw->update(
 139                         $table,
 140                         [ $model_column => $model, $format_column => $format ],
 141                         [ $key => $ids ],
 142                         __METHOD__
 143                 );
 144
 145                 $this->output( "done.\n" );
 146         }
 147
 148         protected function populateRevisionOrArchive( IDatabase $dbw, $table, $ns ) {
 149                 $prefix = $table === 'archive' ? 'ar' : 'rev';
 150                 $model_column = "{$prefix}_content_model";
 151                 $format_column = "{$prefix}_content_format";
 152                 $key = "{$prefix}_id";
 153                 if ( $table === 'archive' ) {
 154                         $selectTables = 'archive';
 155                         $fields = [ 'ar_namespace', 'ar_title' ];
 156                         $join_conds = [];
 157                         $where = $ns === 'all' ? [] : [ 'ar_namespace' => $ns ];
 158                         $page_id_column = 'ar_page_id';
 159                         $rev_id_column = 'ar_rev_id';
 160                 } else { // revision
 161                         $selectTables = [ 'revision', 'page' ];
 162                         $fields = [ 'page_title', 'page_namespace' ];
 163                         $join_conds = [ 'page' => [ 'JOIN', 'rev_page=page_id' ] ];
 164                         $where = $ns === 'all' ? [] : [ 'page_namespace' => $ns ];
 165                         $page_id_column = 'rev_page';
 166                         $rev_id_column = 'rev_id';
 167                 }
 168
 169                 $toSave = [];
 170                 $idsToClear = [];
 171                 $lastId = 0;
 172                 $batchSize = $this->getBatchSize();
 173                 do {
 174                         $rows = $dbw->select(
 175                                 $selectTables,
 176                                 array_merge(
 177                                         $fields,
 178                                         [ $model_column, $format_column, $key, $page_id_column, $rev_id_column ]
 179                                 ),
 180                                 // @todo support populating format if model is already set
 181                                 [
 182                                         $model_column => null,
 183                                         "$key > " . $dbw->addQuotes( $lastId ),
 184                                 ] + $where,
 185                                 __METHOD__,
 186                                 [ 'LIMIT' => $batchSize, 'ORDER BY' => "$key ASC" ],
 187                                 $join_conds
 188                         );
 189                         $this->output( "Fetched {$rows->numRows()} rows.\n" );
 190                         foreach ( $rows as $row ) {
 191                                 if ( $table === 'archive' ) {
 192                                         $title = Title::makeTitle( $row->ar_namespace, $row->ar_title );
 193                                 } else {
 194                                         $title = Title::newFromRow( $row );
 195                                 }
 196                                 $lastId = $row->{$key};
 197                                 try {
 198                                         $handler = ContentHandler::getForTitle( $title );
 199                                 } catch ( MWException $e ) {
 200                                         $this->error( "Invalid content model for $title" );
 201                                         continue;
 202                                 }
 203                                 $defaultModel = $handler->getModelID();
 204                                 $defaultFormat = $handler->getDefaultFormat();
 205                                 $dbModel = $row->{$model_column};
 206                                 $dbFormat = $row->{$format_column};
 207                                 $id = $row->{$key};
 208                                 if ( $dbModel === null && $dbFormat === null ) {
 209                                         // Set the defaults
 210                                         $toSave[$defaultModel][] = $row->{$key};
 211                                         $idsToClear[] = [
 212                                                 'page_id' => $row->{$page_id_column},
 213                                                 'rev_id' => $row->{$rev_id_column},
 214                                         ];
 215                                 } else { // $dbModel === null, $dbFormat set.
 216                                         if ( $dbFormat === $defaultFormat ) {
 217                                                 $toSave[$defaultModel][] = $row->{$key};
 218                                                 $idsToClear[] = [
 219                                                         'page_id' => $row->{$page_id_column},
 220                                                         'rev_id' => $row->{$rev_id_column},
 221                                                 ];
 222                                         } else { // non-default format, just update now
 223                                                 $this->output( "Updating model to match format for $table $id of $title... " );
 224                                                 $dbw->update(
 225                                                         $table,
 226                                                         [ $model_column => $defaultModel ],
 227                                                         [ $key => $id ],
 228                                                         __METHOD__
 229                                                 );
 230                                                 wfWaitForSlaves();
 231                                                 $this->clearCache( $row->{$page_id_column}, $row->{$rev_id_column} );
 232                                                 $this->output( "done.\n" );
 233                                                 continue;
 234                                         }
 235                                 }
 236
 237                                 if ( count( $toSave[$defaultModel] ) >= $batchSize ) {
 238                                         $this->updateRevisionOrArchiveRows( $dbw, $toSave[$defaultModel], $defaultModel, $table );
 239                                         unset( $toSave[$defaultModel] );
 240                                 }
 241                         }
 242                 } while ( $rows->numRows() >= $batchSize );
 243                 foreach ( $toSave as $model => $ids ) {
 244                         $this->updateRevisionOrArchiveRows( $dbw, $ids, $model, $table );
 245                 }
 246
 247                 foreach ( $idsToClear as $idPair ) {
 248                         $this->clearCache( $idPair['page_id'], $idPair['rev_id'] );
 249                 }
 250         }
 251 }
 252
 253 $maintClass = PopulateContentModel::class;
 254 require_once RUN_MAINTENANCE_IF_MAIN;