maintenance/populateContentTables.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Maintenance
  20  */
  21
  22 use MediaWiki\MediaWikiServices;
  23 use MediaWiki\Storage\NameTableStore;
  24 use MediaWiki\Storage\SqlBlobStore;
  25 use Wikimedia\Assert\Assert;
  26 use Wikimedia\Rdbms\IDatabase;
  27 use Wikimedia\Rdbms\ResultWrapper;
  28
  29 require_once __DIR__ . '/Maintenance.php';
  30
  31 /**
  32  * Populate the content and slot tables.
  33  * @since 1.32
  34  */
  35 class PopulateContentTables extends Maintenance {
  36
  37         /** @var IDatabase */
  38         private $dbw;
  39
  40         /** @var NameTableStore */
  41         private $contentModelStore;
  42
  43         /** @var int */
  44         private $mainRoleId;
  45
  46         /** @var array|null Map "{$modelId}:{$address}" to content_id */
  47         private $contentRowMap = null;
  48
  49         private $count = 0, $totalCount = 0;
  50
  51         public function __construct() {
  52                 parent::__construct();
  53
  54                 $this->addDescription( 'Populate content and slot tables' );
  55                 $this->addOption( 'table', 'revision or archive table, or `all` to populate both', false,
  56                         true );
  57                 $this->addOption( 'reuse-content',
  58                         'Reuse content table rows when the address and model are the same. '
  59                         . 'This will increase the script\'s time and memory usage, perhaps significantly.',
  60                         false, false );
  61                 $this->setBatchSize( 500 );
  62         }
  63
  64         private function initServices() {
  65                 $this->dbw = $this->getDB( DB_MASTER );
  66                 $this->contentModelStore = MediaWikiServices::getInstance()->getContentModelStore();
  67                 $this->mainRoleId = MediaWikiServices::getInstance()->getSlotRoleStore()->acquireId( 'main' );
  68         }
  69
  70         public function execute() {
  71                 global $wgMultiContentRevisionSchemaMigrationStage;
  72
  73                 $t0 = microtime( true );
  74
  75                 if ( ( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_NEW ) === 0 ) {
  76                         $this->writeln(
  77                                 '...cannot update while \$wgMultiContentRevisionSchemaMigrationStage '
  78                                 . 'does not have the SCHEMA_COMPAT_WRITE_NEW bit set.'
  79                         );
  80                         return false;
  81                 }
  82
  83                 $this->initServices();
  84
  85                 if ( $this->getOption( 'reuse-content', false ) ) {
  86                         $this->loadContentMap();
  87                 }
  88
  89                 foreach ( $this->getTables() as $table ) {
  90                         $this->populateTable( $table );
  91                 }
  92
  93                 $elapsed = microtime( true ) - $t0;
  94                 $this->writeln( "Done. Processed $this->totalCount rows in $elapsed seconds" );
  95                 return true;
  96         }
  97
  98         /**
  99          * @return string[]
 100          */
 101         private function getTables() {
 102                 $table = $this->getOption( 'table', 'all' );
 103                 $validTableOptions = [ 'all', 'revision', 'archive' ];
 104
 105                 if ( !in_array( $table, $validTableOptions ) ) {
 106                         $this->fatalError( 'Invalid table. Must be either `revision` or `archive` or `all`' );
 107                 }
 108
 109                 if ( $table === 'all' ) {
 110                         $tables = [ 'revision', 'archive' ];
 111                 } else {
 112                         $tables = [ $table ];
 113                 }
 114
 115                 return $tables;
 116         }
 117
 118         private function loadContentMap() {
 119                 $t0 = microtime( true );
 120                 $this->writeln( "Loading existing content table rows..." );
 121                 $this->contentRowMap = [];
 122                 $dbr = $this->getDB( DB_REPLICA );
 123                 $from = false;
 124                 while ( true ) {
 125                         $res = $dbr->select(
 126                                 'content',
 127                                 [ 'content_id', 'content_address', 'content_model' ],
 128                                 $from ? "content_id > $from" : '',
 129                                 __METHOD__,
 130                                 [ 'ORDER BY' => 'content_id', 'LIMIT' => $this->getBatchSize() ]
 131                         );
 132                         if ( !$res || !$res->numRows() ) {
 133                                 break;
 134                         }
 135                         foreach ( $res as $row ) {
 136                                 $from = $row->content_id;
 137                                 $this->contentRowMap["{$row->content_model}:{$row->content_address}"] = $row->content_id;
 138                         }
 139                 }
 140                 $elapsed = microtime( true ) - $t0;
 141                 $this->writeln( "Loaded " . count( $this->contentRowMap ) . " rows in $elapsed seconds" );
 142         }
 143
 144         /**
 145          * @param string $table
 146          */
 147         private function populateTable( $table ) {
 148                 $t0 = microtime( true );
 149                 $this->count = 0;
 150                 $this->writeln( "Populating $table..." );
 151
 152                 if ( $table === 'revision' ) {
 153                         $idField = 'rev_id';
 154                         $tables = [ 'revision', 'slots', 'page' ];
 155                         $fields = [
 156                                 'rev_id',
 157                                 'len' => 'rev_len',
 158                                 'sha1' => 'rev_sha1',
 159                                 'text_id' => 'rev_text_id',
 160                                 'content_model' => 'rev_content_model',
 161                                 'namespace' => 'page_namespace',
 162                                 'title' => 'page_title',
 163                         ];
 164                         $joins = [
 165                                 'slots' => [ 'LEFT JOIN', 'rev_id=slot_revision_id' ],
 166                                 'page' => [ 'LEFT JOIN', 'rev_page=page_id' ],
 167                         ];
 168                 } else {
 169                         $idField = 'ar_rev_id';
 170                         $tables = [ 'archive', 'slots' ];
 171                         $fields = [
 172                                 'rev_id' => 'ar_rev_id',
 173                                 'len' => 'ar_len',
 174                                 'sha1' => 'ar_sha1',
 175                                 'text_id' => 'ar_text_id',
 176                                 'content_model' => 'ar_content_model',
 177                                 'namespace' => 'ar_namespace',
 178                                 'title' => 'ar_title',
 179                         ];
 180                         $joins = [
 181                                 'slots' => [ 'LEFT JOIN', 'ar_rev_id=slot_revision_id' ],
 182                         ];
 183                 }
 184
 185                 $minmax = $this->dbw->selectRow(
 186                         $table,
 187                         [ 'min' => "MIN( $idField )", 'max' => "MAX( $idField )" ],
 188                         '',
 189                         __METHOD__
 190                 );
 191                 if ( !$minmax || !is_numeric( $minmax->min ) || !is_numeric( $minmax->max ) ) {
 192                         // No rows?
 193                         $minmax = (object)[ 'min' => 1, 'max' => 0 ];
 194                 }
 195
 196                 $batchSize = $this->getBatchSize();
 197
 198                 for ( $startId = $minmax->min; $startId <= $minmax->max; $startId += $batchSize ) {
 199                         $endId = min( $startId + $batchSize - 1, $minmax->max );
 200                         $rows = $this->dbw->select(
 201                                 $tables,
 202                                 $fields,
 203                                 [
 204                                         "$idField >= $startId",
 205                                         "$idField <= $endId",
 206                                         'slot_revision_id IS NULL',
 207                                 ],
 208                                 __METHOD__,
 209                                 [ 'ORDER BY' => 'rev_id' ],
 210                                 $joins
 211                         );
 212                         if ( $rows->numRows() !== 0 ) {
 213                                 $this->populateContentTablesForRowBatch( $rows, $startId, $table );
 214                         }
 215
 216                         $elapsed = microtime( true ) - $t0;
 217                         $this->writeln(
 218                                 "... $table processed up to revision id $endId of {$minmax->max}"
 219                                 . " ($this->count rows in $elapsed seconds)"
 220                         );
 221                 }
 222
 223                 $elapsed = microtime( true ) - $t0;
 224                 $this->writeln( "Done populating $table table. Processed $this->count rows in $elapsed seconds" );
 225         }
 226
 227         /**
 228          * @param ResultWrapper $rows
 229          * @param int $startId
 230          * @param string $table
 231          * @return int|null
 232          */
 233         private function populateContentTablesForRowBatch( ResultWrapper $rows, $startId, $table ) {
 234                 $this->beginTransaction( $this->dbw, __METHOD__ );
 235
 236                 if ( $this->contentRowMap === null ) {
 237                         $map = [];
 238                 } else {
 239                         $map = &$this->contentRowMap;
 240                 }
 241                 $contentKeys = [];
 242
 243                 try {
 244                         // Step 1: Figure out content rows needing insertion.
 245                         $contentRows = [];
 246                         foreach ( $rows as $row ) {
 247                                 $revisionId = $row->rev_id;
 248
 249                                 Assert::invariant( $revisionId !== null, 'rev_id must not be null' );
 250
 251                                 $modelId = $this->contentModelStore->acquireId( $this->getContentModel( $row ) );
 252                                 $address = SqlBlobStore::makeAddressFromTextId( $row->text_id );
 253
 254                                 $key = "{$modelId}:{$address}";
 255                                 $contentKeys[$revisionId] = $key;
 256
 257                                 if ( !isset( $map[$key] ) ) {
 258                                         $map[$key] = false;
 259                                         $contentRows[] = [
 260                                                 'content_size' => (int)$row->len,
 261                                                 'content_sha1' => $row->sha1,
 262                                                 'content_model' => $modelId,
 263                                                 'content_address' => $address,
 264                                         ];
 265                                 }
 266                         }
 267
 268                         // Step 2: Insert them, then read them back in for use in the next step.
 269                         if ( $contentRows ) {
 270                                 $id = $this->dbw->selectField( 'content', 'MAX(content_id)', '', __METHOD__ );
 271                                 $this->dbw->insert( 'content', $contentRows, __METHOD__ );
 272                                 $res = $this->dbw->select(
 273                                         'content',
 274                                         [ 'content_id', 'content_model', 'content_address' ],
 275                                         'content_id > ' . (int)$id,
 276                                         __METHOD__
 277                                 );
 278                                 foreach ( $res as $row ) {
 279                                         $key = $row->content_model . ':' . $row->content_address;
 280                                         $map[$key] = $row->content_id;
 281                                 }
 282                         }
 283
 284                         // Step 3: Insert the slot rows.
 285                         $slotRows = [];
 286                         foreach ( $rows as $row ) {
 287                                 $revisionId = $row->rev_id;
 288                                 $contentId = $map[$contentKeys[$revisionId]] ?? false;
 289                                 if ( $contentId === false ) {
 290                                         throw new \RuntimeException( "Content row for $revisionId not found after content insert" );
 291                                 }
 292                                 $slotRows[] = [
 293                                         'slot_revision_id' => $revisionId,
 294                                         'slot_role_id' => $this->mainRoleId,
 295                                         'slot_content_id' => $contentId,
 296                                         // There's no way to really know the previous revision, so assume no inheriting.
 297                                         // rev_parent_id can get changed on undeletions, and deletions can screw up
 298                                         // rev_timestamp ordering.
 299                                         'slot_origin' => $revisionId,
 300                                 ];
 301                         }
 302                         $this->dbw->insert( 'slots', $slotRows, __METHOD__ );
 303                         $this->count += count( $slotRows );
 304                         $this->totalCount += count( $slotRows );
 305                 } catch ( \Exception $e ) {
 306                         $this->rollbackTransaction( $this->dbw, __METHOD__ );
 307                         $this->fatalError( "Failed to populate content table $table row batch starting at $startId "
 308                                 . "due to exception: " . $e->__toString() );
 309                 }
 310
 311                 $this->commitTransaction( $this->dbw, __METHOD__ );
 312         }
 313
 314         /**
 315          * @param \stdClass $row
 316          * @return string
 317          */
 318         private function getContentModel( $row ) {
 319                 if ( isset( $row->content_model ) ) {
 320                         return $row->content_model;
 321                 }
 322
 323                 $title = Title::makeTitle( $row->namespace, $row->title );
 324
 325                 return ContentHandler::getDefaultModelFor( $title );
 326         }
 327
 328         /**
 329          * @param string $msg
 330          */
 331         private function writeln( $msg ) {
 332                 $this->output( "$msg\n" );
 333         }
 334 }
 335
 336 $maintClass = 'PopulateContentTables';
 337 require_once RUN_MAINTENANCE_IF_MAIN;