maintenance/includes/MigrateActors.php

   1 <?php
   2 /**
   3  * Helper for migrating actors from pre-1.31 columns to the 'actor' table
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance
  22  */
  23
  24 use Wikimedia\Rdbms\IDatabase;
  25
  26 require_once __DIR__ . '/../Maintenance.php';
  27
  28 /**
  29  * Maintenance script that migrates actors from pre-1.31 columns to the
  30  * 'actor' table
  31  *
  32  * @ingroup Maintenance
  33  */
  34 class MigrateActors extends LoggedUpdateMaintenance {
  35         public function __construct() {
  36                 parent::__construct();
  37                 $this->addDescription( 'Migrates actors from pre-1.31 columns to the \'actor\' table' );
  38                 $this->setBatchSize( 100 );
  39         }
  40
  41         protected function getUpdateKey() {
  42                 return __CLASS__;
  43         }
  44
  45         protected function doDBUpdates() {
  46                 global $wgActorTableSchemaMigrationStage;
  47
  48                 if ( !( $wgActorTableSchemaMigrationStage & SCHEMA_COMPAT_WRITE_NEW ) ) {
  49                         $this->output(
  50                                 "...cannot update while \$wgActorTableSchemaMigrationStage lacks SCHEMA_COMPAT_WRITE_NEW\n"
  51                         );
  52                         return false;
  53                 }
  54
  55                 $this->output( "Creating actor entries for all registered users\n" );
  56                 $end = 0;
  57                 $dbw = $this->getDB( DB_MASTER );
  58                 $max = $dbw->selectField( 'user', 'MAX(user_id)', '', __METHOD__ );
  59                 $count = 0;
  60                 while ( $end < $max ) {
  61                         $start = $end + 1;
  62                         $end = min( $start + $this->mBatchSize, $max );
  63                         $this->output( "... $start - $end\n" );
  64                         $dbw->insertSelect(
  65                                 'actor',
  66                                 'user',
  67                                 [ 'actor_user' => 'user_id', 'actor_name' => 'user_name' ],
  68                                 [ "user_id >= $start", "user_id <= $end" ],
  69                                 __METHOD__,
  70                                 [ 'IGNORE' ],
  71                                 [ 'ORDER BY' => [ 'user_id' ] ]
  72                         );
  73                         $count += $dbw->affectedRows();
  74                         wfWaitForSlaves();
  75                 }
  76                 $this->output( "Completed actor creation, added $count new actor(s)\n" );
  77
  78                 $errors = 0;
  79                 $errors += $this->migrateToTemp(
  80                         'revision', 'rev_id', [ 'revactor_timestamp' => 'rev_timestamp', 'revactor_page' => 'rev_page' ],
  81                         'rev_user', 'rev_user_text', 'revactor_rev', 'revactor_actor'
  82                 );
  83                 $errors += $this->migrate( 'archive', 'ar_id', 'ar_user', 'ar_user_text', 'ar_actor' );
  84                 $errors += $this->migrate( 'ipblocks', 'ipb_id', 'ipb_by', 'ipb_by_text', 'ipb_by_actor' );
  85                 $errors += $this->migrate( 'image', 'img_name', 'img_user', 'img_user_text', 'img_actor' );
  86                 $errors += $this->migrate(
  87                         'oldimage', [ 'oi_name', 'oi_timestamp' ], 'oi_user', 'oi_user_text', 'oi_actor'
  88                 );
  89                 $errors += $this->migrate( 'filearchive', 'fa_id', 'fa_user', 'fa_user_text', 'fa_actor' );
  90                 $errors += $this->migrate( 'recentchanges', 'rc_id', 'rc_user', 'rc_user_text', 'rc_actor' );
  91                 $errors += $this->migrate( 'logging', 'log_id', 'log_user', 'log_user_text', 'log_actor' );
  92
  93                 $errors += $this->migrateLogSearch();
  94
  95                 return $errors === 0;
  96         }
  97
  98         /**
  99          * Calculate a "next" condition and a display string
 100          * @param IDatabase $dbw
 101          * @param string[] $primaryKey Primary key of the table.
 102          * @param object $row Database row
 103          * @return array [ string $next, string $display ]
 104          */
 105         private function makeNextCond( $dbw, $primaryKey, $row ) {
 106                 $next = '';
 107                 $display = [];
 108                 for ( $i = count( $primaryKey ) - 1; $i >= 0; $i-- ) {
 109                         $field = $primaryKey[$i];
 110                         $display[] = $field . '=' . $row->$field;
 111                         $value = $dbw->addQuotes( $row->$field );
 112                         if ( $next === '' ) {
 113                                 $next = "$field > $value";
 114                         } else {
 115                                 $next = "$field > $value OR $field = $value AND ($next)";
 116                         }
 117                 }
 118                 $display = implode( ' ', array_reverse( $display ) );
 119                 return [ $next, $display ];
 120         }
 121
 122         /**
 123          * Make the subqueries for `actor_id`
 124          * @param IDatabase $dbw
 125          * @param string $userField User ID field name
 126          * @param string $nameField User name field name
 127          * @return string SQL fragment
 128          */
 129         private function makeActorIdSubquery( $dbw, $userField, $nameField ) {
 130                 $idSubquery = $dbw->buildSelectSubquery(
 131                         'actor',
 132                         'actor_id',
 133                         [ "$userField = actor_user" ],
 134                         __METHOD__
 135                 );
 136                 $nameSubquery = $dbw->buildSelectSubquery(
 137                         'actor',
 138                         'actor_id',
 139                         [ "$nameField = actor_name" ],
 140                         __METHOD__
 141                 );
 142                 return "CASE WHEN $userField = 0 OR $userField IS NULL THEN $nameSubquery ELSE $idSubquery END";
 143         }
 144
 145         /**
 146          * Add actors for anons in a set of rows
 147          *
 148          * @suppress SecurityCheck-SQLInjection The array_keys/array_map is too much for static analysis
 149          * @param IDatabase $dbw
 150          * @param string $nameField
 151          * @param object[] &$rows
 152          * @param array &$complainedAboutUsers
 153          * @param int &$countErrors
 154          * @return int Count of actors inserted
 155          */
 156         private function addActorsForRows(
 157                 IDatabase $dbw, $nameField, array &$rows, array &$complainedAboutUsers, &$countErrors
 158         ) {
 159                 $needActors = [];
 160                 $countActors = 0;
 161
 162                 $keep = [];
 163                 foreach ( $rows as $index => $row ) {
 164                         $keep[$index] = true;
 165                         if ( $row->actor_id === null ) {
 166                                 // All registered users should have an actor_id already. So
 167                                 // if we have a usable name here, it means they didn't run
 168                                 // maintenance/cleanupUsersWithNoId.php
 169                                 $name = $row->$nameField;
 170                                 if ( User::isUsableName( $name ) ) {
 171                                         if ( !isset( $complainedAboutUsers[$name] ) ) {
 172                                                 $complainedAboutUsers[$name] = true;
 173                                                 $this->error(
 174                                                         "User name \"$name\" is usable, cannot create an anonymous actor for it."
 175                                                         . " Run maintenance/cleanupUsersWithNoId.php to fix this situation.\n"
 176                                                 );
 177                                         }
 178                                         unset( $keep[$index] );
 179                                         $countErrors++;
 180                                 } else {
 181                                         $needActors[$name] = 0;
 182                                 }
 183                         }
 184                 }
 185                 $rows = array_intersect_key( $rows, $keep );
 186
 187                 if ( $needActors ) {
 188                         $dbw->insert(
 189                                 'actor',
 190                                 array_map( function ( $v ) {
 191                                         return [
 192                                                 'actor_name' => $v,
 193                                         ];
 194                                 }, array_keys( $needActors ) ),
 195                                 __METHOD__
 196                         );
 197                         $countActors += $dbw->affectedRows();
 198
 199                         $res = $dbw->select(
 200                                 'actor',
 201                                 [ 'actor_id', 'actor_name' ],
 202                                 [ 'actor_name' => array_keys( $needActors ) ],
 203                                 __METHOD__
 204                         );
 205                         foreach ( $res as $row ) {
 206                                 $needActors[$row->actor_name] = $row->actor_id;
 207                         }
 208                         foreach ( $rows as $row ) {
 209                                 if ( $row->actor_id === null ) {
 210                                         $row->actor_id = $needActors[$row->$nameField];
 211                                 }
 212                         }
 213                 }
 214
 215                 return $countActors;
 216         }
 217
 218         /**
 219          * Migrate actors in a table.
 220          *
 221          * Assumes any row with the actor field non-zero have already been migrated.
 222          * Blanks the name field when migrating.
 223          *
 224          * @param string $table Table to migrate
 225          * @param string|string[] $primaryKey Primary key of the table.
 226          * @param string $userField User ID field name
 227          * @param string $nameField User name field name
 228          * @param string $actorField Actor field name
 229          * @return int Number of errors
 230          */
 231         protected function migrate( $table, $primaryKey, $userField, $nameField, $actorField ) {
 232                 $complainedAboutUsers = [];
 233
 234                 $primaryKey = (array)$primaryKey;
 235                 $pkFilter = array_flip( $primaryKey );
 236                 $this->output(
 237                         "Beginning migration of $table.$userField and $table.$nameField to $table.$actorField\n"
 238                 );
 239                 wfWaitForSlaves();
 240
 241                 $dbw = $this->getDB( DB_MASTER );
 242                 $actorIdSubquery = $this->makeActorIdSubquery( $dbw, $userField, $nameField );
 243                 $next = '1=1';
 244                 $countUpdated = 0;
 245                 $countActors = 0;
 246                 $countErrors = 0;
 247                 while ( true ) {
 248                         // Fetch the rows needing update
 249                         $res = $dbw->select(
 250                                 $table,
 251                                 array_merge( $primaryKey, [ $userField, $nameField, 'actor_id' => $actorIdSubquery ] ),
 252                                 [
 253                                         $actorField => 0,
 254                                         $next,
 255                                 ],
 256                                 __METHOD__,
 257                                 [
 258                                         'ORDER BY' => $primaryKey,
 259                                         'LIMIT' => $this->mBatchSize,
 260                                 ]
 261                         );
 262                         if ( !$res->numRows() ) {
 263                                 break;
 264                         }
 265
 266                         // Insert new actors for rows that need one
 267                         $rows = iterator_to_array( $res );
 268                         $lastRow = end( $rows );
 269                         $countActors += $this->addActorsForRows(
 270                                 $dbw, $nameField, $rows, $complainedAboutUsers, $countErrors
 271                         );
 272
 273                         // Update the existing rows
 274                         foreach ( $rows as $row ) {
 275                                 if ( !$row->actor_id ) {
 276                                         list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
 277                                         $this->error(
 278                                                 "Could not make actor for row with $display "
 279                                                 . "$userField={$row->$userField} $nameField={$row->$nameField}\n"
 280                                         );
 281                                         $countErrors++;
 282                                         continue;
 283                                 }
 284                                 $dbw->update(
 285                                         $table,
 286                                         [
 287                                                 $actorField => $row->actor_id,
 288                                         ],
 289                                         array_intersect_key( (array)$row, $pkFilter ) + [
 290                                                 $actorField => 0
 291                                         ],
 292                                         __METHOD__
 293                                 );
 294                                 $countUpdated += $dbw->affectedRows();
 295                         }
 296
 297                         list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
 298                         $this->output( "... $display\n" );
 299                         wfWaitForSlaves();
 300                 }
 301
 302                 $this->output(
 303                         "Completed migration, updated $countUpdated row(s) with $countActors new actor(s), "
 304                         . "$countErrors error(s)\n"
 305                 );
 306                 return $countErrors;
 307         }
 308
 309         /**
 310          * Migrate actors in a table to a temporary table.
 311          *
 312          * Assumes the new table is named "{$table}_actor_temp", and it has two
 313          * columns, in order, being the primary key of the original table and the
 314          * actor ID field.
 315          * Blanks the name field when migrating.
 316          *
 317          * @param string $table Table to migrate
 318          * @param string $primaryKey Primary key of the table.
 319          * @param array $extra Extra fields to copy
 320          * @param string $userField User ID field name
 321          * @param string $nameField User name field name
 322          * @param string $newPrimaryKey Primary key of the new table.
 323          * @param string $actorField Actor field name
 324          */
 325         protected function migrateToTemp(
 326                 $table, $primaryKey, $extra, $userField, $nameField, $newPrimaryKey, $actorField
 327         ) {
 328                 $complainedAboutUsers = [];
 329
 330                 $newTable = $table . '_actor_temp';
 331                 $this->output(
 332                         "Beginning migration of $table.$userField and $table.$nameField to $newTable.$actorField\n"
 333                 );
 334                 wfWaitForSlaves();
 335
 336                 $dbw = $this->getDB( DB_MASTER );
 337                 $actorIdSubquery = $this->makeActorIdSubquery( $dbw, $userField, $nameField );
 338                 $next = [];
 339                 $countUpdated = 0;
 340                 $countActors = 0;
 341                 $countErrors = 0;
 342                 while ( true ) {
 343                         // Fetch the rows needing update
 344                         $res = $dbw->select(
 345                                 [ $table, $newTable ],
 346                                 [ $primaryKey, $userField, $nameField, 'actor_id' => $actorIdSubquery ] + $extra,
 347                                 [ $newPrimaryKey => null ] + $next,
 348                                 __METHOD__,
 349                                 [
 350                                         'ORDER BY' => $primaryKey,
 351                                         'LIMIT' => $this->mBatchSize,
 352                                 ],
 353                                 [
 354                                         $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ],
 355                                 ]
 356                         );
 357                         if ( !$res->numRows() ) {
 358                                 break;
 359                         }
 360
 361                         // Insert new actors for rows that need one
 362                         $rows = iterator_to_array( $res );
 363                         $lastRow = end( $rows );
 364                         $countActors += $this->addActorsForRows(
 365                                 $dbw, $nameField, $rows, $complainedAboutUsers, $countErrors
 366                         );
 367
 368                         // Update rows
 369                         if ( $rows ) {
 370                                 $inserts = [];
 371                                 $updates = [];
 372                                 foreach ( $rows as $row ) {
 373                                         if ( !$row->actor_id ) {
 374                                                 list( , $display ) = $this->makeNextCond( $dbw, [ $primaryKey ], $row );
 375                                                 $this->error(
 376                                                         "Could not make actor for row with $display "
 377                                                         . "$userField={$row->$userField} $nameField={$row->$nameField}\n"
 378                                                 );
 379                                                 $countErrors++;
 380                                                 continue;
 381                                         }
 382                                         $ins = [
 383                                                 $newPrimaryKey => $row->$primaryKey,
 384                                                 $actorField => $row->actor_id,
 385                                         ];
 386                                         foreach ( $extra as $to => $from ) {
 387                                                 $ins[$to] = $row->$to; // It's aliased
 388                                         }
 389                                         $inserts[] = $ins;
 390                                         $updates[] = $row->$primaryKey;
 391                                 }
 392                                 $this->beginTransaction( $dbw, __METHOD__ );
 393                                 $dbw->insert( $newTable, $inserts, __METHOD__ );
 394                                 $countUpdated += $dbw->affectedRows();
 395                                 $this->commitTransaction( $dbw, __METHOD__ );
 396                         }
 397
 398                         // Calculate the "next" condition
 399                         list( $n, $display ) = $this->makeNextCond( $dbw, [ $primaryKey ], $lastRow );
 400                         $next = [ $n ];
 401                         $this->output( "... $display\n" );
 402                 }
 403
 404                 $this->output(
 405                         "Completed migration, updated $countUpdated row(s) with $countActors new actor(s), "
 406                         . "$countErrors error(s)\n"
 407                 );
 408                 return $countErrors;
 409         }
 410
 411         /**
 412          * Migrate actors in the log_search table.
 413          * @return int Number of errors
 414          */
 415         protected function migrateLogSearch() {
 416                 $complainedAboutUsers = [];
 417
 418                 $primaryKey = [ 'ls_value', 'ls_log_id' ];
 419                 $this->output( "Beginning migration of log_search\n" );
 420                 wfWaitForSlaves();
 421
 422                 $dbw = $this->getDB( DB_MASTER );
 423                 $countInserted = 0;
 424                 $countActors = 0;
 425                 $countErrors = 0;
 426
 427                 $next = '1=1';
 428                 while ( true ) {
 429                         // Fetch the rows needing update
 430                         $res = $dbw->select(
 431                                 [ 'log_search', 'actor' ],
 432                                 [ 'ls_value', 'ls_log_id', 'actor_id' ],
 433                                 [
 434                                         'ls_field' => 'target_author_id',
 435                                         $next
 436                                 ],
 437                                 __METHOD__,
 438                                 [
 439                                         'ORDER BY' => $primaryKey,
 440                                         'LIMIT' => $this->mBatchSize,
 441                                 ],
 442                                 [ 'actor' => [ 'LEFT JOIN', 'ls_value = ' . $dbw->buildStringCast( 'actor_user' ) ] ]
 443                         );
 444                         if ( !$res->numRows() ) {
 445                                 break;
 446                         }
 447
 448                         // Insert a 'target_author_actor' for each 'target_author_id'
 449                         $ins = [];
 450                         foreach ( $res as $row ) {
 451                                 $lastRow = $row;
 452                                 if ( !$row->actor_id ) {
 453                                         list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
 454                                         $this->error( "No actor for target_author_id row with $display\n" );
 455                                         $countErrors++;
 456                                         continue;
 457                                 }
 458                                 $ins[] = [
 459                                         'ls_field' => 'target_author_actor',
 460                                         'ls_value' => $row->actor_id,
 461                                         'ls_log_id' => $row->ls_log_id,
 462                                 ];
 463                         }
 464                         $dbw->insert( 'log_search', $ins, __METHOD__, [ 'IGNORE' ] );
 465                         $countInserted += $dbw->affectedRows();
 466
 467                         list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
 468                         $this->output( "... target_author_id, $display\n" );
 469                         wfWaitForSlaves();
 470                 }
 471
 472                 $next = '1=1';
 473                 while ( true ) {
 474                         // Fetch the rows needing update
 475                         $res = $dbw->select(
 476                                 [ 'log_search', 'actor' ],
 477                                 [ 'ls_value', 'ls_log_id', 'actor_id' ],
 478                                 [
 479                                         'ls_field' => 'target_author_ip',
 480                                         $next
 481                                 ],
 482                                 __METHOD__,
 483                                 [
 484                                         'ORDER BY' => $primaryKey,
 485                                         'LIMIT' => $this->mBatchSize,
 486                                 ],
 487                                 [ 'actor' => [ 'LEFT JOIN', 'ls_value = actor_name' ] ]
 488                         );
 489                         if ( !$res->numRows() ) {
 490                                 break;
 491                         }
 492
 493                         // Insert new actors for rows that need one
 494                         $rows = iterator_to_array( $res );
 495                         $lastRow = end( $rows );
 496                         $countActors += $this->addActorsForRows(
 497                                 $dbw, 'ls_value', $rows, $complainedAboutUsers, $countErrors
 498                         );
 499
 500                         // Insert a 'target_author_actor' for each 'target_author_ip'
 501                         $ins = [];
 502                         foreach ( $rows as $row ) {
 503                                 if ( !$row->actor_id ) {
 504                                         list( , $display ) = $this->makeNextCond( $dbw, $primaryKey, $row );
 505                                         $this->error( "Could not make actor for target_author_ip row with $display\n" );
 506                                         $countErrors++;
 507                                         continue;
 508                                 }
 509                                 $ins[] = [
 510                                         'ls_field' => 'target_author_actor',
 511                                         'ls_value' => $row->actor_id,
 512                                         'ls_log_id' => $row->ls_log_id,
 513                                 ];
 514                         }
 515                         $dbw->insert( 'log_search', $ins, __METHOD__, [ 'IGNORE' ] );
 516                         $countInserted += $dbw->affectedRows();
 517
 518                         list( $next, $display ) = $this->makeNextCond( $dbw, $primaryKey, $lastRow );
 519                         $this->output( "... target_author_ip, $display\n" );
 520                         wfWaitForSlaves();
 521                 }
 522
 523                 $this->output(
 524                         "Completed migration, inserted $countInserted row(s) with $countActors new actor(s), "
 525                         . "$countErrors error(s)\n"
 526                 );
 527                 return $countErrors;
 528         }
 529 }