X-Git-Url: http://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=blobdiff_plain;f=maintenance%2Frebuildrecentchanges.php;h=dc8bf290b0dde18f0b4318ed41318d6cb130d88c;hp=bbf91f585e5a737e47d892a04e1fca703188279b;hb=cc69954a9df31b3c31d72074ec84c0a0d9464e29;hpb=d0142faf253e069473888dc378979b804fb2b07e diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php index bbf91f585e..dc8bf290b0 100644 --- a/maintenance/rebuildrecentchanges.php +++ b/maintenance/rebuildrecentchanges.php @@ -24,7 +24,9 @@ */ require_once __DIR__ . '/Maintenance.php'; + use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\ILBFactory; /** * Maintenance script that rebuilds recent changes from scratch. @@ -64,11 +66,12 @@ class RebuildRecentchanges extends Maintenance { $this->fatalError( "Both 'from' and 'to' must be given, or neither" ); } - $this->rebuildRecentChangesTablePass1(); - $this->rebuildRecentChangesTablePass2(); - $this->rebuildRecentChangesTablePass3(); - $this->rebuildRecentChangesTablePass4(); - $this->rebuildRecentChangesTablePass5(); + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $this->rebuildRecentChangesTablePass1( $lbFactory ); + $this->rebuildRecentChangesTablePass2( $lbFactory ); + $this->rebuildRecentChangesTablePass3( $lbFactory ); + $this->rebuildRecentChangesTablePass4( $lbFactory ); + $this->rebuildRecentChangesTablePass5( $lbFactory ); if ( !( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) ) { $this->purgeFeeds(); } @@ -78,10 +81,9 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 1: Insert `recentchanges` entries for page revisions. */ - private function rebuildRecentChangesTablePass1() { + private function rebuildRecentChangesTablePass1( ILBFactory $lbFactory ) { $dbw = $this->getDB( DB_MASTER ); - $revCommentStore = new CommentStore( 'rev_comment' ); - $rcCommentStore = new CommentStore( 'rc_comment' ); + $commentStore = CommentStore::getStore(); if ( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) { $this->cutoffFrom = wfTimestamp( TS_UNIX, $this->getOption( 'from' ) ); @@ -111,18 +113,17 @@ class RebuildRecentchanges extends Maintenance { ); foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) { $dbw->delete( 'recentchanges', [ 'rc_id' => $rcidBatch ], __METHOD__ ); - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } $this->output( "Loading from page and revision tables...\n" ); - $commentQuery = $revCommentStore->getJoin(); + $commentQuery = $commentStore->getJoin( 'rev_comment' ); + $actorQuery = ActorMigration::newMigration()->getJoin( 'rev_user' ); $res = $dbw->select( - [ 'revision', 'page' ] + $commentQuery['tables'], + [ 'revision', 'page' ] + $commentQuery['tables'] + $actorQuery['tables'], [ 'rev_timestamp', - 'rev_user', - 'rev_user_text', 'rev_minor_edit', 'rev_id', 'rev_deleted', @@ -130,7 +131,7 @@ class RebuildRecentchanges extends Maintenance { 'page_title', 'page_is_new', 'page_id' - ] + $commentQuery['fields'], + ] + $commentQuery['fields'] + $actorQuery['fields'], [ 'rev_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), 'rev_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) @@ -139,19 +140,19 @@ class RebuildRecentchanges extends Maintenance { [ 'ORDER BY' => 'rev_timestamp DESC' ], [ 'page' => [ 'JOIN', 'rev_page=page_id' ], - ] + $commentQuery['joins'] + ] + $commentQuery['joins'] + $actorQuery['joins'] ); $this->output( "Inserting from page and revision tables...\n" ); $inserted = 0; + $actorMigration = ActorMigration::newMigration(); foreach ( $res as $row ) { - $comment = $revCommentStore->getComment( $row ); + $comment = $commentStore->getComment( 'rev_comment', $row ); + $user = User::newFromAnyId( $row->rev_user, $row->rev_user_text, $row->rev_actor ); $dbw->insert( 'recentchanges', [ 'rc_timestamp' => $row->rev_timestamp, - 'rc_user' => $row->rev_user, - 'rc_user_text' => $row->rev_user_text, 'rc_namespace' => $row->page_namespace, 'rc_title' => $row->page_title, 'rc_minor' => $row->rev_minor_edit, @@ -163,11 +164,12 @@ class RebuildRecentchanges extends Maintenance { 'rc_type' => $row->page_is_new ? RC_NEW : RC_EDIT, 'rc_source' => $row->page_is_new ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT, 'rc_deleted' => $row->rev_deleted - ] + $rcCommentStore->insert( $dbw, $comment ), + ] + $commentStore->insert( $dbw, 'rc_comment', $comment ) + + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ), __METHOD__ ); if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } @@ -176,7 +178,7 @@ class RebuildRecentchanges extends Maintenance { * Rebuild pass 2: Enhance entries for page revisions with references to the previous revision * (rc_last_oldid, rc_new etc.) and size differences (rc_old_len, rc_new_len). */ - private function rebuildRecentChangesTablePass2() { + private function rebuildRecentChangesTablePass2( ILBFactory $lbFactory ) { $dbw = $this->getDB( DB_MASTER ); $this->output( "Updating links and size differences...\n" ); @@ -257,7 +259,7 @@ class RebuildRecentchanges extends Maintenance { $lastSize = $size; if ( ( ++$updated % $this->getBatchSize() ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } @@ -266,22 +268,20 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 3: Insert `recentchanges` entries for action logs. */ - private function rebuildRecentChangesTablePass3() { + private function rebuildRecentChangesTablePass3( ILBFactory $lbFactory ) { global $wgLogTypes, $wgLogRestrictions; $dbw = $this->getDB( DB_MASTER ); - $logCommentStore = new CommentStore( 'log_comment' ); - $rcCommentStore = new CommentStore( 'rc_comment' ); + $commentStore = CommentStore::getStore(); $this->output( "Loading from user, page, and logging tables...\n" ); - $commentQuery = $logCommentStore->getJoin(); + $commentQuery = $commentStore->getJoin( 'log_comment' ); + $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' ); $res = $dbw->select( - [ 'user', 'logging', 'page' ] + $commentQuery['tables'], + [ 'logging', 'page' ] + $commentQuery['tables'] + $actorQuery['tables'], [ 'log_timestamp', - 'log_user', - 'user_name', 'log_namespace', 'log_title', 'page_id', @@ -290,11 +290,10 @@ class RebuildRecentchanges extends Maintenance { 'log_id', 'log_params', 'log_deleted' - ] + $commentQuery['fields'], + ] + $commentQuery['fields'] + $actorQuery['fields'], [ 'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), 'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), - 'log_user=user_id', // Some logs don't go in RC since they are private. // @FIXME: core/extensions also have spammy logs that don't go in RC. 'log_type' => array_diff( $wgLogTypes, array_keys( $wgLogRestrictions ) ), @@ -304,20 +303,20 @@ class RebuildRecentchanges extends Maintenance { [ 'page' => [ 'LEFT JOIN', [ 'log_namespace=page_namespace', 'log_title=page_title' ] ] - ] + $commentQuery['joins'] + ] + $commentQuery['joins'] + $actorQuery['joins'] ); $field = $dbw->fieldInfo( 'recentchanges', 'rc_cur_id' ); $inserted = 0; + $actorMigration = ActorMigration::newMigration(); foreach ( $res as $row ) { - $comment = $logCommentStore->getComment( $row ); + $comment = $commentStore->getComment( 'log_comment', $row ); + $user = User::newFromAnyId( $row->log_user, $row->log_user_text, $row->log_actor ); $dbw->insert( 'recentchanges', [ 'rc_timestamp' => $row->log_timestamp, - 'rc_user' => $row->log_user, - 'rc_user_text' => $row->user_name, 'rc_namespace' => $row->log_namespace, 'rc_title' => $row->log_title, 'rc_minor' => 0, @@ -336,12 +335,13 @@ class RebuildRecentchanges extends Maintenance { 'rc_logid' => $row->log_id, 'rc_params' => $row->log_params, 'rc_deleted' => $row->log_deleted - ] + $rcCommentStore->insert( $dbw, $comment ), + ] + $commentStore->insert( $dbw, 'rc_comment', $comment ) + + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ), __METHOD__ ); if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } @@ -349,13 +349,12 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 4: Mark bot and autopatrolled entries. */ - private function rebuildRecentChangesTablePass4() { + private function rebuildRecentChangesTablePass4( ILBFactory $lbFactory ) { global $wgUseRCPatrol, $wgMiserMode; $dbw = $this->getDB( DB_MASTER ); - list( $recentchanges, $usergroups, $user ) = - $dbw->tableNamesN( 'recentchanges', 'user_groups', 'user' ); + $userQuery = User::getQueryInfo(); # @FIXME: recognize other bot account groups (not the same as users with 'bot' rights) # @NOTE: users with 'bot' rights choose when edits are bot edits or not. That information @@ -365,32 +364,42 @@ class RebuildRecentchanges extends Maintenance { # Flag our recent bot edits if ( $botgroups ) { - $botwhere = $dbw->makeList( $botgroups ); - $this->output( "Flagging bot account edits...\n" ); # Find all users that are bots - $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . - "WHERE ug_group IN($botwhere) AND user_id = ug_user"; - $res = $dbw->query( $sql, __METHOD__ ); + $res = $dbw->select( + array_merge( [ 'user_groups' ], $userQuery['tables'] ), + $userQuery['fields'], + [ 'ug_group' => $botgroups ], + __METHOD__, + [ 'DISTINCT' ], + [ 'user_groups' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins'] + ); $botusers = []; foreach ( $res as $obj ) { - $botusers[] = $obj->user_name; + $botusers[] = User::newFromRow( $obj ); } # Fill in the rc_bot field if ( $botusers ) { - $rcids = $dbw->selectFieldValues( - 'recentchanges', - 'rc_id', - [ - 'rc_user_text' => $botusers, - "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), - "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) - ], - __METHOD__ - ); + $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $botusers, false ); + $rcids = []; + foreach ( $actorQuery['orconds'] as $cond ) { + $rcids = array_merge( $rcids, $dbw->selectFieldValues( + [ 'recentchanges' ] + $actorQuery['tables'], + 'rc_id', + [ + "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + $cond, + ], + __METHOD__, + [], + $actorQuery['joins'] + ) ); + } + $rcids = array_values( array_unique( $rcids ) ); foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) { $dbw->update( @@ -399,35 +408,47 @@ class RebuildRecentchanges extends Maintenance { [ 'rc_id' => $rcidBatch ], __METHOD__ ); - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } # Flag our recent autopatrolled edits if ( !$wgMiserMode && $autopatrolgroups ) { - $patrolwhere = $dbw->makeList( $autopatrolgroups ); $patrolusers = []; $this->output( "Flagging auto-patrolled edits...\n" ); # Find all users in RC with autopatrol rights - $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . - "WHERE ug_group IN($patrolwhere) AND user_id = ug_user"; - $res = $dbw->query( $sql, __METHOD__ ); + $res = $dbw->select( + array_merge( [ 'user_groups' ], $userQuery['tables'] ), + $userQuery['fields'], + [ 'ug_group' => $autopatrolgroups ], + __METHOD__, + [ 'DISTINCT' ], + [ 'user_groups' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins'] + ); foreach ( $res as $obj ) { - $patrolusers[] = $dbw->addQuotes( $obj->user_name ); + $patrolusers[] = User::newFromRow( $obj ); } # Fill in the rc_patrolled field if ( $patrolusers ) { - $patrolwhere = implode( ',', $patrolusers ); - $sql2 = "UPDATE $recentchanges SET rc_patrolled=1 " . - "WHERE rc_user_text IN($patrolwhere) " . - "AND rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ) . ' ' . - "AND rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ); - $dbw->query( $sql2 ); + $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $patrolusers, false ); + foreach ( $actorQuery['orconds'] as $cond ) { + $dbw->update( + 'recentchanges', + [ 'rc_patrolled' => 1 ], + [ + $cond, + 'rc_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + ], + __METHOD__ + ); + $lbFactory->waitForReplication(); + } } } } @@ -436,7 +457,7 @@ class RebuildRecentchanges extends Maintenance { * Rebuild pass 5: Delete duplicate entries where we generate both a page revision and a log entry * for a single action (upload only, at the moment, but potentially also move, protect, ...). */ - private function rebuildRecentChangesTablePass5() { + private function rebuildRecentChangesTablePass5( ILBFactory $lbFactory ) { $dbw = wfGetDB( DB_MASTER ); $this->output( "Removing duplicate revision and logging entries...\n" ); @@ -475,7 +496,7 @@ class RebuildRecentchanges extends Maintenance { ); if ( ( ++$updates % $this->getBatchSize() ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } @@ -495,5 +516,5 @@ class RebuildRecentchanges extends Maintenance { } } -$maintClass = "RebuildRecentchanges"; +$maintClass = RebuildRecentchanges::class; require_once RUN_MAINTENANCE_IF_MAIN;