X-Git-Url: https://git.heureux-cyclage.org/?p=lhc%2Fweb%2Fwiklou.git;a=blobdiff_plain;f=maintenance%2Frebuildrecentchanges.php;h=a259484f7f11fca838fb043aae3154f4b2b61731;hp=b93d1127d67a66e08435a187619f5a547dd49e62;hb=a7064a0883ec6d715c8c8103efe777769a85437d;hpb=a8ec960e9d910acbcd0d50efad3bd73e3ae812aa diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php index b93d1127d6..a259484f7f 100644 --- a/maintenance/rebuildrecentchanges.php +++ b/maintenance/rebuildrecentchanges.php @@ -24,7 +24,9 @@ */ require_once __DIR__ . '/Maintenance.php'; + use MediaWiki\MediaWikiServices; +use Wikimedia\Rdbms\ILBFactory; /** * Maintenance script that rebuilds recent changes from scratch. @@ -61,14 +63,15 @@ class RebuildRecentchanges extends Maintenance { ( $this->hasOption( 'from' ) && !$this->hasOption( 'to' ) ) || ( !$this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) ) { - $this->error( "Both 'from' and 'to' must be given, or neither", 1 ); + $this->fatalError( "Both 'from' and 'to' must be given, or neither" ); } - $this->rebuildRecentChangesTablePass1(); - $this->rebuildRecentChangesTablePass2(); - $this->rebuildRecentChangesTablePass3(); - $this->rebuildRecentChangesTablePass4(); - $this->rebuildRecentChangesTablePass5(); + $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory(); + $this->rebuildRecentChangesTablePass1( $lbFactory ); + $this->rebuildRecentChangesTablePass2( $lbFactory ); + $this->rebuildRecentChangesTablePass3( $lbFactory ); + $this->rebuildRecentChangesTablePass4( $lbFactory ); + $this->rebuildRecentChangesTablePass5( $lbFactory ); if ( !( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) ) { $this->purgeFeeds(); } @@ -78,8 +81,9 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 1: Insert `recentchanges` entries for page revisions. */ - private function rebuildRecentChangesTablePass1() { + private function rebuildRecentChangesTablePass1( ILBFactory $lbFactory ) { $dbw = $this->getDB( DB_MASTER ); + $commentStore = CommentStore::getStore(); if ( $this->hasOption( 'from' ) && $this->hasOption( 'to' ) ) { $this->cutoffFrom = wfTimestamp( TS_UNIX, $this->getOption( 'from' ) ); @@ -107,19 +111,19 @@ class RebuildRecentchanges extends Maintenance { 'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) ] ); - foreach ( array_chunk( $rcids, $this->mBatchSize ) as $rcidBatch ) { + foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) { $dbw->delete( 'recentchanges', [ 'rc_id' => $rcidBatch ], __METHOD__ ); - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } $this->output( "Loading from page and revision tables...\n" ); + + $commentQuery = $commentStore->getJoin( 'rev_comment' ); + $actorQuery = ActorMigration::newMigration()->getJoin( 'rev_user' ); $res = $dbw->select( - [ 'page', 'revision' ], + [ 'revision', 'page' ] + $commentQuery['tables'] + $actorQuery['tables'], [ 'rev_timestamp', - 'rev_user', - 'rev_user_text', - 'rev_comment', 'rev_minor_edit', 'rev_id', 'rev_deleted', @@ -127,28 +131,30 @@ class RebuildRecentchanges extends Maintenance { 'page_title', 'page_is_new', 'page_id' - ], + ] + $commentQuery['fields'] + $actorQuery['fields'], [ 'rev_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), - 'rev_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), - 'rev_page=page_id' + 'rev_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) ], __METHOD__, - [ 'ORDER BY' => 'rev_timestamp DESC' ] + [ 'ORDER BY' => 'rev_timestamp DESC' ], + [ + 'page' => [ 'JOIN', 'rev_page=page_id' ], + ] + $commentQuery['joins'] + $actorQuery['joins'] ); $this->output( "Inserting from page and revision tables...\n" ); $inserted = 0; + $actorMigration = ActorMigration::newMigration(); foreach ( $res as $row ) { + $comment = $commentStore->getComment( 'rev_comment', $row ); + $user = User::newFromAnyId( $row->rev_user, $row->rev_user_text, $row->rev_actor ); $dbw->insert( 'recentchanges', [ 'rc_timestamp' => $row->rev_timestamp, - 'rc_user' => $row->rev_user, - 'rc_user_text' => $row->rev_user_text, 'rc_namespace' => $row->page_namespace, 'rc_title' => $row->page_title, - 'rc_comment' => $row->rev_comment, 'rc_minor' => $row->rev_minor_edit, 'rc_bot' => 0, 'rc_new' => $row->page_is_new, @@ -156,14 +162,14 @@ class RebuildRecentchanges extends Maintenance { 'rc_this_oldid' => $row->rev_id, 'rc_last_oldid' => 0, // is this ok? 'rc_type' => $row->page_is_new ? RC_NEW : RC_EDIT, - 'rc_source' => $row->page_is_new ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT - , + 'rc_source' => $row->page_is_new ? RecentChange::SRC_NEW : RecentChange::SRC_EDIT, 'rc_deleted' => $row->rev_deleted - ], + ] + $commentStore->insert( $dbw, 'rc_comment', $comment ) + + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ), __METHOD__ ); - if ( ( ++$inserted % $this->mBatchSize ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) { + $lbFactory->waitForReplication(); } } } @@ -172,7 +178,7 @@ class RebuildRecentchanges extends Maintenance { * Rebuild pass 2: Enhance entries for page revisions with references to the previous revision * (rc_last_oldid, rc_new etc.) and size differences (rc_old_len, rc_new_len). */ - private function rebuildRecentChangesTablePass2() { + private function rebuildRecentChangesTablePass2( ILBFactory $lbFactory ) { $dbw = $this->getDB( DB_MASTER ); $this->output( "Updating links and size differences...\n" ); @@ -252,8 +258,8 @@ class RebuildRecentchanges extends Maintenance { $lastOldId = intval( $obj->rc_this_oldid ); $lastSize = $size; - if ( ( ++$updated % $this->mBatchSize ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + if ( ( ++$updated % $this->getBatchSize() ) == 0 ) { + $lbFactory->waitForReplication(); } } } @@ -262,33 +268,32 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 3: Insert `recentchanges` entries for action logs. */ - private function rebuildRecentChangesTablePass3() { + private function rebuildRecentChangesTablePass3( ILBFactory $lbFactory ) { global $wgLogTypes, $wgLogRestrictions; $dbw = $this->getDB( DB_MASTER ); + $commentStore = CommentStore::getStore(); $this->output( "Loading from user, page, and logging tables...\n" ); + $commentQuery = $commentStore->getJoin( 'log_comment' ); + $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' ); $res = $dbw->select( - [ 'user', 'logging', 'page' ], + [ 'logging', 'page' ] + $commentQuery['tables'] + $actorQuery['tables'], [ 'log_timestamp', - 'log_user', - 'user_name', 'log_namespace', 'log_title', - 'log_comment', 'page_id', 'log_type', 'log_action', 'log_id', 'log_params', 'log_deleted' - ], + ] + $commentQuery['fields'] + $actorQuery['fields'], [ 'log_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), 'log_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), - 'log_user=user_id', // Some logs don't go in RC since they are private. // @FIXME: core/extensions also have spammy logs that don't go in RC. 'log_type' => array_diff( $wgLogTypes, array_keys( $wgLogRestrictions ) ), @@ -298,22 +303,22 @@ class RebuildRecentchanges extends Maintenance { [ 'page' => [ 'LEFT JOIN', [ 'log_namespace=page_namespace', 'log_title=page_title' ] ] - ] + ] + $commentQuery['joins'] + $actorQuery['joins'] ); $field = $dbw->fieldInfo( 'recentchanges', 'rc_cur_id' ); $inserted = 0; + $actorMigration = ActorMigration::newMigration(); foreach ( $res as $row ) { + $comment = $commentStore->getComment( 'log_comment', $row ); + $user = User::newFromAnyId( $row->log_user, $row->log_user_text, $row->log_actor ); $dbw->insert( 'recentchanges', [ 'rc_timestamp' => $row->log_timestamp, - 'rc_user' => $row->log_user, - 'rc_user_text' => $row->user_name, 'rc_namespace' => $row->log_namespace, 'rc_title' => $row->log_title, - 'rc_comment' => $row->log_comment, 'rc_minor' => 0, 'rc_bot' => 0, 'rc_patrolled' => 1, @@ -330,12 +335,13 @@ class RebuildRecentchanges extends Maintenance { 'rc_logid' => $row->log_id, 'rc_params' => $row->log_params, 'rc_deleted' => $row->log_deleted - ], + ] + $commentStore->insert( $dbw, 'rc_comment', $comment ) + + $actorMigration->getInsertValues( $dbw, 'rc_user', $user ), __METHOD__ ); - if ( ( ++$inserted % $this->mBatchSize ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) { + $lbFactory->waitForReplication(); } } } @@ -343,13 +349,12 @@ class RebuildRecentchanges extends Maintenance { /** * Rebuild pass 4: Mark bot and autopatrolled entries. */ - private function rebuildRecentChangesTablePass4() { + private function rebuildRecentChangesTablePass4( ILBFactory $lbFactory ) { global $wgUseRCPatrol, $wgMiserMode; $dbw = $this->getDB( DB_MASTER ); - list( $recentchanges, $usergroups, $user ) = - $dbw->tableNamesN( 'recentchanges', 'user_groups', 'user' ); + $userQuery = User::getQueryInfo(); # @FIXME: recognize other bot account groups (not the same as users with 'bot' rights) # @NOTE: users with 'bot' rights choose when edits are bot edits or not. That information @@ -359,69 +364,91 @@ class RebuildRecentchanges extends Maintenance { # Flag our recent bot edits if ( $botgroups ) { - $botwhere = $dbw->makeList( $botgroups ); - $this->output( "Flagging bot account edits...\n" ); # Find all users that are bots - $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . - "WHERE ug_group IN($botwhere) AND user_id = ug_user"; - $res = $dbw->query( $sql, __METHOD__ ); + $res = $dbw->select( + array_merge( [ 'user_groups' ], $userQuery['tables'] ), + $userQuery['fields'], + [ 'ug_group' => $botgroups ], + __METHOD__, + [ 'DISTINCT' ], + [ 'user_group' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins'] + ); $botusers = []; foreach ( $res as $obj ) { - $botusers[] = $obj->user_name; + $botusers[] = User::newFromRow( $obj ); } # Fill in the rc_bot field if ( $botusers ) { - $rcids = $dbw->selectFieldValues( - 'recentchanges', - 'rc_id', - [ - 'rc_user_text' => $botusers, - "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), - "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ) - ], - __METHOD__ - ); + $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $botusers, false ); + $rcids = []; + foreach ( $actorQuery['orconds'] as $cond ) { + $rcids = array_merge( $rcids, $dbw->selectFieldValues( + [ 'recentchanges' ] + $actorQuery['tables'], + 'rc_id', + [ + "rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + "rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + $cond, + ], + __METHOD__, + [], + $actorQuery['joins'] + ) ); + } + $rcids = array_values( array_unique( $rcids ) ); - foreach ( array_chunk( $rcids, $this->mBatchSize ) as $rcidBatch ) { + foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) { $dbw->update( 'recentchanges', [ 'rc_bot' => 1 ], [ 'rc_id' => $rcidBatch ], __METHOD__ ); - wfGetLBFactory()->waitForReplication(); + $lbFactory->waitForReplication(); } } } # Flag our recent autopatrolled edits if ( !$wgMiserMode && $autopatrolgroups ) { - $patrolwhere = $dbw->makeList( $autopatrolgroups ); $patrolusers = []; $this->output( "Flagging auto-patrolled edits...\n" ); # Find all users in RC with autopatrol rights - $sql = "SELECT DISTINCT user_name FROM $usergroups, $user " . - "WHERE ug_group IN($patrolwhere) AND user_id = ug_user"; - $res = $dbw->query( $sql, __METHOD__ ); + $res = $dbw->select( + array_merge( [ 'user_groups' ], $userQuery['tables'] ), + $userQuery['fields'], + [ 'ug_group' => $autopatrolgroups ], + __METHOD__, + [ 'DISTINCT' ], + [ 'user_group' => [ 'JOIN', 'user_id = ug_user' ] ] + $userQuery['joins'] + ); foreach ( $res as $obj ) { - $patrolusers[] = $dbw->addQuotes( $obj->user_name ); + $patrolusers[] = User::newFromRow( $obj ); } # Fill in the rc_patrolled field if ( $patrolusers ) { - $patrolwhere = implode( ',', $patrolusers ); - $sql2 = "UPDATE $recentchanges SET rc_patrolled=1 " . - "WHERE rc_user_text IN($patrolwhere) " . - "AND rc_timestamp > " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ) . ' ' . - "AND rc_timestamp < " . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ); - $dbw->query( $sql2 ); + $actorQuery = ActorMigration::newMigration()->getWhere( $dbw, 'rc_user', $patrolusers, false ); + foreach ( $actorQuery['orconds'] as $cond ) { + $dbw->update( + 'recentchanges', + [ 'rc_patrolled' => 1 ], + [ + $cond, + 'rc_timestamp > ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffFrom ) ), + 'rc_timestamp < ' . $dbw->addQuotes( $dbw->timestamp( $this->cutoffTo ) ), + ], + __METHOD__ + ); + $lbFactory->waitForReplication(); + } } } } @@ -430,7 +457,7 @@ class RebuildRecentchanges extends Maintenance { * Rebuild pass 5: Delete duplicate entries where we generate both a page revision and a log entry * for a single action (upload only, at the moment, but potentially also move, protect, ...). */ - private function rebuildRecentChangesTablePass5() { + private function rebuildRecentChangesTablePass5( ILBFactory $lbFactory ) { $dbw = wfGetDB( DB_MASTER ); $this->output( "Removing duplicate revision and logging entries...\n" ); @@ -468,8 +495,8 @@ class RebuildRecentchanges extends Maintenance { __METHOD__ ); - if ( ( ++$updates % $this->mBatchSize ) == 0 ) { - wfGetLBFactory()->waitForReplication(); + if ( ( ++$updates % $this->getBatchSize() ) == 0 ) { + $lbFactory->waitForReplication(); } } } @@ -489,5 +516,5 @@ class RebuildRecentchanges extends Maintenance { } } -$maintClass = "RebuildRecentchanges"; +$maintClass = RebuildRecentchanges::class; require_once RUN_MAINTENANCE_IF_MAIN;