From dac20d0ffab5a850563d3fb02331a927e42a2bd9 Mon Sep 17 00:00:00 2001 From: Max Semenik Date: Sun, 5 Nov 2017 01:09:51 -0700 Subject: [PATCH] Introduce Maintenance::getBatchSize() Just to isolate the internals. Fix most of usages in the core. Change-Id: I8b3e9ca1f42b7c49ee57f17b88ca2fc7b404f342 --- maintenance/Maintenance.php | 11 +++++++++++ maintenance/checkImages.php | 2 +- maintenance/checkUsernames.php | 2 +- maintenance/cleanupBlocks.php | 9 +++++---- maintenance/cleanupEmptyCategories.php | 4 ++-- maintenance/cleanupInvalidDbKeys.php | 2 +- maintenance/cleanupRemovedModules.php | 2 +- maintenance/cleanupUploadStash.php | 4 ++-- maintenance/convertUserOptions.php | 2 +- maintenance/copyFileBackend.php | 8 ++++---- maintenance/copyJobQueue.php | 2 +- maintenance/deleteSelfExternals.php | 2 +- maintenance/dumpCategoriesAsRdf.php | 4 ++-- maintenance/findMissingFiles.php | 7 ++++--- maintenance/findOrphanedFiles.php | 2 +- maintenance/fixDefaultJsonContentPages.php | 4 ++-- maintenance/fixUserRegistration.php | 4 ++-- maintenance/initUserPreference.php | 2 +- maintenance/invalidateUserSessions.php | 2 +- maintenance/makeTestEdits.php | 2 +- maintenance/manageJobs.php | 2 +- maintenance/migrateComments.php | 4 ++-- maintenance/migrateFileRepoLayout.php | 9 +++++---- maintenance/migrateUserGroup.php | 9 +++++---- maintenance/populateBacklinkNamespace.php | 9 +++++---- maintenance/populateContentModel.php | 14 ++++++++------ maintenance/populateFilearchiveSha1.php | 2 +- maintenance/populateImageSha1.php | 2 +- maintenance/populateIpChanges.php | 2 +- maintenance/populateLogSearch.php | 9 +++++---- maintenance/populateLogUsertext.php | 9 +++++---- maintenance/populatePPSortKey.php | 2 +- maintenance/populateParentId.php | 7 ++++--- maintenance/purgeChangedFiles.php | 2 +- maintenance/purgeChangedPages.php | 2 +- maintenance/purgeList.php | 2 +- maintenance/purgeModuleDeps.php | 2 +- maintenance/rebuildrecentchanges.php | 8 ++++---- maintenance/recountCategories.php | 6 +++--- maintenance/refreshFileHeaders.php | 13 ++++++++++--- maintenance/removeInvalidEmails.php | 2 +- maintenance/resetUserTokens.php | 4 ++-- maintenance/syncFileBackend.php | 2 +- maintenance/wrapOldPasswords.php | 2 +- 44 files changed, 115 insertions(+), 87 deletions(-) diff --git a/maintenance/Maintenance.php b/maintenance/Maintenance.php index 174b9732dd..d37b990b2d 100644 --- a/maintenance/Maintenance.php +++ b/maintenance/Maintenance.php @@ -307,6 +307,17 @@ abstract class Maintenance { return $this->hasArg( $argId ) ? $this->mArgs[$argId] : $default; } + /** + * Returns batch size + * + * @since 1.31 + * + * @return int|null + */ + protected function getBatchSize() { + return $this->mBatchSize; + } + /** * Set the batch size. * @param int $s The number of operations to do in a batch diff --git a/maintenance/checkImages.php b/maintenance/checkImages.php index c2f0b2713b..6fe15fb60e 100644 --- a/maintenance/checkImages.php +++ b/maintenance/checkImages.php @@ -47,7 +47,7 @@ class CheckImages extends Maintenance { do { $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name > ' . $dbr->addQuotes( $start ) ], - __METHOD__, [ 'LIMIT' => $this->mBatchSize ], $fileQuery['joins'] ); + __METHOD__, [ 'LIMIT' => $this->getBatchSize() ], $fileQuery['joins'] ); foreach ( $res as $row ) { $numImages++; $start = $row->img_name; diff --git a/maintenance/checkUsernames.php b/maintenance/checkUsernames.php index e6d95477c3..f29f8367c8 100644 --- a/maintenance/checkUsernames.php +++ b/maintenance/checkUsernames.php @@ -50,7 +50,7 @@ class CheckUsernames extends Maintenance { __METHOD__, [ 'ORDER BY' => 'user_id', - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ] ); diff --git a/maintenance/cleanupBlocks.php b/maintenance/cleanupBlocks.php index 37417c73d1..5c5c88f0b0 100644 --- a/maintenance/cleanupBlocks.php +++ b/maintenance/cleanupBlocks.php @@ -44,8 +44,9 @@ class CleanupBlocks extends Maintenance { $max = $db->selectField( 'ipblocks', 'MAX(ipb_user)' ); // Step 1: Clean up any duplicate user blocks - for ( $from = 1; $from <= $max; $from += $this->mBatchSize ) { - $to = min( $max, $from + $this->mBatchSize - 1 ); + $batchSize = $this->getBatchSize(); + for ( $from = 1; $from <= $max; $from += $batchSize ) { + $to = min( $max, $from + $batchSize - 1 ); $this->output( "Cleaning up duplicate ipb_user ($from-$to of $max)\n" ); $delete = []; @@ -118,8 +119,8 @@ class CleanupBlocks extends Maintenance { } // Step 2: Update the user name in any blocks where it doesn't match - for ( $from = 1; $from <= $max; $from += $this->mBatchSize ) { - $to = min( $max, $from + $this->mBatchSize - 1 ); + for ( $from = 1; $from <= $max; $from += $batchSize ) { + $to = min( $max, $from + $batchSize - 1 ); $this->output( "Cleaning up mismatched user name ($from-$to of $max)\n" ); $res = $db->select( diff --git a/maintenance/cleanupEmptyCategories.php b/maintenance/cleanupEmptyCategories.php index 8672223819..2d22704de1 100644 --- a/maintenance/cleanupEmptyCategories.php +++ b/maintenance/cleanupEmptyCategories.php @@ -109,7 +109,7 @@ TEXT __METHOD__, [ 'ORDER BY' => 'page_title', - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ], [ 'category' => [ 'LEFT JOIN', 'page_title = cat_title' ], @@ -161,7 +161,7 @@ TEXT __METHOD__, [ 'ORDER BY' => 'cat_title', - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ], [ 'page' => [ 'LEFT JOIN', [ diff --git a/maintenance/cleanupInvalidDbKeys.php b/maintenance/cleanupInvalidDbKeys.php index b487f896ca..569fd2bbd6 100644 --- a/maintenance/cleanupInvalidDbKeys.php +++ b/maintenance/cleanupInvalidDbKeys.php @@ -161,7 +161,7 @@ TEXT $titleField . $dbr->buildLike( $percent, '_' ), ], LIST_OR ) ], __METHOD__, - [ 'LIMIT' => $this->mBatchSize ] + [ 'LIMIT' => $this->getBatchSize() ] ); $this->outputStatus( "Number of invalid rows: " . $res->numRows() . "\n" ); diff --git a/maintenance/cleanupRemovedModules.php b/maintenance/cleanupRemovedModules.php index dbaf6438c2..a6f10c6b39 100644 --- a/maintenance/cleanupRemovedModules.php +++ b/maintenance/cleanupRemovedModules.php @@ -57,7 +57,7 @@ class CleanupRemovedModules extends Maintenance { $modDeps = $dbw->tableName( 'module_deps' ); $i = 1; - foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) { + foreach ( array_chunk( $rows, $this->getBatchSize() ) as $chunk ) { // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) .. $conds = array_map( function ( stdClass $row ) use ( $dbw ) { return $dbw->makeList( (array)$row, IDatabase::LIST_AND ); diff --git a/maintenance/cleanupUploadStash.php b/maintenance/cleanupUploadStash.php index 95bbe3d3ff..14c6a6b18a 100644 --- a/maintenance/cleanupUploadStash.php +++ b/maintenance/cleanupUploadStash.php @@ -103,7 +103,7 @@ class UploadStashCleanup extends Maintenance { foreach ( $iterator as $file ) { if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; - if ( count( $batch ) >= $this->mBatchSize ) { + if ( count( $batch ) >= $this->getBatchSize() ) { $this->doOperations( $tempRepo, $batch ); $i += count( $batch ); $batch = []; @@ -129,7 +129,7 @@ class UploadStashCleanup extends Maintenance { foreach ( $iterator as $file ) { if ( wfTimestamp( TS_UNIX, $tempRepo->getFileTimestamp( "$dir/$file" ) ) < $cutoff ) { $batch[] = [ 'op' => 'delete', 'src' => "$dir/$file" ]; - if ( count( $batch ) >= $this->mBatchSize ) { + if ( count( $batch ) >= $this->getBatchSize() ) { $this->doOperations( $tempRepo, $batch ); $i += count( $batch ); $batch = []; diff --git a/maintenance/convertUserOptions.php b/maintenance/convertUserOptions.php index 675d0695d9..501f045607 100644 --- a/maintenance/convertUserOptions.php +++ b/maintenance/convertUserOptions.php @@ -61,7 +61,7 @@ class ConvertUserOptions extends Maintenance { __METHOD__, [ 'ORDER BY' => 'user_id', - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ] ); $id = $this->convertOptionBatch( $res, $dbw ); diff --git a/maintenance/copyFileBackend.php b/maintenance/copyFileBackend.php index e2e6c6c366..ee103b8848 100644 --- a/maintenance/copyFileBackend.php +++ b/maintenance/copyFileBackend.php @@ -110,10 +110,10 @@ class CopyFileBackend extends Maintenance { // Check up on the rate file periodically to adjust the concurrency if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { $this->setBatchSize( max( 1, (int)file_get_contents( $rateFile ) ) ); - $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); + $this->output( "\tBatch size is now {$this->getBatchSize()}.\n" ); } $batchPaths[$srcPathRel] = 1; // remove duplicates - if ( count( $batchPaths ) >= $this->mBatchSize ) { + if ( count( $batchPaths ) >= $this->getBatchSize() ) { $this->copyFileBatch( array_keys( $batchPaths ), $backendRel, $src, $dst ); $batchPaths = []; // done } @@ -137,10 +137,10 @@ class CopyFileBackend extends Maintenance { // Check up on the rate file periodically to adjust the concurrency if ( $rateFile && ( !$count || ( $count % 500 ) == 0 ) ) { $this->setBatchSize( max( 1, (int)file_get_contents( $rateFile ) ) ); - $this->output( "\tBatch size is now {$this->mBatchSize}.\n" ); + $this->output( "\tBatch size is now {$this->getBatchSize()}.\n" ); } $batchPaths[$delPathRel] = 1; // remove duplicates - if ( count( $batchPaths ) >= $this->mBatchSize ) { + if ( count( $batchPaths ) >= $this->getBatchSize() ) { $this->delFileBatch( array_keys( $batchPaths ), $backendRel, $dst ); $batchPaths = []; // done } diff --git a/maintenance/copyJobQueue.php b/maintenance/copyJobQueue.php index e1d697d88d..08e40fd648 100644 --- a/maintenance/copyJobQueue.php +++ b/maintenance/copyJobQueue.php @@ -77,7 +77,7 @@ class CopyJobQueue extends Maintenance { foreach ( $jobs as $job ) { ++$total; $batch[] = $job; - if ( count( $batch ) >= $this->mBatchSize ) { + if ( count( $batch ) >= $this->getBatchSize() ) { $dst->push( $batch ); $totalOK += count( $batch ); $batch = []; diff --git a/maintenance/deleteSelfExternals.php b/maintenance/deleteSelfExternals.php index f76325bc32..ab257846bd 100644 --- a/maintenance/deleteSelfExternals.php +++ b/maintenance/deleteSelfExternals.php @@ -44,7 +44,7 @@ class DeleteSelfExternals extends Maintenance { wfWaitForSlaves(); $this->commitTransaction( $db, __METHOD__ ); $q = $db->limitResult( "DELETE /* deleteSelfExternals */ FROM externallinks WHERE el_to" - . $db->buildLike( $wgServer . '/', $db->anyString() ), $this->mBatchSize ); + . $db->buildLike( $wgServer . '/', $db->anyString() ), $this->getBatchSize() ); $this->output( "Deleting a batch\n" ); $db->query( $q ); if ( !$db->affectedRows() ) { diff --git a/maintenance/dumpCategoriesAsRdf.php b/maintenance/dumpCategoriesAsRdf.php index ff50498f0a..282a04b0c3 100644 --- a/maintenance/dumpCategoriesAsRdf.php +++ b/maintenance/dumpCategoriesAsRdf.php @@ -60,7 +60,7 @@ class DumpCategoriesAsRdf extends Maintenance { $dbr, 'page', [ 'page_title' ], - $this->mBatchSize + $this->getBatchSize() ); $it->addConditions( [ 'page_namespace' => NS_CATEGORY, @@ -80,7 +80,7 @@ class DumpCategoriesAsRdf extends Maintenance { $dbr, 'categorylinks', [ 'cl_from', 'cl_to' ], - $this->mBatchSize + $this->getBatchSize() ); $it->addConditions( [ 'cl_type' => 'subcat', diff --git a/maintenance/findMissingFiles.php b/maintenance/findMissingFiles.php index 4ce7ca68ae..8bf2bdb457 100644 --- a/maintenance/findMissingFiles.php +++ b/maintenance/findMissingFiles.php @@ -37,6 +37,7 @@ class FindMissingFiles extends Maintenance { $repo = RepoGroup::singleton()->getLocalRepo(); $dbr = $repo->getReplicaDB(); $be = $repo->getBackend(); + $batchSize = $this->getBatchSize(); $mtime1 = $dbr->timestampOrNull( $this->getOption( 'mtimeafter', null ) ); $mtime2 = $dbr->timestampOrNull( $this->getOption( 'mtimebefore', null ) ); @@ -66,7 +67,7 @@ class FindMissingFiles extends Maintenance { __METHOD__, // DISTINCT causes a pointless filesort [ 'ORDER BY' => 'name', 'GROUP BY' => 'name', - 'LIMIT' => $this->mBatchSize ], + 'LIMIT' => $batchSize ], $joinConds ); @@ -101,7 +102,7 @@ class FindMissingFiles extends Maintenance { $checkPaths[] = $file->getPath(); } - foreach ( array_chunk( $checkPaths, $this->mBatchSize ) as $paths ) { + foreach ( array_chunk( $checkPaths, $batchSize ) as $paths ) { $be->preloadFileStat( [ 'srcs' => $paths ] ); foreach ( $paths as $path ) { if ( $be->fileExists( [ 'src' => $path ] ) === false ) { @@ -110,7 +111,7 @@ class FindMissingFiles extends Maintenance { } } } - } while ( $res->numRows() >= $this->mBatchSize ); + } while ( $res->numRows() >= $batchSize ); } } diff --git a/maintenance/findOrphanedFiles.php b/maintenance/findOrphanedFiles.php index 765fbe4a0a..c4cab71688 100644 --- a/maintenance/findOrphanedFiles.php +++ b/maintenance/findOrphanedFiles.php @@ -61,7 +61,7 @@ class FindOrphanedFiles extends Maintenance { } $pathBatch[] = $path; - if ( count( $pathBatch ) >= $this->mBatchSize ) { + if ( count( $pathBatch ) >= $this->getBatchSize() ) { $this->checkFiles( $repo, $pathBatch, $verbose ); $pathBatch = []; } diff --git a/maintenance/fixDefaultJsonContentPages.php b/maintenance/fixDefaultJsonContentPages.php index 460b553455..7262770b6f 100644 --- a/maintenance/fixDefaultJsonContentPages.php +++ b/maintenance/fixDefaultJsonContentPages.php @@ -64,12 +64,12 @@ class FixDefaultJsonContentPages extends LoggedUpdateMaintenance { 'page_id > ' . $dbr->addQuotes( $lastPage ) ], __METHOD__, - [ 'ORDER BY' => 'page_id', 'LIMIT' => $this->mBatchSize ] + [ 'ORDER BY' => 'page_id', 'LIMIT' => $this->getBatchSize() ] ); foreach ( $rows as $row ) { $this->handleRow( $row ); } - } while ( $rows->numRows() >= $this->mBatchSize ); + } while ( $rows->numRows() >= $this->getBatchSize() ); } return true; diff --git a/maintenance/fixUserRegistration.php b/maintenance/fixUserRegistration.php index 37fd44fb2e..30364c655b 100644 --- a/maintenance/fixUserRegistration.php +++ b/maintenance/fixUserRegistration.php @@ -51,7 +51,7 @@ class FixUserRegistration extends Maintenance { ], __METHOD__, [ - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), 'ORDER BY' => 'user_id', ] ); @@ -83,7 +83,7 @@ class FixUserRegistration extends Maintenance { $this->output( "Waiting for replica DBs..." ); wfWaitForSlaves(); $this->output( " done.\n" ); - } while ( $res->numRows() >= $this->mBatchSize ); + } while ( $res->numRows() >= $this->getBatchSize() ); } } diff --git a/maintenance/initUserPreference.php b/maintenance/initUserPreference.php index f4da570fbb..7b7cf15a76 100644 --- a/maintenance/initUserPreference.php +++ b/maintenance/initUserPreference.php @@ -46,7 +46,7 @@ class InitUserPreference extends Maintenance { $dbr, 'user_properties', [ 'up_user', 'up_property' ], - $this->mBatchSize + $this->getBatchSize() ); $iterator->setFetchColumns( [ 'up_user', 'up_value' ] ); $iterator->addConditions( [ diff --git a/maintenance/invalidateUserSessions.php b/maintenance/invalidateUserSessions.php index 11e3372ce6..8f67acddb9 100644 --- a/maintenance/invalidateUserSessions.php +++ b/maintenance/invalidateUserSessions.php @@ -83,7 +83,7 @@ class InvalidateUserSesssions extends Maintenance { . str_replace( [ "\r", "\n" ], ' ', $e->getMessage() ) . "\n" ); } - if ( $i % $this->mBatchSize ) { + if ( $i % $this->getBatchSize() ) { $lbFactory->waitForReplication(); } } diff --git a/maintenance/makeTestEdits.php b/maintenance/makeTestEdits.php index ca2f7c51c9..1effb61d4e 100644 --- a/maintenance/makeTestEdits.php +++ b/maintenance/makeTestEdits.php @@ -55,7 +55,7 @@ class MakeTestEdits extends Maintenance { $page->doEditContent( $content, $summary, 0, false, $user ); $this->output( "Edited $title\n" ); - if ( $i && ( $i % $this->mBatchSize ) == 0 ) { + if ( $i && ( $i % $this->getBatchSize() ) == 0 ) { wfWaitForSlaves(); } } diff --git a/maintenance/manageJobs.php b/maintenance/manageJobs.php index 32333b768b..5f39a3d5ce 100644 --- a/maintenance/manageJobs.php +++ b/maintenance/manageJobs.php @@ -82,7 +82,7 @@ class ManageJobs extends Maintenance { $queue->push( $job ); ++$count; - if ( ( $count % $this->mBatchSize ) == 0 ) { + if ( ( $count % $this->getBatchSize() ) == 0 ) { $queue->waitForBackups(); } } diff --git a/maintenance/migrateComments.php b/maintenance/migrateComments.php index 9d4e2615c0..01ee9f84fd 100644 --- a/maintenance/migrateComments.php +++ b/maintenance/migrateComments.php @@ -161,7 +161,7 @@ class MigrateComments extends LoggedUpdateMaintenance { __METHOD__, [ 'ORDER BY' => $primaryKey, - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ] ); if ( !$res->numRows() ) { @@ -245,7 +245,7 @@ class MigrateComments extends LoggedUpdateMaintenance { __METHOD__, [ 'ORDER BY' => $primaryKey, - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), ], [ $newTable => [ 'LEFT JOIN', "{$primaryKey}={$newPrimaryKey}" ] ] ); diff --git a/maintenance/migrateFileRepoLayout.php b/maintenance/migrateFileRepoLayout.php index f771fff73d..b2cce3ea58 100644 --- a/maintenance/migrateFileRepoLayout.php +++ b/maintenance/migrateFileRepoLayout.php @@ -69,6 +69,7 @@ class MigrateFileRepoLayout extends Maintenance { $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) ); } + $batchSize = $this->getBatchSize(); $batch = []; $lastName = ''; do { @@ -76,7 +77,7 @@ class MigrateFileRepoLayout extends Maintenance { [ 'img_name', 'img_sha1' ], array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ), __METHOD__, - [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'img_name' ] + [ 'LIMIT' => $batchSize, 'ORDER BY' => 'img_name' ] ); foreach ( $res as $row ) { @@ -143,7 +144,7 @@ class MigrateFileRepoLayout extends Maintenance { 'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ]; } - if ( count( $batch ) >= $this->mBatchSize ) { + if ( count( $batch ) >= $batchSize ) { $this->runBatch( $batch, $be ); $batch = []; } @@ -166,7 +167,7 @@ class MigrateFileRepoLayout extends Maintenance { $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ], array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ), __METHOD__, - [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'fa_id' ] + [ 'LIMIT' => $batchSize, 'ORDER BY' => 'fa_id' ] ); foreach ( $res as $row ) { @@ -201,7 +202,7 @@ class MigrateFileRepoLayout extends Maintenance { $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath, 'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ]; - if ( count( $batch ) >= $this->mBatchSize ) { + if ( count( $batch ) >= $batchSize ) { $this->runBatch( $batch, $be ); $batch = []; } diff --git a/maintenance/migrateUserGroup.php b/maintenance/migrateUserGroup.php index 597a876df9..ad82542fb5 100644 --- a/maintenance/migrateUserGroup.php +++ b/maintenance/migrateUserGroup.php @@ -42,6 +42,7 @@ class MigrateUserGroup extends Maintenance { $oldGroup = $this->getArg( 0 ); $newGroup = $this->getArg( 1 ); $dbw = $this->getDB( DB_MASTER ); + $batchSize = $this->getBatchSize(); $start = $dbw->selectField( 'user_groups', 'MIN(ug_user)', [ 'ug_group' => $oldGroup ], __FUNCTION__ ); $end = $dbw->selectField( 'user_groups', 'MAX(ug_user)', @@ -50,9 +51,9 @@ class MigrateUserGroup extends Maintenance { $this->error( "Nothing to do - no users in the '$oldGroup' group", true ); } # Do remaining chunk - $end += $this->mBatchSize - 1; + $end += $batchSize - 1; $blockStart = $start; - $blockEnd = $start + $this->mBatchSize - 1; + $blockEnd = $start + $batchSize - 1; // Migrate users over in batches... while ( $blockEnd <= $end ) { $affected = 0; @@ -97,8 +98,8 @@ class MigrateUserGroup extends Maintenance { } $count += $affected; - $blockStart += $this->mBatchSize; - $blockEnd += $this->mBatchSize; + $blockStart += $batchSize; + $blockEnd += $batchSize; wfWaitForSlaves(); } $this->output( "Done! $count users in group '$oldGroup' are now in '$newGroup' instead.\n" ); diff --git a/maintenance/populateBacklinkNamespace.php b/maintenance/populateBacklinkNamespace.php index 295dacda99..eb8280613f 100644 --- a/maintenance/populateBacklinkNamespace.php +++ b/maintenance/populateBacklinkNamespace.php @@ -59,11 +59,12 @@ class PopulateBacklinkNamespace extends LoggedUpdateMaintenance { return false; } $end = $db->selectField( 'page', 'MAX(page_id)', false, __METHOD__ ); + $batchSize = $this->getBatchSize(); # Do remaining chunk - $end += $this->mBatchSize - 1; + $end += $batchSize - 1; $blockStart = $start; - $blockEnd = $start + $this->mBatchSize - 1; + $blockEnd = $start + $batchSize - 1; while ( $blockEnd <= $end ) { $this->output( "...doing page_id from $blockStart to $blockEnd\n" ); $cond = "page_id BETWEEN $blockStart AND $blockEnd"; @@ -85,8 +86,8 @@ class PopulateBacklinkNamespace extends LoggedUpdateMaintenance { __METHOD__ ); } - $blockStart += $this->mBatchSize - 1; - $blockEnd += $this->mBatchSize - 1; + $blockStart += $batchSize - 1; + $blockEnd += $batchSize - 1; wfWaitForSlaves(); } return true; diff --git a/maintenance/populateContentModel.php b/maintenance/populateContentModel.php index d99f70a10e..7cc829d99e 100644 --- a/maintenance/populateContentModel.php +++ b/maintenance/populateContentModel.php @@ -97,6 +97,7 @@ class PopulateContentModel extends Maintenance { $toSave = []; $lastId = 0; $nsCondition = $ns === 'all' ? [] : [ 'page_namespace' => $ns ]; + $batchSize = $this->getBatchSize(); do { $rows = $dbw->select( 'page', @@ -106,20 +107,20 @@ class PopulateContentModel extends Maintenance { 'page_id > ' . $dbw->addQuotes( $lastId ), ] + $nsCondition, __METHOD__, - [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'page_id ASC' ] + [ 'LIMIT' => $batchSize, 'ORDER BY' => 'page_id ASC' ] ); $this->output( "Fetched {$rows->numRows()} rows.\n" ); foreach ( $rows as $row ) { $title = Title::newFromRow( $row ); $model = ContentHandler::getDefaultModelFor( $title ); $toSave[$model][] = $row->page_id; - if ( count( $toSave[$model] ) >= $this->mBatchSize ) { + if ( count( $toSave[$model] ) >= $batchSize ) { $this->updatePageRows( $dbw, $toSave[$model], $model ); unset( $toSave[$model] ); } $lastId = $row->page_id; } - } while ( $rows->numRows() >= $this->mBatchSize ); + } while ( $rows->numRows() >= $batchSize ); foreach ( $toSave as $model => $pages ) { $this->updatePageRows( $dbw, $pages, $model ); } @@ -168,6 +169,7 @@ class PopulateContentModel extends Maintenance { $toSave = []; $idsToClear = []; $lastId = 0; + $batchSize = $this->getBatchSize(); do { $rows = $dbw->select( $selectTables, @@ -181,7 +183,7 @@ class PopulateContentModel extends Maintenance { "$key > " . $dbw->addQuotes( $lastId ), ] + $where, __METHOD__, - [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => "$key ASC" ], + [ 'LIMIT' => $batchSize, 'ORDER BY' => "$key ASC" ], $join_conds ); $this->output( "Fetched {$rows->numRows()} rows.\n" ); @@ -232,12 +234,12 @@ class PopulateContentModel extends Maintenance { } } - if ( count( $toSave[$defaultModel] ) >= $this->mBatchSize ) { + if ( count( $toSave[$defaultModel] ) >= $batchSize ) { $this->updateRevisionOrArchiveRows( $dbw, $toSave[$defaultModel], $defaultModel, $table ); unset( $toSave[$defaultModel] ); } } - } while ( $rows->numRows() >= $this->mBatchSize ); + } while ( $rows->numRows() >= $batchSize ); foreach ( $toSave as $model => $ids ) { $this->updateRevisionOrArchiveRows( $dbw, $ids, $model, $table ); } diff --git a/maintenance/populateFilearchiveSha1.php b/maintenance/populateFilearchiveSha1.php index 7557a42ff4..cbd969db06 100644 --- a/maintenance/populateFilearchiveSha1.php +++ b/maintenance/populateFilearchiveSha1.php @@ -58,7 +58,7 @@ class PopulateFilearchiveSha1 extends LoggedUpdateMaintenance { $this->output( "Populating fa_sha1 field from fa_storage_key\n" ); $endId = $dbw->selectField( $table, 'MAX(fa_id)', false, __METHOD__ ); - $batchSize = $this->mBatchSize; + $batchSize = $this->getBatchSize(); $done = 0; do { diff --git a/maintenance/populateImageSha1.php b/maintenance/populateImageSha1.php index b581d6614c..2735a1ed8d 100644 --- a/maintenance/populateImageSha1.php +++ b/maintenance/populateImageSha1.php @@ -119,7 +119,7 @@ class PopulateImageSha1 extends LoggedUpdateMaintenance { $numRows = $res->numRows(); $i = 0; foreach ( $res as $row ) { - if ( $i % $this->mBatchSize == 0 ) { + if ( $i % $this->getBatchSize() == 0 ) { $this->output( sprintf( "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) ); wfWaitForSlaves(); diff --git a/maintenance/populateIpChanges.php b/maintenance/populateIpChanges.php index 40a596c8f2..178c49a73e 100644 --- a/maintenance/populateIpChanges.php +++ b/maintenance/populateIpChanges.php @@ -78,7 +78,7 @@ TEXT $this->output( "Copying IP revisions to ip_changes, from rev_id $start to rev_id $end\n" ); while ( $blockStart <= $end ) { - $blockEnd = min( $blockStart + $this->mBatchSize, $end ); + $blockEnd = min( $blockStart + $this->getBatchSize(), $end ); $rows = $dbr->select( 'revision', [ 'rev_id', 'rev_timestamp', 'rev_user_text' ], diff --git a/maintenance/populateLogSearch.php b/maintenance/populateLogSearch.php index e75b84d530..113cc63950 100644 --- a/maintenance/populateLogSearch.php +++ b/maintenance/populateLogSearch.php @@ -53,6 +53,7 @@ class PopulateLogSearch extends LoggedUpdateMaintenance { } protected function doDBUpdates() { + $batchSize = $this->getBatchSize(); $db = $this->getDB( DB_MASTER ); if ( !$db->tableExists( 'log_search' ) ) { $this->error( "log_search does not exist" ); @@ -68,9 +69,9 @@ class PopulateLogSearch extends LoggedUpdateMaintenance { $end = $db->selectField( 'logging', 'MAX(log_id)', false, __FUNCTION__ ); # Do remaining chunk - $end += $this->mBatchSize - 1; + $end += $batchSize - 1; $blockStart = $start; - $blockEnd = $start + $this->mBatchSize - 1; + $blockEnd = $start + $batchSize - 1; $delTypes = [ 'delete', 'suppress' ]; // revisiondelete types while ( $blockEnd <= $end ) { @@ -158,8 +159,8 @@ class PopulateLogSearch extends LoggedUpdateMaintenance { $log->addRelations( 'target_author_ip', $userIPs, $row->log_id ); } } - $blockStart += $this->mBatchSize; - $blockEnd += $this->mBatchSize; + $blockStart += $batchSize; + $blockEnd += $batchSize; wfWaitForSlaves(); } $this->output( "Done populating log_search table.\n" ); diff --git a/maintenance/populateLogUsertext.php b/maintenance/populateLogUsertext.php index dd120fe003..c5c079ac0a 100644 --- a/maintenance/populateLogUsertext.php +++ b/maintenance/populateLogUsertext.php @@ -48,6 +48,7 @@ class PopulateLogUsertext extends LoggedUpdateMaintenance { } protected function doDBUpdates() { + $batchSize = $this->getBatchSize(); $db = $this->getDB( DB_MASTER ); $start = $db->selectField( 'logging', 'MIN(log_id)', false, __METHOD__ ); if ( !$start ) { @@ -58,9 +59,9 @@ class PopulateLogUsertext extends LoggedUpdateMaintenance { $end = $db->selectField( 'logging', 'MAX(log_id)', false, __METHOD__ ); # Do remaining chunk - $end += $this->mBatchSize - 1; + $end += $batchSize - 1; $blockStart = $start; - $blockEnd = $start + $this->mBatchSize - 1; + $blockEnd = $start + $batchSize - 1; while ( $blockEnd <= $end ) { $this->output( "...doing log_id from $blockStart to $blockEnd\n" ); $cond = "log_id BETWEEN $blockStart AND $blockEnd AND log_user = user_id"; @@ -73,8 +74,8 @@ class PopulateLogUsertext extends LoggedUpdateMaintenance { [ 'log_id' => $row->log_id ], __METHOD__ ); } $this->commitTransaction( $db, __METHOD__ ); - $blockStart += $this->mBatchSize; - $blockEnd += $this->mBatchSize; + $blockStart += $batchSize; + $blockEnd += $batchSize; wfWaitForSlaves(); } $this->output( "Done populating log_user_text field.\n" ); diff --git a/maintenance/populatePPSortKey.php b/maintenance/populatePPSortKey.php index 366905a6a9..98d535aa29 100644 --- a/maintenance/populatePPSortKey.php +++ b/maintenance/populatePPSortKey.php @@ -57,7 +57,7 @@ class PopulatePPSortKey extends LoggedUpdateMaintenance { __METHOD__, [ 'ORDER BY' => 'pp_page, pp_propname', - 'LIMIT' => $this->mBatchSize + 'LIMIT' => $this->getBatchSize() ] ); diff --git a/maintenance/populateParentId.php b/maintenance/populateParentId.php index 457033a4fa..99c2b23014 100644 --- a/maintenance/populateParentId.php +++ b/maintenance/populateParentId.php @@ -46,6 +46,7 @@ class PopulateParentId extends LoggedUpdateMaintenance { } protected function doDBUpdates() { + $batchSize = $this->getBatchSize(); $db = $this->getDB( DB_MASTER ); if ( !$db->tableExists( 'revision' ) ) { $this->error( "revision table does not exist" ); @@ -62,7 +63,7 @@ class PopulateParentId extends LoggedUpdateMaintenance { } # Do remaining chunk $blockStart = intval( $start ); - $blockEnd = intval( $start ) + $this->mBatchSize - 1; + $blockEnd = intval( $start ) + $batchSize - 1; $count = 0; $changed = 0; while ( $blockStart <= $end ) { @@ -116,8 +117,8 @@ class PopulateParentId extends LoggedUpdateMaintenance { __METHOD__ ); $count++; } - $blockStart += $this->mBatchSize; - $blockEnd += $this->mBatchSize; + $blockStart += $batchSize; + $blockEnd += $batchSize; wfWaitForSlaves(); } $this->output( "rev_parent_id population complete ... {$count} rows [{$changed} changed]\n" ); diff --git a/maintenance/purgeChangedFiles.php b/maintenance/purgeChangedFiles.php index 3c0fc7e572..98054523bc 100644 --- a/maintenance/purgeChangedFiles.php +++ b/maintenance/purgeChangedFiles.php @@ -200,7 +200,7 @@ class PurgeChangedFiles extends Maintenance { $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" ); - if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) { + if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->getBatchSize() ) { $bSize = 0; // sleep-per-batch is milliseconds, usleep wants micro seconds. usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) ); diff --git a/maintenance/purgeChangedPages.php b/maintenance/purgeChangedPages.php index cf65c69315..cce9b036a5 100644 --- a/maintenance/purgeChangedPages.php +++ b/maintenance/purgeChangedPages.php @@ -79,7 +79,7 @@ class PurgeChangedPages extends Maintenance { $stuckCount = 0; // loop breaker while ( true ) { // Adjust bach size if we are stuck in a second that had many changes - $bSize = $this->mBatchSize + ( $stuckCount * $this->mBatchSize ); + $bSize = ( $stuckCount + 1 ) * $this->getBatchSize(); $res = $dbr->select( [ 'page', 'revision' ], diff --git a/maintenance/purgeList.php b/maintenance/purgeList.php index 5ca7918ea9..8cf217f95f 100644 --- a/maintenance/purgeList.php +++ b/maintenance/purgeList.php @@ -99,7 +99,7 @@ class PurgeList extends Maintenance { $conds + [ 'page_id > ' . $dbr->addQuotes( $startId ) ], __METHOD__, [ - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), 'ORDER BY' => 'page_id' ] diff --git a/maintenance/purgeModuleDeps.php b/maintenance/purgeModuleDeps.php index feeeb65b65..591f7ee744 100644 --- a/maintenance/purgeModuleDeps.php +++ b/maintenance/purgeModuleDeps.php @@ -48,7 +48,7 @@ class PurgeModuleDeps extends Maintenance { $modDeps = $dbw->tableName( 'module_deps' ); $i = 1; - foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) { + foreach ( array_chunk( $rows, $this->getBatchSize() ) as $chunk ) { // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) .. $conds = array_map( function ( stdClass $row ) use ( $dbw ) { return $dbw->makeList( (array)$row, IDatabase::LIST_AND ); diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php index a2cf3c5bca..8003ee0a60 100644 --- a/maintenance/rebuildrecentchanges.php +++ b/maintenance/rebuildrecentchanges.php @@ -256,7 +256,7 @@ class RebuildRecentchanges extends Maintenance { $lastOldId = intval( $obj->rc_this_oldid ); $lastSize = $size; - if ( ( ++$updated % $this->mBatchSize ) == 0 ) { + if ( ( ++$updated % $this->getBatchSize() ) == 0 ) { wfGetLBFactory()->waitForReplication(); } } @@ -340,7 +340,7 @@ class RebuildRecentchanges extends Maintenance { __METHOD__ ); - if ( ( ++$inserted % $this->mBatchSize ) == 0 ) { + if ( ( ++$inserted % $this->getBatchSize() ) == 0 ) { wfGetLBFactory()->waitForReplication(); } } @@ -392,7 +392,7 @@ class RebuildRecentchanges extends Maintenance { __METHOD__ ); - foreach ( array_chunk( $rcids, $this->mBatchSize ) as $rcidBatch ) { + foreach ( array_chunk( $rcids, $this->getBatchSize() ) as $rcidBatch ) { $dbw->update( 'recentchanges', [ 'rc_bot' => 1 ], @@ -474,7 +474,7 @@ class RebuildRecentchanges extends Maintenance { __METHOD__ ); - if ( ( ++$updates % $this->mBatchSize ) == 0 ) { + if ( ( ++$updates % $this->getBatchSize() ) == 0 ) { wfGetLBFactory()->waitForReplication(); } } diff --git a/maintenance/recountCategories.php b/maintenance/recountCategories.php index a4bfa98992..b4d75c7a1a 100644 --- a/maintenance/recountCategories.php +++ b/maintenance/recountCategories.php @@ -97,8 +97,8 @@ TEXT } protected function doWork() { - $this->output( "Finding up to {$this->mBatchSize} drifted rows " . - "starting at cat_id {$this->minimumId}...\n" ); + $this->output( "Finding up to {$this->getBatchSize()} drifted rows " . + "starting at cat_id {$this->getBatchSize()}...\n" ); $countingConds = [ 'cl_to = cat_title' ]; if ( $this->mode === 'subcats' ) { @@ -124,7 +124,7 @@ TEXT "cat_{$this->mode} != ($countingSubquery)" ], __METHOD__, - [ 'LIMIT' => $this->mBatchSize ] + [ 'LIMIT' => $this->getBatchSize() ] ); if ( !$idsToUpdate ) { return false; diff --git a/maintenance/refreshFileHeaders.php b/maintenance/refreshFileHeaders.php index e123de7fdb..bd625ba129 100644 --- a/maintenance/refreshFileHeaders.php +++ b/maintenance/refreshFileHeaders.php @@ -78,8 +78,15 @@ class RefreshFileHeaders extends Maintenance { $conds[] = "img_minor_mime = {$dbr->addQuotes( $minor_mime )}"; } - $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], $conds, - __METHOD__, [ 'LIMIT' => $this->mBatchSize, 'ORDER BY' => 'img_name ASC' ], $fileQuery['joins'] + $res = $dbr->select( $fileQuery['tables'], + $fileQuery['fields'], + $conds, + __METHOD__, + [ + 'LIMIT' => $this->getBatchSize(), + 'ORDER BY' => 'img_name ASC' + ], + $fileQuery['joins'] ); if ( $res->numRows() > 0 ) { @@ -122,7 +129,7 @@ class RefreshFileHeaders extends Maintenance { $this->output( "Updating headers for {$backendOperationsCount} file(s).\n" ); $this->updateFileHeaders( $repo, $backendOperations ); - } while ( $res->numRows() === $this->mBatchSize ); + } while ( $res->numRows() === $this->getBatchSize() ); $this->output( "Done. Updated headers for $count file(s).\n" ); } diff --git a/maintenance/removeInvalidEmails.php b/maintenance/removeInvalidEmails.php index 1034005aa0..91025bf109 100644 --- a/maintenance/removeInvalidEmails.php +++ b/maintenance/removeInvalidEmails.php @@ -36,7 +36,7 @@ class RemoveInvalidEmails extends Maintenance { 'user_email_authenticated IS NULL' ], __METHOD__, - [ 'LIMIT' => $this->mBatchSize ] + [ 'LIMIT' => $this->getBatchSize() ] ); $count = $rows->numRows(); $badIds = []; diff --git a/maintenance/resetUserTokens.php b/maintenance/resetUserTokens.php index 1c8b4b9d0b..710198b870 100644 --- a/maintenance/resetUserTokens.php +++ b/maintenance/resetUserTokens.php @@ -80,7 +80,7 @@ class ResetUserTokens extends Maintenance { $maxid = $dbr->selectField( 'user', 'MAX(user_id)', [], __METHOD__ ); $min = 0; - $max = $this->mBatchSize; + $max = $this->getBatchSize(); do { $result = $dbr->select( 'user', @@ -99,7 +99,7 @@ class ResetUserTokens extends Maintenance { } $min = $max; - $max = $min + $this->mBatchSize; + $max = $min + $this->getBatchSize(); wfWaitForSlaves(); } while ( $min <= $maxid ); diff --git a/maintenance/syncFileBackend.php b/maintenance/syncFileBackend.php index 82149a6ddd..154f54e53f 100644 --- a/maintenance/syncFileBackend.php +++ b/maintenance/syncFileBackend.php @@ -161,7 +161,7 @@ class SyncFileBackend extends Maintenance { $next = null; do { - $limit = min( $this->mBatchSize, $end - $start + 1 ); // don't go pass ending ID + $limit = min( $this->getBatchSize(), $end - $start + 1 ); // don't go pass ending ID $this->output( "Doing id $start to " . ( $start + $limit - 1 ) . "...\n" ); $entries = $src->getJournal()->getChangeEntries( $start, $limit, $next ); diff --git a/maintenance/wrapOldPasswords.php b/maintenance/wrapOldPasswords.php index 85fa7805e6..981143ecf7 100644 --- a/maintenance/wrapOldPasswords.php +++ b/maintenance/wrapOldPasswords.php @@ -87,7 +87,7 @@ class WrapOldPasswords extends Maintenance { __METHOD__, [ 'ORDER BY' => 'user_id', - 'LIMIT' => $this->mBatchSize, + 'LIMIT' => $this->getBatchSize(), 'LOCK IN SHARE MODE', ] ); -- 2.20.1