From: Erik Bernhardson Date: Thu, 30 Jul 2015 22:09:11 +0000 (-0700) Subject: Import BatchRowUpdate classes from Echo X-Git-Tag: 1.31.0-rc.0~10434^2 X-Git-Url: http://git.heureux-cyclage.org/?a=commitdiff_plain;h=85d5626d6c9e6da2640813f26f0850ee2f6e2ce7;p=lhc%2Fweb%2Fwiklou.git Import BatchRowUpdate classes from Echo This is a set of classes written for Echo to simplify writing maintenance scripts that iterate over an entire table and update some of those rows. This has shown to be reusable elsewhere, especially the BatchRowIterator class and will be useful to have generally avilable in core. The Echo classes are all prefixed with the Echo name so there wont be any conflict is both are installed. Change-Id: I64c1751106caf34f41af799dbaf8794115537f06 --- diff --git a/autoload.php b/autoload.php index a8940c47e3..d2577216ef 100644 --- a/autoload.php +++ b/autoload.php @@ -157,6 +157,9 @@ $wgAutoloadLocalClasses = array( 'BagOStuff' => __DIR__ . '/includes/libs/objectcache/BagOStuff.php', 'BaseDump' => __DIR__ . '/maintenance/backupPrefetch.inc', 'BaseTemplate' => __DIR__ . '/includes/skins/BaseTemplate.php', + 'BatchRowIterator' => __DIR__ . '/includes/utils/BatchRowIterator.php', + 'BatchRowUpdate' => __DIR__ . '/includes/utils/BatchRowUpdate.php', + 'BatchRowWriter' => __DIR__ . '/includes/utils/BatchRowWriter.php', 'BatchedQueryRunner' => __DIR__ . '/maintenance/runBatchedQuery.php', 'BcryptPassword' => __DIR__ . '/includes/password/BcryptPassword.php', 'BenchHttpHttps' => __DIR__ . '/maintenance/benchmarks/bench_HTTP_HTTPS.php', @@ -572,6 +575,7 @@ $wgAutoloadLocalClasses = array( 'InstallerOverrides' => __DIR__ . '/mw-config/overrides.php', 'Interwiki' => __DIR__ . '/includes/interwiki/Interwiki.php', 'InvalidPassword' => __DIR__ . '/includes/password/InvalidPassword.php', + 'IteratorDecorator' => __DIR__ . '/includes/utils/iterators/IteratorDecorator.php', 'IuConverter' => __DIR__ . '/languages/classes/LanguageIu.php', 'JSCompilerContext' => __DIR__ . '/includes/libs/jsminplus.php', 'JSMinPlus' => __DIR__ . '/includes/libs/jsminplus.php', @@ -814,6 +818,7 @@ $wgAutoloadLocalClasses = array( 'NewPagesPager' => __DIR__ . '/includes/specials/SpecialNewpages.php', 'NewUsersLogFormatter' => __DIR__ . '/includes/logging/NewUsersLogFormatter.php', 'NolinesImageGallery' => __DIR__ . '/includes/gallery/NolinesImageGallery.php', + 'NotRecursiveIterator' => __DIR__ . '/includes/utils/iterators/NotRecursiveIterator.php', 'NukeNS' => __DIR__ . '/maintenance/nukeNS.php', 'NukePage' => __DIR__ . '/maintenance/nukePage.php', 'NullFileJournal' => __DIR__ . '/includes/filebackend/filejournal/FileJournal.php', @@ -1051,6 +1056,7 @@ $wgAutoloadLocalClasses = array( 'RightsLogFormatter' => __DIR__ . '/includes/logging/RightsLogFormatter.php', 'RollbackAction' => __DIR__ . '/includes/actions/RollbackAction.php', 'RollbackEdits' => __DIR__ . '/maintenance/rollbackEdits.php', + 'RowUpdateGenerator' => __DIR__ . '/includes/utils/RowUpdateGenerator.php', 'RunJobs' => __DIR__ . '/maintenance/runJobs.php', 'RunningStat' => __DIR__ . '/includes/libs/RunningStat.php', 'SQLiteField' => __DIR__ . '/includes/db/DatabaseSqlite.php', diff --git a/includes/utils/BatchRowIterator.php b/includes/utils/BatchRowIterator.php new file mode 100644 index 0000000000..9441608aad --- /dev/null +++ b/includes/utils/BatchRowIterator.php @@ -0,0 +1,278 @@ +primaryKey + */ + protected $orderBy; + + /** + * @var array $current The current iterator value + */ + private $current = array(); + + /** + * @var integer key 0-indexed number of pages fetched since self::reset() + */ + private $key; + + /** + * @param DatabaseBase $db The database to read from + * @param string $table The name of the table to read from + * @param string|array $primaryKey The name or names of the primary key columns + * @param integer $batchSize The number of rows to fetch per iteration + * @throws MWException + */ + public function __construct( DatabaseBase $db, $table, $primaryKey, $batchSize ) { + if ( $batchSize < 1 ) { + throw new MWException( 'Batch size must be at least 1 row.' ); + } + $this->db = $db; + $this->table = $table; + $this->primaryKey = (array) $primaryKey; + $this->fetchColumns = $this->primaryKey; + $this->orderBy = implode( ' ASC,', $this->primaryKey ) . ' ASC'; + $this->batchSize = $batchSize; + } + + /** + * @param array $condition Query conditions suitable for use with + * DatabaseBase::select + */ + public function addConditions( array $conditions ) { + $this->conditions = array_merge( $this->conditions, $conditions ); + } + + /** + * @param array $condition Query join conditions suitable for use + * with DatabaseBase::select + */ + public function addJoinConditions( array $conditions ) { + $this->joinConditions = array_merge( $this->joinConditions, $conditions ); + } + + /** + * @param array $columns List of column names to select from the + * table suitable for use with DatabaseBase::select() + */ + public function setFetchColumns( array $columns ) { + // If it's not the all column selector merge in the primary keys we need + if ( count( $columns ) === 1 && reset( $columns ) === '*' ) { + $this->fetchColumns = $columns; + } else { + $this->fetchColumns = array_unique( array_merge( + $this->primaryKey, + $columns + ) ); + } + } + + /** + * Extracts the primary key(s) from a database row. + * + * @param stdClass $row An individual database row from this iterator + * @return array Map of primary key column to value within the row + */ + public function extractPrimaryKeys( $row ) { + $pk = array(); + foreach ( $this->primaryKey as $column ) { + $pk[$column] = $row->$column; + } + return $pk; + } + + /** + * @return array The most recently fetched set of rows from the database + */ + public function current() { + return $this->current; + } + + /** + * @return integer 0-indexed count of the page number fetched + */ + public function key() { + return $this->key; + } + + /** + * Reset the iterator to the begining of the table. + */ + public function rewind() { + $this->key = -1; // self::next() will turn this into 0 + $this->current = array(); + $this->next(); + } + + /** + * @return boolean True when the iterator is in a valid state + */ + public function valid() { + return (bool) $this->current; + } + + /** + * @return boolean True when this result set has rows + */ + public function hasChildren() { + return $this->current && count( $this->current ); + } + + /** + * @return RecursiveIterator + */ + public function getChildren() { + return new NotRecursiveIterator( new ArrayIterator( $this->current ) ); + } + + /** + * Fetch the next set of rows from the database. + */ + public function next() { + $res = $this->db->select( + $this->table, + $this->fetchColumns, + $this->buildConditions(), + __METHOD__, + array( + 'LIMIT' => $this->batchSize, + 'ORDER BY' => $this->orderBy, + ), + $this->joinConditions + ); + + // The iterator is converted to an array because in addition to + // returning it in self::current() we need to use the end value + // in self::buildConditions() + $this->current = iterator_to_array( $res ); + $this->key++; + } + + /** + * Uses the primary key list and the maximal result row from the + * previous iteration to build an SQL condition sufficient for + * selecting the next page of results. All except the final key use + * `=` conditions while the final key uses a `>` condition + * + * Example output: + * array( '( foo = 42 AND bar > 7 ) OR ( foo > 42 )' ) + * + * @return array The SQL conditions necessary to select the next set + * of rows in the batched query + */ + protected function buildConditions() { + if ( !$this->current ) { + return $this->conditions; + } + + $maxRow = end( $this->current ); + $maximumValues = array(); + foreach ( $this->primaryKey as $column ) { + $maximumValues[$column] = $this->db->addQuotes( $maxRow->$column ); + } + + $pkConditions = array(); + // For example: If we have 3 primary keys + // first run through will generate + // col1 = 4 AND col2 = 7 AND col3 > 1 + // second run through will generate + // col1 = 4 AND col2 > 7 + // and the final run through will generate + // col1 > 4 + while ( $maximumValues ) { + $pkConditions[] = $this->buildGreaterThanCondition( $maximumValues ); + array_pop( $maximumValues ); + } + + $conditions = $this->conditions; + $conditions[] = sprintf( '( %s )', implode( ' ) OR ( ', $pkConditions ) ); + + return $conditions; + } + + /** + * Given an array of column names and their maximum value generate + * an SQL condition where all keys except the last match $quotedMaximumValues + * exactly and the last column is greater than the matching value in + * $quotedMaximumValues + * + * @param array $quotedMaximumValues The maximum values quoted with + * $this->db->addQuotes() + * @return string An SQL condition that will select rows where all + * columns match the maximum value exactly except the last column + * which must be greater than the provided maximum value + */ + protected function buildGreaterThanCondition( array $quotedMaximumValues ) { + $keys = array_keys( $quotedMaximumValues ); + $lastColumn = end( $keys ); + $lastValue = array_pop( $quotedMaximumValues ); + $conditions = array(); + foreach ( $quotedMaximumValues as $column => $value ) { + $conditions[] = "$column = $value"; + } + $conditions[] = "$lastColumn > $lastValue"; + + return implode( ' AND ', $conditions ); + } +} diff --git a/includes/utils/BatchRowUpdate.php b/includes/utils/BatchRowUpdate.php new file mode 100644 index 0000000000..a4257a5346 --- /dev/null +++ b/includes/utils/BatchRowUpdate.php @@ -0,0 +1,133 @@ +execute(); + * + * An example maintenance script utilizing the BatchRowUpdate can be + * located in the Echo extension file maintenance/updateSchema.php + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ +class BatchRowUpdate { + /** + * @var BatchRowIterator $reader Iterator that returns an array of + * database rows + */ + protected $reader; + + /** + * @var BatchRowWriter $writer Writer capable of pushing row updates + * to the database + */ + protected $writer; + + /** + * @var RowUpdateGenerator $generator Generates single row updates + * based on the rows content + */ + protected $generator; + + /** + * @var callable $output Output callback + */ + protected $output; + + /** + * @param BatchRowIterator $reader Iterator that returns an + * array of database rows + * @param BatchRowWriter $writer Writer capable of pushing + * row updates to the database + * @param RowUpdateGenerator $generator Generates single row updates + * based on the rows content + */ + public function __construct( BatchRowIterator $reader, BatchRowWriter $writer, RowUpdateGenerator $generator ) { + $this->reader = $reader; + $this->writer = $writer; + $this->generator = $generator; + $this->output = function() { + }; // nop + } + + /** + * Runs the batch update process + */ + public function execute() { + foreach ( $this->reader as $rows ) { + $updates = array(); + foreach ( $rows as $row ) { + $update = $this->generator->update( $row ); + if ( $update ) { + $updates[] = array( + 'primaryKey' => $this->reader->extractPrimaryKeys( $row ), + 'changes' => $update, + ); + } + } + + if ( $updates ) { + $this->output( "Processing " . count( $updates ) . " rows\n" ); + $this->writer->write( $updates ); + } + } + + $this->output( "Completed\n" ); + } + + /** + * Accepts a callable which will receive a single parameter + * containing string status updates + * + * @param callable $output A callback taking a single string + * parameter to output + * + * @throws MWException + */ + public function setOutput( $output ) { + if ( !is_callable( $output ) ) { + throw new MWException( + 'Provided $output param is required to be callable.' + ); + } + $this->output = $output; + } + + /** + * Write out a status update + * + * @param string $text The value to print + */ + protected function output( $text ) { + call_user_func( $this->output, $text ); + } +} diff --git a/includes/utils/BatchRowWriter.php b/includes/utils/BatchRowWriter.php new file mode 100644 index 0000000000..9daeab14c1 --- /dev/null +++ b/includes/utils/BatchRowWriter.php @@ -0,0 +1,71 @@ +db = $db; + $this->table = $table; + $this->clusterName = $clusterName; + } + + /** + * @param array $updates Array of arrays each containing two keys, 'primaryKey' + * and 'changes'. primaryKey must contain a map of column names to values + * sufficient to uniquely identify the row changes must contain a map of column + * names to update values to apply to the row. + */ + public function write( array $updates ) { + $this->db->begin(); + + foreach ( $updates as $update ) { + $this->db->update( + $this->table, + $update['changes'], + $update['primaryKey'], + __METHOD__ + ); + } + + $this->db->commit(); + wfWaitForSlaves( false, false, $this->clusterName ); + } +} diff --git a/includes/utils/RowUpdateGenerator.php b/includes/utils/RowUpdateGenerator.php new file mode 100644 index 0000000000..6a4792cb56 --- /dev/null +++ b/includes/utils/RowUpdateGenerator.php @@ -0,0 +1,39 @@ + 'new value', + * 'other_col' => 99, + * ); + * + * @param stdClass $row A row from the database + * @return array Map of column names to updated value within the + * database row. When no update is required returns an empty array. + */ + public function update( $row ); +} diff --git a/includes/utils/iterators/IteratorDecorator.php b/includes/utils/iterators/IteratorDecorator.php new file mode 100644 index 0000000000..288a311320 --- /dev/null +++ b/includes/utils/iterators/IteratorDecorator.php @@ -0,0 +1,50 @@ +iterator = $iterator; + } + + public function current() { + return $this->iterator->current(); + } + + public function key() { + return $this->iterator->key(); + } + + public function next() { + return $this->iterator->next(); + } + + public function rewind() { + return $this->iterator->rewind(); + } + + public function valid() { + return $this->iterator->valid(); + } +} diff --git a/includes/utils/iterators/NotRecursiveIterator.php b/includes/utils/iterators/NotRecursiveIterator.php new file mode 100644 index 0000000000..52ca61b449 --- /dev/null +++ b/includes/utils/iterators/NotRecursiveIterator.php @@ -0,0 +1,35 @@ +mockDb(); + $writer = new BatchRowWriter( $db, 'echo_event' ); + + $updates = array( + self::mockUpdate( array( 'something' => 'changed' ) ), + self::mockUpdate( array( 'otherthing' => 'changed' ) ), + self::mockUpdate( array( 'and' => 'something', 'else' => 'changed' ) ), + ); + + $db->expects( $this->exactly( count( $updates ) ) ) + ->method( 'update' ); + + $writer->write( $updates ); + } + + static protected function mockUpdate( array $changes ) { + static $i = 0; + return array( + 'primaryKey' => array( 'event_id' => $i++ ), + 'changes' => $changes, + ); + } + + public function testReaderBasicIterate() { + $db = $this->mockDb(); + $batchSize = 2; + $reader = new BatchRowIterator( $db, 'some_table', 'id_field', $batchSize ); + + $response = $this->genSelectResult( $batchSize, /*numRows*/ 5, function() { + static $i = 0; + return array( 'id_field' => ++$i ); + } ); + $db->expects( $this->exactly( count( $response ) ) ) + ->method( 'select' ) + ->will( $this->consecutivelyReturnFromSelect( $response ) ); + + $pos = 0; + foreach ( $reader as $rows ) { + $this->assertEquals( $response[$pos], $rows, "Testing row in position $pos" ); + $pos++; + } + // -1 is because the final array() marks the end and isnt included + $this->assertEquals( count( $response ) - 1, $pos ); + } + + static public function provider_readerGetPrimaryKey() { + $row = array( + 'id_field' => 42, + 'some_col' => 'dvorak', + 'other_col' => 'samurai', + ); + return array( + + array( + 'Must return single column pk when requested', + array( 'id_field' => 42 ), + $row + ), + + array( + 'Must return multiple column pks when requested', + array( 'id_field' => 42, 'other_col' => 'samurai' ), + $row + ), + + ); + } + + /** + * @dataProvider provider_readerGetPrimaryKey + */ + public function testReaderGetPrimaryKey( $message, array $expected, array $row ) { + $reader = new BatchRowIterator( $this->mockDb(), 'some_table', array_keys( $expected ), 8675309 ); + $this->assertEquals( $expected, $reader->extractPrimaryKeys( (object) $row ), $message ); + } + + static public function provider_readerSetFetchColumns() { + return array( + + array( + 'Must merge primary keys into select conditions', + // Expected column select + array( 'foo', 'bar' ), + // primary keys + array( 'foo' ), + // setFetchColumn + array( 'bar' ) + ), + + array( + 'Must not merge primary keys into the all columns selector', + // Expected column select + array( '*' ), + // primary keys + array( 'foo' ), + // setFetchColumn + array( '*' ), + ), + + array( + 'Must not duplicate primary keys into column selector', + // Expected column select. + // TODO: figure out how to only assert the array_values portion and not the keys + array( 0 => 'foo', 1 => 'bar', 3 => 'baz' ), + // primary keys + array( 'foo', 'bar', ), + // setFetchColumn + array( 'bar', 'baz' ), + ), + ); + } + + /** + * @dataProvider provider_readerSetFetchColumns + */ + public function testReaderSetFetchColumns( $message, array $columns, array $primaryKeys, array $fetchColumns ) { + $db = $this->mockDb(); + $db->expects( $this->once() ) + ->method( 'select' ) + ->with( 'some_table', $columns ) // only testing second parameter of DatabaseBase::select + ->will( $this->returnValue( new ArrayIterator( array() ) ) ); + + $reader = new BatchRowIterator( $db, 'some_table', $primaryKeys, 22 ); + $reader->setFetchColumns( $fetchColumns ); + // triggers first database select + $reader->rewind(); + } + + static public function provider_readerSelectConditions() { + return array( + + array( + "With single primary key must generate id > 'value'", + // Expected second iteration + array( "( id_field > '3' )" ), + // Primary key(s) + 'id_field', + ), + + array( + 'With multiple primary keys the first conditions must use >= and the final condition must use >', + // Expected second iteration + array( "( id_field = '3' AND foo > '103' ) OR ( id_field > '3' )" ), + // Primary key(s) + array( 'id_field', 'foo' ), + ), + + ); + } + + /** + * Slightly hackish to use reflection, but asserting different parameters + * to consecutive calls of DatabaseBase::select in phpunit is error prone + * + * @dataProvider provider_readerSelectConditions + */ + public function testReaderSelectConditionsMultiplePrimaryKeys( $message, $expectedSecondIteration, $primaryKeys, $batchSize = 3 ) { + $results = $this->genSelectResult( $batchSize, $batchSize * 3, function() { + static $i = 0, $j = 100, $k = 1000; + return array( 'id_field' => ++$i, 'foo' => ++$j, 'bar' => ++$k ); + } ); + $db = $this->mockDbConsecutiveSelect( $results ); + + $conditions = array( 'bar' => 42, 'baz' => 'hai' ); + $reader = new BatchRowIterator( $db, 'some_table', $primaryKeys, $batchSize ); + $reader->addConditions( $conditions ); + + $buildConditions = new ReflectionMethod( $reader, 'buildConditions' ); + $buildConditions->setAccessible( true ); + + // On first iteration only the passed conditions must be used + $this->assertEquals( $conditions, $buildConditions->invoke( $reader ), + 'First iteration must return only the conditions passed in addConditions' ); + $reader->rewind(); + + // Second iteration must use the maximum primary key of last set + $this->assertEquals( + $conditions + $expectedSecondIteration, + $buildConditions->invoke( $reader ), + $message + ); + } + + protected function mockDbConsecutiveSelect( array $retvals ) { + $db = $this->mockDb(); + $db->expects( $this->any() ) + ->method( 'select' ) + ->will( $this->consecutivelyReturnFromSelect( $retvals ) ); + $db->expects( $this->any() ) + ->method( 'addQuotes' ) + ->will( $this->returnCallback( function( $value ) { + return "'$value'"; // not real quoting: doesn't matter in test + } ) ); + + return $db; + } + + protected function consecutivelyReturnFromSelect( array $results ) { + $retvals = array(); + foreach ( $results as $rows ) { + // The DatabaseBase::select method returns iterators, so we do too. + $retvals[] = $this->returnValue( new ArrayIterator( $rows ) ); + } + + return call_user_func_array( array( $this, 'onConsecutiveCalls' ), $retvals ); + } + + + protected function genSelectResult( $batchSize, $numRows, $rowGenerator ) { + $res = array(); + for ( $i = 0; $i < $numRows; $i += $batchSize ) { + $rows = array(); + for ( $j = 0; $j < $batchSize && $i + $j < $numRows; $j++ ) { + $rows [] = (object) call_user_func( $rowGenerator ); + } + $res[] = $rows; + } + $res[] = array(); // termination condition requires empty result for last row + return $res; + } + + protected function mockDb() { + // Cant mock from DatabaseType or DatabaseBase, they dont + // have the full gamut of methods + $databaseMysql = $this->getMockBuilder( 'DatabaseMysql' ) + ->disableOriginalConstructor() + ->getMock(); + $databaseMysql->expects( $this->any() ) + ->method( 'isOpen' ) + ->will( $this->returnValue( true ) ); + return $databaseMysql; + } +}