'BagOStuff' => __DIR__ . '/includes/libs/objectcache/BagOStuff.php',
'BaseDump' => __DIR__ . '/maintenance/backupPrefetch.inc',
'BaseTemplate' => __DIR__ . '/includes/skins/BaseTemplate.php',
+ 'BatchRowIterator' => __DIR__ . '/includes/utils/BatchRowIterator.php',
+ 'BatchRowUpdate' => __DIR__ . '/includes/utils/BatchRowUpdate.php',
+ 'BatchRowWriter' => __DIR__ . '/includes/utils/BatchRowWriter.php',
'BatchedQueryRunner' => __DIR__ . '/maintenance/runBatchedQuery.php',
'BcryptPassword' => __DIR__ . '/includes/password/BcryptPassword.php',
'BenchHttpHttps' => __DIR__ . '/maintenance/benchmarks/bench_HTTP_HTTPS.php',
'InstallerOverrides' => __DIR__ . '/mw-config/overrides.php',
'Interwiki' => __DIR__ . '/includes/interwiki/Interwiki.php',
'InvalidPassword' => __DIR__ . '/includes/password/InvalidPassword.php',
+ 'IteratorDecorator' => __DIR__ . '/includes/utils/iterators/IteratorDecorator.php',
'IuConverter' => __DIR__ . '/languages/classes/LanguageIu.php',
'JSCompilerContext' => __DIR__ . '/includes/libs/jsminplus.php',
'JSMinPlus' => __DIR__ . '/includes/libs/jsminplus.php',
'NewPagesPager' => __DIR__ . '/includes/specials/SpecialNewpages.php',
'NewUsersLogFormatter' => __DIR__ . '/includes/logging/NewUsersLogFormatter.php',
'NolinesImageGallery' => __DIR__ . '/includes/gallery/NolinesImageGallery.php',
+ 'NotRecursiveIterator' => __DIR__ . '/includes/utils/iterators/NotRecursiveIterator.php',
'NukeNS' => __DIR__ . '/maintenance/nukeNS.php',
'NukePage' => __DIR__ . '/maintenance/nukePage.php',
'NullFileJournal' => __DIR__ . '/includes/filebackend/filejournal/FileJournal.php',
'RightsLogFormatter' => __DIR__ . '/includes/logging/RightsLogFormatter.php',
'RollbackAction' => __DIR__ . '/includes/actions/RollbackAction.php',
'RollbackEdits' => __DIR__ . '/maintenance/rollbackEdits.php',
+ 'RowUpdateGenerator' => __DIR__ . '/includes/utils/RowUpdateGenerator.php',
'RunJobs' => __DIR__ . '/maintenance/runJobs.php',
'RunningStat' => __DIR__ . '/includes/libs/RunningStat.php',
'SQLiteField' => __DIR__ . '/includes/db/DatabaseSqlite.php',
--- /dev/null
+<?php
+/**
+ * Allows iterating a large number of rows in batches transparently.
+ * By default when iterated over returns the full query result as an
+ * array of rows. Can be wrapped in RecursiveIteratorIterator to
+ * collapse those arrays into a single stream of rows queried in batches.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowIterator implements RecursiveIterator {
+
+ /**
+ * @var DatabaseBase $db The database to read from
+ */
+ protected $db;
+
+ /**
+ * @var string $table The name of the table to read from
+ */
+ protected $table;
+
+ /**
+ * @var array $primaryKey The name of the primary key(s)
+ */
+ protected $primaryKey;
+
+ /**
+ * @var integer $batchSize The number of rows to fetch per iteration
+ */
+ protected $batchSize;
+
+ /**
+ * @var array $conditions Array of strings containing SQL conditions
+ * to add to the query
+ */
+ protected $conditions = array();
+
+ /**
+ * @var array $joinConditions
+ */
+ protected $joinConditions = array();
+
+ /**
+ * @var array $fetchColumns List of column names to select from the
+ * table suitable for use with DatabaseBase::select()
+ */
+ protected $fetchColumns;
+
+ /**
+ * @var string $orderBy SQL Order by condition generated from $this->primaryKey
+ */
+ protected $orderBy;
+
+ /**
+ * @var array $current The current iterator value
+ */
+ private $current = array();
+
+ /**
+ * @var integer key 0-indexed number of pages fetched since self::reset()
+ */
+ private $key;
+
+ /**
+ * @param DatabaseBase $db The database to read from
+ * @param string $table The name of the table to read from
+ * @param string|array $primaryKey The name or names of the primary key columns
+ * @param integer $batchSize The number of rows to fetch per iteration
+ * @throws MWException
+ */
+ public function __construct( DatabaseBase $db, $table, $primaryKey, $batchSize ) {
+ if ( $batchSize < 1 ) {
+ throw new MWException( 'Batch size must be at least 1 row.' );
+ }
+ $this->db = $db;
+ $this->table = $table;
+ $this->primaryKey = (array) $primaryKey;
+ $this->fetchColumns = $this->primaryKey;
+ $this->orderBy = implode( ' ASC,', $this->primaryKey ) . ' ASC';
+ $this->batchSize = $batchSize;
+ }
+
+ /**
+ * @param array $condition Query conditions suitable for use with
+ * DatabaseBase::select
+ */
+ public function addConditions( array $conditions ) {
+ $this->conditions = array_merge( $this->conditions, $conditions );
+ }
+
+ /**
+ * @param array $condition Query join conditions suitable for use
+ * with DatabaseBase::select
+ */
+ public function addJoinConditions( array $conditions ) {
+ $this->joinConditions = array_merge( $this->joinConditions, $conditions );
+ }
+
+ /**
+ * @param array $columns List of column names to select from the
+ * table suitable for use with DatabaseBase::select()
+ */
+ public function setFetchColumns( array $columns ) {
+ // If it's not the all column selector merge in the primary keys we need
+ if ( count( $columns ) === 1 && reset( $columns ) === '*' ) {
+ $this->fetchColumns = $columns;
+ } else {
+ $this->fetchColumns = array_unique( array_merge(
+ $this->primaryKey,
+ $columns
+ ) );
+ }
+ }
+
+ /**
+ * Extracts the primary key(s) from a database row.
+ *
+ * @param stdClass $row An individual database row from this iterator
+ * @return array Map of primary key column to value within the row
+ */
+ public function extractPrimaryKeys( $row ) {
+ $pk = array();
+ foreach ( $this->primaryKey as $column ) {
+ $pk[$column] = $row->$column;
+ }
+ return $pk;
+ }
+
+ /**
+ * @return array The most recently fetched set of rows from the database
+ */
+ public function current() {
+ return $this->current;
+ }
+
+ /**
+ * @return integer 0-indexed count of the page number fetched
+ */
+ public function key() {
+ return $this->key;
+ }
+
+ /**
+ * Reset the iterator to the begining of the table.
+ */
+ public function rewind() {
+ $this->key = -1; // self::next() will turn this into 0
+ $this->current = array();
+ $this->next();
+ }
+
+ /**
+ * @return boolean True when the iterator is in a valid state
+ */
+ public function valid() {
+ return (bool) $this->current;
+ }
+
+ /**
+ * @return boolean True when this result set has rows
+ */
+ public function hasChildren() {
+ return $this->current && count( $this->current );
+ }
+
+ /**
+ * @return RecursiveIterator
+ */
+ public function getChildren() {
+ return new NotRecursiveIterator( new ArrayIterator( $this->current ) );
+ }
+
+ /**
+ * Fetch the next set of rows from the database.
+ */
+ public function next() {
+ $res = $this->db->select(
+ $this->table,
+ $this->fetchColumns,
+ $this->buildConditions(),
+ __METHOD__,
+ array(
+ 'LIMIT' => $this->batchSize,
+ 'ORDER BY' => $this->orderBy,
+ ),
+ $this->joinConditions
+ );
+
+ // The iterator is converted to an array because in addition to
+ // returning it in self::current() we need to use the end value
+ // in self::buildConditions()
+ $this->current = iterator_to_array( $res );
+ $this->key++;
+ }
+
+ /**
+ * Uses the primary key list and the maximal result row from the
+ * previous iteration to build an SQL condition sufficient for
+ * selecting the next page of results. All except the final key use
+ * `=` conditions while the final key uses a `>` condition
+ *
+ * Example output:
+ * array( '( foo = 42 AND bar > 7 ) OR ( foo > 42 )' )
+ *
+ * @return array The SQL conditions necessary to select the next set
+ * of rows in the batched query
+ */
+ protected function buildConditions() {
+ if ( !$this->current ) {
+ return $this->conditions;
+ }
+
+ $maxRow = end( $this->current );
+ $maximumValues = array();
+ foreach ( $this->primaryKey as $column ) {
+ $maximumValues[$column] = $this->db->addQuotes( $maxRow->$column );
+ }
+
+ $pkConditions = array();
+ // For example: If we have 3 primary keys
+ // first run through will generate
+ // col1 = 4 AND col2 = 7 AND col3 > 1
+ // second run through will generate
+ // col1 = 4 AND col2 > 7
+ // and the final run through will generate
+ // col1 > 4
+ while ( $maximumValues ) {
+ $pkConditions[] = $this->buildGreaterThanCondition( $maximumValues );
+ array_pop( $maximumValues );
+ }
+
+ $conditions = $this->conditions;
+ $conditions[] = sprintf( '( %s )', implode( ' ) OR ( ', $pkConditions ) );
+
+ return $conditions;
+ }
+
+ /**
+ * Given an array of column names and their maximum value generate
+ * an SQL condition where all keys except the last match $quotedMaximumValues
+ * exactly and the last column is greater than the matching value in
+ * $quotedMaximumValues
+ *
+ * @param array $quotedMaximumValues The maximum values quoted with
+ * $this->db->addQuotes()
+ * @return string An SQL condition that will select rows where all
+ * columns match the maximum value exactly except the last column
+ * which must be greater than the provided maximum value
+ */
+ protected function buildGreaterThanCondition( array $quotedMaximumValues ) {
+ $keys = array_keys( $quotedMaximumValues );
+ $lastColumn = end( $keys );
+ $lastValue = array_pop( $quotedMaximumValues );
+ $conditions = array();
+ foreach ( $quotedMaximumValues as $column => $value ) {
+ $conditions[] = "$column = $value";
+ }
+ $conditions[] = "$lastColumn > $lastValue";
+
+ return implode( ' AND ', $conditions );
+ }
+}
--- /dev/null
+<?php
+/*
+ * Ties together the batch update components to provide a composable
+ * method of batch updating rows in a database. To use create a class
+ * implementing the RowUpdateGenerator interface and configure the
+ * BatchRowIterator and BatchRowWriter for access to the correct table.
+ * The components will handle reading, writing, and waiting for slaves
+ * while the generator implementation handles generating update arrays
+ * for singular rows.
+ *
+ * Instantiate:
+ * $updater = new BatchRowUpdate(
+ * new BatchRowIterator( $dbr, 'some_table', 'primary_key_column', 500 ),
+ * new BatchRowWriter( $dbw, 'some_table', 'clusterName' ),
+ * new MyImplementationOfRowUpdateGenerator
+ * );
+ *
+ * Run:
+ * $updater->execute();
+ *
+ * An example maintenance script utilizing the BatchRowUpdate can be
+ * located in the Echo extension file maintenance/updateSchema.php
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowUpdate {
+ /**
+ * @var BatchRowIterator $reader Iterator that returns an array of
+ * database rows
+ */
+ protected $reader;
+
+ /**
+ * @var BatchRowWriter $writer Writer capable of pushing row updates
+ * to the database
+ */
+ protected $writer;
+
+ /**
+ * @var RowUpdateGenerator $generator Generates single row updates
+ * based on the rows content
+ */
+ protected $generator;
+
+ /**
+ * @var callable $output Output callback
+ */
+ protected $output;
+
+ /**
+ * @param BatchRowIterator $reader Iterator that returns an
+ * array of database rows
+ * @param BatchRowWriter $writer Writer capable of pushing
+ * row updates to the database
+ * @param RowUpdateGenerator $generator Generates single row updates
+ * based on the rows content
+ */
+ public function __construct( BatchRowIterator $reader, BatchRowWriter $writer, RowUpdateGenerator $generator ) {
+ $this->reader = $reader;
+ $this->writer = $writer;
+ $this->generator = $generator;
+ $this->output = function() {
+ }; // nop
+ }
+
+ /**
+ * Runs the batch update process
+ */
+ public function execute() {
+ foreach ( $this->reader as $rows ) {
+ $updates = array();
+ foreach ( $rows as $row ) {
+ $update = $this->generator->update( $row );
+ if ( $update ) {
+ $updates[] = array(
+ 'primaryKey' => $this->reader->extractPrimaryKeys( $row ),
+ 'changes' => $update,
+ );
+ }
+ }
+
+ if ( $updates ) {
+ $this->output( "Processing " . count( $updates ) . " rows\n" );
+ $this->writer->write( $updates );
+ }
+ }
+
+ $this->output( "Completed\n" );
+ }
+
+ /**
+ * Accepts a callable which will receive a single parameter
+ * containing string status updates
+ *
+ * @param callable $output A callback taking a single string
+ * parameter to output
+ *
+ * @throws MWException
+ */
+ public function setOutput( $output ) {
+ if ( !is_callable( $output ) ) {
+ throw new MWException(
+ 'Provided $output param is required to be callable.'
+ );
+ }
+ $this->output = $output;
+ }
+
+ /**
+ * Write out a status update
+ *
+ * @param string $text The value to print
+ */
+ protected function output( $text ) {
+ call_user_func( $this->output, $text );
+ }
+}
--- /dev/null
+<?php
+/**
+ * Updates database rows by primary key in batches.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class BatchRowWriter {
+ /**
+ * @var DatabaseBase $db The database to write to
+ */
+ protected $db;
+
+ /**
+ * @var string $table The name of the table to update
+ */
+ protected $table;
+
+ /**
+ * @var string $clusterName A cluster name valid for use with LBFactory
+ */
+ protected $clusterName;
+
+ /**
+ * @param DatabaseBase $db The database to write to
+ * @param string $table The name of the table to update
+ * @param string|bool $clusterName A cluster name valid for use with LBFactory
+ */
+ public function __construct( DatabaseBase $db, $table, $clusterName = false ) {
+ $this->db = $db;
+ $this->table = $table;
+ $this->clusterName = $clusterName;
+ }
+
+ /**
+ * @param array $updates Array of arrays each containing two keys, 'primaryKey'
+ * and 'changes'. primaryKey must contain a map of column names to values
+ * sufficient to uniquely identify the row changes must contain a map of column
+ * names to update values to apply to the row.
+ */
+ public function write( array $updates ) {
+ $this->db->begin();
+
+ foreach ( $updates as $update ) {
+ $this->db->update(
+ $this->table,
+ $update['changes'],
+ $update['primaryKey'],
+ __METHOD__
+ );
+ }
+
+ $this->db->commit();
+ wfWaitForSlaves( false, false, $this->clusterName );
+ }
+}
--- /dev/null
+<?php
+/**
+ * Interface for generating updates to single rows in the database.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+interface RowUpdateGenerator {
+ /**
+ * Given a database row, generates an array mapping column names to
+ * updated value within the database row.
+ *
+ * Sample Response:
+ * return array(
+ * 'some_col' => 'new value',
+ * 'other_col' => 99,
+ * );
+ *
+ * @param stdClass $row A row from the database
+ * @return array Map of column names to updated value within the
+ * database row. When no update is required returns an empty array.
+ */
+ public function update( $row );
+}
--- /dev/null
+<?php
+/**
+ * Allows extending classes to decorate an Iterator with
+ * reduced boilerplate.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+abstract class IteratorDecorator implements Iterator {
+ protected $iterator;
+
+ public function __construct( Iterator $iterator ) {
+ $this->iterator = $iterator;
+ }
+
+ public function current() {
+ return $this->iterator->current();
+ }
+
+ public function key() {
+ return $this->iterator->key();
+ }
+
+ public function next() {
+ return $this->iterator->next();
+ }
+
+ public function rewind() {
+ return $this->iterator->rewind();
+ }
+
+ public function valid() {
+ return $this->iterator->valid();
+ }
+}
--- /dev/null
+<?php
+/**
+ * Wraps a non-recursive iterator with methods to be recursive
+ * without children.
+ *
+ * Alternatively wraps a recursive iterator to prevent recursing deeper
+ * than the wrapped iterator.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+class NotRecursiveIterator extends IteratorDecorator implements RecursiveIterator {
+ public function hasChildren() {
+ return false;
+ }
+
+ public function getChildren() {
+ return null;
+ }
+}
--- /dev/null
+<?php
+
+/**
+ * Tests for BatchRowUpdate and its components
+ *
+ * @group db
+ */
+class BatchRowUpdateTest extends MediaWikiTestCase {
+
+ public function testWriterBasicFunctionality() {
+ $db = $this->mockDb();
+ $writer = new BatchRowWriter( $db, 'echo_event' );
+
+ $updates = array(
+ self::mockUpdate( array( 'something' => 'changed' ) ),
+ self::mockUpdate( array( 'otherthing' => 'changed' ) ),
+ self::mockUpdate( array( 'and' => 'something', 'else' => 'changed' ) ),
+ );
+
+ $db->expects( $this->exactly( count( $updates ) ) )
+ ->method( 'update' );
+
+ $writer->write( $updates );
+ }
+
+ static protected function mockUpdate( array $changes ) {
+ static $i = 0;
+ return array(
+ 'primaryKey' => array( 'event_id' => $i++ ),
+ 'changes' => $changes,
+ );
+ }
+
+ public function testReaderBasicIterate() {
+ $db = $this->mockDb();
+ $batchSize = 2;
+ $reader = new BatchRowIterator( $db, 'some_table', 'id_field', $batchSize );
+
+ $response = $this->genSelectResult( $batchSize, /*numRows*/ 5, function() {
+ static $i = 0;
+ return array( 'id_field' => ++$i );
+ } );
+ $db->expects( $this->exactly( count( $response ) ) )
+ ->method( 'select' )
+ ->will( $this->consecutivelyReturnFromSelect( $response ) );
+
+ $pos = 0;
+ foreach ( $reader as $rows ) {
+ $this->assertEquals( $response[$pos], $rows, "Testing row in position $pos" );
+ $pos++;
+ }
+ // -1 is because the final array() marks the end and isnt included
+ $this->assertEquals( count( $response ) - 1, $pos );
+ }
+
+ static public function provider_readerGetPrimaryKey() {
+ $row = array(
+ 'id_field' => 42,
+ 'some_col' => 'dvorak',
+ 'other_col' => 'samurai',
+ );
+ return array(
+
+ array(
+ 'Must return single column pk when requested',
+ array( 'id_field' => 42 ),
+ $row
+ ),
+
+ array(
+ 'Must return multiple column pks when requested',
+ array( 'id_field' => 42, 'other_col' => 'samurai' ),
+ $row
+ ),
+
+ );
+ }
+
+ /**
+ * @dataProvider provider_readerGetPrimaryKey
+ */
+ public function testReaderGetPrimaryKey( $message, array $expected, array $row ) {
+ $reader = new BatchRowIterator( $this->mockDb(), 'some_table', array_keys( $expected ), 8675309 );
+ $this->assertEquals( $expected, $reader->extractPrimaryKeys( (object) $row ), $message );
+ }
+
+ static public function provider_readerSetFetchColumns() {
+ return array(
+
+ array(
+ 'Must merge primary keys into select conditions',
+ // Expected column select
+ array( 'foo', 'bar' ),
+ // primary keys
+ array( 'foo' ),
+ // setFetchColumn
+ array( 'bar' )
+ ),
+
+ array(
+ 'Must not merge primary keys into the all columns selector',
+ // Expected column select
+ array( '*' ),
+ // primary keys
+ array( 'foo' ),
+ // setFetchColumn
+ array( '*' ),
+ ),
+
+ array(
+ 'Must not duplicate primary keys into column selector',
+ // Expected column select.
+ // TODO: figure out how to only assert the array_values portion and not the keys
+ array( 0 => 'foo', 1 => 'bar', 3 => 'baz' ),
+ // primary keys
+ array( 'foo', 'bar', ),
+ // setFetchColumn
+ array( 'bar', 'baz' ),
+ ),
+ );
+ }
+
+ /**
+ * @dataProvider provider_readerSetFetchColumns
+ */
+ public function testReaderSetFetchColumns( $message, array $columns, array $primaryKeys, array $fetchColumns ) {
+ $db = $this->mockDb();
+ $db->expects( $this->once() )
+ ->method( 'select' )
+ ->with( 'some_table', $columns ) // only testing second parameter of DatabaseBase::select
+ ->will( $this->returnValue( new ArrayIterator( array() ) ) );
+
+ $reader = new BatchRowIterator( $db, 'some_table', $primaryKeys, 22 );
+ $reader->setFetchColumns( $fetchColumns );
+ // triggers first database select
+ $reader->rewind();
+ }
+
+ static public function provider_readerSelectConditions() {
+ return array(
+
+ array(
+ "With single primary key must generate id > 'value'",
+ // Expected second iteration
+ array( "( id_field > '3' )" ),
+ // Primary key(s)
+ 'id_field',
+ ),
+
+ array(
+ 'With multiple primary keys the first conditions must use >= and the final condition must use >',
+ // Expected second iteration
+ array( "( id_field = '3' AND foo > '103' ) OR ( id_field > '3' )" ),
+ // Primary key(s)
+ array( 'id_field', 'foo' ),
+ ),
+
+ );
+ }
+
+ /**
+ * Slightly hackish to use reflection, but asserting different parameters
+ * to consecutive calls of DatabaseBase::select in phpunit is error prone
+ *
+ * @dataProvider provider_readerSelectConditions
+ */
+ public function testReaderSelectConditionsMultiplePrimaryKeys( $message, $expectedSecondIteration, $primaryKeys, $batchSize = 3 ) {
+ $results = $this->genSelectResult( $batchSize, $batchSize * 3, function() {
+ static $i = 0, $j = 100, $k = 1000;
+ return array( 'id_field' => ++$i, 'foo' => ++$j, 'bar' => ++$k );
+ } );
+ $db = $this->mockDbConsecutiveSelect( $results );
+
+ $conditions = array( 'bar' => 42, 'baz' => 'hai' );
+ $reader = new BatchRowIterator( $db, 'some_table', $primaryKeys, $batchSize );
+ $reader->addConditions( $conditions );
+
+ $buildConditions = new ReflectionMethod( $reader, 'buildConditions' );
+ $buildConditions->setAccessible( true );
+
+ // On first iteration only the passed conditions must be used
+ $this->assertEquals( $conditions, $buildConditions->invoke( $reader ),
+ 'First iteration must return only the conditions passed in addConditions' );
+ $reader->rewind();
+
+ // Second iteration must use the maximum primary key of last set
+ $this->assertEquals(
+ $conditions + $expectedSecondIteration,
+ $buildConditions->invoke( $reader ),
+ $message
+ );
+ }
+
+ protected function mockDbConsecutiveSelect( array $retvals ) {
+ $db = $this->mockDb();
+ $db->expects( $this->any() )
+ ->method( 'select' )
+ ->will( $this->consecutivelyReturnFromSelect( $retvals ) );
+ $db->expects( $this->any() )
+ ->method( 'addQuotes' )
+ ->will( $this->returnCallback( function( $value ) {
+ return "'$value'"; // not real quoting: doesn't matter in test
+ } ) );
+
+ return $db;
+ }
+
+ protected function consecutivelyReturnFromSelect( array $results ) {
+ $retvals = array();
+ foreach ( $results as $rows ) {
+ // The DatabaseBase::select method returns iterators, so we do too.
+ $retvals[] = $this->returnValue( new ArrayIterator( $rows ) );
+ }
+
+ return call_user_func_array( array( $this, 'onConsecutiveCalls' ), $retvals );
+ }
+
+
+ protected function genSelectResult( $batchSize, $numRows, $rowGenerator ) {
+ $res = array();
+ for ( $i = 0; $i < $numRows; $i += $batchSize ) {
+ $rows = array();
+ for ( $j = 0; $j < $batchSize && $i + $j < $numRows; $j++ ) {
+ $rows [] = (object) call_user_func( $rowGenerator );
+ }
+ $res[] = $rows;
+ }
+ $res[] = array(); // termination condition requires empty result for last row
+ return $res;
+ }
+
+ protected function mockDb() {
+ // Cant mock from DatabaseType or DatabaseBase, they dont
+ // have the full gamut of methods
+ $databaseMysql = $this->getMockBuilder( 'DatabaseMysql' )
+ ->disableOriginalConstructor()
+ ->getMock();
+ $databaseMysql->expects( $this->any() )
+ ->method( 'isOpen' )
+ ->will( $this->returnValue( true ) );
+ return $databaseMysql;
+ }
+}
dépôts / lhc / web / wiklou.git / commitdiff