3 * Helper class for the --prefetch option of dumpTextPass.php
5 * Copyright © 2005 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
24 * @ingroup Maintenance
28 * Readahead helper for making large MediaWiki data dumps;
29 * reads in a previous XML dump to sequentially prefetch text
30 * records already normalized and decompressed.
32 * This can save load on the external database servers, hopefully.
34 * Assumes that dumps will be recorded in the canonical order:
35 * - ascending by page_id
36 * - ascending by rev_id within each page
37 * - text contents are immutable and should not change once
38 * recorded, so the previous dump is a reliable source
40 * @ingroup Maintenance
45 var $atPageEnd = false;
49 function BaseDump( $infile ) {
50 $this->reader = new XMLReader();
51 $this->reader->open( $infile );
55 * Attempts to fetch the text of a particular page revision
56 * from the dump stream. May return null if the page is
59 * @param $page Integer: ID number of page to read
60 * @param $rev Integer: ID number of revision to read
61 * @return string or null
63 function prefetch( $page, $rev ) {
64 $page = intval( $page );
65 $rev = intval( $rev );
66 while ( $this->lastPage < $page && !$this->atEnd ) {
67 $this->debug( "BaseDump::prefetch at page $this->lastPage, looking for $page" );
70 if ( $this->lastPage > $page || $this->atEnd ) {
71 $this->debug( "BaseDump::prefetch already past page $page looking for rev $rev [$this->lastPage, $this->lastRev]" );
74 while ( $this->lastRev < $rev && !$this->atEnd && !$this->atPageEnd ) {
75 $this->debug( "BaseDump::prefetch at page $this->lastPage, rev $this->lastRev, looking for $page, $rev" );
78 if ( $this->lastRev == $rev && !$this->atEnd ) {
79 $this->debug( "BaseDump::prefetch hit on $page, $rev [$this->lastPage, $this->lastRev]" );
80 return $this->nextText();
82 $this->debug( "BaseDump::prefetch already past rev $rev on page $page [$this->lastPage, $this->lastRev]" );
87 function debug( $str ) {
88 wfDebug( $str . "\n" );
90 // $dumper->progress( $str );
97 if ( $this->skipTo( 'page', 'mediawiki' ) ) {
98 if ( $this->skipTo( 'id' ) ) {
99 $this->lastPage = intval( $this->nodeContents() );
101 $this->atPageEnd = false;
112 if ( $this->skipTo( 'revision' ) ) {
113 if ( $this->skipTo( 'id' ) ) {
114 $this->lastRev = intval( $this->nodeContents() );
117 $this->atPageEnd = true;
124 function nextText() {
125 $this->skipTo( 'text' );
126 return strval( $this->nodeContents() );
132 function skipTo( $name, $parent = 'page' ) {
133 if ( $this->atEnd ) {
136 while ( $this->reader->read() ) {
137 if ( $this->reader->nodeType == XMLReader::ELEMENT &&
138 $this->reader->name == $name ) {
141 if ( $this->reader->nodeType == XMLReader::END_ELEMENT &&
142 $this->reader->name == $parent ) {
143 $this->debug( "BaseDump::skipTo found </$parent> searching for <$name>" );
147 return $this->close();
151 * Shouldn't something like this be built-in to XMLReader?
152 * Fetches text contents of the current element, assuming
153 * no sub-elements or such scary things.
158 function nodeContents() {
159 if ( $this->atEnd ) {
162 if ( $this->reader->isEmptyElement ) {
166 while ( $this->reader->read() ) {
167 switch( $this->reader->nodeType ) {
168 case XMLReader::TEXT:
169 // case XMLReader::WHITESPACE:
170 case XMLReader::SIGNIFICANT_WHITESPACE:
171 $buffer .= $this->reader->value;
173 case XMLReader::END_ELEMENT:
177 return $this->close();
184 $this->reader->close();