maintenance/dumpTextPass.php

   1 <?php
   2 /**
   3  * Script that postprocesses XML dumps from dumpBackup.php to add page text
   4  *
   5  * Copyright © 2005 Brion Vibber <brion@pobox.com>, 2010 Alexandre Emsenhuber
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Maintenance
  25  */
  26
  27 $originalDir = getcwd();
  28
  29 require_once( dirname( __FILE__ ) . '/commandLine.inc' );
  30 require_once( 'backup.inc' );
  31
  32 /**
  33  * @ingroup Maintenance
  34  */
  35 class TextPassDumper extends BackupDumper {
  36         var $prefetch = null;
  37         var $input = "php://stdin";
  38         var $fetchCount = 0;
  39         var $prefetchCount = 0;
  40         var $lastTime = 0;
  41         var $pageCountLast = 0;
  42         var $revCountLast = 0;
  43         var $prefetchCountLast = 0;
  44         var $fetchCountLast = 0;
  45
  46         var $failures = 0;
  47         var $maxFailures = 5;
  48         var $failedTextRetrievals = 0;
  49         var $maxConsecutiveFailedTextRetrievals = 200;
  50         var $failureTimeout = 5; // Seconds to sleep after db failure
  51
  52         var $php = "php";
  53         var $spawn = false;
  54         var $spawnProc = false;
  55         var $spawnWrite = false;
  56         var $spawnRead = false;
  57         var $spawnErr = false;
  58
  59         var $ID = 0;
  60
  61         function initProgress( $history ) {
  62                 parent::initProgress();
  63                 $this->ID = getmypid();
  64                 $this->lastTime = $this->startTime;
  65         }
  66
  67         function dump( $history, $text = WikiExporter::TEXT ) {
  68                 # This shouldn't happen if on console... ;)
  69                 header( 'Content-type: text/html; charset=UTF-8' );
  70
  71                 # Notice messages will foul up your XML output even if they're
  72                 # relatively harmless.
  73                 if ( ini_get( 'display_errors' ) )
  74                         ini_set( 'display_errors', 'stderr' );
  75
  76                 $this->initProgress( $history );
  77
  78                 $this->db = $this->backupDb();
  79
  80                 $this->readDump();
  81
  82                 if ( $this->spawnProc ) {
  83                         $this->closeSpawn();
  84                 }
  85
  86                 $this->report( true );
  87         }
  88
  89         function processOption( $opt, $val, $param ) {
  90                 global $IP;
  91                 $url = $this->processFileOpt( $val, $param );
  92
  93                 switch( $opt ) {
  94                 case 'prefetch':
  95                         require_once "$IP/maintenance/backupPrefetch.inc";
  96                         $this->prefetch = new BaseDump( $url );
  97                         break;
  98                 case 'stub':
  99                         $this->input = $url;
 100                         break;
 101                 case 'spawn':
 102                         $this->spawn = true;
 103                         if ( $val ) {
 104                                 $this->php = $val;
 105                         }
 106                         break;
 107                 }
 108         }
 109
 110         function processFileOpt( $val, $param ) {
 111                 $fileURIs = explode(';',$param);
 112                 foreach ( $fileURIs as $URI ) {
 113                         switch( $val ) {
 114                                 case "file":
 115                                         $newURI = $URI;
 116                                         break;
 117                                 case "gzip":
 118                                         $newURI = "compress.zlib://$URI";
 119                                         break;
 120                                 case "bzip2":
 121                                         $newURI = "compress.bzip2://$URI";
 122                                         break;
 123                                 case "7zip":
 124                                         $newURI = "mediawiki.compress.7z://$URI";
 125                                         break;
 126                                 default:
 127                                         $newURI = $URI;
 128                         }
 129                         $newFileURIs[] = $newURI;
 130                 }
 131                 $val = implode( ';', $newFileURIs );
 132                 return $val;
 133         }
 134
 135         /**
 136          * Overridden to include prefetch ratio if enabled.
 137          */
 138         function showReport() {
 139                 if ( !$this->prefetch ) {
 140                         return parent::showReport();
 141                 }
 142
 143                 if ( $this->reporting ) {
 144                         $now = wfTimestamp( TS_DB );
 145                         $deltaAll = wfTime() - $this->startTime;
 146                         $deltaPart = wfTime() - $this->lastTime;
 147                         $this->pageCountPart = $this->pageCount - $this->pageCountLast;
 148                         $this->revCountPart = $this->revCount - $this->revCountLast;
 149
 150                         if ( $deltaAll ) {
 151                                 $portion = $this->revCount / $this->maxCount;
 152                                 $eta = $this->startTime + $deltaAll / $portion;
 153                                 $etats = wfTimestamp( TS_DB, intval( $eta ) );
 154                                 if ( $this->fetchCount ) {
 155                                         $fetchRate = 100.0 * $this->prefetchCount / $this->fetchCount;
 156                                 }
 157                                 else {
 158                                         $fetchRate = '-';
 159                                 }
 160                                 $pageRate = $this->pageCount / $deltaAll;
 161                                 $revRate = $this->revCount / $deltaAll;
 162                         } else {
 163                                 $pageRate = '-';
 164                                 $revRate = '-';
 165                                 $etats = '-';
 166                                 $fetchRate = '-';
 167                         }
 168                         if ( $deltaPart ) {
 169                                 if ( $this->fetchCountLast ) {
 170                                         $fetchRatePart = 100.0 * $this->prefetchCountLast / $this->fetchCountLast;
 171                                 }
 172                                 else {
 173                                         $fetchRatePart = '-';
 174                                 }
 175                                 $pageRatePart = $this->pageCountPart / $deltaPart;
 176                                 $revRatePart = $this->revCountPart / $deltaPart;
 177
 178                         } else {
 179                                 $fetchRatePart = '-';
 180                                 $pageRatePart = '-';
 181                                 $revRatePart = '-';
 182                         }
 183                         $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), %0.1f%%|%0.1f%% prefetched (all|curr), ETA %s [max %d]",-
 184                                         $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, $pageRatePart, $this->revCount, $revRate, $revRatePart, $fetchRate, $fetchRatePart, $etats, $this->maxCount ) );
 185                         $this->lastTime = $now;
 186                         $this->partCountLast = $this->partCount;
 187                         $this->revCountLast = $this->revCount;
 188                         $this->prefetchCountLast = $this->prefetchCount;
 189                         $this->fetchCountLast = $this->fetchCount;
 190                 }
 191         }
 192
 193         function readDump() {
 194                 $state = '';
 195                 $lastName = '';
 196                 $this->thisPage = 0;
 197                 $this->thisRev = 0;
 198
 199                 $reader = new XMLReader();
 200                 $reader->open( $this->input );
 201                 $writer = new XMLWriter();
 202                 $writer->openMemory();
 203
 204
 205                 while ( $reader->read() ) {
 206                         $tag = $reader->name;
 207                         $type = $reader->nodeType;
 208
 209                         if ( $type == XmlReader::END_ELEMENT ) {
 210                                 $writer->endElement();
 211
 212                                 if ( $tag == 'revision' ) {
 213                                         $this->revCount();
 214                                         $this->thisRev = '';
 215                                 } elseif ( $tag == 'page' ) {
 216                                         $this->reportPage();
 217                                         $this->thisPage = '';
 218                                 }
 219                         } elseif ( $type == XmlReader::ELEMENT ) {
 220                                 $attribs = array();
 221                                 if ( $reader->hasAttributes ) {
 222                                         for ( $i = 0; $reader->moveToAttributeNo( $i ); $i++ ) {
 223                                                 $attribs[$reader->name] = $reader->value;
 224                                         }
 225                                 }
 226
 227                                 if ( $reader->isEmptyElement && $tag == 'text' && isset( $attribs['id'] ) ) {
 228                                         $writer->startElement( 'text' );
 229                                         $writer->writeAttribute( 'xml:space', 'preserve' );
 230                                         $text = $this->getText( $attribs['id'] );
 231                                         if ( strlen( $text ) ) {
 232                                                 $writer->text( $text );
 233                                         }
 234                                         $writer->endElement();
 235                                 } else {
 236                                         $writer->startElement( $tag );
 237                                         foreach( $attribs as $name => $val ) {
 238                                                 $writer->writeAttribute( $name, $val );
 239                                         }
 240                                         if ( $reader->isEmptyElement ) {
 241                                                 $writer->endElement();
 242                                         }
 243                                 }
 244
 245                                 $lastName = $tag;
 246                                 if ( $tag == 'revision' ) {
 247                                         $state = 'revision';
 248                                 } elseif ( $tag == 'page' ) {
 249                                         $state = 'page';
 250                                 }
 251                         } elseif ( $type == XMLReader::SIGNIFICANT_WHITESPACE || $type == XMLReader::TEXT ) {
 252                                 if ( $lastName == 'id' ) {
 253                                         if ( $state == 'revision' ) {
 254                                                 $this->thisRev .= $reader->value;
 255                                         } elseif ( $state == 'page' ) {
 256                                                 $this->thisPage .= $reader->value;
 257                                         }
 258                                 }
 259                                 $writer->text( $reader->value );
 260                         }
 261                         $this->sink->write( $writer->outputMemory() );
 262                 }
 263         }
 264
 265         function getText( $id ) {
 266                 $this->fetchCount++;
 267                 if ( isset( $this->prefetch ) ) {
 268                         $text = $this->prefetch->prefetch( $this->thisPage, $this->thisRev );
 269                         if ( $text !== null ) { // Entry missing from prefetch dump
 270                                 $dbr = wfGetDB( DB_SLAVE );
 271                                 $revID = intval( $this->thisRev );
 272                                 $revLength = $dbr->selectField( 'revision', 'rev_len', array( 'rev_id' => $revID ) );
 273                                 // if length of rev text in file doesn't match length in db, we reload
 274                                 // this avoids carrying forward broken data from previous xml dumps
 275                                 if( strlen( $text ) == $revLength ) {
 276                                         $this->prefetchCount++;
 277                                         return $text;
 278                                 }
 279                         }
 280                 }
 281                 return $this->doGetText( $id );
 282         }
 283
 284         private function doGetText( $id ) {
 285                 $id = intval( $id );
 286                 $this->failures = 0;
 287                 $ex = new MWException( "Graceful storage failure" );
 288                 while (true) {
 289                         if ( $this->spawn ) {
 290                                 if ($this->failures) {
 291                                         // we don't know why it failed, could be the child process
 292                                         // borked, could be db entry busted, could be db server out to lunch,
 293                                         // so cover all bases
 294                                         $this->closeSpawn();
 295                                         $this->openSpawn();
 296                                 }
 297                                 $text =  $this->getTextSpawned( $id );
 298                         } else {
 299                                 $text =  $this->getTextDbSafe( $id );
 300                         }
 301                         if ( $text === false ) {
 302                                 $this->failures++;
 303                                 if ( $this->failures > $this->maxFailures) {
 304                                         $this->progress( "Failed to retrieve revision text for text id ".
 305                                                                          "$id after $this->maxFailures tries, giving up" );
 306                                         // were there so many bad retrievals in a row we want to bail?
 307                                         // at some point we have to declare the dump irretrievably broken
 308                                         $this->failedTextRetrievals++;
 309                                         if ($this->failedTextRetrievals > $this->maxConsecutiveFailedTextRetrievals) {
 310                                                 throw $ex;
 311                                         }
 312                                         else {
 313                                                 // would be nice to return something better to the caller someday,
 314                                                 // log what we know about the failure and about the revision
 315                                                 return("");
 316                                         }
 317                                 } else {
 318                                         $this->progress( "Error $this->failures " .
 319                                                                  "of allowed $this->maxFailures retrieving revision text for text id $id! " .
 320                                                                  "Pausing $this->failureTimeout seconds before retry..." );
 321                                         sleep( $this->failureTimeout );
 322                                 }
 323                         } else {
 324                                 $this->failedTextRetrievals= 0;
 325                                 return( $text );
 326                         }
 327                 }
 328
 329         }
 330
 331         /**
 332          * Fetch a text revision from the database, retrying in case of failure.
 333          * This may survive some transitory errors by reconnecting, but
 334          * may not survive a long-term server outage.
 335          */
 336         private function getTextDbSafe( $id ) {
 337                 while ( true ) {
 338                         try {
 339                                 $text = $this->getTextDb( $id );
 340                         } catch ( DBQueryError $ex ) {
 341                                 $text = false;
 342                         }
 343                         return $text;
 344                 }
 345         }
 346
 347         /**
 348          * May throw a database error if, say, the server dies during query.
 349          */
 350         private function getTextDb( $id ) {
 351                 global $wgContLang;
 352                 $row = $this->db->selectRow( 'text',
 353                         array( 'old_text', 'old_flags' ),
 354                         array( 'old_id' => $id ),
 355                         __METHOD__ );
 356                 $text = Revision::getRevisionText( $row );
 357                 if ( $text === false ) {
 358                         return false;
 359                 }
 360                 $stripped = str_replace( "\r", "", $text );
 361                 $normalized = $wgContLang->normalize( $stripped );
 362                 return $normalized;
 363         }
 364
 365         private function getTextSpawned( $id ) {
 366                 wfSuppressWarnings();
 367                 if ( !$this->spawnProc ) {
 368                         // First time?
 369                         $this->openSpawn();
 370                 }
 371                 $text = $this->getTextSpawnedOnce( $id );
 372                 wfRestoreWarnings();
 373                 return $text;
 374         }
 375
 376         function openSpawn() {
 377                 global $IP;
 378
 379                 $cmd = implode( " ",
 380                         array_map( 'wfEscapeShellArg',
 381                                 array(
 382                                         $this->php,
 383                                         "$IP/maintenance/fetchText.php",
 384                                         '--wiki', wfWikiID() ) ) );
 385                 $spec = array(
 386                         0 => array( "pipe", "r" ),
 387                         1 => array( "pipe", "w" ),
 388                         2 => array( "file", "/dev/null", "a" ) );
 389                 $pipes = array();
 390
 391                 $this->progress( "Spawning database subprocess: $cmd" );
 392                 $this->spawnProc = proc_open( $cmd, $spec, $pipes );
 393                 if ( !$this->spawnProc ) {
 394                         // shit
 395                         $this->progress( "Subprocess spawn failed." );
 396                         return false;
 397                 }
 398                 list(
 399                         $this->spawnWrite, // -> stdin
 400                         $this->spawnRead,  // <- stdout
 401                 ) = $pipes;
 402
 403                 return true;
 404         }
 405
 406         private function closeSpawn() {
 407                 wfSuppressWarnings();
 408                 if ( $this->spawnRead )
 409                         fclose( $this->spawnRead );
 410                 $this->spawnRead = false;
 411                 if ( $this->spawnWrite )
 412                         fclose( $this->spawnWrite );
 413                 $this->spawnWrite = false;
 414                 if ( $this->spawnErr )
 415                         fclose( $this->spawnErr );
 416                 $this->spawnErr = false;
 417                 if ( $this->spawnProc )
 418                         pclose( $this->spawnProc );
 419                 $this->spawnProc = false;
 420                 wfRestoreWarnings();
 421         }
 422
 423         private function getTextSpawnedOnce( $id ) {
 424                 global $wgContLang;
 425
 426                 $ok = fwrite( $this->spawnWrite, "$id\n" );
 427                 // $this->progress( ">> $id" );
 428                 if ( !$ok ) return false;
 429
 430                 $ok = fflush( $this->spawnWrite );
 431                 // $this->progress( ">> [flush]" );
 432                 if ( !$ok ) return false;
 433
 434                 // check that the text id they are sending is the one we asked for
 435                 // this avoids out of sync revision text errors we have encountered in the past
 436                 $newId = fgets( $this->spawnRead );
 437                 if ( $newId === false ) {
 438                         return false;
 439                 }
 440                 if ( $id != intval( $newId ) ) {
 441                         return false;
 442                 }
 443
 444                 $len = fgets( $this->spawnRead );
 445                 // $this->progress( "<< " . trim( $len ) );
 446                 if ( $len === false ) return false;
 447
 448                 $nbytes = intval( $len );
 449                 // actual error, not zero-length text
 450                 if ($nbytes < 0 ) return false;
 451
 452                 $text = "";
 453
 454                 // Subprocess may not send everything at once, we have to loop.
 455                 while ( $nbytes > strlen( $text ) ) {
 456                         $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) );
 457                         if ( $buffer === false ) break;
 458                         $text .= $buffer;
 459                 }
 460
 461                 $gotbytes = strlen( $text );
 462                 if ( $gotbytes != $nbytes ) {
 463                         $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes " );
 464                         return false;
 465                 }
 466
 467                 // Do normalization in the dump thread...
 468                 $stripped = str_replace( "\r", "", $text );
 469                 $normalized = $wgContLang->normalize( $stripped );
 470                 return $normalized;
 471         }
 472 }
 473
 474
 475 $dumper = new TextPassDumper( $argv );
 476
 477 if ( !isset( $options['help'] ) ) {
 478         $dumper->dump( WikiExporter::FULL );
 479 } else {
 480         $dumper->progress( <<<ENDS
 481 This script postprocesses XML dumps from dumpBackup.php to add
 482 page text which was stubbed out (using --stub).
 483
 484 XML input is accepted on stdin.
 485 XML output is sent to stdout; progress reports are sent to stderr.
 486
 487 Usage: php dumpTextPass.php [<options>]
 488 Options:
 489   --stub=<type>:<file> To load a compressed stub dump instead of stdin
 490   --prefetch=<type>:<file> Use a prior dump file as a text source, to save
 491                           pressure on the database.
 492   --quiet         Don't dump status reports to stderr.
 493   --report=n  Report position and speed after every n pages processed.
 494                           (Default: 100)
 495   --server=h  Force reading from MySQL server h
 496   --output=<type>:<file> Write to a file instead of stdout
 497                            <type>s: file, gzip, bzip2, 7zip
 498   --current       Base ETA on number of pages in database instead of all revisions
 499   --spawn         Spawn a subprocess for loading text records
 500   --help      Display this help message
 501 ENDS
 502 );
 503 }
 504
 505