includes/export/WikiExporter.php

   1 <?php
   2 /**
   3  * Base class for exporting
   4  *
   5  * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  */
  25
  26 /**
  27  * @defgroup Dump Dump
  28  */
  29
  30 use MediaWiki\MediaWikiServices as MediaWikiServicesAlias;
  31 use Wikimedia\Rdbms\IResultWrapper;
  32 use Wikimedia\Rdbms\IDatabase;
  33
  34 /**
  35  * @ingroup SpecialPage Dump
  36  */
  37 class WikiExporter {
  38         /** @var bool Return distinct author list (when not returning full history) */
  39         public $list_authors = false;
  40
  41         /** @var bool */
  42         public $dumpUploads = false;
  43
  44         /** @var bool */
  45         public $dumpUploadFileContents = false;
  46
  47         /** @var string */
  48         public $author_list = "";
  49
  50         const FULL = 1;
  51         const CURRENT = 2;
  52         const STABLE = 4; // extension defined
  53         const LOGS = 8;
  54         const RANGE = 16;
  55
  56         const TEXT = XmlDumpWriter::WRITE_CONTENT;
  57         const STUB = XmlDumpWriter::WRITE_STUB;
  58
  59         const BATCH_SIZE = 50000;
  60
  61         /** @var int */
  62         public $text;
  63
  64         /** @var DumpOutput */
  65         public $sink;
  66
  67         /** @var XmlDumpWriter */
  68         private $writer;
  69
  70         /**
  71          * Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion.
  72          * @return string
  73          */
  74         public static function schemaVersion() {
  75                 global $wgXmlDumpSchemaVersion;
  76                 return $wgXmlDumpSchemaVersion;
  77         }
  78
  79         /**
  80          * @param IDatabase $db
  81          * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
  82          *   WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
  83          *   - offset: non-inclusive offset at which to start the query
  84          *   - limit: maximum number of rows to return
  85          *   - dir: "asc" or "desc" timestamp order
  86          * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
  87          * @param null|array $limitNamespaces Comma-separated list of namespace numbers
  88          *   to limit results
  89          */
  90         function __construct( $db, $history = self::CURRENT, $text = self::TEXT,
  91                         $limitNamespaces = null
  92         ) {
  93                 $this->db = $db;
  94                 $this->history = $history;
  95                 $this->writer = new XmlDumpWriter( $text, self::schemaVersion() );
  96                 $this->sink = new DumpOutput();
  97                 $this->text = $text;
  98                 $this->limitNamespaces = $limitNamespaces;
  99         }
 100
 101         /**
 102          * @param string $schemaVersion which schema version the generated XML should comply to.
 103          * One of the values from self::$supportedSchemas, using the XML_DUMP_SCHEMA_VERSION_XX
 104          * constants.
 105          */
 106         public function setSchemaVersion( $schemaVersion ) {
 107                 $this->writer = new XmlDumpWriter( $this->text, $schemaVersion );
 108         }
 109
 110         /**
 111          * Set the DumpOutput or DumpFilter object which will receive
 112          * various row objects and XML output for filtering. Filters
 113          * can be chained or used as callbacks.
 114          *
 115          * @param DumpOutput &$sink
 116          */
 117         public function setOutputSink( &$sink ) {
 118                 $this->sink =& $sink;
 119         }
 120
 121         public function openStream() {
 122                 $output = $this->writer->openStream();
 123                 $this->sink->writeOpenStream( $output );
 124         }
 125
 126         public function closeStream() {
 127                 $output = $this->writer->closeStream();
 128                 $this->sink->writeCloseStream( $output );
 129         }
 130
 131         /**
 132          * Dumps a series of page and revision records for all pages
 133          * in the database, either including complete history or only
 134          * the most recent version.
 135          */
 136         public function allPages() {
 137                 $this->dumpFrom( '' );
 138         }
 139
 140         /**
 141          * Dumps a series of page and revision records for those pages
 142          * in the database falling within the page_id range given.
 143          * @param int $start Inclusive lower limit (this id is included)
 144          * @param int $end Exclusive upper limit (this id is not included)
 145          *   If 0, no upper limit.
 146          * @param bool $orderRevs order revisions within pages in ascending order
 147          */
 148         public function pagesByRange( $start, $end, $orderRevs ) {
 149                 if ( $orderRevs ) {
 150                         $condition = 'rev_page >= ' . intval( $start );
 151                         if ( $end ) {
 152                                 $condition .= ' AND rev_page < ' . intval( $end );
 153                         }
 154                 } else {
 155                         $condition = 'page_id >= ' . intval( $start );
 156                         if ( $end ) {
 157                                 $condition .= ' AND page_id < ' . intval( $end );
 158                         }
 159                 }
 160                 $this->dumpFrom( $condition, $orderRevs );
 161         }
 162
 163         /**
 164          * Dumps a series of page and revision records for those pages
 165          * in the database with revisions falling within the rev_id range given.
 166          * @param int $start Inclusive lower limit (this id is included)
 167          * @param int $end Exclusive upper limit (this id is not included)
 168          *   If 0, no upper limit.
 169          */
 170         public function revsByRange( $start, $end ) {
 171                 $condition = 'rev_id >= ' . intval( $start );
 172                 if ( $end ) {
 173                         $condition .= ' AND rev_id < ' . intval( $end );
 174                 }
 175                 $this->dumpFrom( $condition );
 176         }
 177
 178         /**
 179          * @param Title $title
 180          */
 181         public function pageByTitle( $title ) {
 182                 $this->dumpFrom(
 183                         'page_namespace=' . $title->getNamespace() .
 184                         ' AND page_title=' . $this->db->addQuotes( $title->getDBkey() ) );
 185         }
 186
 187         /**
 188          * @param string $name
 189          * @throws MWException
 190          */
 191         public function pageByName( $name ) {
 192                 $title = Title::newFromText( $name );
 193                 if ( is_null( $title ) ) {
 194                         throw new MWException( "Can't export invalid title" );
 195                 } else {
 196                         $this->pageByTitle( $title );
 197                 }
 198         }
 199
 200         /**
 201          * @param array $names
 202          */
 203         public function pagesByName( $names ) {
 204                 foreach ( $names as $name ) {
 205                         $this->pageByName( $name );
 206                 }
 207         }
 208
 209         public function allLogs() {
 210                 $this->dumpFrom( '' );
 211         }
 212
 213         /**
 214          * @param int $start
 215          * @param int $end
 216          */
 217         public function logsByRange( $start, $end ) {
 218                 $condition = 'log_id >= ' . intval( $start );
 219                 if ( $end ) {
 220                         $condition .= ' AND log_id < ' . intval( $end );
 221                 }
 222                 $this->dumpFrom( $condition );
 223         }
 224
 225         /**
 226          * Generates the distinct list of authors of an article
 227          * Not called by default (depends on $this->list_authors)
 228          * Can be set by Special:Export when not exporting whole history
 229          *
 230          * @param array $cond
 231          */
 232         protected function do_list_authors( $cond ) {
 233                 $this->author_list = "<contributors>";
 234                 // rev_deleted
 235
 236                 $revQuery = Revision::getQueryInfo( [ 'page' ] );
 237                 $res = $this->db->select(
 238                         $revQuery['tables'],
 239                         [
 240                                 'rev_user_text' => $revQuery['fields']['rev_user_text'],
 241                                 'rev_user' => $revQuery['fields']['rev_user'],
 242                         ],
 243                         [
 244                                 $this->db->bitAnd( 'rev_deleted', Revision::DELETED_USER ) . ' = 0',
 245                                 $cond,
 246                         ],
 247                         __METHOD__,
 248                         [ 'DISTINCT' ],
 249                         $revQuery['joins']
 250                 );
 251
 252                 foreach ( $res as $row ) {
 253                         $this->author_list .= "<contributor>" .
 254                                 "<username>" .
 255                                 htmlspecialchars( $row->rev_user_text ) .
 256                                 "</username>" .
 257                                 "<id>" .
 258                                 ( (int)$row->rev_user ) .
 259                                 "</id>" .
 260                                 "</contributor>";
 261                 }
 262                 $this->author_list .= "</contributors>";
 263         }
 264
 265         /**
 266          * @param string $cond
 267          * @param bool $orderRevs
 268          * @throws MWException
 269          * @throws Exception
 270          */
 271         protected function dumpFrom( $cond = '', $orderRevs = false ) {
 272                 if ( $this->history & self::LOGS ) {
 273                         $this->dumpLogs( $cond );
 274                 } else {
 275                         $this->dumpPages( $cond, $orderRevs );
 276                 }
 277         }
 278
 279         /**
 280          * @param string $cond
 281          * @throws Exception
 282          */
 283         protected function dumpLogs( $cond ) {
 284                 $where = [];
 285                 # Hide private logs
 286                 $hideLogs = LogEventsList::getExcludeClause( $this->db );
 287                 if ( $hideLogs ) {
 288                         $where[] = $hideLogs;
 289                 }
 290                 # Add on any caller specified conditions
 291                 if ( $cond ) {
 292                         $where[] = $cond;
 293                 }
 294                 # Get logging table name for logging.* clause
 295                 $logging = $this->db->tableName( 'logging' );
 296
 297                 $result = null; // Assuring $result is not undefined, if exception occurs early
 298
 299                 $commentQuery = CommentStore::getStore()->getJoin( 'log_comment' );
 300                 $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' );
 301
 302                 $lastLogId = 0;
 303                 while ( true ) {
 304                         $result = $this->db->select(
 305                                 array_merge( [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ] ),
 306                                 [ "{$logging}.*", 'user_name' ] + $commentQuery['fields'] + $actorQuery['fields'],
 307                                 array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
 308                                 __METHOD__,
 309                                 [
 310                                         'ORDER BY' => 'log_id',
 311                                         'USE INDEX' => [ 'logging' => 'PRIMARY' ],
 312                                         'LIMIT' => self::BATCH_SIZE,
 313                                 ],
 314                                 [
 315                                         'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
 316                                 ] + $commentQuery['joins'] + $actorQuery['joins']
 317                         );
 318
 319                         if ( !$result->numRows() ) {
 320                                 break;
 321                         }
 322
 323                         $lastLogId = $this->outputLogStream( $result );
 324                 }
 325         }
 326
 327         /**
 328          * @param string $cond
 329          * @param bool $orderRevs
 330          * @throws MWException
 331          * @throws Exception
 332          */
 333         protected function dumpPages( $cond, $orderRevs ) {
 334                 global $wgMultiContentRevisionSchemaMigrationStage;
 335                 if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD ) ) {
 336                         // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
 337                         throw new MWException(
 338                                 'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
 339                                 . ' Support for dumping from the new schema is not implemented yet!'
 340                         );
 341                 }
 342
 343                 $revQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getQueryInfo(
 344                         [ 'page' ]
 345                 );
 346                 $slotQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getSlotsQueryInfo(
 347                         [ 'content' ]
 348                 );
 349
 350                 // We want page primary rather than revision.
 351                 // We also want to join in the slots and content tables.
 352                 // NOTE: This means we may get multiple rows per revision, and more rows
 353                 // than the batch size! Should be ok, since the max number of slots is
 354                 // fixed and low (dozens at worst).
 355                 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
 356                 $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
 357                 $join = $revQuery['joins'] + [
 358                                 'revision' => $revQuery['joins']['page'],
 359                                 'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
 360                                 'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
 361                         ];
 362                 unset( $join['page'] );
 363
 364                 $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
 365                 $fields[] = 'page_restrictions';
 366
 367                 if ( $this->text != self::STUB ) {
 368                         $fields['_load_content'] = '1';
 369                 }
 370
 371                 $conds = [];
 372                 if ( $cond !== '' ) {
 373                         $conds[] = $cond;
 374                 }
 375                 $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
 376                 $opts['USE INDEX'] = [];
 377
 378                 $op = '>';
 379                 if ( is_array( $this->history ) ) {
 380                         # Time offset/limit for all pages/history...
 381                         # Set time order
 382                         if ( $this->history['dir'] == 'asc' ) {
 383                                 $opts['ORDER BY'] = 'rev_timestamp ASC';
 384                         } else {
 385                                 $op = '<';
 386                                 $opts['ORDER BY'] = 'rev_timestamp DESC';
 387                         }
 388                         # Set offset
 389                         if ( !empty( $this->history['offset'] ) ) {
 390                                 $conds[] = "rev_timestamp $op " .
 391                                         $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) );
 392                         }
 393                         # Set query limit
 394                         if ( !empty( $this->history['limit'] ) ) {
 395                                 $maxRowCount = intval( $this->history['limit'] );
 396                         }
 397                 } elseif ( $this->history & self::FULL ) {
 398                         # Full history dumps...
 399                         # query optimization for history stub dumps
 400                         if ( $this->text == self::STUB ) {
 401                                 $opts[] = 'STRAIGHT_JOIN';
 402                                 $opts['USE INDEX']['revision'] = 'rev_page_id';
 403                                 unset( $join['revision'] );
 404                                 $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
 405                         }
 406                 } elseif ( $this->history & self::CURRENT ) {
 407                         # Latest revision dumps...
 408                         if ( $this->list_authors && $cond != '' ) { // List authors, if so desired
 409                                 $this->do_list_authors( $cond );
 410                         }
 411                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 412                         $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
 413                 } elseif ( $this->history & self::STABLE ) {
 414                         # "Stable" revision dumps...
 415                         # Default JOIN, to be overridden...
 416                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 417                         # One, and only one hook should set this, and return false
 418                         if ( Hooks::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
 419                                 throw new MWException( __METHOD__ . " given invalid history dump type." );
 420                         }
 421                 } elseif ( $this->history & self::RANGE ) {
 422                         # Dump of revisions within a specified range.  Condition already set in revsByRange().
 423                 } else {
 424                         # Unknown history specification parameter?
 425                         throw new MWException( __METHOD__ . " given invalid history dump type." );
 426                 }
 427
 428                 $result = null; // Assuring $result is not undefined, if exception occurs early
 429                 $done = false;
 430                 $lastRow = null;
 431                 $revPage = 0;
 432                 $revId = 0;
 433                 $rowCount = 0;
 434
 435                 $opts['LIMIT'] = self::BATCH_SIZE;
 436
 437                 Hooks::run( 'ModifyExportQuery',
 438                         [ $this->db, &$tables, &$cond, &$opts, &$join ] );
 439
 440                 while ( !$done ) {
 441                         // If necessary, impose the overall maximum and stop looping after this iteration.
 442                         if ( !empty( $maxRowCount ) && $rowCount + self::BATCH_SIZE > $maxRowCount ) {
 443                                 $opts['LIMIT'] = $maxRowCount - $rowCount;
 444                                 $done = true;
 445                         }
 446
 447                         $queryConds = $conds;
 448                         $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
 449                                 intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
 450
 451                         # Do the query and process any results, remembering max ids for the next iteration.
 452                         $result = $this->db->select(
 453                                 $tables,
 454                                 $fields,
 455                                 $queryConds,
 456                                 __METHOD__,
 457                                 $opts,
 458                                 $join
 459                         );
 460                         if ( $result->numRows() > 0 ) {
 461                                 $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
 462                                 $rowCount += $result->numRows();
 463                                 $revPage = $lastRow->rev_page;
 464                                 $revId = $lastRow->rev_id;
 465                         } else {
 466                                 $done = true;
 467                         }
 468
 469                         // If we are finished, close off final page element (if any).
 470                         if ( $done && $lastRow ) {
 471                                 $this->finishPageStreamOutput( $lastRow );
 472                         }
 473                 }
 474         }
 475
 476         /**
 477          * Runs through a query result set dumping page, revision, and slot records.
 478          * The result set should join the page, revision, slots, and content tables,
 479          * and be sorted/grouped by page and revision to avoid duplicate page records in the output.
 480          *
 481          * @param IResultWrapper $results
 482          * @param object $lastRow the last row output from the previous call (or null if none)
 483          * @return object the last row processed
 484          */
 485         protected function outputPageStreamBatch( $results, $lastRow ) {
 486                 $rowCarry = null;
 487                 while ( true ) {
 488                         $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
 489
 490                         if ( !$slotRows ) {
 491                                 break;
 492                         }
 493
 494                         // All revision info is present in all slot rows.
 495                         // Use the first slot row as the revision row.
 496                         $revRow = $slotRows[0];
 497
 498                         if ( $this->limitNamespaces &&
 499                                 !in_array( $revRow->page_namespace, $this->limitNamespaces ) ) {
 500                                 $lastRow = $revRow;
 501                                 continue;
 502                         }
 503
 504                         if ( $lastRow === null ||
 505                                 $lastRow->page_namespace !== $revRow->page_namespace ||
 506                                 $lastRow->page_title !== $revRow->page_title ) {
 507                                 if ( $lastRow !== null ) {
 508                                         $output = '';
 509                                         if ( $this->dumpUploads ) {
 510                                                 $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 511                                         }
 512                                         $output .= $this->writer->closePage();
 513                                         $this->sink->writeClosePage( $output );
 514                                 }
 515                                 $output = $this->writer->openPage( $revRow );
 516                                 $this->sink->writeOpenPage( $revRow, $output );
 517                         }
 518                         $output = $this->writer->writeRevision( $revRow, $slotRows );
 519                         $this->sink->writeRevision( $revRow, $output );
 520                         $lastRow = $revRow;
 521                 }
 522
 523                 if ( $rowCarry ) {
 524                         throw new LogicException( 'Error while processing a stream of slot rows' );
 525                 }
 526
 527                 return $lastRow;
 528         }
 529
 530         /**
 531          * Returns all slot rows for a revision.
 532          * Takes and returns a carry row from the last batch;
 533          *
 534          * @param IResultWrapper|array $results
 535          * @param null|object &$carry A row carried over from the last call to getSlotRowBatch()
 536          *
 537          * @return object[]
 538          */
 539         protected function getSlotRowBatch( $results, &$carry = null ) {
 540                 $slotRows = [];
 541                 $prev = null;
 542
 543                 if ( $carry ) {
 544                         $slotRows[] = $carry;
 545                         $prev = $carry;
 546                         $carry = null;
 547                 }
 548
 549                 while ( $row = $results->fetchObject() ) {
 550                         if ( $prev && $prev->rev_id !== $row->rev_id ) {
 551                                 $carry = $row;
 552                                 break;
 553                         }
 554                         $slotRows[] = $row;
 555                         $prev = $row;
 556                 }
 557
 558                 return $slotRows;
 559         }
 560
 561         /**
 562          * Final page stream output, after all batches are complete
 563          *
 564          * @param object $lastRow the last row output from the last batch (or null if none)
 565          */
 566         protected function finishPageStreamOutput( $lastRow ) {
 567                 $output = '';
 568                 if ( $this->dumpUploads ) {
 569                         $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 570                 }
 571                 $output .= $this->author_list;
 572                 $output .= $this->writer->closePage();
 573                 $this->sink->writeClosePage( $output );
 574         }
 575
 576         /**
 577          * @param IResultWrapper $resultset
 578          * @return int the log_id value of the last item output, or null if none
 579          */
 580         protected function outputLogStream( $resultset ) {
 581                 foreach ( $resultset as $row ) {
 582                         $output = $this->writer->writeLogItem( $row );
 583                         $this->sink->writeLogItem( $row, $output );
 584                 }
 585                 return isset( $row ) ? $row->log_id : null;
 586         }
 587 }