includes/export/WikiExporter.php

   1 <?php
   2 /**
   3  * Base class for exporting
   4  *
   5  * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  */
  25
  26 /**
  27  * @defgroup Dump Dump
  28  */
  29
  30 use MediaWiki\MediaWikiServices as MediaWikiServicesAlias;
  31 use MediaWiki\Storage\RevisionRecord;
  32 use Wikimedia\Rdbms\IResultWrapper;
  33 use Wikimedia\Rdbms\IDatabase;
  34
  35 /**
  36  * @ingroup SpecialPage Dump
  37  */
  38 class WikiExporter {
  39         /** @var bool Return distinct author list (when not returning full history) */
  40         public $list_authors = false;
  41
  42         /** @var bool */
  43         public $dumpUploads = false;
  44
  45         /** @var bool */
  46         public $dumpUploadFileContents = false;
  47
  48         /** @var string */
  49         public $author_list = "";
  50
  51         const FULL = 1;
  52         const CURRENT = 2;
  53         const STABLE = 4; // extension defined
  54         const LOGS = 8;
  55         const RANGE = 16;
  56
  57         const TEXT = XmlDumpWriter::WRITE_CONTENT;
  58         const STUB = XmlDumpWriter::WRITE_STUB;
  59
  60         const BATCH_SIZE = 50000;
  61
  62         /** @var int */
  63         public $text;
  64
  65         /** @var DumpOutput */
  66         public $sink;
  67
  68         /** @var XmlDumpWriter */
  69         private $writer;
  70
  71         /** @var IDatabase */
  72         protected $db;
  73
  74         /** @var array|int */
  75         protected $history;
  76
  77         /** @var array|null */
  78         protected $limitNamespaces;
  79
  80         /**
  81          * Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion.
  82          * @return string
  83          */
  84         public static function schemaVersion() {
  85                 global $wgXmlDumpSchemaVersion;
  86                 return $wgXmlDumpSchemaVersion;
  87         }
  88
  89         /**
  90          * @param IDatabase $db
  91          * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
  92          *   WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
  93          *   - offset: non-inclusive offset at which to start the query
  94          *   - limit: maximum number of rows to return
  95          *   - dir: "asc" or "desc" timestamp order
  96          * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
  97          * @param null|array $limitNamespaces Comma-separated list of namespace numbers
  98          *   to limit results
  99          */
 100         function __construct(
 101                 $db,
 102                 $history = self::CURRENT,
 103                 $text = self::TEXT,
 104                 $limitNamespaces = null
 105         ) {
 106                 $this->db = $db;
 107                 $this->history = $history;
 108                 $this->writer = new XmlDumpWriter( $text, self::schemaVersion() );
 109                 $this->sink = new DumpOutput();
 110                 $this->text = $text;
 111                 $this->limitNamespaces = $limitNamespaces;
 112         }
 113
 114         /**
 115          * @param string $schemaVersion which schema version the generated XML should comply to.
 116          * One of the values from self::$supportedSchemas, using the XML_DUMP_SCHEMA_VERSION_XX
 117          * constants.
 118          */
 119         public function setSchemaVersion( $schemaVersion ) {
 120                 $this->writer = new XmlDumpWriter( $this->text, $schemaVersion );
 121         }
 122
 123         /**
 124          * Set the DumpOutput or DumpFilter object which will receive
 125          * various row objects and XML output for filtering. Filters
 126          * can be chained or used as callbacks.
 127          *
 128          * @param DumpOutput|DumpFilter &$sink
 129          */
 130         public function setOutputSink( &$sink ) {
 131                 $this->sink =& $sink;
 132         }
 133
 134         public function openStream() {
 135                 $output = $this->writer->openStream();
 136                 $this->sink->writeOpenStream( $output );
 137         }
 138
 139         public function closeStream() {
 140                 $output = $this->writer->closeStream();
 141                 $this->sink->writeCloseStream( $output );
 142         }
 143
 144         /**
 145          * Dumps a series of page and revision records for all pages
 146          * in the database, either including complete history or only
 147          * the most recent version.
 148          */
 149         public function allPages() {
 150                 $this->dumpFrom( '' );
 151         }
 152
 153         /**
 154          * Dumps a series of page and revision records for those pages
 155          * in the database falling within the page_id range given.
 156          * @param int $start Inclusive lower limit (this id is included)
 157          * @param int $end Exclusive upper limit (this id is not included)
 158          *   If 0, no upper limit.
 159          * @param bool $orderRevs order revisions within pages in ascending order
 160          */
 161         public function pagesByRange( $start, $end, $orderRevs ) {
 162                 if ( $orderRevs ) {
 163                         $condition = 'rev_page >= ' . intval( $start );
 164                         if ( $end ) {
 165                                 $condition .= ' AND rev_page < ' . intval( $end );
 166                         }
 167                 } else {
 168                         $condition = 'page_id >= ' . intval( $start );
 169                         if ( $end ) {
 170                                 $condition .= ' AND page_id < ' . intval( $end );
 171                         }
 172                 }
 173                 $this->dumpFrom( $condition, $orderRevs );
 174         }
 175
 176         /**
 177          * Dumps a series of page and revision records for those pages
 178          * in the database with revisions falling within the rev_id range given.
 179          * @param int $start Inclusive lower limit (this id is included)
 180          * @param int $end Exclusive upper limit (this id is not included)
 181          *   If 0, no upper limit.
 182          */
 183         public function revsByRange( $start, $end ) {
 184                 $condition = 'rev_id >= ' . intval( $start );
 185                 if ( $end ) {
 186                         $condition .= ' AND rev_id < ' . intval( $end );
 187                 }
 188                 $this->dumpFrom( $condition );
 189         }
 190
 191         /**
 192          * @param Title $title
 193          */
 194         public function pageByTitle( $title ) {
 195                 $this->dumpFrom(
 196                         'page_namespace=' . $title->getNamespace() .
 197                         ' AND page_title=' . $this->db->addQuotes( $title->getDBkey() ) );
 198         }
 199
 200         /**
 201          * @param string $name
 202          * @throws MWException
 203          */
 204         public function pageByName( $name ) {
 205                 $title = Title::newFromText( $name );
 206                 if ( is_null( $title ) ) {
 207                         throw new MWException( "Can't export invalid title" );
 208                 } else {
 209                         $this->pageByTitle( $title );
 210                 }
 211         }
 212
 213         /**
 214          * @param array $names
 215          */
 216         public function pagesByName( $names ) {
 217                 foreach ( $names as $name ) {
 218                         $this->pageByName( $name );
 219                 }
 220         }
 221
 222         public function allLogs() {
 223                 $this->dumpFrom( '' );
 224         }
 225
 226         /**
 227          * @param int $start
 228          * @param int $end
 229          */
 230         public function logsByRange( $start, $end ) {
 231                 $condition = 'log_id >= ' . intval( $start );
 232                 if ( $end ) {
 233                         $condition .= ' AND log_id < ' . intval( $end );
 234                 }
 235                 $this->dumpFrom( $condition );
 236         }
 237
 238         /**
 239          * Generates the distinct list of authors of an article
 240          * Not called by default (depends on $this->list_authors)
 241          * Can be set by Special:Export when not exporting whole history
 242          *
 243          * @param string $cond
 244          */
 245         protected function do_list_authors( $cond ) {
 246                 $this->author_list = "<contributors>";
 247                 // rev_deleted
 248
 249                 $revQuery = Revision::getQueryInfo( [ 'page' ] );
 250                 $res = $this->db->select(
 251                         $revQuery['tables'],
 252                         [
 253                                 'rev_user_text' => $revQuery['fields']['rev_user_text'],
 254                                 'rev_user' => $revQuery['fields']['rev_user'],
 255                         ],
 256                         [
 257                                 $this->db->bitAnd( 'rev_deleted', RevisionRecord::DELETED_USER ) . ' = 0',
 258                                 $cond,
 259                         ],
 260                         __METHOD__,
 261                         [ 'DISTINCT' ],
 262                         $revQuery['joins']
 263                 );
 264
 265                 foreach ( $res as $row ) {
 266                         $this->author_list .= "<contributor>" .
 267                                 "<username>" .
 268                                 htmlspecialchars( $row->rev_user_text ) .
 269                                 "</username>" .
 270                                 "<id>" .
 271                                 ( (int)$row->rev_user ) .
 272                                 "</id>" .
 273                                 "</contributor>";
 274                 }
 275                 $this->author_list .= "</contributors>";
 276         }
 277
 278         /**
 279          * @param string $cond
 280          * @param bool $orderRevs
 281          * @throws MWException
 282          * @throws Exception
 283          */
 284         protected function dumpFrom( $cond = '', $orderRevs = false ) {
 285                 if ( $this->history & self::LOGS ) {
 286                         $this->dumpLogs( $cond );
 287                 } else {
 288                         $this->dumpPages( $cond, $orderRevs );
 289                 }
 290         }
 291
 292         /**
 293          * @param string $cond
 294          * @throws Exception
 295          */
 296         protected function dumpLogs( $cond ) {
 297                 $where = [];
 298                 # Hide private logs
 299                 $hideLogs = LogEventsList::getExcludeClause( $this->db );
 300                 if ( $hideLogs ) {
 301                         $where[] = $hideLogs;
 302                 }
 303                 # Add on any caller specified conditions
 304                 if ( $cond ) {
 305                         $where[] = $cond;
 306                 }
 307                 $result = null; // Assuring $result is not undefined, if exception occurs early
 308
 309                 $commentQuery = CommentStore::getStore()->getJoin( 'log_comment' );
 310                 $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' );
 311
 312                 $tables = array_merge(
 313                         [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ]
 314                 );
 315                 $fields = [
 316                         'log_id', 'log_type', 'log_action', 'log_timestamp', 'log_namespace',
 317                         'log_title', 'log_params', 'log_deleted', 'user_name'
 318                 ] + $commentQuery['fields'] + $actorQuery['fields'];
 319                 $options = [
 320                         'ORDER BY' => 'log_id',
 321                         'USE INDEX' => [ 'logging' => 'PRIMARY' ],
 322                         'LIMIT' => self::BATCH_SIZE,
 323                 ];
 324                 $joins = [
 325                         'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
 326                 ] + $commentQuery['joins'] + $actorQuery['joins'];
 327
 328                 $lastLogId = 0;
 329                 while ( true ) {
 330                         $result = $this->db->select(
 331                                 $tables,
 332                                 $fields,
 333                                 array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
 334                                 __METHOD__,
 335                                 $options,
 336                                 $joins
 337                         );
 338
 339                         if ( !$result->numRows() ) {
 340                                 break;
 341                         }
 342
 343                         $lastLogId = $this->outputLogStream( $result );
 344                 }
 345         }
 346
 347         /**
 348          * @param string $cond
 349          * @param bool $orderRevs
 350          * @throws MWException
 351          * @throws Exception
 352          */
 353         protected function dumpPages( $cond, $orderRevs ) {
 354                 global $wgMultiContentRevisionSchemaMigrationStage;
 355                 if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD ) ) {
 356                         // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
 357                         throw new MWException(
 358                                 'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
 359                                 . ' Support for dumping from the new schema is not implemented yet!'
 360                         );
 361                 }
 362
 363                 $revQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getQueryInfo(
 364                         [ 'page' ]
 365                 );
 366                 $slotQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getSlotsQueryInfo(
 367                         [ 'content' ]
 368                 );
 369
 370                 // We want page primary rather than revision.
 371                 // We also want to join in the slots and content tables.
 372                 // NOTE: This means we may get multiple rows per revision, and more rows
 373                 // than the batch size! Should be ok, since the max number of slots is
 374                 // fixed and low (dozens at worst).
 375                 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
 376                 $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
 377                 $join = $revQuery['joins'] + [
 378                                 'revision' => $revQuery['joins']['page'],
 379                                 'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
 380                                 'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
 381                         ];
 382                 unset( $join['page'] );
 383
 384                 $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
 385                 $fields[] = 'page_restrictions';
 386
 387                 if ( $this->text != self::STUB ) {
 388                         $fields['_load_content'] = '1';
 389                 }
 390
 391                 $conds = [];
 392                 if ( $cond !== '' ) {
 393                         $conds[] = $cond;
 394                 }
 395                 $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
 396                 $opts['USE INDEX'] = [];
 397
 398                 $op = '>';
 399                 if ( is_array( $this->history ) ) {
 400                         # Time offset/limit for all pages/history...
 401                         # Set time order
 402                         if ( $this->history['dir'] == 'asc' ) {
 403                                 $opts['ORDER BY'] = 'rev_timestamp ASC';
 404                         } else {
 405                                 $op = '<';
 406                                 $opts['ORDER BY'] = 'rev_timestamp DESC';
 407                         }
 408                         # Set offset
 409                         if ( !empty( $this->history['offset'] ) ) {
 410                                 $conds[] = "rev_timestamp $op " .
 411                                         $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) );
 412                         }
 413                         # Set query limit
 414                         if ( !empty( $this->history['limit'] ) ) {
 415                                 $maxRowCount = intval( $this->history['limit'] );
 416                         }
 417                 } elseif ( $this->history & self::FULL ) {
 418                         # Full history dumps...
 419                         # query optimization for history stub dumps
 420                         if ( $this->text == self::STUB ) {
 421                                 $opts[] = 'STRAIGHT_JOIN';
 422                                 $opts['USE INDEX']['revision'] = 'rev_page_id';
 423                                 unset( $join['revision'] );
 424                                 $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
 425                         }
 426                 } elseif ( $this->history & self::CURRENT ) {
 427                         # Latest revision dumps...
 428                         if ( $this->list_authors && $cond != '' ) { // List authors, if so desired
 429                                 $this->do_list_authors( $cond );
 430                         }
 431                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 432                         $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
 433                 } elseif ( $this->history & self::STABLE ) {
 434                         # "Stable" revision dumps...
 435                         # Default JOIN, to be overridden...
 436                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 437                         # One, and only one hook should set this, and return false
 438                         if ( Hooks::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
 439                                 throw new MWException( __METHOD__ . " given invalid history dump type." );
 440                         }
 441                 } elseif ( $this->history & self::RANGE ) {
 442                         # Dump of revisions within a specified range.  Condition already set in revsByRange().
 443                 } else {
 444                         # Unknown history specification parameter?
 445                         throw new MWException( __METHOD__ . " given invalid history dump type." );
 446                 }
 447
 448                 $result = null; // Assuring $result is not undefined, if exception occurs early
 449                 $done = false;
 450                 $lastRow = null;
 451                 $revPage = 0;
 452                 $revId = 0;
 453                 $rowCount = 0;
 454
 455                 $opts['LIMIT'] = self::BATCH_SIZE;
 456
 457                 Hooks::run( 'ModifyExportQuery',
 458                         [ $this->db, &$tables, &$cond, &$opts, &$join ] );
 459
 460                 while ( !$done ) {
 461                         // If necessary, impose the overall maximum and stop looping after this iteration.
 462                         if ( !empty( $maxRowCount ) && $rowCount + self::BATCH_SIZE > $maxRowCount ) {
 463                                 $opts['LIMIT'] = $maxRowCount - $rowCount;
 464                                 $done = true;
 465                         }
 466
 467                         $queryConds = $conds;
 468                         $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
 469                                 intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
 470
 471                         # Do the query and process any results, remembering max ids for the next iteration.
 472                         $result = $this->db->select(
 473                                 $tables,
 474                                 $fields,
 475                                 $queryConds,
 476                                 __METHOD__,
 477                                 $opts,
 478                                 $join
 479                         );
 480                         if ( $result->numRows() > 0 ) {
 481                                 $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
 482                                 $rowCount += $result->numRows();
 483                                 $revPage = $lastRow->rev_page;
 484                                 $revId = $lastRow->rev_id;
 485                         } else {
 486                                 $done = true;
 487                         }
 488
 489                         // If we are finished, close off final page element (if any).
 490                         if ( $done && $lastRow ) {
 491                                 $this->finishPageStreamOutput( $lastRow );
 492                         }
 493                 }
 494         }
 495
 496         /**
 497          * Runs through a query result set dumping page, revision, and slot records.
 498          * The result set should join the page, revision, slots, and content tables,
 499          * and be sorted/grouped by page and revision to avoid duplicate page records in the output.
 500          *
 501          * @param IResultWrapper $results
 502          * @param object $lastRow the last row output from the previous call (or null if none)
 503          * @return object the last row processed
 504          */
 505         protected function outputPageStreamBatch( $results, $lastRow ) {
 506                 $rowCarry = null;
 507                 while ( true ) {
 508                         $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
 509
 510                         if ( !$slotRows ) {
 511                                 break;
 512                         }
 513
 514                         // All revision info is present in all slot rows.
 515                         // Use the first slot row as the revision row.
 516                         $revRow = $slotRows[0];
 517
 518                         if ( $this->limitNamespaces &&
 519                                 !in_array( $revRow->page_namespace, $this->limitNamespaces ) ) {
 520                                 $lastRow = $revRow;
 521                                 continue;
 522                         }
 523
 524                         if ( $lastRow === null ||
 525                                 $lastRow->page_namespace !== $revRow->page_namespace ||
 526                                 $lastRow->page_title !== $revRow->page_title ) {
 527                                 if ( $lastRow !== null ) {
 528                                         $output = '';
 529                                         if ( $this->dumpUploads ) {
 530                                                 $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 531                                         }
 532                                         $output .= $this->writer->closePage();
 533                                         $this->sink->writeClosePage( $output );
 534                                 }
 535                                 $output = $this->writer->openPage( $revRow );
 536                                 $this->sink->writeOpenPage( $revRow, $output );
 537                         }
 538                         $output = $this->writer->writeRevision( $revRow, $slotRows );
 539                         $this->sink->writeRevision( $revRow, $output );
 540                         $lastRow = $revRow;
 541                 }
 542
 543                 if ( $rowCarry ) {
 544                         throw new LogicException( 'Error while processing a stream of slot rows' );
 545                 }
 546
 547                 return $lastRow;
 548         }
 549
 550         /**
 551          * Returns all slot rows for a revision.
 552          * Takes and returns a carry row from the last batch;
 553          *
 554          * @param IResultWrapper|array $results
 555          * @param null|object &$carry A row carried over from the last call to getSlotRowBatch()
 556          *
 557          * @return object[]
 558          */
 559         protected function getSlotRowBatch( $results, &$carry = null ) {
 560                 $slotRows = [];
 561                 $prev = null;
 562
 563                 if ( $carry ) {
 564                         $slotRows[] = $carry;
 565                         $prev = $carry;
 566                         $carry = null;
 567                 }
 568
 569                 while ( $row = $results->fetchObject() ) {
 570                         if ( $prev && $prev->rev_id !== $row->rev_id ) {
 571                                 $carry = $row;
 572                                 break;
 573                         }
 574                         $slotRows[] = $row;
 575                         $prev = $row;
 576                 }
 577
 578                 return $slotRows;
 579         }
 580
 581         /**
 582          * Final page stream output, after all batches are complete
 583          *
 584          * @param object $lastRow the last row output from the last batch (or null if none)
 585          */
 586         protected function finishPageStreamOutput( $lastRow ) {
 587                 $output = '';
 588                 if ( $this->dumpUploads ) {
 589                         $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 590                 }
 591                 $output .= $this->author_list;
 592                 $output .= $this->writer->closePage();
 593                 $this->sink->writeClosePage( $output );
 594         }
 595
 596         /**
 597          * @param IResultWrapper $resultset
 598          * @return int the log_id value of the last item output, or null if none
 599          */
 600         protected function outputLogStream( $resultset ) {
 601                 foreach ( $resultset as $row ) {
 602                         $output = $this->writer->writeLogItem( $row );
 603                         $this->sink->writeLogItem( $row, $output );
 604                 }
 605                 return isset( $row ) ? $row->log_id : null;
 606         }
 607 }