includes/export/WikiExporter.php

   1 <?php
   2 /**
   3  * Base class for exporting
   4  *
   5  * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  */
  25
  26 /**
  27  * @defgroup Dump Dump
  28  */
  29
  30 use MediaWiki\MediaWikiServices as MediaWikiServicesAlias;
  31 use MediaWiki\Storage\RevisionRecord;
  32 use Wikimedia\Rdbms\IResultWrapper;
  33 use Wikimedia\Rdbms\IDatabase;
  34
  35 /**
  36  * @ingroup SpecialPage Dump
  37  */
  38 class WikiExporter {
  39         /** @var bool Return distinct author list (when not returning full history) */
  40         public $list_authors = false;
  41
  42         /** @var bool */
  43         public $dumpUploads = false;
  44
  45         /** @var bool */
  46         public $dumpUploadFileContents = false;
  47
  48         /** @var string */
  49         public $author_list = "";
  50
  51         const FULL = 1;
  52         const CURRENT = 2;
  53         const STABLE = 4; // extension defined
  54         const LOGS = 8;
  55         const RANGE = 16;
  56
  57         const TEXT = XmlDumpWriter::WRITE_CONTENT;
  58         const STUB = XmlDumpWriter::WRITE_STUB;
  59
  60         const BATCH_SIZE = 50000;
  61
  62         /** @var int */
  63         public $text;
  64
  65         /** @var DumpOutput */
  66         public $sink;
  67
  68         /** @var XmlDumpWriter */
  69         private $writer;
  70
  71         /** @var IDatabase */
  72         protected $db;
  73
  74         /** @var array|int */
  75         protected $history;
  76
  77         /** @var array|null */
  78         protected $limitNamespaces;
  79
  80         /**
  81          * Returns the default export schema version, as defined by $wgXmlDumpSchemaVersion.
  82          * @return string
  83          */
  84         public static function schemaVersion() {
  85                 global $wgXmlDumpSchemaVersion;
  86                 return $wgXmlDumpSchemaVersion;
  87         }
  88
  89         /**
  90          * @param IDatabase $db
  91          * @param int|array $history One of WikiExporter::FULL, WikiExporter::CURRENT,
  92          *   WikiExporter::RANGE or WikiExporter::STABLE, or an associative array:
  93          *   - offset: non-inclusive offset at which to start the query
  94          *   - limit: maximum number of rows to return
  95          *   - dir: "asc" or "desc" timestamp order
  96          * @param int $text One of WikiExporter::TEXT or WikiExporter::STUB
  97          * @param null|array $limitNamespaces Comma-separated list of namespace numbers
  98          *   to limit results
  99          */
 100         function __construct(
 101                 $db,
 102                 $history = self::CURRENT,
 103                 $text = self::TEXT,
 104                 $limitNamespaces = null
 105         ) {
 106                 $this->db = $db;
 107                 $this->history = $history;
 108                 $this->writer = new XmlDumpWriter( $text, self::schemaVersion() );
 109                 $this->sink = new DumpOutput();
 110                 $this->text = $text;
 111                 $this->limitNamespaces = $limitNamespaces;
 112         }
 113
 114         /**
 115          * @param string $schemaVersion which schema version the generated XML should comply to.
 116          * One of the values from self::$supportedSchemas, using the XML_DUMP_SCHEMA_VERSION_XX
 117          * constants.
 118          */
 119         public function setSchemaVersion( $schemaVersion ) {
 120                 $this->writer = new XmlDumpWriter( $this->text, $schemaVersion );
 121         }
 122
 123         /**
 124          * Set the DumpOutput or DumpFilter object which will receive
 125          * various row objects and XML output for filtering. Filters
 126          * can be chained or used as callbacks.
 127          *
 128          * @param DumpOutput|DumpFilter &$sink
 129          */
 130         public function setOutputSink( &$sink ) {
 131                 $this->sink =& $sink;
 132         }
 133
 134         public function openStream() {
 135                 $output = $this->writer->openStream();
 136                 $this->sink->writeOpenStream( $output );
 137         }
 138
 139         public function closeStream() {
 140                 $output = $this->writer->closeStream();
 141                 $this->sink->writeCloseStream( $output );
 142         }
 143
 144         /**
 145          * Dumps a series of page and revision records for all pages
 146          * in the database, either including complete history or only
 147          * the most recent version.
 148          */
 149         public function allPages() {
 150                 $this->dumpFrom( '' );
 151         }
 152
 153         /**
 154          * Dumps a series of page and revision records for those pages
 155          * in the database falling within the page_id range given.
 156          * @param int $start Inclusive lower limit (this id is included)
 157          * @param int $end Exclusive upper limit (this id is not included)
 158          *   If 0, no upper limit.
 159          * @param bool $orderRevs order revisions within pages in ascending order
 160          */
 161         public function pagesByRange( $start, $end, $orderRevs ) {
 162                 if ( $orderRevs ) {
 163                         $condition = 'rev_page >= ' . intval( $start );
 164                         if ( $end ) {
 165                                 $condition .= ' AND rev_page < ' . intval( $end );
 166                         }
 167                 } else {
 168                         $condition = 'page_id >= ' . intval( $start );
 169                         if ( $end ) {
 170                                 $condition .= ' AND page_id < ' . intval( $end );
 171                         }
 172                 }
 173                 $this->dumpFrom( $condition, $orderRevs );
 174         }
 175
 176         /**
 177          * Dumps a series of page and revision records for those pages
 178          * in the database with revisions falling within the rev_id range given.
 179          * @param int $start Inclusive lower limit (this id is included)
 180          * @param int $end Exclusive upper limit (this id is not included)
 181          *   If 0, no upper limit.
 182          */
 183         public function revsByRange( $start, $end ) {
 184                 $condition = 'rev_id >= ' . intval( $start );
 185                 if ( $end ) {
 186                         $condition .= ' AND rev_id < ' . intval( $end );
 187                 }
 188                 $this->dumpFrom( $condition );
 189         }
 190
 191         /**
 192          * @param Title $title
 193          */
 194         public function pageByTitle( $title ) {
 195                 $this->dumpFrom(
 196                         'page_namespace=' . $title->getNamespace() .
 197                         ' AND page_title=' . $this->db->addQuotes( $title->getDBkey() ) );
 198         }
 199
 200         /**
 201          * @param string $name
 202          * @throws MWException
 203          */
 204         public function pageByName( $name ) {
 205                 $title = Title::newFromText( $name );
 206                 if ( is_null( $title ) ) {
 207                         throw new MWException( "Can't export invalid title" );
 208                 } else {
 209                         $this->pageByTitle( $title );
 210                 }
 211         }
 212
 213         /**
 214          * @param array $names
 215          */
 216         public function pagesByName( $names ) {
 217                 foreach ( $names as $name ) {
 218                         $this->pageByName( $name );
 219                 }
 220         }
 221
 222         public function allLogs() {
 223                 $this->dumpFrom( '' );
 224         }
 225
 226         /**
 227          * @param int $start
 228          * @param int $end
 229          */
 230         public function logsByRange( $start, $end ) {
 231                 $condition = 'log_id >= ' . intval( $start );
 232                 if ( $end ) {
 233                         $condition .= ' AND log_id < ' . intval( $end );
 234                 }
 235                 $this->dumpFrom( $condition );
 236         }
 237
 238         /**
 239          * Generates the distinct list of authors of an article
 240          * Not called by default (depends on $this->list_authors)
 241          * Can be set by Special:Export when not exporting whole history
 242          *
 243          * @param string $cond
 244          */
 245         protected function do_list_authors( $cond ) {
 246                 $this->author_list = "<contributors>";
 247                 // rev_deleted
 248
 249                 $revQuery = Revision::getQueryInfo( [ 'page' ] );
 250                 $res = $this->db->select(
 251                         $revQuery['tables'],
 252                         [
 253                                 'rev_user_text' => $revQuery['fields']['rev_user_text'],
 254                                 'rev_user' => $revQuery['fields']['rev_user'],
 255                         ],
 256                         [
 257                                 $this->db->bitAnd( 'rev_deleted', RevisionRecord::DELETED_USER ) . ' = 0',
 258                                 $cond,
 259                         ],
 260                         __METHOD__,
 261                         [ 'DISTINCT' ],
 262                         $revQuery['joins']
 263                 );
 264
 265                 foreach ( $res as $row ) {
 266                         $this->author_list .= "<contributor>" .
 267                                 "<username>" .
 268                                 htmlspecialchars( $row->rev_user_text ) .
 269                                 "</username>" .
 270                                 "<id>" .
 271                                 ( (int)$row->rev_user ) .
 272                                 "</id>" .
 273                                 "</contributor>";
 274                 }
 275                 $this->author_list .= "</contributors>";
 276         }
 277
 278         /**
 279          * @param string $cond
 280          * @param bool $orderRevs
 281          * @throws MWException
 282          * @throws Exception
 283          */
 284         protected function dumpFrom( $cond = '', $orderRevs = false ) {
 285                 if ( $this->history & self::LOGS ) {
 286                         $this->dumpLogs( $cond );
 287                 } else {
 288                         $this->dumpPages( $cond, $orderRevs );
 289                 }
 290         }
 291
 292         /**
 293          * @param string $cond
 294          * @throws Exception
 295          */
 296         protected function dumpLogs( $cond ) {
 297                 $where = [];
 298                 # Hide private logs
 299                 $hideLogs = LogEventsList::getExcludeClause( $this->db );
 300                 if ( $hideLogs ) {
 301                         $where[] = $hideLogs;
 302                 }
 303                 # Add on any caller specified conditions
 304                 if ( $cond ) {
 305                         $where[] = $cond;
 306                 }
 307                 # Get logging table name for logging.* clause
 308                 $logging = $this->db->tableName( 'logging' );
 309
 310                 $result = null; // Assuring $result is not undefined, if exception occurs early
 311
 312                 $commentQuery = CommentStore::getStore()->getJoin( 'log_comment' );
 313                 $actorQuery = ActorMigration::newMigration()->getJoin( 'log_user' );
 314
 315                 $lastLogId = 0;
 316                 while ( true ) {
 317                         $result = $this->db->select(
 318                                 array_merge( [ 'logging' ], $commentQuery['tables'], $actorQuery['tables'], [ 'user' ] ),
 319                                 [ "{$logging}.*", 'user_name' ] + $commentQuery['fields'] + $actorQuery['fields'],
 320                                 array_merge( $where, [ 'log_id > ' . intval( $lastLogId ) ] ),
 321                                 __METHOD__,
 322                                 [
 323                                         'ORDER BY' => 'log_id',
 324                                         'USE INDEX' => [ 'logging' => 'PRIMARY' ],
 325                                         'LIMIT' => self::BATCH_SIZE,
 326                                 ],
 327                                 [
 328                                         'user' => [ 'JOIN', 'user_id = ' . $actorQuery['fields']['log_user'] ]
 329                                 ] + $commentQuery['joins'] + $actorQuery['joins']
 330                         );
 331
 332                         if ( !$result->numRows() ) {
 333                                 break;
 334                         }
 335
 336                         $lastLogId = $this->outputLogStream( $result );
 337                 }
 338         }
 339
 340         /**
 341          * @param string $cond
 342          * @param bool $orderRevs
 343          * @throws MWException
 344          * @throws Exception
 345          */
 346         protected function dumpPages( $cond, $orderRevs ) {
 347                 global $wgMultiContentRevisionSchemaMigrationStage;
 348                 if ( !( $wgMultiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_OLD ) ) {
 349                         // TODO: Make XmlDumpWriter use a RevisionStore! (see T198706 and T174031)
 350                         throw new MWException(
 351                                 'Cannot use WikiExporter with SCHEMA_COMPAT_WRITE_OLD mode disabled!'
 352                                 . ' Support for dumping from the new schema is not implemented yet!'
 353                         );
 354                 }
 355
 356                 $revQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getQueryInfo(
 357                         [ 'page' ]
 358                 );
 359                 $slotQuery = MediaWikiServicesAlias::getInstance()->getRevisionStore()->getSlotsQueryInfo(
 360                         [ 'content' ]
 361                 );
 362
 363                 // We want page primary rather than revision.
 364                 // We also want to join in the slots and content tables.
 365                 // NOTE: This means we may get multiple rows per revision, and more rows
 366                 // than the batch size! Should be ok, since the max number of slots is
 367                 // fixed and low (dozens at worst).
 368                 $tables = array_merge( [ 'page' ], array_diff( $revQuery['tables'], [ 'page' ] ) );
 369                 $tables = array_merge( $tables, array_diff( $slotQuery['tables'], $tables ) );
 370                 $join = $revQuery['joins'] + [
 371                                 'revision' => $revQuery['joins']['page'],
 372                                 'slots' => [ 'JOIN', [ 'slot_revision_id = rev_id' ] ],
 373                                 'content' => [ 'JOIN', [ 'content_id = slot_content_id' ] ],
 374                         ];
 375                 unset( $join['page'] );
 376
 377                 $fields = array_merge( $revQuery['fields'], $slotQuery['fields'] );
 378                 $fields[] = 'page_restrictions';
 379
 380                 if ( $this->text != self::STUB ) {
 381                         $fields['_load_content'] = '1';
 382                 }
 383
 384                 $conds = [];
 385                 if ( $cond !== '' ) {
 386                         $conds[] = $cond;
 387                 }
 388                 $opts = [ 'ORDER BY' => [ 'rev_page ASC', 'rev_id ASC' ] ];
 389                 $opts['USE INDEX'] = [];
 390
 391                 $op = '>';
 392                 if ( is_array( $this->history ) ) {
 393                         # Time offset/limit for all pages/history...
 394                         # Set time order
 395                         if ( $this->history['dir'] == 'asc' ) {
 396                                 $opts['ORDER BY'] = 'rev_timestamp ASC';
 397                         } else {
 398                                 $op = '<';
 399                                 $opts['ORDER BY'] = 'rev_timestamp DESC';
 400                         }
 401                         # Set offset
 402                         if ( !empty( $this->history['offset'] ) ) {
 403                                 $conds[] = "rev_timestamp $op " .
 404                                         $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) );
 405                         }
 406                         # Set query limit
 407                         if ( !empty( $this->history['limit'] ) ) {
 408                                 $maxRowCount = intval( $this->history['limit'] );
 409                         }
 410                 } elseif ( $this->history & self::FULL ) {
 411                         # Full history dumps...
 412                         # query optimization for history stub dumps
 413                         if ( $this->text == self::STUB ) {
 414                                 $opts[] = 'STRAIGHT_JOIN';
 415                                 $opts['USE INDEX']['revision'] = 'rev_page_id';
 416                                 unset( $join['revision'] );
 417                                 $join['page'] = [ 'JOIN', 'rev_page=page_id' ];
 418                         }
 419                 } elseif ( $this->history & self::CURRENT ) {
 420                         # Latest revision dumps...
 421                         if ( $this->list_authors && $cond != '' ) { // List authors, if so desired
 422                                 $this->do_list_authors( $cond );
 423                         }
 424                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 425                         $opts[ 'ORDER BY' ] = [ 'page_id ASC' ];
 426                 } elseif ( $this->history & self::STABLE ) {
 427                         # "Stable" revision dumps...
 428                         # Default JOIN, to be overridden...
 429                         $join['revision'] = [ 'JOIN', 'page_id=rev_page AND page_latest=rev_id' ];
 430                         # One, and only one hook should set this, and return false
 431                         if ( Hooks::run( 'WikiExporter::dumpStableQuery', [ &$tables, &$opts, &$join ] ) ) {
 432                                 throw new MWException( __METHOD__ . " given invalid history dump type." );
 433                         }
 434                 } elseif ( $this->history & self::RANGE ) {
 435                         # Dump of revisions within a specified range.  Condition already set in revsByRange().
 436                 } else {
 437                         # Unknown history specification parameter?
 438                         throw new MWException( __METHOD__ . " given invalid history dump type." );
 439                 }
 440
 441                 $result = null; // Assuring $result is not undefined, if exception occurs early
 442                 $done = false;
 443                 $lastRow = null;
 444                 $revPage = 0;
 445                 $revId = 0;
 446                 $rowCount = 0;
 447
 448                 $opts['LIMIT'] = self::BATCH_SIZE;
 449
 450                 Hooks::run( 'ModifyExportQuery',
 451                         [ $this->db, &$tables, &$cond, &$opts, &$join ] );
 452
 453                 while ( !$done ) {
 454                         // If necessary, impose the overall maximum and stop looping after this iteration.
 455                         if ( !empty( $maxRowCount ) && $rowCount + self::BATCH_SIZE > $maxRowCount ) {
 456                                 $opts['LIMIT'] = $maxRowCount - $rowCount;
 457                                 $done = true;
 458                         }
 459
 460                         $queryConds = $conds;
 461                         $queryConds[] = 'rev_page>' . intval( $revPage ) . ' OR (rev_page=' .
 462                                 intval( $revPage ) . ' AND rev_id' . $op . intval( $revId ) . ')';
 463
 464                         # Do the query and process any results, remembering max ids for the next iteration.
 465                         $result = $this->db->select(
 466                                 $tables,
 467                                 $fields,
 468                                 $queryConds,
 469                                 __METHOD__,
 470                                 $opts,
 471                                 $join
 472                         );
 473                         if ( $result->numRows() > 0 ) {
 474                                 $lastRow = $this->outputPageStreamBatch( $result, $lastRow );
 475                                 $rowCount += $result->numRows();
 476                                 $revPage = $lastRow->rev_page;
 477                                 $revId = $lastRow->rev_id;
 478                         } else {
 479                                 $done = true;
 480                         }
 481
 482                         // If we are finished, close off final page element (if any).
 483                         if ( $done && $lastRow ) {
 484                                 $this->finishPageStreamOutput( $lastRow );
 485                         }
 486                 }
 487         }
 488
 489         /**
 490          * Runs through a query result set dumping page, revision, and slot records.
 491          * The result set should join the page, revision, slots, and content tables,
 492          * and be sorted/grouped by page and revision to avoid duplicate page records in the output.
 493          *
 494          * @param IResultWrapper $results
 495          * @param object $lastRow the last row output from the previous call (or null if none)
 496          * @return object the last row processed
 497          */
 498         protected function outputPageStreamBatch( $results, $lastRow ) {
 499                 $rowCarry = null;
 500                 while ( true ) {
 501                         $slotRows = $this->getSlotRowBatch( $results, $rowCarry );
 502
 503                         if ( !$slotRows ) {
 504                                 break;
 505                         }
 506
 507                         // All revision info is present in all slot rows.
 508                         // Use the first slot row as the revision row.
 509                         $revRow = $slotRows[0];
 510
 511                         if ( $this->limitNamespaces &&
 512                                 !in_array( $revRow->page_namespace, $this->limitNamespaces ) ) {
 513                                 $lastRow = $revRow;
 514                                 continue;
 515                         }
 516
 517                         if ( $lastRow === null ||
 518                                 $lastRow->page_namespace !== $revRow->page_namespace ||
 519                                 $lastRow->page_title !== $revRow->page_title ) {
 520                                 if ( $lastRow !== null ) {
 521                                         $output = '';
 522                                         if ( $this->dumpUploads ) {
 523                                                 $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 524                                         }
 525                                         $output .= $this->writer->closePage();
 526                                         $this->sink->writeClosePage( $output );
 527                                 }
 528                                 $output = $this->writer->openPage( $revRow );
 529                                 $this->sink->writeOpenPage( $revRow, $output );
 530                         }
 531                         $output = $this->writer->writeRevision( $revRow, $slotRows );
 532                         $this->sink->writeRevision( $revRow, $output );
 533                         $lastRow = $revRow;
 534                 }
 535
 536                 if ( $rowCarry ) {
 537                         throw new LogicException( 'Error while processing a stream of slot rows' );
 538                 }
 539
 540                 return $lastRow;
 541         }
 542
 543         /**
 544          * Returns all slot rows for a revision.
 545          * Takes and returns a carry row from the last batch;
 546          *
 547          * @param IResultWrapper|array $results
 548          * @param null|object &$carry A row carried over from the last call to getSlotRowBatch()
 549          *
 550          * @return object[]
 551          */
 552         protected function getSlotRowBatch( $results, &$carry = null ) {
 553                 $slotRows = [];
 554                 $prev = null;
 555
 556                 if ( $carry ) {
 557                         $slotRows[] = $carry;
 558                         $prev = $carry;
 559                         $carry = null;
 560                 }
 561
 562                 while ( $row = $results->fetchObject() ) {
 563                         if ( $prev && $prev->rev_id !== $row->rev_id ) {
 564                                 $carry = $row;
 565                                 break;
 566                         }
 567                         $slotRows[] = $row;
 568                         $prev = $row;
 569                 }
 570
 571                 return $slotRows;
 572         }
 573
 574         /**
 575          * Final page stream output, after all batches are complete
 576          *
 577          * @param object $lastRow the last row output from the last batch (or null if none)
 578          */
 579         protected function finishPageStreamOutput( $lastRow ) {
 580                 $output = '';
 581                 if ( $this->dumpUploads ) {
 582                         $output .= $this->writer->writeUploads( $lastRow, $this->dumpUploadFileContents );
 583                 }
 584                 $output .= $this->author_list;
 585                 $output .= $this->writer->closePage();
 586                 $this->sink->writeClosePage( $output );
 587         }
 588
 589         /**
 590          * @param IResultWrapper $resultset
 591          * @return int the log_id value of the last item output, or null if none
 592          */
 593         protected function outputLogStream( $resultset ) {
 594                 foreach ( $resultset as $row ) {
 595                         $output = $this->writer->writeLogItem( $row );
 596                         $this->sink->writeLogItem( $row, $output );
 597                 }
 598                 return isset( $row ) ? $row->log_id : null;
 599         }
 600 }