includes/export/XmlDumpWriter.php

   1 <?php
   2 /**
   3  * XmlDumpWriter
   4  *
   5  * Copyright © 2003, 2005, 2006 Brion Vibber <brion@pobox.com>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  */
  25 use MediaWiki\MediaWikiServices;
  26 use MediaWiki\Revision\RevisionStore;
  27 use MediaWiki\Storage\SqlBlobStore;
  28
  29 /**
  30  * @ingroup Dump
  31  */
  32 class XmlDumpWriter {
  33
  34         /**
  35          * Title of the currently processed page
  36          *
  37          * @var Title|null
  38          */
  39         private $currentTitle = null;
  40
  41         /**
  42          * Opens the XML output stream's root "<mediawiki>" element.
  43          * This does not include an xml directive, so is safe to include
  44          * as a subelement in a larger XML stream. Namespace and XML Schema
  45          * references are included.
  46          *
  47          * Output will be encoded in UTF-8.
  48          *
  49          * @return string
  50          */
  51         function openStream() {
  52                 $ver = WikiExporter::schemaVersion();
  53                 return Xml::element( 'mediawiki', [
  54                         'xmlns'              => "http://www.mediawiki.org/xml/export-$ver/",
  55                         'xmlns:xsi'          => "http://www.w3.org/2001/XMLSchema-instance",
  56                         /*
  57                          * When a new version of the schema is created, it needs staging on mediawiki.org.
  58                          * This requires a change in the operations/mediawiki-config git repo.
  59                          *
  60                          * Create a changeset like https://gerrit.wikimedia.org/r/#/c/149643/ in which
  61                          * you copy in the new xsd file.
  62                          *
  63                          * After it is reviewed, merged and deployed (sync-docroot), the index.html needs purging.
  64                          * echo "https://www.mediawiki.org/xml/index.html" | mwscript purgeList.php --wiki=aawiki
  65                          */
  66                         'xsi:schemaLocation' => "http://www.mediawiki.org/xml/export-$ver/ " .
  67                                 "http://www.mediawiki.org/xml/export-$ver.xsd",
  68                         'version' => $ver,
  69                         'xml:lang' => MediaWikiServices::getInstance()->getContentLanguage()->getHtmlCode() ],
  70                         null ) .
  71                         "\n" .
  72                         $this->siteInfo();
  73         }
  74
  75         /**
  76          * @return string
  77          */
  78         function siteInfo() {
  79                 $info = [
  80                         $this->sitename(),
  81                         $this->dbname(),
  82                         $this->homelink(),
  83                         $this->generator(),
  84                         $this->caseSetting(),
  85                         $this->namespaces() ];
  86                 return "  <siteinfo>\n    " .
  87                         implode( "\n    ", $info ) .
  88                         "\n  </siteinfo>\n";
  89         }
  90
  91         /**
  92          * @return string
  93          */
  94         function sitename() {
  95                 global $wgSitename;
  96                 return Xml::element( 'sitename', [], $wgSitename );
  97         }
  98
  99         /**
 100          * @return string
 101          */
 102         function dbname() {
 103                 global $wgDBname;
 104                 return Xml::element( 'dbname', [], $wgDBname );
 105         }
 106
 107         /**
 108          * @return string
 109          */
 110         function generator() {
 111                 global $wgVersion;
 112                 return Xml::element( 'generator', [], "MediaWiki $wgVersion" );
 113         }
 114
 115         /**
 116          * @return string
 117          */
 118         function homelink() {
 119                 return Xml::element( 'base', [], Title::newMainPage()->getCanonicalURL() );
 120         }
 121
 122         /**
 123          * @return string
 124          */
 125         function caseSetting() {
 126                 global $wgCapitalLinks;
 127                 // "case-insensitive" option is reserved for future
 128                 $sensitivity = $wgCapitalLinks ? 'first-letter' : 'case-sensitive';
 129                 return Xml::element( 'case', [], $sensitivity );
 130         }
 131
 132         /**
 133          * @return string
 134          */
 135         function namespaces() {
 136                 $spaces = "<namespaces>\n";
 137                 foreach (
 138                         MediaWikiServices::getInstance()->getContentLanguage()->getFormattedNamespaces()
 139                         as $ns => $title
 140                 ) {
 141                         $spaces .= '      ' .
 142                                 Xml::element( 'namespace',
 143                                         [
 144                                                 'key' => $ns,
 145                                                 'case' => MWNamespace::isCapitalized( $ns ) ? 'first-letter' : 'case-sensitive',
 146                                         ], $title ) . "\n";
 147                 }
 148                 $spaces .= "    </namespaces>";
 149                 return $spaces;
 150         }
 151
 152         /**
 153          * Closes the output stream with the closing root element.
 154          * Call when finished dumping things.
 155          *
 156          * @return string
 157          */
 158         function closeStream() {
 159                 return "</mediawiki>\n";
 160         }
 161
 162         /**
 163          * Opens a "<page>" section on the output stream, with data
 164          * from the given database row.
 165          *
 166          * @param object $row
 167          * @return string
 168          */
 169         public function openPage( $row ) {
 170                 $out = "  <page>\n";
 171                 $this->currentTitle = Title::makeTitle( $row->page_namespace, $row->page_title );
 172                 $canonicalTitle = self::canonicalTitle( $this->currentTitle );
 173                 $out .= '    ' . Xml::elementClean( 'title', [], $canonicalTitle ) . "\n";
 174                 $out .= '    ' . Xml::element( 'ns', [], strval( $row->page_namespace ) ) . "\n";
 175                 $out .= '    ' . Xml::element( 'id', [], strval( $row->page_id ) ) . "\n";
 176                 if ( $row->page_is_redirect ) {
 177                         $page = WikiPage::factory( $this->currentTitle );
 178                         $redirect = $page->getRedirectTarget();
 179                         if ( $redirect instanceof Title && $redirect->isValidRedirectTarget() ) {
 180                                 $out .= '    ';
 181                                 $out .= Xml::element( 'redirect', [ 'title' => self::canonicalTitle( $redirect ) ] );
 182                                 $out .= "\n";
 183                         }
 184                 }
 185
 186                 if ( $row->page_restrictions != '' ) {
 187                         $out .= '    ' . Xml::element( 'restrictions', [],
 188                                 strval( $row->page_restrictions ) ) . "\n";
 189                 }
 190
 191                 Hooks::run( 'XmlDumpWriterOpenPage', [ $this, &$out, $row, $this->currentTitle ] );
 192
 193                 return $out;
 194         }
 195
 196         /**
 197          * Closes a "<page>" section on the output stream.
 198          *
 199          * @private
 200          * @return string
 201          */
 202         function closePage() {
 203                 return "  </page>\n";
 204         }
 205
 206         /**
 207          * @return RevisionStore
 208          */
 209         private function getRevisionStore() {
 210                 return MediaWikiServices::getInstance()->getRevisionStore();
 211         }
 212
 213         /**
 214          * @return SqlBlobStore
 215          */
 216         private function getBlobStore() {
 217                 return MediaWikiServices::getInstance()->getBlobStore();
 218         }
 219
 220         /**
 221          * Dumps a "<revision>" section on the output stream, with
 222          * data filled in from the given database row.
 223          *
 224          * @param object $row
 225          * @return string
 226          * @private
 227          */
 228         function writeRevision( $row ) {
 229                 $out = "    <revision>\n";
 230                 $out .= "      " . Xml::element( 'id', null, strval( $row->rev_id ) ) . "\n";
 231                 if ( isset( $row->rev_parent_id ) && $row->rev_parent_id ) {
 232                         $out .= "      " . Xml::element( 'parentid', null, strval( $row->rev_parent_id ) ) . "\n";
 233                 }
 234
 235                 $out .= $this->writeTimestamp( $row->rev_timestamp );
 236
 237                 if ( isset( $row->rev_deleted ) && ( $row->rev_deleted & Revision::DELETED_USER ) ) {
 238                         $out .= "      " . Xml::element( 'contributor', [ 'deleted' => 'deleted' ] ) . "\n";
 239                 } else {
 240                         $out .= $this->writeContributor( $row->rev_user, $row->rev_user_text );
 241                 }
 242
 243                 if ( isset( $row->rev_minor_edit ) && $row->rev_minor_edit ) {
 244                         $out .= "      <minor/>\n";
 245                 }
 246                 if ( isset( $row->rev_deleted ) && ( $row->rev_deleted & Revision::DELETED_COMMENT ) ) {
 247                         $out .= "      " . Xml::element( 'comment', [ 'deleted' => 'deleted' ] ) . "\n";
 248                 } else {
 249                         $comment = CommentStore::getStore()->getComment( 'rev_comment', $row )->text;
 250                         if ( $comment != '' ) {
 251                                 $out .= "      " . Xml::elementClean( 'comment', [], strval( $comment ) ) . "\n";
 252                         }
 253                 }
 254
 255                 // TODO: rev_content_model no longer exists with MCR, see T174031
 256                 if ( isset( $row->rev_content_model ) && !is_null( $row->rev_content_model ) ) {
 257                         $content_model = strval( $row->rev_content_model );
 258                 } else {
 259                         // probably using $wgContentHandlerUseDB = false;
 260                         $content_model = ContentHandler::getDefaultModelFor( $this->currentTitle );
 261                 }
 262
 263                 $content_handler = ContentHandler::getForModelID( $content_model );
 264
 265                 // TODO: rev_content_format no longer exists with MCR, see T174031
 266                 if ( isset( $row->rev_content_format ) && !is_null( $row->rev_content_format ) ) {
 267                         $content_format = strval( $row->rev_content_format );
 268                 } else {
 269                         // probably using $wgContentHandlerUseDB = false;
 270                         $content_format = $content_handler->getDefaultFormat();
 271                 }
 272
 273                 $out .= "      " . Xml::element( 'model', null, strval( $content_model ) ) . "\n";
 274                 $out .= "      " . Xml::element( 'format', null, strval( $content_format ) ) . "\n";
 275
 276                 $text = '';
 277                 if ( isset( $row->rev_deleted ) && ( $row->rev_deleted & Revision::DELETED_TEXT ) ) {
 278                         $out .= "      " . Xml::element( 'text', [ 'deleted' => 'deleted' ] ) . "\n";
 279                 } elseif ( isset( $row->old_text ) ) {
 280                         // Raw text from the database may have invalid chars
 281                         $text = strval( Revision::getRevisionText( $row ) );
 282                         $text = $content_handler->exportTransform( $text, $content_format );
 283                         $out .= "      " . Xml::elementClean( 'text',
 284                                 [ 'xml:space' => 'preserve', 'bytes' => intval( $row->rev_len ) ],
 285                                 strval( $text ) ) . "\n";
 286                 } elseif ( isset( $row->_load_content ) ) {
 287                         // TODO: make this fully MCR aware, see T174031
 288                         $rev = $this->getRevisionStore()->newRevisionFromRow( $row, 0, $this->currentTitle );
 289                         $slot = $rev->getSlot( 'main' );
 290                         $content = $slot->getContent();
 291
 292                         if ( $content instanceof TextContent ) {
 293                                 // HACK: For text based models, bypass the serialization step.
 294                                 // This allows extensions (like Flow)that use incompatible combinations
 295                                 // of serialization format and content model.
 296                                 $text = $content->getNativeData();
 297                         } else {
 298                                 $text = $content->serialize( $content_format );
 299                         }
 300
 301                         $text = $content_handler->exportTransform( $text, $content_format );
 302                         $out .= "      " . Xml::elementClean( 'text',
 303                                 [ 'xml:space' => 'preserve', 'bytes' => intval( $slot->getSize() ) ],
 304                                 strval( $text ) ) . "\n";
 305                 } elseif ( isset( $row->rev_text_id ) ) {
 306                         // Stub output for pre-MCR schema
 307                         // TODO: MCR: rev_text_id only exists in the pre-MCR schema. Remove this when
 308                         // we drop support for the old schema.
 309                         $out .= "      " . Xml::element( 'text',
 310                                 [ 'id' => $row->rev_text_id, 'bytes' => intval( $row->rev_len ) ],
 311                                 "" ) . "\n";
 312                 } else {
 313                         // Backwards-compatible stub output for MCR aware schema
 314                         // TODO: MCR: emit content addresses instead of text ids, see T174031, T199121
 315                         $rev = $this->getRevisionStore()->newRevisionFromRow( $row, 0, $this->currentTitle );
 316                         $slot = $rev->getSlot( 'main' );
 317
 318                         // Note that this is currently the ONLY reason we have a BlobStore here at all.
 319                         // When removing this line, check whether the BlobStore has become unused.
 320                         $textId = $this->getBlobStore()->getTextIdFromAddress( $slot->getAddress() );
 321                         $out .= "      " . Xml::element( 'text',
 322                                         [ 'id' => $textId, 'bytes' => intval( $slot->getSize() ) ],
 323                                         "" ) . "\n";
 324                 }
 325
 326                 if ( isset( $row->rev_sha1 )
 327                         && $row->rev_sha1
 328                         && !( $row->rev_deleted & Revision::DELETED_TEXT )
 329                 ) {
 330                         $out .= "      " . Xml::element( 'sha1', null, strval( $row->rev_sha1 ) ) . "\n";
 331                 } else {
 332                         $out .= "      <sha1/>\n";
 333                 }
 334
 335                 // Avoid PHP 7.1 warning from passing $this by reference
 336                 $writer = $this;
 337                 Hooks::run( 'XmlDumpWriterWriteRevision', [ &$writer, &$out, $row, $text ] );
 338
 339                 $out .= "    </revision>\n";
 340
 341                 return $out;
 342         }
 343
 344         /**
 345          * Dumps a "<logitem>" section on the output stream, with
 346          * data filled in from the given database row.
 347          *
 348          * @param object $row
 349          * @return string
 350          * @private
 351          */
 352         function writeLogItem( $row ) {
 353                 $out = "  <logitem>\n";
 354                 $out .= "    " . Xml::element( 'id', null, strval( $row->log_id ) ) . "\n";
 355
 356                 $out .= $this->writeTimestamp( $row->log_timestamp, "    " );
 357
 358                 if ( $row->log_deleted & LogPage::DELETED_USER ) {
 359                         $out .= "    " . Xml::element( 'contributor', [ 'deleted' => 'deleted' ] ) . "\n";
 360                 } else {
 361                         $out .= $this->writeContributor( $row->log_user, $row->user_name, "    " );
 362                 }
 363
 364                 if ( $row->log_deleted & LogPage::DELETED_COMMENT ) {
 365                         $out .= "    " . Xml::element( 'comment', [ 'deleted' => 'deleted' ] ) . "\n";
 366                 } else {
 367                         $comment = CommentStore::getStore()->getComment( 'log_comment', $row )->text;
 368                         if ( $comment != '' ) {
 369                                 $out .= "    " . Xml::elementClean( 'comment', null, strval( $comment ) ) . "\n";
 370                         }
 371                 }
 372
 373                 $out .= "    " . Xml::element( 'type', null, strval( $row->log_type ) ) . "\n";
 374                 $out .= "    " . Xml::element( 'action', null, strval( $row->log_action ) ) . "\n";
 375
 376                 if ( $row->log_deleted & LogPage::DELETED_ACTION ) {
 377                         $out .= "    " . Xml::element( 'text', [ 'deleted' => 'deleted' ] ) . "\n";
 378                 } else {
 379                         $title = Title::makeTitle( $row->log_namespace, $row->log_title );
 380                         $out .= "    " . Xml::elementClean( 'logtitle', null, self::canonicalTitle( $title ) ) . "\n";
 381                         $out .= "    " . Xml::elementClean( 'params',
 382                                 [ 'xml:space' => 'preserve' ],
 383                                 strval( $row->log_params ) ) . "\n";
 384                 }
 385
 386                 $out .= "  </logitem>\n";
 387
 388                 return $out;
 389         }
 390
 391         /**
 392          * @param string $timestamp
 393          * @param string $indent Default to six spaces
 394          * @return string
 395          */
 396         function writeTimestamp( $timestamp, $indent = "      " ) {
 397                 $ts = wfTimestamp( TS_ISO_8601, $timestamp );
 398                 return $indent . Xml::element( 'timestamp', null, $ts ) . "\n";
 399         }
 400
 401         /**
 402          * @param int $id
 403          * @param string $text
 404          * @param string $indent Default to six spaces
 405          * @return string
 406          */
 407         function writeContributor( $id, $text, $indent = "      " ) {
 408                 $out = $indent . "<contributor>\n";
 409                 if ( $id || !IP::isValid( $text ) ) {
 410                         $out .= $indent . "  " . Xml::elementClean( 'username', null, strval( $text ) ) . "\n";
 411                         $out .= $indent . "  " . Xml::element( 'id', null, strval( $id ) ) . "\n";
 412                 } else {
 413                         $out .= $indent . "  " . Xml::elementClean( 'ip', null, strval( $text ) ) . "\n";
 414                 }
 415                 $out .= $indent . "</contributor>\n";
 416                 return $out;
 417         }
 418
 419         /**
 420          * Warning! This data is potentially inconsistent. :(
 421          * @param object $row
 422          * @param bool $dumpContents
 423          * @return string
 424          */
 425         function writeUploads( $row, $dumpContents = false ) {
 426                 if ( $row->page_namespace == NS_FILE ) {
 427                         $img = wfLocalFile( $row->page_title );
 428                         if ( $img && $img->exists() ) {
 429                                 $out = '';
 430                                 foreach ( array_reverse( $img->getHistory() ) as $ver ) {
 431                                         $out .= $this->writeUpload( $ver, $dumpContents );
 432                                 }
 433                                 $out .= $this->writeUpload( $img, $dumpContents );
 434                                 return $out;
 435                         }
 436                 }
 437                 return '';
 438         }
 439
 440         /**
 441          * @param File $file
 442          * @param bool $dumpContents
 443          * @return string
 444          */
 445         function writeUpload( $file, $dumpContents = false ) {
 446                 if ( $file->isOld() ) {
 447                         $archiveName = "      " .
 448                                 Xml::element( 'archivename', null, $file->getArchiveName() ) . "\n";
 449                 } else {
 450                         $archiveName = '';
 451                 }
 452                 if ( $dumpContents ) {
 453                         $be = $file->getRepo()->getBackend();
 454                         # Dump file as base64
 455                         # Uses only XML-safe characters, so does not need escaping
 456                         # @todo Too bad this loads the contents into memory (script might swap)
 457                         $contents = '      <contents encoding="base64">' .
 458                                 chunk_split( base64_encode(
 459                                         $be->getFileContents( [ 'src' => $file->getPath() ] ) ) ) .
 460                                 "      </contents>\n";
 461                 } else {
 462                         $contents = '';
 463                 }
 464                 if ( $file->isDeleted( File::DELETED_COMMENT ) ) {
 465                         $comment = Xml::element( 'comment', [ 'deleted' => 'deleted' ] );
 466                 } else {
 467                         $comment = Xml::elementClean( 'comment', null, strval( $file->getDescription() ) );
 468                 }
 469                 return "    <upload>\n" .
 470                         $this->writeTimestamp( $file->getTimestamp() ) .
 471                         $this->writeContributor( $file->getUser( 'id' ), $file->getUser( 'text' ) ) .
 472                         "      " . $comment . "\n" .
 473                         "      " . Xml::element( 'filename', null, $file->getName() ) . "\n" .
 474                         $archiveName .
 475                         "      " . Xml::element( 'src', null, $file->getCanonicalUrl() ) . "\n" .
 476                         "      " . Xml::element( 'size', null, $file->getSize() ) . "\n" .
 477                         "      " . Xml::element( 'sha1base36', null, $file->getSha1() ) . "\n" .
 478                         "      " . Xml::element( 'rel', null, $file->getRel() ) . "\n" .
 479                         $contents .
 480                         "    </upload>\n";
 481         }
 482
 483         /**
 484          * Return prefixed text form of title, but using the content language's
 485          * canonical namespace. This skips any special-casing such as gendered
 486          * user namespaces -- which while useful, are not yet listed in the
 487          * XML "<siteinfo>" data so are unsafe in export.
 488          *
 489          * @param Title $title
 490          * @return string
 491          * @since 1.18
 492          */
 493         public static function canonicalTitle( Title $title ) {
 494                 if ( $title->isExternal() ) {
 495                         return $title->getPrefixedText();
 496                 }
 497
 498                 $prefix = MediaWikiServices::getInstance()->getContentLanguage()->
 499                         getFormattedNsText( $title->getNamespace() );
 500
 501                 // @todo Emit some kind of warning to the user if $title->getNamespace() !==
 502                 // NS_MAIN and $prefix === '' (viz. pages in an unregistered namespace)
 503
 504                 if ( $prefix !== '' ) {
 505                         $prefix .= ':';
 506                 }
 507
 508                 return $prefix . $title->getText();
 509         }
 510 }