includes/Export.php

   1 <?php
   2 # Copyright (C) 2003, 2005 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19 /**
  20  *
  21  * @package MediaWiki
  22  * @subpackage SpecialPage
  23  */
  24
  25 /** */
  26 require_once( 'Revision.php' );
  27
  28 define( 'MW_EXPORT_FULL',     0 );
  29 define( 'MW_EXPORT_CURRENT',  1 );
  30
  31 define( 'MW_EXPORT_BUFFER',   0 );
  32 define( 'MW_EXPORT_STREAM',   1 );
  33
  34 define( 'MW_EXPORT_TEXT',     0 );
  35 define( 'MW_EXPORT_STUB',     1 );
  36
  37
  38 /**
  39  * @package MediaWiki
  40  * @subpackage SpecialPage
  41  */
  42 class WikiExporter {
  43         /**
  44          * If using MW_EXPORT_STREAM to stream a large amount of data,
  45          * provide a database connection which is not managed by
  46          * LoadBalancer to read from: some history blob types will
  47          * make additional queries to pull source data while the
  48          * main query is still running.
  49          *
  50          * @param Database $db
  51          * @param int $history one of MW_EXPORT_FULL or MW_EXPORT_CURRENT
  52          * @param int $buffer one of MW_EXPORT_BUFFER or MW_EXPORT_STREAM
  53          */
  54         function WikiExporter( &$db, $history = MW_EXPORT_CURRENT,
  55                         $buffer = MW_EXPORT_BUFFER, $text = MW_EXPORT_TEXT ) {
  56                 $this->db =& $db;
  57                 $this->history = $history;
  58                 $this->buffer  = $buffer;
  59                 $this->writer  = new XmlDumpWriter();
  60                 $this->sink    = new DumpOutput();
  61                 $this->text    = $text;
  62         }
  63
  64         /**
  65          * Set the DumpOutput or DumpFilter object which will receive
  66          * various row objects and XML output for filtering. Filters
  67          * can be chained or used as callbacks.
  68          *
  69          * @param mixed $callback
  70          */
  71         function setOutputSink( &$sink ) {
  72                 $this->sink =& $sink;
  73         }
  74
  75         function openStream() {
  76                 $output = $this->writer->openStream();
  77                 $this->sink->writeOpenStream( $output );
  78         }
  79
  80         function closeStream() {
  81                 $output = $this->writer->closeStream();
  82                 $this->sink->writeCloseStream( $output );
  83         }
  84
  85         /**
  86          * Dumps a series of page and revision records for all pages
  87          * in the database, either including complete history or only
  88          * the most recent version.
  89          */
  90         function allPages() {
  91                 return $this->dumpFrom( '' );
  92         }
  93
  94         /**
  95          * Dumps a series of page and revision records for those pages
  96          * in the database falling within the page_id range given.
  97          * @param int $start Inclusive lower limit (this id is included)
  98          * @param int $end   Exclusive upper limit (this id is not included)
  99          *                   If 0, no upper limit.
 100          */
 101         function pagesByRange( $start, $end ) {
 102                 $condition = 'page_id >= ' . intval( $start );
 103                 if( $end ) {
 104                         $condition .= ' AND page_id < ' . intval( $end );
 105                 }
 106                 return $this->dumpFrom( $condition );
 107         }
 108
 109         /**
 110          * @param Title $title
 111          */
 112         function pageByTitle( $title ) {
 113                 return $this->dumpFrom(
 114                         'page_namespace=' . $title->getNamespace() .
 115                         ' AND page_title=' . $this->db->addQuotes( $title->getDbKey() ) );
 116         }
 117
 118         function pageByName( $name ) {
 119                 $title = Title::newFromText( $name );
 120                 if( is_null( $title ) ) {
 121                         return new WikiError( "Can't export invalid title" );
 122                 } else {
 123                         return $this->pageByTitle( $title );
 124                 }
 125         }
 126
 127         function pagesByName( $names ) {
 128                 foreach( $names as $name ) {
 129                         $this->pageByName( $name );
 130                 }
 131         }
 132
 133
 134         // -------------------- private implementation below --------------------
 135
 136         function dumpFrom( $cond = '' ) {
 137                 $fname = 'WikiExporter::dumpFrom';
 138                 wfProfileIn( $fname );
 139
 140                 $page     = $this->db->tableName( 'page' );
 141                 $revision = $this->db->tableName( 'revision' );
 142                 $text     = $this->db->tableName( 'text' );
 143
 144                 if( $this->history == MW_EXPORT_FULL ) {
 145                         $join = 'page_id=rev_page';
 146                 } elseif( $this->history == MW_EXPORT_CURRENT ) {
 147                         $join = 'page_id=rev_page AND page_latest=rev_id';
 148                 } else {
 149                         wfProfileOut( $fname );
 150                         return new WikiError( "$fname given invalid history dump type." );
 151                 }
 152                 $where = ( $cond == '' ) ? '' : "$cond AND";
 153
 154                 if( $this->buffer == MW_EXPORT_STREAM ) {
 155                         $prev = $this->db->bufferResults( false );
 156                 }
 157                 if( $cond == '' ) {
 158                         // Optimization hack for full-database dump
 159                         $pageindex = 'FORCE INDEX (PRIMARY)';
 160                         $revindex = 'FORCE INDEX(page_timestamp)';
 161                 } else {
 162                         $pageindex = '';
 163                         $revindex = '';
 164                 }
 165                 if( $this->text == MW_EXPORT_STUB ) {
 166                         $sql = "SELECT * FROM
 167                                         $page $pageindex,
 168                                         $revision $revindex
 169                                         WHERE $where $join
 170                                         ORDER BY page_id";
 171                 } else {
 172                         $sql = "SELECT * FROM
 173                                         $page $pageindex,
 174                                         $revision $revindex,
 175                                         $text
 176                                         WHERE $where $join AND rev_text_id=old_id
 177                                         ORDER BY page_id";
 178                 }
 179                 $result = $this->db->query( $sql, $fname );
 180                 $wrapper = $this->db->resultObject( $result );
 181                 $this->outputStream( $wrapper );
 182
 183                 if( $this->buffer == MW_EXPORT_STREAM ) {
 184                         $this->db->bufferResults( $prev );
 185                 }
 186
 187                 wfProfileOut( $fname );
 188         }
 189
 190         /**
 191          * Runs through a query result set dumping page and revision records.
 192          * The result set should be sorted/grouped by page to avoid duplicate
 193          * page records in the output.
 194          *
 195          * The result set will be freed once complete. Should be safe for
 196          * streaming (non-buffered) queries, as long as it was made on a
 197          * separate database connection not managed by LoadBalancer; some
 198          * blob storage types will make queries to pull source data.
 199          *
 200          * @param ResultWrapper $resultset
 201          * @access private
 202          */
 203         function outputStream( $resultset ) {
 204                 $last = null;
 205                 while( $row = $resultset->fetchObject() ) {
 206                         if( is_null( $last ) ||
 207                                 $last->page_namespace != $row->page_namespace ||
 208                                 $last->page_title     != $row->page_title ) {
 209                                 if( isset( $last ) ) {
 210                                         $output = $this->writer->closePage();
 211                                         $this->sink->writeClosePage( $output );
 212                                 }
 213                                 $output = $this->writer->openPage( $row );
 214                                 $this->sink->writeOpenPage( $row, $output );
 215                                 $last = $row;
 216                         }
 217                         $output = $this->writer->writeRevision( $row );
 218                         $this->sink->writeRevision( $row, $output );
 219                 }
 220                 if( isset( $last ) ) {
 221                         $output = $this->writer->closePage();
 222                         $this->sink->writeClosePage( $output );
 223                 }
 224                 $resultset->free();
 225         }
 226 }
 227
 228 class XmlDumpWriter {
 229
 230         /**
 231          * Returns the export schema version.
 232          * @return string
 233          */
 234         function schemaVersion() {
 235                 return "0.3";
 236         }
 237
 238         /**
 239          * Opens the XML output stream's root <mediawiki> element.
 240          * This does not include an xml directive, so is safe to include
 241          * as a subelement in a larger XML stream. Namespace and XML Schema
 242          * references are included.
 243          *
 244          * Output will be encoded in UTF-8.
 245          *
 246          * @return string
 247          */
 248         function openStream() {
 249                 global $wgContLanguageCode;
 250                 $ver = $this->schemaVersion();
 251                 return wfElement( 'mediawiki', array(
 252                         'xmlns'              => "http://www.mediawiki.org/xml/export-$ver/",
 253                         'xmlns:xsi'          => "http://www.w3.org/2001/XMLSchema-instance",
 254                         'xsi:schemaLocation' => "http://www.mediawiki.org/xml/export-$ver/ " .
 255                                                 "http://www.mediawiki.org/xml/export-$ver.xsd",
 256                         'version'            => $ver,
 257                         'xml:lang'           => $wgContLanguageCode ),
 258                         null ) .
 259                         "\n" .
 260                         $this->siteInfo();
 261         }
 262
 263         function siteInfo() {
 264                 $info = array(
 265                         $this->sitename(),
 266                         $this->homelink(),
 267                         $this->generator(),
 268                         $this->caseSetting(),
 269                         $this->namespaces() );
 270                 return "  <siteinfo>\n    " .
 271                         implode( "\n    ", $info ) .
 272                         "\n  </siteinfo>\n";
 273         }
 274
 275         function sitename() {
 276                 global $wgSitename;
 277                 return wfElement( 'sitename', array(), $wgSitename );
 278         }
 279
 280         function generator() {
 281                 global $wgVersion;
 282                 return wfElement( 'generator', array(), "MediaWiki $wgVersion" );
 283         }
 284
 285         function homelink() {
 286                 $page = Title::newFromText( wfMsgForContent( 'mainpage' ) );
 287                 return wfElement( 'base', array(), $page->getFullUrl() );
 288         }
 289
 290         function caseSetting() {
 291                 global $wgCapitalLinks;
 292                 // "case-insensitive" option is reserved for future
 293                 $sensitivity = $wgCapitalLinks ? 'first-letter' : 'case-sensitive';
 294                 return wfElement( 'case', array(), $sensitivity );
 295         }
 296
 297         function namespaces() {
 298                 global $wgContLang;
 299                 $spaces = "  <namespaces>\n";
 300                 foreach( $wgContLang->getFormattedNamespaces() as $ns => $title ) {
 301                         $spaces .= '      ' . wfElement( 'namespace', array( 'key' => $ns ), $title ) . "\n";
 302                 }
 303                 $spaces .= "    </namespaces>";
 304                 return $spaces;
 305         }
 306
 307         /**
 308          * Closes the output stream with the closing root element.
 309          * Call when finished dumping things.
 310          */
 311         function closeStream() {
 312                 return "</mediawiki>\n";
 313         }
 314
 315
 316         /**
 317          * Opens a <page> section on the output stream, with data
 318          * from the given database row.
 319          *
 320          * @param object $row
 321          * @return string
 322          * @access private
 323          */
 324         function openPage( $row ) {
 325                 $out = "  <page>\n";
 326                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 327                 $out .= '    ' . wfElementClean( 'title', array(), $title->getPrefixedText() ) . "\n";
 328                 $out .= '    ' . wfElement( 'id', array(), strval( $row->page_id ) ) . "\n";
 329                 if( '' != $row->page_restrictions ) {
 330                         $out .= '    ' . wfElement( 'restrictions', array(),
 331                                 strval( $row->page_restrictions ) ) . "\n";
 332                 }
 333                 return $out;
 334         }
 335
 336         /**
 337          * Closes a <page> section on the output stream.
 338          *
 339          * @access private
 340          */
 341         function closePage() {
 342                 return "  </page>\n";
 343         }
 344
 345         /**
 346          * Dumps a <revision> section on the output stream, with
 347          * data filled in from the given database row.
 348          *
 349          * @param object $row
 350          * @return string
 351          * @access private
 352          */
 353         function writeRevision( $row ) {
 354                 $fname = 'WikiExporter::dumpRev';
 355                 wfProfileIn( $fname );
 356
 357                 $out  = "    <revision>\n";
 358                 $out .= "      " . wfElement( 'id', null, strval( $row->rev_id ) ) . "\n";
 359
 360                 $ts = wfTimestamp( TS_ISO_8601, $row->rev_timestamp );
 361                 $out .= "      " . wfElement( 'timestamp', null, $ts ) . "\n";
 362
 363                 $out .= "      <contributor>\n";
 364                 if( $row->rev_user ) {
 365                         $out .= "        " . wfElementClean( 'username', null, strval( $row->rev_user_text ) ) . "\n";
 366                         $out .= "        " . wfElement( 'id', null, strval( $row->rev_user ) ) . "\n";
 367                 } else {
 368                         $out .= "        " . wfElementClean( 'ip', null, strval( $row->rev_user_text ) ) . "\n";
 369                 }
 370                 $out .= "      </contributor>\n";
 371
 372                 if( $row->rev_minor_edit ) {
 373                         $out .=  "      <minor/>\n";
 374                 }
 375                 if( $row->rev_comment != '' ) {
 376                         $out .= "      " . wfElementClean( 'comment', null, strval( $row->rev_comment ) ) . "\n";
 377                 }
 378
 379                 if( isset( $row->old_text ) ) {
 380                         // Raw text from the database may have invalid chars
 381                         $text = strval( Revision::getRevisionText( $row ) );
 382                         $out .= "      " . wfElementClean( 'text',
 383                                 array( 'xml:space' => 'preserve' ),
 384                                 strval( $text ) ) . "\n";
 385                 } else {
 386                         // Stub output
 387                         $out .= "      " . wfElement( 'text',
 388                                 array( 'id' => $row->rev_text_id ),
 389                                 "" ) . "\n";
 390                 }
 391
 392                 $out .= "    </revision>\n";
 393
 394                 wfProfileOut( $fname );
 395                 return $out;
 396         }
 397
 398 }
 399
 400
 401 /**
 402  * Base class for output stream; prints to stdout or buffer or whereever.
 403  */
 404 class DumpOutput {
 405         function writeOpenStream( $string ) {
 406                 $this->write( $string );
 407         }
 408
 409         function writeCloseStream( $string ) {
 410                 $this->write( $string );
 411         }
 412
 413         function writeOpenPage( $page, $string ) {
 414                 $this->write( $string );
 415         }
 416
 417         function writeClosePage( $string ) {
 418                 $this->write( $string );
 419         }
 420
 421         function writeRevision( $rev, $string ) {
 422                 $this->write( $string );
 423         }
 424
 425         /**
 426          * Override to write to a different stream type.
 427          * @return bool
 428          */
 429         function write( $string ) {
 430                 print $string;
 431         }
 432 }
 433
 434 /**
 435  * Stream outputter to send data to a file.
 436  */
 437 class DumpFileOutput extends DumpOutput {
 438         var $handle;
 439
 440         function DumpFileOutput( $file ) {
 441                 $this->handle = fopen( $file, "wt" );
 442         }
 443
 444         function write( $string ) {
 445                 fputs( $this->handle, $string );
 446         }
 447 }
 448
 449 /**
 450  * Stream outputter to send data to a file via some filter program.
 451  * Even if compression is available in a library, using a separate
 452  * program can allow us to make use of a multi-processor system.
 453  */
 454 class DumpPipeOutput extends DumpFileOutput {
 455         function DumpPipeOutput( $command, $file = null ) {
 456                 if( !is_null( $file ) ) {
 457                         $command .=  " > " . wfEscapeShellArg( $file );
 458                 }
 459                 $this->handle = popen( $command, "w" );
 460         }
 461 }
 462
 463 /**
 464  * Sends dump output via the gzip compressor.
 465  */
 466 class DumpGZipOutput extends DumpPipeOutput {
 467         function DumpGZipOutput( $file ) {
 468                 parent::DumpPipeOutput( "gzip", $file );
 469         }
 470 }
 471
 472 /**
 473  * Sends dump output via the bgzip2 compressor.
 474  */
 475 class DumpBZip2Output extends DumpPipeOutput {
 476         function DumpBZip2Output( $file ) {
 477                 parent::DumpPipeOutput( "bzip2", $file );
 478         }
 479 }
 480
 481 /**
 482  * Sends dump output via the p7zip compressor.
 483  */
 484 class Dump7ZipOutput extends DumpPipeOutput {
 485         function Dump7ZipOutput( $file ) {
 486                 $command = "7za a -si " . wfEscapeShellArg( $file );
 487                 parent::DumpPipeOutput( $command );
 488         }
 489 }
 490
 491
 492
 493 /**
 494  * Dump output filter class.
 495  * This just does output filtering and streaming; XML formatting is done
 496  * higher up, so be careful in what you do.
 497  */
 498 class DumpFilter {
 499         function DumpFilter( &$sink ) {
 500                 $this->sink =& $sink;
 501         }
 502
 503         function writeOpenStream( $string ) {
 504                 $this->sink->writeOpenStream( $string );
 505         }
 506
 507         function writeCloseStream( $string ) {
 508                 $this->sink->writeCloseStream( $string );
 509         }
 510
 511         function writeOpenPage( $page, $string ) {
 512                 $this->sendingThisPage = $this->pass( $page, $string );
 513                 if( $this->sendingThisPage ) {
 514                         $this->sink->writeOpenPage( $page, $string );
 515                 }
 516         }
 517
 518         function writeClosePage( $string ) {
 519                 if( $this->sendingThisPage ) {
 520                         $this->sink->writeClosePage( $string );
 521                         $this->sendingThisPage = false;
 522                 }
 523         }
 524
 525         function writeRevision( $rev, $string ) {
 526                 if( $this->sendingThisPage ) {
 527                         $this->sink->writeRevision( $rev, $string );
 528                 }
 529         }
 530
 531         /**
 532          * Override for page-based filter types.
 533          * @return bool
 534          */
 535         function pass( $page, $string ) {
 536                 return true;
 537         }
 538 }
 539
 540 /**
 541  * Simple dump output filter to exclude all talk pages.
 542  */
 543 class DumpNotalkFilter extends DumpFilter {
 544         function pass( $page ) {
 545                 return !Namespace::isTalk( $page->page_namespace );
 546         }
 547 }
 548
 549 /**
 550  * Dump output filter to include or exclude pages in a given set of namespaces.
 551  */
 552 class DumpNamespaceFilter extends DumpFilter {
 553         var $invert = false;
 554         var $namespaces = array();
 555
 556         function DumpNamespaceFilter( &$sink, $param ) {
 557                 parent::DumpFilter( $sink );
 558
 559                 $constants = array(
 560                         "NS_MAIN"           => NS_MAIN,
 561                         "NS_TALK"           => NS_TALK,
 562                         "NS_USER"           => NS_USER,
 563                         "NS_USER_TALK"      => NS_USER_TALK,
 564                         "NS_PROJECT"        => NS_PROJECT,
 565                         "NS_PROJECT_TALK"   => NS_PROJECT_TALK,
 566                         "NS_IMAGE"          => NS_IMAGE,
 567                         "NS_IMAGE_TALK"     => NS_IMAGE_TALK,
 568                         "NS_MEDIAWIKI"      => NS_MEDIAWIKI,
 569                         "NS_MEDIAWIKI_TALK" => NS_MEDIAWIKI_TALK,
 570                         "NS_TEMPLATE"       => NS_TEMPLATE,
 571                         "NS_TEMPLATE_TALK"  => NS_TEMPLATE_TALK,
 572                         "NS_HELP"           => NS_HELP,
 573                         "NS_HELP_TALK"      => NS_HELP_TALK,
 574                         "NS_CATEGORY"       => NS_CATEGORY,
 575                         "NS_CATEGORY_TALK"  => NS_CATEGORY_TALK );
 576
 577                 if( $param{0} == '!' ) {
 578                         $this->invert = true;
 579                         $param = substr( $param, 1 );
 580                 }
 581
 582                 foreach( explode( ',', $param ) as $key ) {
 583                         $key = trim( $key );
 584                         if( isset( $constants[$key] ) ) {
 585                                 $ns = $constants[$key];
 586                                 $this->namespaces[$ns] = true;
 587                         } elseif( is_numeric( $key ) ) {
 588                                 $ns = intval( $key );
 589                                 $this->namespaces[$ns] = true;
 590                         } else {
 591                                 die( "Unrecognized namespace key '$key'\n" );
 592                         }
 593                 }
 594         }
 595
 596         function pass( $page ) {
 597                 $match = isset( $this->namespaces[$page->page_namespace] );
 598                 return $this->invert xor $match;
 599         }
 600 }
 601
 602
 603 /**
 604  * Dump output filter to include only the last revision in each page sequence.
 605  */
 606 class DumpLatestFilter extends DumpFilter {
 607         var $page, $pageString, $rev, $revString;
 608
 609         function writeOpenPage( $page, $string ) {
 610                 $this->page = $page;
 611                 $this->pageString = $string;
 612         }
 613
 614         function writeClosePage( $string ) {
 615                 if( $this->rev ) {
 616                         $this->sink->writeOpenPage( $this->page, $this->pageString );
 617                         $this->sink->writeRevision( $this->rev, $this->revString );
 618                         $this->sink->writeClosePage( $string );
 619                 }
 620                 $this->rev = null;
 621                 $this->revString = null;
 622                 $this->page = null;
 623                 $this->pageString = null;
 624         }
 625
 626         function writeRevision( $rev, $string ) {
 627                 if( $rev->rev_id == $this->page->page_latest ) {
 628                         $this->rev = $rev;
 629                         $this->revString = $string;
 630                 }
 631         }
 632 }
 633
 634 /**
 635  * Base class for output stream; prints to stdout or buffer or whereever.
 636  */
 637 class DumpMultiWriter {
 638         function DumpMultiWriter( $sinks ) {
 639                 $this->sinks = $sinks;
 640                 $this->count = count( $sinks );
 641         }
 642
 643         function writeOpenStream( $string ) {
 644                 for( $i = 0; $i < $this->count; $i++ ) {
 645                         $this->sinks[$i]->writeOpenStream( $string );
 646                 }
 647         }
 648
 649         function writeCloseStream( $string ) {
 650                 for( $i = 0; $i < $this->count; $i++ ) {
 651                         $this->sinks[$i]->writeCloseStream( $string );
 652                 }
 653         }
 654
 655         function writeOpenPage( $page, $string ) {
 656                 for( $i = 0; $i < $this->count; $i++ ) {
 657                         $this->sinks[$i]->writeOpenPage( $page, $string );
 658                 }
 659         }
 660
 661         function writeClosePage( $string ) {
 662                 for( $i = 0; $i < $this->count; $i++ ) {
 663                         $this->sinks[$i]->writeClosePage( $string );
 664                 }
 665         }
 666
 667         function writeRevision( $rev, $string ) {
 668                 for( $i = 0; $i < $this->count; $i++ ) {
 669                         $this->sinks[$i]->writeRevision( $rev, $string );
 670                 }
 671         }
 672 }
 673
 674 function xmlsafe( $string ) {
 675         $fname = 'xmlsafe';
 676         wfProfileIn( $fname );
 677
 678         /**
 679          * The page may contain old data which has not been properly normalized.
 680          * Invalid UTF-8 sequences or forbidden control characters will make our
 681          * XML output invalid, so be sure to strip them out.
 682          */
 683         $string = UtfNormal::cleanUp( $string );
 684
 685         $string = htmlspecialchars( $string );
 686         wfProfileOut( $fname );
 687         return $string;
 688 }
 689
 690 ?>