3 * MediaWiki page data importer
4 * Copyright (C) 2003,2005 Brion Vibber <brion@pobox.com>
5 * http://www.mediawiki.org/
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 * http://www.gnu.org/copyleft/gpl.html
22 * @addtogroup SpecialPage
28 function wfSpecialImport( $page = '' ) {
29 global $wgUser, $wgOut, $wgRequest, $wgTitle, $wgImportSources;
30 global $wgImportTargetNamespace;
33 $namespace = $wgImportTargetNamespace;
38 $wgOut->readOnlyPage();
42 if( $wgRequest->wasPosted() && $wgRequest->getVal( 'action' ) == 'submit') {
44 $namespace = $wgRequest->getIntOrNull( 'namespace' );
46 switch( $wgRequest->getVal( "source" ) ) {
49 if( $wgUser->isAllowed( 'importupload' ) ) {
50 $source = ImportStreamSource
::newFromUpload( "xmlimport" );
52 return $wgOut->permissionRequired( 'importupload' );
56 $interwiki = $wgRequest->getVal( 'interwiki' );
57 $history = $wgRequest->getCheck( 'interwikiHistory' );
58 $frompage = $wgRequest->getText( "frompage" );
59 $source = ImportStreamSource
::newFromInterwiki(
65 $source = new WikiErrorMsg( "importunknownsource" );
68 if( WikiError
::isError( $source ) ) {
69 $wgOut->addWikiText( wfEscapeWikiText( $source->getMessage() ) );
71 $wgOut->addWikiText( wfMsg( "importstart" ) );
73 $importer = new WikiImporter( $source );
74 if( !is_null( $namespace ) ) {
75 $importer->setTargetNamespace( $namespace );
77 $reporter = new ImportReporter( $importer, $isUpload, $interwiki );
80 $result = $importer->doImport();
83 if( WikiError
::isError( $result ) ) {
84 $wgOut->addWikiText( wfMsg( "importfailed",
85 wfEscapeWikiText( $result->getMessage() ) ) );
88 $wgOut->addWikiText( wfMsg( "importsuccess" ) );
93 $action = $wgTitle->getLocalUrl( 'action=submit' );
95 if( $wgUser->isAllowed( 'importupload' ) ) {
96 $wgOut->addWikiText( wfMsg( "importtext" ) );
99 <legend>" . wfMsgHtml('upload') . "</legend>
100 <form enctype='multipart/form-data' method='post' action=\"$action\">
101 <input type='hidden' name='action' value='submit' />
102 <input type='hidden' name='source' value='upload' />
103 <input type='file' name='xmlimport' value='' size='30' />
104 <input type='submit' value=\"" . wfMsgHtml( "uploadbtn" ) . "\" />
109 if( empty( $wgImportSources ) ) {
110 $wgOut->addWikiText( wfMsg( 'importnosources' ) );
114 if( !empty( $wgImportSources ) ) {
117 <legend>" . wfMsgHtml('importinterwiki') . "</legend>
118 <form method='post' action=\"$action\">" .
119 $wgOut->parse( wfMsg( 'import-interwiki-text' ) ) . "
120 <input type='hidden' name='action' value='submit' />
121 <input type='hidden' name='source' value='interwiki' />
125 <select name='interwiki'>" );
126 foreach( $wgImportSources as $prefix ) {
127 $iw = htmlspecialchars( $prefix );
128 $selected = ($interwiki === $prefix) ?
' selected="selected"' : '';
129 $wgOut->addHTML( "<option value=\"$iw\"$selected>$iw</option>\n" );
135 wfInput( 'frompage', 50, $frompage ) .
141 wfCheckLabel( wfMsg( 'import-interwiki-history' ),
142 'interwikiHistory', 'interwikiHistory', $history ) .
148 " . wfMsgHtml( 'import-interwiki-namespace' ) . " " .
149 HTMLnamespaceselector( $namespace, '' ) . "
155 wfSubmitButton( wfMsg( 'import-interwiki-submit' ) ) .
167 * @addtogroup SpecialPage
169 class ImportReporter
{
170 function __construct( $importer, $upload, $interwiki ) {
171 $importer->setPageOutCallback( array( $this, 'reportPage' ) );
172 $this->mPageCount
= 0;
173 $this->mIsUpload
= $upload;
174 $this->mInterwiki
= $interwiki;
179 $wgOut->addHtml( "<ul>\n" );
182 function reportPage( $title, $origTitle, $revisionCount, $successCount ) {
183 global $wgOut, $wgUser, $wgLang, $wgContLang;
185 $skin = $wgUser->getSkin();
189 $localCount = $wgLang->formatNum( $successCount );
190 $contentCount = $wgContLang->formatNum( $successCount );
192 $wgOut->addHtml( "<li>" . $skin->makeKnownLinkObj( $title ) .
194 wfMsgExt( 'import-revision-count', array( 'parsemag', 'escape' ), $localCount ) .
197 if( $successCount > 0 ) {
198 $log = new LogPage( 'import' );
199 if( $this->mIsUpload
) {
200 $detail = wfMsgExt( 'import-logentry-upload-detail', array( 'content', 'parsemag' ),
202 $log->addEntry( 'upload', $title, $detail );
204 $interwiki = '[[:' . $this->mInterwiki
. ':' .
205 $origTitle->getPrefixedText() . ']]';
206 $detail = wfMsgExt( 'import-logentry-interwiki-detail', array( 'content', 'parsemag' ),
207 $contentCount, $interwiki );
208 $log->addEntry( 'interwiki', $title, $detail );
211 $comment = $detail; // quick
212 $dbw = wfGetDB( DB_MASTER
);
213 $nullRevision = Revision
::newNullRevision(
214 $dbw, $title->getArticleId(), $comment, true );
215 $nullRevision->insertOn( $dbw );
217 $article = new Article( $title );
218 $article->updateRevisionOn( $dbw, $nullRevision );
224 if( $this->mPageCount
== 0 ) {
225 $wgOut->addHtml( "<li>" . wfMsgHtml( 'importnopages' ) . "</li>\n" );
227 $wgOut->addHtml( "</ul>\n" );
233 * @addtogroup SpecialPage
238 var $timestamp = "20010115000000";
245 function setTitle( $title ) {
246 if( is_object( $title ) ) {
247 $this->title
= $title;
248 } elseif( is_null( $title ) ) {
249 throw new MWException( "WikiRevision given a null title in import. You may need to adjust \$wgLegalTitleChars." );
251 throw new MWException( "WikiRevision given non-object title in import." );
255 function setID( $id ) {
259 function setTimestamp( $ts ) {
260 # 2003-08-05T18:30:02Z
261 $this->timestamp
= wfTimestamp( TS_MW
, $ts );
264 function setUsername( $user ) {
265 $this->user_text
= $user;
268 function setUserIP( $ip ) {
269 $this->user_text
= $ip;
272 function setText( $text ) {
276 function setComment( $text ) {
277 $this->comment
= $text;
280 function setMinor( $minor ) {
281 $this->minor
= (bool)$minor;
284 function getTitle() {
292 function getTimestamp() {
293 return $this->timestamp
;
297 return $this->user_text
;
304 function getComment() {
305 return $this->comment
;
308 function getMinor() {
312 function importOldRevision() {
313 $dbw = wfGetDB( DB_MASTER
);
315 # Sneak a single revision into place
316 $user = User
::newFromName( $this->getUser() );
318 $userId = intval( $user->getId() );
319 $userText = $user->getName();
322 $userText = $this->getUser();
325 // avoid memory leak...?
326 $linkCache =& LinkCache
::singleton();
329 $article = new Article( $this->title
);
330 $pageId = $article->getId();
332 # must create the page...
333 $pageId = $article->insertOn( $dbw );
338 $prior = Revision
::loadFromTimestamp( $dbw, $this->title
, $this->timestamp
);
339 if( !is_null( $prior ) ) {
340 // FIXME: this could fail slightly for multiple matches :P
341 wfDebug( __METHOD__
. ": skipping existing revision for [[" .
342 $this->title
->getPrefixedText() . "]], timestamp " .
343 $this->timestamp
. "\n" );
348 # FIXME: Use original rev_id optionally
349 # FIXME: blah blah blah
351 #if( $numrows > 0 ) {
352 # return wfMsg( "importhistoryconflict" );
356 $revision = new Revision( array(
358 'text' => $this->getText(),
359 'comment' => $this->getComment(),
361 'user_text' => $userText,
362 'timestamp' => $this->timestamp
,
363 'minor_edit' => $this->minor
,
365 $revId = $revision->insertOn( $dbw );
366 $changed = $article->updateIfNewerOn( $dbw, $revision );
369 wfDebug( __METHOD__
. ": running onArticleCreate\n" );
370 Article
::onArticleCreate( $this->title
);
372 wfDebug( __METHOD__
. ": running create updates\n" );
373 $article->createUpdates( $revision );
375 } elseif( $changed ) {
376 wfDebug( __METHOD__
. ": running onArticleEdit\n" );
377 Article
::onArticleEdit( $this->title
);
379 wfDebug( __METHOD__
. ": running edit updates\n" );
380 $article->editUpdates(
394 * implements Special:Import
395 * @addtogroup SpecialPage
399 var $mPageCallback = null;
400 var $mPageOutCallback = null;
401 var $mRevisionCallback = null;
402 var $mTargetNamespace = null;
405 function WikiImporter( $source ) {
406 $this->setRevisionCallback( array( &$this, "importRevision" ) );
407 $this->mSource
= $source;
410 function throwXmlError( $err ) {
411 $this->debug( "FAILURE: $err" );
412 wfDebug( "WikiImporter XML error: $err\n" );
417 function doImport() {
418 if( empty( $this->mSource
) ) {
419 return new WikiErrorMsg( "importnotext" );
422 $parser = xml_parser_create( "UTF-8" );
424 # case folding violates XML standard, turn it off
425 xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING
, false );
427 xml_set_object( $parser, $this );
428 xml_set_element_handler( $parser, "in_start", "" );
430 $offset = 0; // for context extraction on error reporting
432 $chunk = $this->mSource
->readChunk();
433 if( !xml_parse( $parser, $chunk, $this->mSource
->atEnd() ) ) {
434 wfDebug( "WikiImporter::doImport encountered XML parsing error\n" );
435 return new WikiXmlError( $parser, 'XML import parse failure', $chunk, $offset );
437 $offset +
= strlen( $chunk );
438 } while( $chunk !== false && !$this->mSource
->atEnd() );
439 xml_parser_free( $parser );
444 function debug( $data ) {
445 #wfDebug( "IMPORT: $data\n" );
448 function notice( $data ) {
449 global $wgCommandLineMode;
450 if( $wgCommandLineMode ) {
454 $wgOut->addHTML( "<li>" . htmlspecialchars( $data ) . "</li>\n" );
459 * Sets the action to perform as each new page in the stream is reached.
460 * @param callable $callback
463 function setPageCallback( $callback ) {
464 $previous = $this->mPageCallback
;
465 $this->mPageCallback
= $callback;
470 * Sets the action to perform as each page in the stream is completed.
471 * Callback accepts the page title (as a Title object), a second object
472 * with the original title form (in case it's been overridden into a
473 * local namespace), and a count of revisions.
475 * @param callable $callback
478 function setPageOutCallback( $callback ) {
479 $previous = $this->mPageOutCallback
;
480 $this->mPageOutCallback
= $callback;
485 * Sets the action to perform as each page revision is reached.
486 * @param callable $callback
489 function setRevisionCallback( $callback ) {
490 $previous = $this->mRevisionCallback
;
491 $this->mRevisionCallback
= $callback;
496 * Set a target namespace to override the defaults
498 function setTargetNamespace( $namespace ) {
499 if( is_null( $namespace ) ) {
500 // Don't override namespaces
501 $this->mTargetNamespace
= null;
502 } elseif( $namespace >= 0 ) {
503 // FIXME: Check for validity
504 $this->mTargetNamespace
= intval( $namespace );
511 * Default per-revision callback, performs the import.
512 * @param WikiRevision $revision
515 function importRevision( &$revision ) {
516 $dbw = wfGetDB( DB_MASTER
);
517 return $dbw->deadlockLoop( array( &$revision, 'importOldRevision' ) );
521 * Alternate per-revision callback, for debugging.
522 * @param WikiRevision $revision
525 function debugRevisionHandler( &$revision ) {
526 $this->debug( "Got revision:" );
527 if( is_object( $revision->title
) ) {
528 $this->debug( "-- Title: " . $revision->title
->getPrefixedText() );
530 $this->debug( "-- Title: <invalid>" );
532 $this->debug( "-- User: " . $revision->user_text
);
533 $this->debug( "-- Timestamp: " . $revision->timestamp
);
534 $this->debug( "-- Comment: " . $revision->comment
);
535 $this->debug( "-- Text: " . $revision->text
);
539 * Notify the callback function when a new <page> is reached.
540 * @param Title $title
543 function pageCallback( $title ) {
544 if( is_callable( $this->mPageCallback
) ) {
545 call_user_func( $this->mPageCallback
, $title );
550 * Notify the callback function when a </page> is closed.
551 * @param Title $title
552 * @param Title $origTitle
553 * @param int $revisionCount
554 * @param int $successCount number of revisions for which callback returned true
557 function pageOutCallback( $title, $origTitle, $revisionCount, $successCount ) {
558 if( is_callable( $this->mPageOutCallback
) ) {
559 call_user_func( $this->mPageOutCallback
, $title, $origTitle,
560 $revisionCount, $successCount );
565 # XML parser callbacks from here out -- beware!
566 function donothing( $parser, $x, $y="" ) {
567 #$this->debug( "donothing" );
570 function in_start( $parser, $name, $attribs ) {
571 $this->debug( "in_start $name" );
572 if( $name != "mediawiki" ) {
573 return $this->throwXMLerror( "Expected <mediawiki>, got <$name>" );
575 xml_set_element_handler( $parser, "in_mediawiki", "out_mediawiki" );
578 function in_mediawiki( $parser, $name, $attribs ) {
579 $this->debug( "in_mediawiki $name" );
580 if( $name == 'siteinfo' ) {
581 xml_set_element_handler( $parser, "in_siteinfo", "out_siteinfo" );
582 } elseif( $name == 'page' ) {
583 $this->workRevisionCount
= 0;
584 $this->workSuccessCount
= 0;
585 xml_set_element_handler( $parser, "in_page", "out_page" );
587 return $this->throwXMLerror( "Expected <page>, got <$name>" );
590 function out_mediawiki( $parser, $name ) {
591 $this->debug( "out_mediawiki $name" );
592 if( $name != "mediawiki" ) {
593 return $this->throwXMLerror( "Expected </mediawiki>, got </$name>" );
595 xml_set_element_handler( $parser, "donothing", "donothing" );
599 function in_siteinfo( $parser, $name, $attribs ) {
601 $this->debug( "in_siteinfo $name" );
611 return $this->throwXMLerror( "Element <$name> not allowed in <siteinfo>." );
615 function out_siteinfo( $parser, $name ) {
616 if( $name == "siteinfo" ) {
617 xml_set_element_handler( $parser, "in_mediawiki", "out_mediawiki" );
622 function in_page( $parser, $name, $attribs ) {
623 $this->debug( "in_page $name" );
628 $this->appendfield
= $name;
629 $this->appenddata
= "";
630 $this->parenttag
= "page";
631 xml_set_element_handler( $parser, "in_nothing", "out_append" );
632 xml_set_character_data_handler( $parser, "char_append" );
635 if( is_object( $this->pageTitle
) ) {
636 $this->workRevision
= new WikiRevision
;
637 $this->workRevision
->setTitle( $this->pageTitle
);
638 $this->workRevisionCount++
;
640 // Skipping items due to invalid page title
641 $this->workRevision
= null;
643 xml_set_element_handler( $parser, "in_revision", "out_revision" );
646 return $this->throwXMLerror( "Element <$name> not allowed in a <page>." );
650 function out_page( $parser, $name ) {
651 $this->debug( "out_page $name" );
652 if( $name != "page" ) {
653 return $this->throwXMLerror( "Expected </page>, got </$name>" );
655 xml_set_element_handler( $parser, "in_mediawiki", "out_mediawiki" );
657 $this->pageOutCallback( $this->pageTitle
, $this->origTitle
,
658 $this->workRevisionCount
, $this->workSuccessCount
);
660 $this->workTitle
= null;
661 $this->workRevision
= null;
662 $this->workRevisionCount
= 0;
663 $this->workSuccessCount
= 0;
664 $this->pageTitle
= null;
665 $this->origTitle
= null;
668 function in_nothing( $parser, $name, $attribs ) {
669 $this->debug( "in_nothing $name" );
670 return $this->throwXMLerror( "No child elements allowed here; got <$name>" );
672 function char_append( $parser, $data ) {
673 $this->debug( "char_append '$data'" );
674 $this->appenddata
.= $data;
676 function out_append( $parser, $name ) {
677 $this->debug( "out_append $name" );
678 if( $name != $this->appendfield
) {
679 return $this->throwXMLerror( "Expected </{$this->appendfield}>, got </$name>" );
681 xml_set_element_handler( $parser, "in_$this->parenttag", "out_$this->parenttag" );
682 xml_set_character_data_handler( $parser, "donothing" );
684 switch( $this->appendfield
) {
686 $this->workTitle
= $this->appenddata
;
687 $this->origTitle
= Title
::newFromText( $this->workTitle
);
688 if( !is_null( $this->mTargetNamespace
) && !is_null( $this->origTitle
) ) {
689 $this->pageTitle
= Title
::makeTitle( $this->mTargetNamespace
,
690 $this->origTitle
->getDBkey() );
692 $this->pageTitle
= Title
::newFromText( $this->workTitle
);
694 if( is_null( $this->pageTitle
) ) {
695 // Invalid page title? Ignore the page
696 $this->notice( "Skipping invalid page title '$this->workTitle'" );
698 $this->pageCallback( $this->workTitle
);
702 if ( $this->parenttag
== 'revision' ) {
703 if( $this->workRevision
)
704 $this->workRevision
->setID( $this->appenddata
);
708 if( $this->workRevision
)
709 $this->workRevision
->setText( $this->appenddata
);
712 if( $this->workRevision
)
713 $this->workRevision
->setUsername( $this->appenddata
);
716 if( $this->workRevision
)
717 $this->workRevision
->setUserIP( $this->appenddata
);
720 if( $this->workRevision
)
721 $this->workRevision
->setTimestamp( $this->appenddata
);
724 if( $this->workRevision
)
725 $this->workRevision
->setComment( $this->appenddata
);
728 if( $this->workRevision
)
729 $this->workRevision
->setMinor( true );
732 $this->debug( "Bad append: {$this->appendfield}" );
734 $this->appendfield
= "";
735 $this->appenddata
= "";
738 function in_revision( $parser, $name, $attribs ) {
739 $this->debug( "in_revision $name" );
746 $this->parenttag
= "revision";
747 $this->appendfield
= $name;
748 xml_set_element_handler( $parser, "in_nothing", "out_append" );
749 xml_set_character_data_handler( $parser, "char_append" );
752 xml_set_element_handler( $parser, "in_contributor", "out_contributor" );
755 return $this->throwXMLerror( "Element <$name> not allowed in a <revision>." );
759 function out_revision( $parser, $name ) {
760 $this->debug( "out_revision $name" );
761 if( $name != "revision" ) {
762 return $this->throwXMLerror( "Expected </revision>, got </$name>" );
764 xml_set_element_handler( $parser, "in_page", "out_page" );
766 if( $this->workRevision
) {
767 $ok = call_user_func_array( $this->mRevisionCallback
,
768 array( &$this->workRevision
, &$this ) );
770 $this->workSuccessCount++
;
775 function in_contributor( $parser, $name, $attribs ) {
776 $this->debug( "in_contributor $name" );
781 $this->parenttag
= "contributor";
782 $this->appendfield
= $name;
783 xml_set_element_handler( $parser, "in_nothing", "out_append" );
784 xml_set_character_data_handler( $parser, "char_append" );
787 $this->throwXMLerror( "Invalid tag <$name> in <contributor>" );
791 function out_contributor( $parser, $name ) {
792 $this->debug( "out_contributor $name" );
793 if( $name != "contributor" ) {
794 return $this->throwXMLerror( "Expected </contributor>, got </$name>" );
796 xml_set_element_handler( $parser, "in_revision", "out_revision" );
802 * @todo document (e.g. one-sentence class description).
803 * @addtogroup SpecialPage
805 class ImportStringSource
{
806 function ImportStringSource( $string ) {
807 $this->mString
= $string;
808 $this->mRead
= false;
815 function readChunk() {
816 if( $this->atEnd() ) {
820 return $this->mString
;
826 * @todo document (e.g. one-sentence class description).
827 * @addtogroup SpecialPage
829 class ImportStreamSource
{
830 function ImportStreamSource( $handle ) {
831 $this->mHandle
= $handle;
835 return feof( $this->mHandle
);
838 function readChunk() {
839 return fread( $this->mHandle
, 32768 );
842 static function newFromFile( $filename ) {
843 $file = @fopen
( $filename, 'rt' );
845 return new WikiErrorMsg( "importcantopen" );
847 return new ImportStreamSource( $file );
850 static function newFromUpload( $fieldname = "xmlimport" ) {
851 $upload =& $_FILES[$fieldname];
853 if( !isset( $upload ) ||
!$upload['name'] ) {
854 return new WikiErrorMsg( 'importnofile' );
856 if( !empty( $upload['error'] ) ) {
857 switch($upload['error']){
858 case 1: # The uploaded file exceeds the upload_max_filesize directive in php.ini.
859 return new WikiErrorMsg( 'importuploaderrorsize' );
860 case 2: # The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form.
861 return new WikiErrorMsg( 'importuploaderrorsize' );
862 case 3: # The uploaded file was only partially uploaded
863 return new WikiErrorMsg( 'importuploaderrorpartial' );
864 case 6: #Missing a temporary folder. Introduced in PHP 4.3.10 and PHP 5.0.3.
865 return new WikiErrorMsg( 'importuploaderrortemp' );
866 # case else: # Currently impossible
870 $fname = $upload['tmp_name'];
871 if( is_uploaded_file( $fname ) ) {
872 return ImportStreamSource
::newFromFile( $fname );
874 return new WikiErrorMsg( 'importnofile' );
878 function newFromURL( $url, $method = 'GET' ) {
879 wfDebug( __METHOD__
. ": opening $url\n" );
880 # Use the standard HTTP fetch function; it times out
881 # quicker and sorts out user-agent problems which might
882 # otherwise prevent importing from large sites, such
883 # as the Wikimedia cluster, etc.
884 $data = Http
::request( $method, $url );
885 if( $data !== false ) {
887 fwrite( $file, $data );
890 return new ImportStreamSource( $file );
892 return new WikiErrorMsg( 'importcantopen' );
896 public static function newFromInterwiki( $interwiki, $page, $history=false ) {
897 $link = Title
::newFromText( "$interwiki:Special:Export/$page" );
898 if( is_null( $link ) ||
$link->getInterwiki() == '' ) {
899 return new WikiErrorMsg( 'importbadinterwiki' );
901 $params = $history ?
'history=1' : '';
902 $url = $link->getFullUrl( $params );
903 # For interwikis, use POST to avoid redirects.
904 return ImportStreamSource
::newFromURL( $url, "POST" );