X-Git-Url: http://git.heureux-cyclage.org/?a=blobdiff_plain;f=maintenance%2FimportDump.php;h=904b6247de0a541620d29cf7f66a628760c548f6;hb=7dcf4e29e7bf1c4324cfb446113ba4508a6b6af8;hp=714d76d875ec20f4504e0c98f7f6a081cc874ac9;hpb=f85251e66654f7631cb24188ba55b1ba9a8438b0;p=lhc%2Fweb%2Fwiklou.git diff --git a/maintenance/importDump.php b/maintenance/importDump.php index 714d76d875..904b6247de 100644 --- a/maintenance/importDump.php +++ b/maintenance/importDump.php @@ -1,6 +1,8 @@ + * Import XML dump files into the current wiki. + * + * Copyright © 2005 Brion Vibber * http://www.mediawiki.org/ * * This program is free software; you can redistribute it and/or modify @@ -22,54 +24,164 @@ * @ingroup Maintenance */ -$optionsWithArgs = array( 'report' ); - -require_once( dirname(__FILE__) . '/commandLine.inc' ); +require_once( __DIR__ . '/Maintenance.php' ); /** + * Maintenance script that imports XML dump files into the current wiki. + * * @ingroup Maintenance */ -class BackupReader { - var $reportingInterval = 100; - var $reporting = true; - var $pageCount = 0; - var $revCount = 0; - var $dryRun = false; - var $debug = false; - var $uploads = false; - - function BackupReader() { +class BackupReader extends Maintenance { + public $reportingInterval = 100; + public $pageCount = 0; + public $revCount = 0; + public $dryRun = false; + public $uploads = false; + public $imageBasePath = false; + public $nsFilter = false; + + function __construct() { + parent::__construct(); + $gz = in_array('compress.zlib', stream_get_wrappers()) ? 'ok' : '(disabled; requires PHP zlib module)'; + $bz2 = in_array('compress.bzip2', stream_get_wrappers()) ? 'ok' : '(disabled; requires PHP bzip2 module)'; + + $this->mDescription = << +TEXT; $this->stderr = fopen( "php://stderr", "wt" ); + $this->addOption( 'report', + 'Report position and speed after every n pages processed', false, true ); + $this->addOption( 'namespaces', + 'Import only the pages from namespaces belonging to the list of ' . + 'pipe-separated namespace names or namespace indexes', false, true ); + $this->addOption( 'dry-run', 'Parse dump without actually importing pages' ); + $this->addOption( 'debug', 'Output extra verbose debug information' ); + $this->addOption( 'uploads', 'Process file upload data if included (experimental)' ); + $this->addOption( 'no-updates', 'Disable link table updates. Is faster but leaves the wiki in an inconsistent state' ); + $this->addOption( 'image-base-path', 'Import files from a specified path', false, true ); + $this->addArg( 'file', 'Dump file to import [else use stdin]', false ); + } + + public function execute() { + if( wfReadOnly() ) { + $this->error( "Wiki is in read-only mode; you'll need to disable it for import to work.", true ); + } + + $this->reportingInterval = intval( $this->getOption( 'report', 100 ) ); + if ( !$this->reportingInterval ) { + $this->reportingInterval = 100; // avoid division by zero + } + + $this->dryRun = $this->hasOption( 'dry-run' ); + $this->uploads = $this->hasOption( 'uploads' ); // experimental! + if ( $this->hasOption( 'image-base-path' ) ) { + $this->imageBasePath = $this->getOption( 'image-base-path' ); + } + if ( $this->hasOption( 'namespaces' ) ) { + $this->setNsfilter( explode( '|', $this->getOption( 'namespaces' ) ) ); + } + + if( $this->hasArg() ) { + $this->importFromFile( $this->getArg() ); + } else { + $this->importFromStdin(); + } + + $this->output( "Done!\n" ); + $this->output( "You might want to run rebuildrecentchanges.php to regenerate RecentChanges\n" ); + } + + function setNsfilter( array $namespaces ) { + if ( count( $namespaces ) == 0 ) { + $this->nsFilter = false; + return; + } + $this->nsFilter = array_unique( array_map( array( $this, 'getNsIndex' ), $namespaces ) ); + } + + private function getNsIndex( $namespace ) { + global $wgContLang; + if ( ( $result = $wgContLang->getNsIndex( $namespace ) ) !== false ) { + return $result; + } + $ns = intval( $namespace ); + if ( strval( $ns ) === $namespace && $wgContLang->getNsText( $ns ) !== false ) { + return $ns; + } + $this->error( "Unknown namespace text / index specified: $namespace", true ); + } + + /** + * @param $obj Title|Revision + * @return bool + */ + private function skippedNamespace( $obj ) { + if ( $obj instanceof Title ) { + $ns = $obj->getNamespace(); + } elseif ( $obj instanceof Revision ) { + $ns = $obj->getTitle()->getNamespace(); + } elseif ( $obj instanceof WikiRevision ) { + $ns = $obj->title->getNamespace(); + } else { + echo wfBacktrace(); + $this->error( "Cannot get namespace of object in " . __METHOD__, true ); + } + return is_array( $this->nsFilter ) && !in_array( $ns, $this->nsFilter ); } function reportPage( $page ) { $this->pageCount++; } + /** + * @param $rev Revision + * @return mixed + */ function handleRevision( $rev ) { $title = $rev->getTitle(); - if( !$title ) { + if ( !$title ) { $this->progress( "Got bogus revision with null title!" ); return; } + if ( $this->skippedNamespace( $title ) ) { + return; + } + $this->revCount++; $this->report(); - if( !$this->dryRun ) { + if ( !$this->dryRun ) { call_user_func( $this->importCallback, $rev ); } } - + + /** + * @param $revision Revision + * @return bool + */ function handleUpload( $revision ) { - if( $this->uploads ) { + if ( $this->uploads ) { + if ( $this->skippedNamespace( $revision ) ) { + return; + } $this->uploadCount++; - //$this->report(); + // $this->report(); $this->progress( "upload: " . $revision->getFilename() ); - - if( !$this->dryRun ) { + + if ( !$this->dryRun ) { // bluuuh hack - //call_user_func( $this->uploadCallback, $revision ); + // call_user_func( $this->uploadCallback, $revision ); $dbw = wfGetDB( DB_MASTER ); return $dbw->deadlockLoop( array( $revision, 'importUpload' ) ); } @@ -77,37 +189,43 @@ class BackupReader { } function handleLogItem( $rev ) { + if ( $this->skippedNamespace( $rev ) ) { + return; + } $this->revCount++; $this->report(); - if( !$this->dryRun ) { + if ( !$this->dryRun ) { call_user_func( $this->logItemCallback, $rev ); } } function report( $final = false ) { - if( $final xor ( $this->pageCount % $this->reportingInterval == 0 ) ) { + if ( $final xor ( $this->pageCount % $this->reportingInterval == 0 ) ) { $this->showReport(); } } function showReport() { - if( $this->reporting ) { - $delta = wfTime() - $this->startTime; - if( $delta ) { - $rate = sprintf("%.2f", $this->pageCount / $delta); - $revrate = sprintf("%.2f", $this->revCount / $delta); + if ( !$this->mQuiet ) { + $delta = microtime( true ) - $this->startTime; + if ( $delta ) { + $rate = sprintf( "%.2f", $this->pageCount / $delta ); + $revrate = sprintf( "%.2f", $this->revCount / $delta ); } else { $rate = '-'; $revrate = '-'; } # Logs dumps don't have page tallies - if( $this->pageCount ) + if ( $this->pageCount ) { $this->progress( "$this->pageCount ($rate pages/sec $revrate revs/sec)" ); - else + } else { $this->progress( "$this->revCount ($revrate revs/sec)" ); + } } - wfWaitForSlaves(5); + wfWaitForSlaves(); + // XXX: Don't let deferred jobs array get absurdly large (bug 24375) + DeferredUpdates::doUpdates( 'commit' ); } function progress( $string ) { @@ -115,34 +233,38 @@ class BackupReader { } function importFromFile( $filename ) { - $t = true; - if( preg_match( '/\.gz$/', $filename ) ) { + if ( preg_match( '/\.gz$/', $filename ) ) { $filename = 'compress.zlib://' . $filename; - } - elseif( preg_match( '/\.bz2$/', $filename ) ) { + } elseif ( preg_match( '/\.bz2$/', $filename ) ) { $filename = 'compress.bzip2://' . $filename; - } - elseif( preg_match( '/\.7z$/', $filename ) ) { + } elseif ( preg_match( '/\.7z$/', $filename ) ) { $filename = 'mediawiki.compress.7z://' . $filename; - $t = false; } - $file = fopen( $filename, $t ? 'rt' : 't' ); //our 7zip wrapper uses popen, which seems not to like two-letter modes + $file = fopen( $filename, 'rt' ); return $this->importFromHandle( $file ); } function importFromStdin() { $file = fopen( 'php://stdin', 'rt' ); + if( self::posix_isatty( $file ) ) { + $this->maybeHelp( true ); + } return $this->importFromHandle( $file ); } function importFromHandle( $handle ) { - $this->startTime = wfTime(); + $this->startTime = microtime( true ); $source = new ImportStreamSource( $handle ); $importer = new WikiImporter( $source ); - $importer->setDebug( $this->debug ); + if( $this->hasOption( 'debug' ) ) { + $importer->setDebug( true ); + } + if ( $this->hasOption( 'no-updates' ) ) { + $importer->setNoUpdates( true ); + } $importer->setPageCallback( array( &$this, 'reportPage' ) ); $this->importCallback = $importer->setRevisionCallback( array( &$this, 'handleRevision' ) ); @@ -150,44 +272,20 @@ class BackupReader { array( &$this, 'handleUpload' ) ); $this->logItemCallback = $importer->setLogItemCallback( array( &$this, 'handleLogItem' ) ); + if ( $this->uploads ) { + $importer->setImportUploads( true ); + } + if ( $this->imageBasePath ) { + $importer->setImageBasePath( $this->imageBasePath ); + } + + if ( $this->dryRun ) { + $importer->setPageOutCallback( null ); + } return $importer->doImport(); } } -if( wfReadOnly() ) { - wfDie( "Wiki is in read-only mode; you'll need to disable it for import to work.\n" ); -} - -$reader = new BackupReader(); -if( isset( $options['quiet'] ) ) { - $reader->reporting = false; -} -if( isset( $options['report'] ) ) { - $reader->reportingInterval = intval( $options['report'] ); -} -if( isset( $options['dry-run'] ) ) { - $reader->dryRun = true; -} -if( isset( $options['debug'] ) ) { - $reader->debug = true; -} -if( isset( $options['uploads'] ) ) { - $reader->uploads = true; // experimental! -} - -if( isset( $args[0] ) ) { - $result = $reader->importFromFile( $args[0] ); -} else { - $result = $reader->importFromStdin(); -} - -if( WikiError::isError( $result ) ) { - echo $result->getMessage() . "\n"; -} else { - echo "Done!\n"; - echo "You might want to run rebuildrecentchanges.php to regenerate\n"; - echo "the recentchanges page.\n"; -} - - +$maintClass = 'BackupReader'; +require_once( RUN_MAINTENANCE_IF_MAIN );