X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=maintenance%2Fbackup.inc;h=814d0c09d06273157b2fd1826f18b098c732d3a1;hb=025d7105bac7fa28c892ac221c8b82f90968ff9a;hp=d40636a6f3ad1c910e7a646c6a3ab37661b7dff2;hpb=e174a4ddfb96feee0a8305c60aade97c0ee30d3c;p=lhc%2Fweb%2Fwiklou.git diff --git a/maintenance/backup.inc b/maintenance/backup.inc index d40636a6f3..814d0c09d0 100644 --- a/maintenance/backup.inc +++ b/maintenance/backup.inc @@ -1,6 +1,8 @@ + * Base classes for database dumpers + * + * Copyright © 2005 Brion Vibber * http://www.mediawiki.org/ * * This program is free software; you can redistribute it and/or modify @@ -18,16 +20,22 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * http://www.gnu.org/copyleft/gpl.html * - * @package MediaWiki - * @subpackage SpecialPage + * @file + * @ingroup Dump Maintenance */ +/** + * @ingroup Dump Maintenance + */ class DumpDBZip2Output extends DumpPipeOutput { function DumpDBZip2Output( $file ) { - parent::DumpPipeOutput( "dbzip2", $file ); + parent::__construct( "dbzip2", $file ); } } +/** + * @ingroup Dump Maintenance + */ class BackupDumper { var $reportingInterval = 100; var $reporting = true; @@ -41,8 +49,21 @@ class BackupDumper { var $endId = 0; var $sink = null; // Output filters var $stubText = false; // include rev_text_id instead of text; for 2-pass dump + var $dumpUploads = false; + var $dumpUploadFileContents = false; + var $lastTime = 0; + var $pageCountLast = 0; + var $revCountLast = 0; + var $ID = 0; + + var $outputTypes = array(), $filterTypes = array(); + + /** + * @var LoadBalancer + */ + protected $lb; - function BackupDumper( $args ) { + function __construct( $args ) { $this->stderr = fopen( "php://stderr", "wt" ); // Built-in output and filter plugins @@ -60,16 +81,16 @@ class BackupDumper { } /** - * @param string $name - * @param string $class name of output filter plugin class + * @param $name String + * @param $class String: name of output filter plugin class */ function registerOutput( $name, $class ) { $this->outputTypes[$name] = $class; } /** - * @param string $name - * @param string $class name of filter plugin class + * @param $name String + * @param $class String: name of filter plugin class */ function registerFilter( $name, $class ) { $this->filterTypes[$name] = $class; @@ -77,12 +98,13 @@ class BackupDumper { /** * Load a plugin and register it - * @param string $class Name of plugin class; must have a static 'register' - * method that takes a BackupDumper as a parameter. - * @param string $file Full or relative path to the PHP file to load, or empty + * + * @param $class String: name of plugin class; must have a static 'register' + * method that takes a BackupDumper as a parameter. + * @param $file String: full or relative path to the PHP file to load, or empty */ function loadPlugin( $class, $file ) { - if( $file != '' ) { + if ( $file != '' ) { require_once( $file ); } $register = array( $class, 'register' ); @@ -90,37 +112,36 @@ class BackupDumper { } /** - * @param array $args - * @return array - * @static + * @param $args Array + * @return Array */ function processArgs( $args ) { $sink = null; $sinks = array(); - foreach( $args as $arg ) { - if( preg_match( '/^--(.+?)(?:=(.+?)(?::(.+?))?)?$/', $arg, $matches ) ) { - @list( $full, $opt, $val, $param ) = $matches; + foreach ( $args as $arg ) { + $matches = array(); + if ( preg_match( '/^--(.+?)(?:=(.+?)(?::(.+?))?)?$/', $arg, $matches ) ) { + @list( /* $full */ , $opt, $val, $param ) = $matches; switch( $opt ) { case "plugin": $this->loadPlugin( $val, $param ); break; case "output": - if( !is_null( $sink ) ) { + if ( !is_null( $sink ) ) { $sinks[] = $sink; } - if( !isset( $this->outputTypes[$val] ) ) { - wfDie( "Unrecognized output sink type '$val'\n" ); + if ( !isset( $this->outputTypes[$val] ) ) { + $this->fatalError( "Unrecognized output sink type '$val'" ); } $type = $this->outputTypes[$val]; $sink = new $type( $param ); break; case "filter": - if( is_null( $sink ) ) { - $this->progress( "Warning: assuming stdout for filter output\n" ); + if ( is_null( $sink ) ) { $sink = new DumpOutput(); } - if( !isset( $this->filterTypes[$val] ) ) { - wfDie( "Unrecognized filter type '$val'\n" ); + if ( !isset( $this->filterTypes[$val] ) ) { + $this->fatalError( "Unrecognized filter type '$val'" ); } $type = $this->filterTypes[$val]; $filter = new $type( $sink, $param ); @@ -137,11 +158,11 @@ class BackupDumper { $this->server = $val; break; case "force-normal": - if( !function_exists( 'utf8_normalize' ) ) { - dl( "php_utfnormal.so" ); - if( !function_exists( 'utf8_normalize' ) ) { - wfDie( "Failed to load UTF-8 normalization extension. " . - "Install or remove --force-normal parameter to use slower code.\n" ); + if ( !function_exists( 'utf8_normalize' ) ) { + wfDl( "php_utfnormal.so" ); + if ( !function_exists( 'utf8_normalize' ) ) { + $this->fatalError( "Failed to load UTF-8 normalization extension. " . + "Install or remove --force-normal parameter to use slower code." ); } } break; @@ -151,12 +172,12 @@ class BackupDumper { } } - if( is_null( $sink ) ) { + if ( is_null( $sink ) ) { $sink = new DumpOutput(); } $sinks[] = $sink; - if( count( $sinks ) > 1 ) { + if ( count( $sinks ) > 1 ) { return new DumpMultiWriter( $sinks ); } else { return $sink; @@ -167,64 +188,91 @@ class BackupDumper { // extension point for subclasses to add options } - function dump( $history, $text = MW_EXPORT_TEXT ) { + function dump( $history, $text = WikiExporter::TEXT ) { # Notice messages will foul up your XML output even if they're # relatively harmless. - ini_set( 'display_errors', false ); + if ( ini_get( 'display_errors' ) ) + ini_set( 'display_errors', 'stderr' ); $this->initProgress( $history ); - $db =& $this->backupDb(); - $exporter = new WikiExporter( $db, $history, MW_EXPORT_STREAM, $text ); + $db = $this->backupDb(); + $exporter = new WikiExporter( $db, $history, WikiExporter::STREAM, $text ); + $exporter->dumpUploads = $this->dumpUploads; + $exporter->dumpUploadFileContents = $this->dumpUploadFileContents; $wrapper = new ExportProgressFilter( $this->sink, $this ); $exporter->setOutputSink( $wrapper ); - if( !$this->skipHeader ) + if ( !$this->skipHeader ) $exporter->openStream(); - - if( is_null( $this->pages ) ) { - if( $this->startId || $this->endId ) { + # Log item dumps: all or by range + if ( $history & WikiExporter::LOGS ) { + if ( $this->startId || $this->endId ) { + $exporter->logsByRange( $this->startId, $this->endId ); + } else { + $exporter->allLogs(); + } + # Page dumps: all or by page ID range + } else if ( is_null( $this->pages ) ) { + if ( $this->startId || $this->endId ) { $exporter->pagesByRange( $this->startId, $this->endId ); + } elseif ( $this->revStartId || $this->revEndId ) { + $exporter->revsByRange( $this->revStartId, $this->revEndId ); } else { $exporter->allPages(); } + # Dump of specific pages } else { $exporter->pagesByName( $this->pages ); } - if( !$this->skipFooter ) + if ( !$this->skipFooter ) $exporter->closeStream(); $this->report( true ); } - + /** * Initialise starting time and maximum revision count. * We'll make ETA calculations based an progress, assuming relatively * constant per-revision rate. - * @param int $history MW_EXPORT_CURRENT or MW_EXPORT_FULL + * @param $history Integer: WikiExporter::CURRENT or WikiExporter::FULL */ - function initProgress( $history = MW_EXPORT_FULL ) { - $table = ($history == MW_EXPORT_CURRENT) ? 'page' : 'revision'; - $field = ($history == MW_EXPORT_CURRENT) ? 'page_id' : 'rev_id'; - - $dbr =& wfGetDB( DB_SLAVE ); - $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', 'BackupDumper::dump' ); + function initProgress( $history = WikiExporter::FULL ) { + $table = ( $history == WikiExporter::CURRENT ) ? 'page' : 'revision'; + $field = ( $history == WikiExporter::CURRENT ) ? 'page_id' : 'rev_id'; + + $dbr = wfGetDB( DB_SLAVE ); + $this->maxCount = $dbr->selectField( $table, "MAX($field)", '', __METHOD__ ); $this->startTime = wfTime(); + $this->lastTime = $this->startTime; + $this->ID = getmypid(); } - function &backupDb() { - global $wgDBadminuser, $wgDBadminpassword; - global $wgDBname, $wgDebugDumpSql; - $flags = ($wgDebugDumpSql ? DBO_DEBUG : 0) | DBO_DEFAULT; // god-damn hack - $db = new Database( $this->backupServer(), $wgDBadminuser, $wgDBadminpassword, $wgDBname, false, $flags ); - $timeout = 3600 * 24; - $db->query( "SET net_read_timeout=$timeout" ); - $db->query( "SET net_write_timeout=$timeout" ); + /** + * @todo Fixme: the --server parameter is currently not respected, as it + * doesn't seem terribly easy to ask the load balancer for a particular + * connection by name. + * @return DatabaseBase + */ + function backupDb() { + $this->lb = wfGetLBFactory()->newMainLB(); + $db = $this->lb->getConnection( DB_SLAVE, 'backup' ); + + // Discourage the server from disconnecting us if it takes a long time + // to read out the big ol' batch query. + $db->setTimeout( 3600 * 24 ); + return $db; } + function __destruct() { + if ( isset( $this->lb ) ) { + $this->lb->closeAll(); + } + } + function backupServer() { global $wgDBserver; return $this->server @@ -242,39 +290,58 @@ class BackupDumper { } function report( $final = false ) { - if( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) { + if ( $final xor ( $this->revCount % $this->reportingInterval == 0 ) ) { $this->showReport(); } } function showReport() { - if( $this->reporting ) { - $delta = wfTime() - $this->startTime; + if ( $this->reporting ) { $now = wfTimestamp( TS_DB ); - if( $delta ) { - $rate = $this->pageCount / $delta; - $revrate = $this->revCount / $delta; + $nowts = wfTime(); + $deltaAll = wfTime() - $this->startTime; + $deltaPart = wfTime() - $this->lastTime; + $this->pageCountPart = $this->pageCount - $this->pageCountLast; + $this->revCountPart = $this->revCount - $this->revCountLast; + + if ( $deltaAll ) { $portion = $this->revCount / $this->maxCount; - $eta = $this->startTime + $delta / $portion; + $eta = $this->startTime + $deltaAll / $portion; $etats = wfTimestamp( TS_DB, intval( $eta ) ); + $pageRate = $this->pageCount / $deltaAll; + $revRate = $this->revCount / $deltaAll; } else { - $rate = '-'; - $revrate = '-'; + $pageRate = '-'; + $revRate = '-'; $etats = '-'; } - $this->progress( sprintf( "%s: %s %d pages (%0.3f/sec), %d revs (%0.3f/sec), ETA %s [max %d]", - $now, wfWikiID(), $this->pageCount, $rate, $this->revCount, $revrate, $etats, $this->maxCount ) ); + if ( $deltaPart ) { + $pageRatePart = $this->pageCountPart / $deltaPart; + $revRatePart = $this->revCountPart / $deltaPart; + } else { + $pageRatePart = '-'; + $revRatePart = '-'; + } + $this->progress( sprintf( "%s: %s (ID %d) %d pages (%0.1f|%0.1f/sec all|curr), %d revs (%0.1f|%0.1f/sec all|curr), ETA %s [max %d]", + $now, wfWikiID(), $this->ID, $this->pageCount, $pageRate, $pageRatePart, $this->revCount, $revRate, $revRatePart, $etats, $this->maxCount ) ); + $this->lastTime = $nowts; + $this->revCountLast = $this->revCount; } } function progress( $string ) { fwrite( $this->stderr, $string . "\n" ); } + + function fatalError( $msg ) { + $this->progress( "$msg\n" ); + die(1); + } } class ExportProgressFilter extends DumpFilter { - function ExportProgressFilter( &$sink, &$progress ) { - parent::DumpFilter( $sink ); + function __construct( &$sink, &$progress ) { + parent::__construct( $sink ); $this->progress = $progress; } @@ -288,5 +355,3 @@ class ExportProgressFilter extends DumpFilter { $this->progress->revCount(); } } - -?>