Live fixes: lots of scary magic runes; php 5 fixes; reporting; etc

diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc
index efd45b3..c1fd704 100644 (file)
--- a/maintenance/storage/compressOld.inc
+++ b/maintenance/storage/compressOld.inc
@@ -103,10 +103,13 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh
         */
 
        # For each article, get a list of revisions which fit the criteria
+       
        # No recompression, use a condition on old_flags
+       # Don't compress object type entities, because that might produce data loss when
+       # overwriting bulk storage concat rows. Don't compress external references, because
+       # the script doesn't yet delete rows from external storage.
        $conds = array(
-               "old_flags NOT LIKE '%object%' " .
-               " AND (old_flags NOT LIKE '%external%' OR old_text NOT LIKE 'DB://%/%/%')");
+               "old_flags NOT LIKE '%object%' AND old_flags NOT LIKE '%external%'");
 
        if ( $beginDate ) {
                $conds[] = "rev_timestamp>'" . $beginDate . "'";
@@ -133,8 +136,12 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh
        $totalMatchingRevisions = 0;
        $masterPos = false;
        for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
-               wfWaitForSlaves( 10 );
+               wfWaitForSlaves( 5 );
 
+               # Wake up
+               $dbr->ping();           
+
+               # Get the page row
                $pageRes = $dbr->select( 'page', array('page_id', 'page_namespace', 'page_title'),
                        $pageConds + array('page_id' => $pageId), $fname );
                if ( $dbr->numRows( $pageRes ) == 0 ) {
@@ -213,7 +220,6 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh
                                                $stub = $chunk->addItem( $text );
                                                $stub->setLocation( $primaryOldid );
                                                $stub->setReferrer( $oldid );
-                                               $hash = $stub->getHash();
                                                print '.';
                                                $usedChunk = true;
                                        }
@@ -277,6 +283,7 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh
                        print "/";
                        $dbw->commit();
                        $i += $thisChunkSize;
+                       wfWaitForSlaves( 5 );
                }
                print "\n";
        }

diff --git a/maintenance/storage/compressOld.php b/maintenance/storage/compressOld.php
index 9cee055..d597f1d 100644 (file)
--- a/maintenance/storage/compressOld.php
+++ b/maintenance/storage/compressOld.php
@@ -45,7 +45,7 @@ $defaults = array(
        't' => 'concat',
        'c' => 20,
        's' => 0,
-       'f' => 3,
+       'f' => 5,
        'h' => 100,
        'b' => '',
     'e' => '',

diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php
index aabc5e6..0b46f70 100644 (file)
--- a/maintenance/storage/moveToExternal.php
+++ b/maintenance/storage/moveToExternal.php
@@ -1,7 +1,6 @@
 <?php
 
 define( 'REPORTING_INTERVAL', 100 );
-define( 'STUB_HEADER', 'O:15:"historyblobstub"' );
 
 if ( !defined( 'MEDIAWIKI' ) ) {
        $optionsWithArgs = array( 'm' );
@@ -53,24 +52,42 @@ function moveToExternal( $cluster, $maxID ) {
                }
 
                # Resolve stubs
-               $flags = explode( ',', $row->old_flags );
-               if ( in_array( 'object', $flags )
-                       && substr( $row->old_text, 0, strlen( STUB_HEADER ) ) === STUB_HEADER )
-               {
-                       resolveStub( $id, $row->old_text, $row->old_flags );
+               $text = $row->old_text;
+               if ( $row->old_flags === '' ) {
+                       $flags = 'external';
+               } else {
+                       $flags = "{$row->old_flags},external";
+               }
+               
+               if ( strpos( $flags, 'object' ) !== false ) {
+                       $obj = unserialize( $text );
+                       $className = strtolower( get_class( $obj ) );
+                       if ( $className == 'historyblobstub' ) {
+                               resolveStub( $id, $row->old_text, $row->old_flags );
+                               continue;
+                       } elseif ( $className == 'historyblobcurstub' ) {
+                               $text = gzdeflate( $obj->getText() );
+                               $flags = 'utf-8,gzip,external';
+                       } elseif ( $className == 'concatenatedgziphistoryblob' ) {
+                               // Do nothing
+                       } else {
+                               print "Warning: unrecognised object class \"$className\"\n";
+                               continue;
+                       }
+               }
+
+               if ( strlen( $text ) < 100 ) {
+                       // Don't move tiny revisions
                        continue;
                }
 
-               $url = $ext->store( $cluster, $row->old_text );
+               #print "Storing "  . strlen( $text ) . " bytes to $url\n";
+
+               $url = $ext->store( $cluster, $text );
                if ( !$url ) {
                        print "Error writing to external storage\n";
                        exit;
                }
-               if ( $row->old_flags === '' ) {
-                       $flags = 'external';
-               } else {
-                       $flags = "{$row->old_flags},external";
-               }
                $dbw->update( 'text',
                        array( 'old_flags' => $flags, 'old_text' => $url ),
                        array( 'old_id' => $id ), $fname );

diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php
index 5833418..e93d5c9 100644 (file)
--- a/maintenance/storage/resolveStubs.php
+++ b/maintenance/storage/resolveStubs.php
@@ -18,45 +18,37 @@ if ( !defined( 'MEDIAWIKI' ) ) {
 function resolveStubs() {
        $fname = 'resolveStubs';
 
-       print "Retrieving stub rows...\n";
        $dbr =& wfGetDB( DB_SLAVE );
+       $dbw =& wfGetDB( DB_MASTER );
        $maxID = $dbr->selectField( 'text', 'MAX(old_id)', false, $fname );
-       $stubs = array();
-       $flagsArray = array();
-
-       # Do it in 100 blocks
-       for ( $b = 0; $b < 100; $b++ ) {
-               print "$b%\r";
-               $start = intval($maxID / 100) * $b + 1;
-               $end = intval($maxID / 100) * ($b + 1);
-
+       $blockSize = 10000;
+       $numBlocks = intval( $maxID / $blockSize ) + 1;
+
+       for ( $b = 0; $b < $numBlocks; $b++ ) {
+               wfWaitForSlaves( 5 );
+               
+               printf( "%5.2f%%\n", $b / $numBlocks * 100 );
+               $start = intval($maxID / $numBlocks) * $b + 1;
+               $end = intval($maxID / $numBlocks) * ($b + 1);
+               $stubs = array();
+               $flagsArray = array();
+
+               
                $res = $dbr->select( 'text', array( 'old_id', 'old_text', 'old_flags' ),
-                       "old_id>=$start AND old_id<=$end AND old_flags like '%object%' ".
+                       "old_id>=$start AND old_id<=$end " .
+                       # Using a more restrictive flag set for now, until I do some more analysis -- TS
+                       #"AND old_flags LIKE '%object%' AND old_flags NOT LIKE '%external%' ".
+                       
+                       "AND old_flags='object' " .
                        "AND old_text LIKE 'O:15:\"historyblobstub\"%'", $fname );
                while ( $row = $dbr->fetchObject( $res ) ) {
-                       $stubs[$row->old_id] = $row->old_text;
-                       $flagsArray[$row->old_id] = $row->old_flags;
+                       resolveStub( $row->old_id, $row->old_text, $row->old_flags );
                }
                $dbr->freeResult( $res );
-       }
-       print "100%\n";
-
-       print "\nConverting " . count( $stubs ) . " rows ...\n";
 
-       # Get master database, no transactions
-       $dbw =& wfGetDB( DB_MASTER );
-       $dbw->clearFlag( DBO_TRX );
-       $dbw->immediateCommit();
-
-       $i = 0;
-       foreach( $stubs as $id => $stub ) {
-               if ( !(++$i % REPORTING_INTERVAL) ) {
-                       print "$i\n";
-                       wfWaitForSlaves( 5 );
-               }
-
-               resolveStub( $id, $stub, $flagsArray[$id] );
+               
        }
+       print "100%\n";
 }
 
 /**
@@ -71,8 +63,8 @@ function resolveStub( $id, $stubText, $flags ) {
        $dbr =& wfGetDB( DB_SLAVE );
        $dbw =& wfGetDB( DB_MASTER );
 
-       if ( get_class( $stub ) !== 'historyblobstub' ) {
-               print "Error, invalid stub object\n";
+       if ( strtolower( get_class( $stub ) ) !== 'historyblobstub' ) {
+               print "Error found object of class " . get_class( $stub ) . ", expecting historyblobstub\n";
                return;
        }
 
@@ -84,7 +76,7 @@ function resolveStub( $id, $stubText, $flags ) {
 
        if ( !$externalRow ) {
                # Object wasn't external
-               continue;
+               return;
        }
 
        # Preserve the legacy encoding flag, but switch from object to external