maintenance/storage/moveToExternal.php

   1 <?php
   2
   3 define( 'REPORTING_INTERVAL', 1 );
   4
   5 if ( !defined( 'MEDIAWIKI' ) ) {
   6         $optionsWithArgs = array( 'm', 's' );
   7
   8         require_once( dirname(__FILE__) . '/../commandLine.inc' );
   9         require_once( 'ExternalStoreDB.php' );
  10         require_once( 'resolveStubs.php' );
  11
  12         $fname = 'moveToExternal';
  13
  14         if ( !isset( $args[0] ) ) {
  15                 print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n";
  16                 exit;
  17         }
  18
  19         $cluster = $args[0];
  20         $dbw = wfGetDB( DB_MASTER );
  21
  22         if ( isset( $options['e'] ) ) {
  23                 $maxID = $options['e'];
  24         } else {
  25                 $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
  26         }
  27         $minID = isset( $options['s'] ) ? $options['s'] : 1;
  28
  29         moveToExternal( $cluster, $maxID, $minID );
  30 }
  31
  32
  33
  34 function moveToExternal( $cluster, $maxID, $minID = 1 ) {
  35         $fname = 'moveToExternal';
  36         $dbw = wfGetDB( DB_MASTER );
  37         $dbr = wfGetDB( DB_SLAVE );
  38
  39         $count = $maxID - $minID + 1;
  40         $blockSize = 1000;
  41         $numBlocks = ceil( $count / $blockSize );
  42         print "Moving text rows from $minID to $maxID to external storage\n";
  43         $ext = new ExternalStoreDB;
  44         $numMoved = 0;
  45         $numStubs = 0;
  46
  47         for ( $block = 0; $block < $numBlocks; $block++ ) {
  48                 $blockStart = $block * $blockSize + $minID;
  49                 $blockEnd = $blockStart + $blockSize - 1;
  50
  51                 if ( !($block % REPORTING_INTERVAL) ) {
  52                         print "oldid=$blockStart, moved=$numMoved\n";
  53                         wfWaitForSlaves( 2 );
  54                 }
  55
  56                 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
  57                         array(
  58                                 "old_id BETWEEN $blockStart AND $blockEnd",
  59                                 "old_flags NOT LIKE '%external%'",
  60                         ), $fname );
  61                 while ( $row = $dbr->fetchObject( $res ) ) {
  62                         # Resolve stubs
  63                         $text = $row->old_text;
  64                         $id = $row->old_id;
  65                         if ( $row->old_flags === '' ) {
  66                                 $flags = 'external';
  67                         } else {
  68                                 $flags = "{$row->old_flags},external";
  69                         }
  70
  71                         if ( strpos( $flags, 'object' ) !== false ) {
  72                                 $obj = unserialize( $text );
  73                                 $className = strtolower( get_class( $obj ) );
  74                                 if ( $className == 'historyblobstub' ) {
  75                                         #resolveStub( $id, $row->old_text, $row->old_flags );
  76                                         #$numStubs++;
  77                                         continue;
  78                                 } elseif ( $className == 'historyblobcurstub' ) {
  79                                         $text = gzdeflate( $obj->getText() );
  80                                         $flags = 'utf-8,gzip,external';
  81                                 } elseif ( $className == 'concatenatedgziphistoryblob' ) {
  82                                         // Do nothing
  83                                 } else {
  84                                         print "Warning: unrecognised object class \"$className\"\n";
  85                                         continue;
  86                                 }
  87                         } else {
  88                                 $className = false;
  89                         }
  90
  91                         if ( strlen( $text ) < 100 && $className === false ) {
  92                                 // Don't move tiny revisions
  93                                 continue;
  94                         }
  95
  96                         #print "Storing "  . strlen( $text ) . " bytes to $url\n";
  97                         #print "old_id=$id\n";
  98
  99                         $url = $ext->store( $cluster, $text );
 100                         if ( !$url ) {
 101                                 print "Error writing to external storage\n";
 102                                 exit;
 103                         }
 104                         $dbw->update( 'text',
 105                                 array( 'old_flags' => $flags, 'old_text' => $url ),
 106                                 array( 'old_id' => $id ), $fname );
 107                         $numMoved++;
 108                 }
 109                 $dbr->freeResult( $res );
 110         }
 111 }
 112
 113