maintenance/storage/moveToExternal.php

   1 <?php
   2 /**
   3  * Move revision's text to external storage
   4  *
   5  * @file
   6  * @ingroup Maintenance ExternalStorage
   7  */
   8
   9 define( 'REPORTING_INTERVAL', 1 );
  10
  11 if ( !defined( 'MEDIAWIKI' ) ) {
  12         $optionsWithArgs = array( 'e', 's' );
  13
  14         require_once( dirname( __FILE__ ) . '/../commandLine.inc' );
  15         require_once( 'ExternalStoreDB.php' );
  16         require_once( 'resolveStubs.php' );
  17
  18         $fname = 'moveToExternal';
  19
  20         if ( !isset( $args[0] ) ) {
  21                 print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n";
  22                 exit;
  23         }
  24
  25         $cluster = $args[0];
  26         $dbw = wfGetDB( DB_MASTER );
  27
  28         if ( isset( $options['e'] ) ) {
  29                 $maxID = $options['e'];
  30         } else {
  31                 $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
  32         }
  33         $minID = isset( $options['s'] ) ? $options['s'] : 1;
  34
  35         moveToExternal( $cluster, $maxID, $minID );
  36 }
  37
  38
  39
  40 function moveToExternal( $cluster, $maxID, $minID = 1 ) {
  41         $fname = 'moveToExternal';
  42         $dbw = wfGetDB( DB_MASTER );
  43         $dbr = wfGetDB( DB_SLAVE );
  44
  45         $count = $maxID - $minID + 1;
  46         $blockSize = 1000;
  47         $numBlocks = ceil( $count / $blockSize );
  48         print "Moving text rows from $minID to $maxID to external storage\n";
  49         $ext = new ExternalStoreDB;
  50         $numMoved = 0;
  51         $numStubs = 0;
  52
  53         for ( $block = 0; $block < $numBlocks; $block++ ) {
  54                 $blockStart = $block * $blockSize + $minID;
  55                 $blockEnd = $blockStart + $blockSize - 1;
  56
  57                 if ( !( $block % REPORTING_INTERVAL ) ) {
  58                         print "oldid=$blockStart, moved=$numMoved\n";
  59                         wfWaitForSlaves( 2 );
  60                 }
  61
  62                 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
  63                         array(
  64                                 "old_id BETWEEN $blockStart AND $blockEnd",
  65                                 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
  66                         ), $fname );
  67                 while ( $row = $dbr->fetchObject( $res ) ) {
  68                         # Resolve stubs
  69                         $text = $row->old_text;
  70                         $id = $row->old_id;
  71                         if ( $row->old_flags === '' ) {
  72                                 $flags = 'external';
  73                         } else {
  74                                 $flags = "{$row->old_flags},external";
  75                         }
  76
  77                         if ( strpos( $flags, 'object' ) !== false ) {
  78                                 $obj = unserialize( $text );
  79                                 $className = strtolower( get_class( $obj ) );
  80                                 if ( $className == 'historyblobstub' ) {
  81                                         # resolveStub( $id, $row->old_text, $row->old_flags );
  82                                         # $numStubs++;
  83                                         continue;
  84                                 } elseif ( $className == 'historyblobcurstub' ) {
  85                                         $text = gzdeflate( $obj->getText() );
  86                                         $flags = 'utf-8,gzip,external';
  87                                 } elseif ( $className == 'concatenatedgziphistoryblob' ) {
  88                                         // Do nothing
  89                                 } else {
  90                                         print "Warning: unrecognised object class \"$className\"\n";
  91                                         continue;
  92                                 }
  93                         } else {
  94                                 $className = false;
  95                         }
  96
  97                         if ( strlen( $text ) < 100 && $className === false ) {
  98                                 // Don't move tiny revisions
  99                                 continue;
 100                         }
 101
 102                         # print "Storing "  . strlen( $text ) . " bytes to $url\n";
 103                         # print "old_id=$id\n";
 104
 105                         $url = $ext->store( $cluster, $text );
 106                         if ( !$url ) {
 107                                 print "Error writing to external storage\n";
 108                                 exit;
 109                         }
 110                         $dbw->update( 'text',
 111                                 array( 'old_flags' => $flags, 'old_text' => $url ),
 112                                 array( 'old_id' => $id ), $fname );
 113                         $numMoved++;
 114                 }
 115                 $dbr->freeResult( $res );
 116         }
 117 }
 118
 119