maintenance/storage/moveToExternal.php

   1 <?php
   2 /**
   3  * Move revision's text to external storage
   4  *
   5  * @file
   6  * @ingroup Maintenance ExternalStorage
   7  */
   8
   9 define( 'REPORTING_INTERVAL', 1 );
  10
  11 if ( !defined( 'MEDIAWIKI' ) ) {
  12         require_once( dirname( __FILE__ ) . '/../commandLine.inc' );
  13         require_once( dirname( __FILE__ ) . '/../../includes/ExternalStoreDB.php' );
  14         require_once( 'resolveStubs.php' );
  15
  16         $fname = 'moveToExternal';
  17
  18         if ( !isset( $args[0] ) ) {
  19                 print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n";
  20                 exit;
  21         }
  22
  23         $cluster = $args[0];
  24         $dbw = wfGetDB( DB_MASTER );
  25
  26         if ( isset( $options['e'] ) ) {
  27                 $maxID = $options['e'];
  28         } else {
  29                 $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
  30         }
  31         $minID = isset( $options['s'] ) ? $options['s'] : 1;
  32
  33         moveToExternal( $cluster, $maxID, $minID );
  34 }
  35
  36 function moveToExternal( $cluster, $maxID, $minID = 1 ) {
  37         $fname = 'moveToExternal';
  38         $dbw = wfGetDB( DB_MASTER );
  39         $dbr = wfGetDB( DB_SLAVE );
  40
  41         $count = $maxID - $minID + 1;
  42         $blockSize = 1000;
  43         $numBlocks = ceil( $count / $blockSize );
  44         print "Moving text rows from $minID to $maxID to external storage\n";
  45         $ext = new ExternalStoreDB;
  46         $numMoved = 0;
  47
  48         for ( $block = 0; $block < $numBlocks; $block++ ) {
  49                 $blockStart = $block * $blockSize + $minID;
  50                 $blockEnd = $blockStart + $blockSize - 1;
  51
  52                 if ( !( $block % REPORTING_INTERVAL ) ) {
  53                         print "oldid=$blockStart, moved=$numMoved\n";
  54                         wfWaitForSlaves( 2 );
  55                 }
  56
  57                 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
  58                         array(
  59                                 "old_id BETWEEN $blockStart AND $blockEnd",
  60                                 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
  61                         ), $fname );
  62                 foreach ( $res as $row ) {
  63                         # Resolve stubs
  64                         $text = $row->old_text;
  65                         $id = $row->old_id;
  66                         if ( $row->old_flags === '' ) {
  67                                 $flags = 'external';
  68                         } else {
  69                                 $flags = "{$row->old_flags},external";
  70                         }
  71
  72                         if ( strpos( $flags, 'object' ) !== false ) {
  73                                 $obj = unserialize( $text );
  74                                 $className = strtolower( get_class( $obj ) );
  75                                 if ( $className == 'historyblobstub' ) {
  76                                         # resolveStub( $id, $row->old_text, $row->old_flags );
  77                                         # $numStubs++;
  78                                         continue;
  79                                 } elseif ( $className == 'historyblobcurstub' ) {
  80                                         $text = gzdeflate( $obj->getText() );
  81                                         $flags = 'utf-8,gzip,external';
  82                                 } elseif ( $className == 'concatenatedgziphistoryblob' ) {
  83                                         // Do nothing
  84                                 } else {
  85                                         print "Warning: unrecognised object class \"$className\"\n";
  86                                         continue;
  87                                 }
  88                         } else {
  89                                 $className = false;
  90                         }
  91
  92                         if ( strlen( $text ) < 100 && $className === false ) {
  93                                 // Don't move tiny revisions
  94                                 continue;
  95                         }
  96
  97                         # print "Storing "  . strlen( $text ) . " bytes to $url\n";
  98                         # print "old_id=$id\n";
  99
 100                         $url = $ext->store( $cluster, $text );
 101                         if ( !$url ) {
 102                                 print "Error writing to external storage\n";
 103                                 exit;
 104                         }
 105                         $dbw->update( 'text',
 106                                 array( 'old_flags' => $flags, 'old_text' => $url ),
 107                                 array( 'old_id' => $id ), $fname );
 108                         $numMoved++;
 109                 }
 110         }
 111 }
 112
 113