maintenance/storage/moveToExternal.php

   1 <?php
   2 /**
   3  * Move revision's text to external storage
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Maintenance ExternalStorage
  22  */
  23
  24 use MediaWiki\MediaWikiServices;
  25
  26 define( 'REPORTING_INTERVAL', 1 );
  27
  28 if ( !defined( 'MEDIAWIKI' ) ) {
  29         $optionsWithArgs = [ 'e', 's' ];
  30         require_once __DIR__ . '/../commandLine.inc';
  31         require_once 'resolveStubs.php';
  32
  33         $fname = 'moveToExternal';
  34
  35         if ( !isset( $args[1] ) ) {
  36                 print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <type> <location>\n";
  37                 exit;
  38         }
  39
  40         $type = $args[0]; // e.g. "DB" or "mwstore"
  41         $location = $args[1]; // e.g. "cluster12" or "global-swift"
  42         $dbw = wfGetDB( DB_MASTER );
  43
  44         $maxID = $options['e'] ?? $dbw->selectField( 'text', 'MAX(old_id)', '', $fname );
  45         $minID = $options['s'] ?? 1;
  46
  47         moveToExternal( $type, $location, $maxID, $minID );
  48 }
  49
  50 function moveToExternal( $type, $location, $maxID, $minID = 1 ) {
  51         $fname = 'moveToExternal';
  52         $dbw = wfGetDB( DB_MASTER );
  53         $dbr = wfGetDB( DB_REPLICA );
  54
  55         $count = $maxID - $minID + 1;
  56         $blockSize = 1000;
  57         $numBlocks = ceil( $count / $blockSize );
  58         print "Moving text rows from $minID to $maxID to external storage\n";
  59
  60         $esFactory = MediaWikiServices::getInstance()->getExternalStoreFactory();
  61         $extStore = $esFactory->getStore( $type );
  62         $numMoved = 0;
  63
  64         for ( $block = 0; $block < $numBlocks; $block++ ) {
  65                 $blockStart = $block * $blockSize + $minID;
  66                 $blockEnd = $blockStart + $blockSize - 1;
  67
  68                 if ( !( $block % REPORTING_INTERVAL ) ) {
  69                         print "oldid=$blockStart, moved=$numMoved\n";
  70                         wfWaitForSlaves();
  71                 }
  72
  73                 $res = $dbr->select( 'text', [ 'old_id', 'old_flags', 'old_text' ],
  74                         [
  75                                 "old_id BETWEEN $blockStart AND $blockEnd",
  76                                 'old_flags NOT ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
  77                         ], $fname );
  78                 foreach ( $res as $row ) {
  79                         # Resolve stubs
  80                         $text = $row->old_text;
  81                         $id = $row->old_id;
  82                         if ( $row->old_flags === '' ) {
  83                                 $flags = 'external';
  84                         } else {
  85                                 $flags = "{$row->old_flags},external";
  86                         }
  87
  88                         if ( strpos( $flags, 'object' ) !== false ) {
  89                                 $obj = unserialize( $text );
  90                                 $className = strtolower( get_class( $obj ) );
  91                                 if ( $className == 'historyblobstub' ) {
  92                                         # resolveStub( $id, $row->old_text, $row->old_flags );
  93                                         # $numStubs++;
  94                                         continue;
  95                                 } elseif ( $className == 'historyblobcurstub' ) {
  96                                         $text = gzdeflate( $obj->getText() );
  97                                         $flags = 'utf-8,gzip,external';
  98                                 } elseif ( $className == 'concatenatedgziphistoryblob' ) {
  99                                         // Do nothing
 100                                 } else {
 101                                         print "Warning: unrecognised object class \"$className\"\n";
 102                                         continue;
 103                                 }
 104                         } else {
 105                                 $className = false;
 106                         }
 107
 108                         if ( strlen( $text ) < 100 && $className === false ) {
 109                                 // Don't move tiny revisions
 110                                 continue;
 111                         }
 112
 113                         # print "Storing "  . strlen( $text ) . " bytes to $url\n";
 114                         # print "old_id=$id\n";
 115
 116                         $url = $extStore->store( $location, $text );
 117                         if ( !$url ) {
 118                                 print "Error writing to external storage\n";
 119                                 exit;
 120                         }
 121                         $dbw->update( 'text',
 122                                 [ 'old_flags' => $flags, 'old_text' => $url ],
 123                                 [ 'old_id' => $id ], $fname );
 124                         $numMoved++;
 125                 }
 126         }
 127 }