maintenance/storage/checkStorage.php

   1 <?php
   2
   3 /**
   4  * Fsck for MediaWiki
   5  */
   6
   7 define( 'CONCAT_HEADER', 'O:27:"concatenatedgziphistoryblob"' );
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         require_once( dirname(__FILE__) . '/../commandLine.inc' );
  11         require_once('ExternalStore.php');
  12         require_once( 'ExternalStoreDB.php' );
  13
  14         checkStorage();
  15 }
  16
  17
  18 //----------------------------------------------------------------------------------
  19
  20 function checkError( $msg, $ids ) {
  21         global $oldIdMap;
  22         if ( is_array( $ids ) && count( $ids ) == 1 ) {
  23                 $ids = reset( $ids );
  24         }
  25         if ( is_array( $ids ) ) {
  26                 $revIds = array();
  27                 foreach ( $ids as $id ) {
  28                         $revIds = array_merge( $revIds, array_keys( $oldIdMap, $id ) );
  29                 }
  30                 print "$msg in text rows " . implode( ', ', $ids ) .
  31                         ", revisions " . implode( ', ', $revIds ) . "\n";
  32         } else {
  33                 $id = $ids;
  34                 $revIds = array_keys( $oldIdMap, $id );
  35                 if ( count( $revIds ) == 1 ) {
  36                         print "$msg in old_id $id, rev_id {$revIds[0]}\n";
  37                 } else {
  38                         print "$msg in old_id $id, revisions " . implode( ', ', $revIds ) . "\n";
  39                 }
  40         }
  41 }
  42
  43 function checkStorage() {
  44         global $oldIdMap;
  45
  46         $fname = 'checkStorage';
  47         $dbr =& wfGetDB( DB_SLAVE );
  48         $maxRevId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, $fname );
  49         $chunkSize = 1000;
  50         $flagStats = array();
  51         $objectStats = array();
  52         $knownFlags = array( 'external', 'gzip', 'object', 'utf-8' );
  53         $dbStore = null;
  54
  55         for ( $chunkStart = 1 ; $chunkStart < $maxRevId; $chunkStart += $chunkSize ) {
  56                 $chunkEnd = $chunkStart + $chunkSize - 1;
  57                 //print "$chunkStart to $chunkEnd of $maxRevId\n";
  58
  59                 // Fetch revision rows
  60                 $oldIdMap = array();
  61                 $res = $dbr->select( 'revision', array( 'rev_id', 'rev_text_id' ),
  62                         array( "rev_id BETWEEN $chunkStart AND $chunkEnd" ), $fname );
  63                 while ( $row = $dbr->fetchObject( $res ) ) {
  64                         $oldIdMap[$row->rev_id] = $row->rev_text_id;
  65                 }
  66                 $dbr->freeResult( $res );
  67
  68                 // Fetch old_flags
  69                 $missingTextRows = array_flip( $oldIdMap );
  70                 $externalRevs = array();
  71                 $objectRevs = array();
  72                 $flagsFields = array();
  73                 $res = $dbr->select( 'text', array( 'old_id', 'old_flags' ),
  74                         'old_id IN (' . implode( ',', $oldIdMap ) . ')', $fname );
  75                 while ( $row = $dbr->fetchObject( $res ) ) {
  76                         $flags = $row->old_flags;
  77                         $id = $row->old_id;
  78
  79                         // Create flagStats row if it doesn't exist
  80                         $flagStats = $flagStats + array( $flags => 0 );
  81                         // Increment counter
  82                         $flagStats[$flags]++;
  83
  84                         // Not missing
  85                         unset( $missingTextRows[$row->old_id] );
  86
  87                         // Check for external or object
  88                         if ( $flags == '' ) {
  89                                 $flagArray = array();
  90                         } else {
  91                                 $flagArray = explode( ',', $flags );
  92                         }
  93                         if ( in_array( 'external', $flagArray ) ) {
  94                                 $flagsFields[$id] = $flags; // is this needed?
  95                                 $externalRevs[] = $id;
  96                         } elseif ( in_array( 'object', $flagArray ) ) {
  97                                 $flagsFields[$id] = $flags; // is this needed?
  98                                 $objectRevs[] = $id;
  99                         }
 100
 101                         // Check for unrecognised flags
 102                         if ( count( array_diff( $flagArray, $knownFlags ) ) ) {
 103                                 print_r( array_diff( $flagArray, $knownFlags ) );
 104                                 checkError( "Warning: invalid flags field \"$flags\"", $id );
 105                         }
 106                 }
 107                 $dbr->freeResult( $res );
 108
 109                 // Output errors for any missing text rows
 110                 foreach ( $missingTextRows as $oldId => $revId ) {
 111                         print "Error: missing text row $oldId for revision $revId\n";
 112                 }
 113
 114                 // Verify external revisions
 115                 $externalConcatBlobs = array();
 116                 $externalNormalBlobs = array();
 117                 if ( count( $externalRevs ) ) {
 118                         $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
 119                                 array( 'old_id IN (' . implode( ',', $externalRevs ) . ')' ), $fname );
 120                         while ( $row = $dbr->fetchObject( $res ) ) {
 121                                 $urlParts = explode( '://', $row->old_text, 2 );
 122                                 if ( count( $urlParts ) !== 2 || $urlParts[1] == '' ) {
 123                                         checkError( "Error: invalid URL \"{$row->old_text}\"", $row->old_id );
 124                                         continue;
 125                                 }
 126                                 list( $proto, $path ) = $urlParts;
 127                                 if ( $proto != 'DB' ) {
 128                                         checkError( "Error: invalid external protocol \"$proto\"", $row->old_id );
 129                                         continue;
 130                                 }
 131                                 $path = explode( '/', $row->old_text );
 132                                 $cluster = $path[2];
 133                                 $id = $path[3];
 134                                 if ( isset( $path[4] ) ) {
 135                                         $externalConcatBlobs[$cluster][$id][] = $row->old_id;
 136                                 } else {
 137                                         $externalNormalBlobs[$cluster][$id][] = $row->old_id;
 138                                 }
 139                         }
 140                         $dbr->freeResult( $res );
 141                 }
 142
 143                 // Check external concat blobs for the right header
 144                 checkExternalConcatBlobs( $externalConcatBlobs );
 145
 146
 147                 // Check external normal blobs for existence
 148                 if ( count( $externalNormalBlobs ) ) {
 149                         if ( is_null( $dbStore ) ) {
 150                                 $dbStore = new ExternalStoreDB;
 151                         }
 152                         foreach ( $externalConcatBlobs as $cluster => $xBlobIds ) {
 153                                 $blobIds = array_keys( $xBlobIds );
 154                                 $extDb =& $dbStore->getSlave( $cluster );
 155                                 $blobsTable = $dbStore->getTable( $extDb );
 156                                 $res = $extDb->select( $blobsTable,
 157                                         array( 'blob_id' ),
 158                                         array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), $fname );
 159                                 while ( $row = $extDb->fetchObject( $res ) ) {
 160                                         unset( $xBlobIds[$row->blob_id] );
 161                                 }
 162                                 $extDb->freeResult( $res );
 163                                 // Print errors for missing blobs rows
 164                                 foreach ( $xBlobIds as $blobId => $oldId ) {
 165                                         checkError( "Error: missing target $blobId for one-part ES URL", $oldId );
 166                                 }
 167                         }
 168                 }
 169
 170                 // Check local objects
 171                 $dbr->ping();
 172                 $concatBlobs = array();
 173                 $curIds = array();
 174                 if ( count( $objectRevs ) ) {
 175                         $headerLength = 300;
 176                         $res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ),
 177                                 array( 'old_id IN (' . implode( ',', $objectRevs ) . ')' ), $fname );
 178                         while ( $row = $dbr->fetchObject( $res ) ) {
 179                                 $oldId = $row->old_id;
 180                                 if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) {
 181                                         checkError( "Error: invalid object header", $oldId );
 182                                         continue;
 183                                 }
 184
 185                                 $className = $matches[2];
 186                                 if ( strlen( $className ) != $matches[1] ) {
 187                                         checkError( "Error: invalid object header, wrong class name length", $oldId );
 188                                         continue;
 189                                 }
 190
 191                                 $objectStats = $objectStats + array( $className => 0 );
 192                                 $objectStats[$className]++;
 193
 194                                 switch ( $className ) {
 195                                         case 'concatenatedgziphistoryblob':
 196                                                 // Good
 197                                                 break;
 198                                         case 'historyblobstub':
 199                                         case 'historyblobcurstub':
 200                                                 if ( strlen( $row->header ) == $headerLength ) {
 201                                                         checkError( "Error: overlong stub header", $oldId );
 202                                                         continue;
 203                                                 }
 204                                                 $stubObj = unserialize( $row->header );
 205                                                 if ( !is_object( $stubObj ) ) {
 206                                                         checkError( "Error: unable to unserialize stub object", $oldId );
 207                                                         continue;
 208                                                 }
 209                                                 if ( $className == 'historyblobstub' ) {
 210                                                         $concatBlobs[$stubObj->mOldId][] = $oldId;
 211                                                 } else {
 212                                                         $curIds[$stubObj->mCurId][] = $oldId;
 213                                                 }
 214                                                 break;
 215                                         default:
 216                                                 checkError( "Error: unrecognised object class \"$className\"", $oldId );
 217                                 }
 218                         }
 219                         $dbr->freeResult( $res );
 220                 }
 221
 222                 // Check local concat blob validity
 223                 $externalConcatBlobs = array();
 224                 if ( count( $concatBlobs ) ) {
 225                         $headerLength = 300;
 226                         $res = $dbr->select( 'text', array( 'old_id', 'old_flags', "LEFT(old_text, $headerLength) AS header" ),
 227                                 array( 'old_id IN (' . implode( ',', array_keys( $concatBlobs ) ) . ')' ), $fname );
 228                         while ( $row = $dbr->fetchObject( $res ) ) {
 229                                 $flags = explode( ',', $row->old_flags );
 230                                 if ( in_array( 'external', $flags ) ) {
 231                                         // Concat blob is in external storage?
 232                                         if ( in_array( 'object', $flags ) ) {
 233                                                 $urlParts = explode( '/', $row->header );
 234                                                 if ( $urlParts[0] != 'DB:' ) {
 235                                                         checkError( "Error: unrecognised external storage type \"{$urlParts[0]}", $row->old_id );
 236                                                 } else {
 237                                                         $cluster = $urlParts[2];
 238                                                         $id = $urlParts[3];
 239                                                         if ( !isset( $externalConcatBlobs[$cluster][$id] ) ) {
 240                                                                 $externalConcatBlobs[$cluster][$id] = array();
 241                                                         }
 242                                                         $externalConcatBlobs[$cluster][$id] = array_merge(
 243                                                                 $externalConcatBlobs[$cluster][$id], $concatBlobs[$row->old_id]
 244                                                         );
 245                                                 }
 246                                         } else {
 247                                                 checkError( "Error: invalid flags \"{$row->old_flags}\" on concat bulk row {$row->old_id}",
 248                                                         $concatBlobs[$row->old_id] );
 249                                         }
 250                                 } elseif ( substr( $row->header, 0, strlen( CONCAT_HEADER ) ) != CONCAT_HEADER ) {
 251                                         checkError( "Error: Incorrect object header for concat bulk row {$row->old_id}",
 252                                                 $concatBlobs[$row->old_id] );
 253                                 } # else good
 254
 255                                 unset( $concatBlobs[$row->old_id] );
 256                         }
 257                         $dbr->freeResult( $res );
 258                 }
 259
 260                 // Check targets of unresolved stubs
 261                 checkExternalConcatBlobs( $externalConcatBlobs );
 262                 $dbr->ping();
 263
 264                 // next chunk
 265         }
 266
 267         print "\n\nFlag statistics:\n";
 268         $total = array_sum( $flagStats );
 269         foreach ( $flagStats as $flag => $count ) {
 270                 printf( "%-30s %10d %5.2f%%\n", $flag, $count, $count / $total * 100 );
 271         }
 272         print "\nObject statistics:\n";
 273         $total = array_sum( $objectStats );
 274         foreach ( $objectStats as $className => $count ) {
 275                 printf( "%-30s %10d %5.2f%%\n", $className, $count, $count / $total * 100 );
 276         }
 277 }
 278
 279
 280 function checkExternalConcatBlobs( $externalConcatBlobs ) {
 281         static $dbStore = null;
 282         $fname = 'checkExternalConcatBlobs';
 283         if ( !count( $externalConcatBlobs ) ) {
 284                 return;
 285         }
 286
 287         if ( is_null( $dbStore ) ) {
 288                 $dbStore = new ExternalStoreDB;
 289         }
 290
 291         foreach ( $externalConcatBlobs as $cluster => $oldIds ) {
 292                 $blobIds = array_keys( $oldIds );
 293                 $extDb =& $dbStore->getSlave( $cluster );
 294                 $blobsTable = $dbStore->getTable( $extDb );
 295                 $headerLength = strlen( CONCAT_HEADER );
 296                 $res = $extDb->select( $blobsTable,
 297                         array( 'blob_id', "LEFT(blob_text, $headerLength) AS header" ),
 298                         array( 'blob_id IN( ' . implode( ',', $blobIds ) . ')' ), $fname );
 299                 while ( $row = $extDb->fetchObject( $res ) ) {
 300                         unset( $oldIds[$row->blob_id] );
 301                         if ( $row->header != CONCAT_HEADER ) {
 302                                 checkError( "Error: invalid header on target of two-part ES URL",
 303                                         $oldIds[$row->blob_id] );
 304                         }
 305                 }
 306                 $extDb->freeResult( $res );
 307
 308                 // Print errors for missing blobs rows
 309                 foreach ( $oldIds as $blobId => $oldIds ) {
 310                         checkError( "Error: missing target $blobId for two-part ES URL", $oldIds );
 311                 }
 312         }
 313 }
 314
 315 ?>