includes/HistoryBlob.php

   1 <?php
   2
   3 /**
   4  * Base class for general text storage via the "object" flag in old_flags, or
   5  * two-part external storage URLs. Used for represent efficient concatenated
   6  * storage, and migration-related pointer objects.
   7  */
   8 interface HistoryBlob
   9 {
  10         /**
  11          * Adds an item of text, returns a stub object which points to the item.
  12          * You must call setLocation() on the stub object before storing it to the
  13          * database
  14          * Returns the key for getItem()
  15          */
  16         public function addItem( $text );
  17
  18         /**
  19          * Get item by key, or false if the key is not present
  20          */
  21         public function getItem( $key );
  22
  23         /**
  24          * Set the "default text"
  25          * This concept is an odd property of the current DB schema, whereby each text item has a revision
  26          * associated with it. The default text is the text of the associated revision. There may, however,
  27          * be other revisions in the same object.
  28          *
  29          * Default text is not required for two-part external storage URLs.
  30          */
  31         public function setText( $text );
  32
  33         /**
  34          * Get default text. This is called from Revision::getRevisionText()
  35          */
  36         function getText();
  37 }
  38
  39 /**
  40  * Concatenated gzip (CGZ) storage
  41  * Improves compression ratio by concatenating like objects before gzipping
  42  */
  43 class ConcatenatedGzipHistoryBlob implements HistoryBlob
  44 {
  45         public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
  46         public $mFast = 0, $mSize = 0;
  47
  48         /** Constructor */
  49         public function ConcatenatedGzipHistoryBlob() {
  50                 if ( !function_exists( 'gzdeflate' ) ) {
  51                         throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
  52                 }
  53         }
  54
  55         public function addItem( $text ) {
  56                 $this->uncompress();
  57                 $hash = md5( $text );
  58                 $this->mItems[$hash] = $text;
  59                 $this->mSize += strlen( $text );
  60
  61                 return $hash;
  62         }
  63
  64         public function getItem( $hash ) {
  65                 $this->uncompress();
  66                 if ( array_key_exists( $hash, $this->mItems ) ) {
  67                         return $this->mItems[$hash];
  68                 } else {
  69                         return false;
  70                 }
  71         }
  72
  73         public function setText( $text ) {
  74                 $this->uncompress();
  75                 $stub = $this->addItem( $text );
  76                 $this->mDefaultHash = $stub->mHash;
  77         }
  78
  79         public function getText() {
  80                 $this->uncompress();
  81                 return $this->getItem( $this->mDefaultHash );
  82         }
  83
  84         /**
  85          * Remove an item
  86          */
  87         public function removeItem( $hash ) {
  88                 $this->mSize -= strlen( $this->mItems[$hash] );
  89                 unset( $this->mItems[$hash] );
  90         }
  91
  92         /**
  93          * Compress the bulk data in the object
  94          */
  95         public function compress() {
  96                 if ( !$this->mCompressed  ) {
  97                         $this->mItems = gzdeflate( serialize( $this->mItems ) );
  98                         $this->mCompressed = true;
  99                 }
 100         }
 101
 102         /**
 103          * Uncompress bulk data
 104          */
 105         public function uncompress() {
 106                 if ( $this->mCompressed ) {
 107                         $this->mItems = unserialize( gzinflate( $this->mItems ) );
 108                         $this->mCompressed = false;
 109                 }
 110         }
 111
 112
 113         function __sleep() {
 114                 $this->compress();
 115                 return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
 116         }
 117
 118         function __wakeup() {
 119                 $this->uncompress();
 120         }
 121
 122         /**
 123          * Helper function for compression jobs
 124          * Returns true until the object is "full" and ready to be committed
 125          */
 126         public function isHappy( $maxFactor, $factorThreshold ) {
 127                 if ( count( $this->mItems ) == 0 ) {
 128                         return true;
 129                 }
 130                 if ( !$this->mFast ) {
 131                         $this->uncompress();
 132                         $record = serialize( $this->mItems );
 133                         $size = strlen( $record );
 134                         $avgUncompressed = $size / count( $this->mItems );
 135                         $compressed = strlen( gzdeflate( $record ) );
 136
 137                         if ( $compressed < $factorThreshold * 1024 ) {
 138                                 return true;
 139                         } else {
 140                                 return $avgUncompressed * $maxFactor < $compressed;
 141                         }
 142                 } else {
 143                         return count( $this->mItems ) <= 10;
 144                 }
 145         }
 146 }
 147
 148
 149 /**
 150  * One-step cache variable to hold base blobs; operations that
 151  * pull multiple revisions may often pull multiple times from
 152  * the same blob. By keeping the last-used one open, we avoid
 153  * redundant unserialization and decompression overhead.
 154  */
 155 global $wgBlobCache;
 156 $wgBlobCache = array();
 157
 158
 159 /**
 160  * Pointer object for an item within a CGZ blob stored in the text table.
 161  */
 162 class HistoryBlobStub {
 163         var $mOldId, $mHash, $mRef;
 164
 165         /**
 166          * @param string $hash The content hash of the text
 167          * @param integer $oldid The old_id for the CGZ object
 168          */
 169         function HistoryBlobStub( $hash = '', $oldid = 0 ) {
 170                 $this->mHash = $hash;
 171         }
 172
 173         /**
 174          * Sets the location (old_id) of the main object to which this object
 175          * points
 176          */
 177         function setLocation( $id ) {
 178                 $this->mOldId = $id;
 179         }
 180
 181         /**
 182          * Sets the location (old_id) of the referring object
 183          */
 184         function setReferrer( $id ) {
 185                 $this->mRef = $id;
 186         }
 187
 188         /**
 189          * Gets the location of the referring object
 190          */
 191         function getReferrer() {
 192                 return $this->mRef;
 193         }
 194
 195         function getText() {
 196                 $fname = 'HistoryBlobStub::getText';
 197                 global $wgBlobCache;
 198                 if( isset( $wgBlobCache[$this->mOldId] ) ) {
 199                         $obj = $wgBlobCache[$this->mOldId];
 200                 } else {
 201                         $dbr = wfGetDB( DB_SLAVE );
 202                         $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId ) );
 203                         if( !$row ) {
 204                                 return false;
 205                         }
 206                         $flags = explode( ',', $row->old_flags );
 207                         if( in_array( 'external', $flags ) ) {
 208                                 $url=$row->old_text;
 209                                 @list( /* $proto */ ,$path)=explode('://',$url,2);
 210                                 if ($path=="") {
 211                                         wfProfileOut( $fname );
 212                                         return false;
 213                                 }
 214                                 $row->old_text=ExternalStore::fetchFromUrl($url);
 215
 216                         }
 217                         if( !in_array( 'object', $flags ) ) {
 218                                 return false;
 219                         }
 220
 221                         if( in_array( 'gzip', $flags ) ) {
 222                                 // This shouldn't happen, but a bug in the compress script
 223                                 // may at times gzip-compress a HistoryBlob object row.
 224                                 $obj = unserialize( gzinflate( $row->old_text ) );
 225                         } else {
 226                                 $obj = unserialize( $row->old_text );
 227                         }
 228
 229                         if( !is_object( $obj ) ) {
 230                                 // Correct for old double-serialization bug.
 231                                 $obj = unserialize( $obj );
 232                         }
 233
 234                         // Save this item for reference; if pulling many
 235                         // items in a row we'll likely use it again.
 236                         $obj->uncompress();
 237                         $wgBlobCache = array( $this->mOldId => $obj );
 238                 }
 239                 return $obj->getItem( $this->mHash );
 240         }
 241
 242         /**
 243          * Get the content hash
 244          */
 245         function getHash() {
 246                 return $this->mHash;
 247         }
 248 }
 249
 250
 251 /**
 252  * To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the
 253  * leftover cur table as the backend. This avoids expensively copying hundreds
 254  * of megabytes of data during the conversion downtime.
 255  *
 256  * Serialized HistoryBlobCurStub objects will be inserted into the text table
 257  * on conversion if $wgFastSchemaUpgrades is set to true.
 258  */
 259 class HistoryBlobCurStub {
 260         var $mCurId;
 261
 262         /**
 263          * @param integer $curid The cur_id pointed to
 264          */
 265         function HistoryBlobCurStub( $curid = 0 ) {
 266                 $this->mCurId = $curid;
 267         }
 268
 269         /**
 270          * Sets the location (cur_id) of the main object to which this object
 271          * points
 272          */
 273         function setLocation( $id ) {
 274                 $this->mCurId = $id;
 275         }
 276
 277         function getText() {
 278                 $dbr = wfGetDB( DB_SLAVE );
 279                 $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId ) );
 280                 if( !$row ) {
 281                         return false;
 282                 }
 283                 return $row->cur_text;
 284         }
 285 }
 286
 287 /**
 288  * Diff-based history compression
 289  * Requires xdiff 1.5+ and zlib
 290  */
 291 class DiffHistoryBlob implements HistoryBlob {
 292         /** Uncompressed item cache */
 293         var $mItems = array();
 294
 295         /**
 296          * Array of diffs, where $this->mDiffs[0] is the diff between
 297          * $this->mDiffs[0] and $this->mDiffs[1]
 298          */
 299         var $mDiffs = array();
 300
 301         /**
 302          * The key for getText()
 303          */
 304         var $mDefaultKey;
 305
 306         /**
 307          * Compressed storage
 308          */
 309         var $mCompressed;
 310
 311         /**
 312          * True if the object is locked against further writes
 313          */
 314         var $mFrozen = false;
 315
 316
 317         function __construct() {
 318                 if ( !function_exists( 'xdiff_string_bdiff' ) ){
 319                         throw new MWException( "Need xdiff 1.5+ support to read or write DiffHistoryBlob\n" );
 320                 }
 321                 if ( !function_exists( 'gzdeflate' ) ) {
 322                         throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
 323                 }
 324         }
 325
 326         function addItem( $text ) {
 327                 if ( $this->mFrozen ) {
 328                         throw new MWException( __METHOD__.": Cannot add more items after sleep/wakeup" );
 329                 }
 330
 331                 $this->mItems[] = $text;
 332                 $i = count( $this->mItems ) - 1;
 333                 if ( $i > 0 ) {
 334                         # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 335                         # "String is not zero-terminated"
 336                         wfSuppressWarnings();
 337                         $this->mDiffs[] = xdiff_string_bdiff( $this->mItems[$i-1], $text ) . '';
 338                         wfRestoreWarnings();
 339                 }
 340                 return $i;
 341         }
 342
 343         function getItem( $key ) {
 344                 if ( $key > count( $this->mDiffs ) + 1 ) {
 345                         return false;
 346                 }
 347                 $key = intval( $key );
 348                 if ( $key == 0 ) {
 349                         return $this->mItems[0];
 350                 }
 351
 352                 $last = count( $this->mItems ) - 1;
 353                 for ( $i = $last + 1; $i <= $key; $i++ ) {
 354                         # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 355                         # "String is not zero-terminated"
 356                         wfSuppressWarnings();
 357                         $this->mItems[$i] = xdiff_string_bpatch( $this->mItems[$i - 1], $this->mDiffs[$i - 1] ) . '';
 358                         wfRestoreWarnings();
 359                 }
 360                 return $this->mItems[$key];
 361         }
 362
 363         function setText( $text ) {
 364                 $this->mDefaultKey = $this->addItem( $text );
 365         }
 366
 367         function getText() {
 368                 return $this->getItem( $this->mDefaultKey );
 369         }
 370
 371         function __sleep() {
 372                 if ( !isset( $this->mItems[0] ) ) {
 373                         // Empty object
 374                         $info = false;
 375                 } else {
 376                         $info = array(
 377                                 'base' => $this->mItems[0],
 378                                 'diffs' => $this->mDiffs
 379                         );
 380                 }
 381                 if ( isset( $this->mDefaultKey ) ) {
 382                         $info['default'] = $this->mDefaultKey;
 383                 }
 384                 $this->mCompressed = gzdeflate( serialize( $info ) );
 385                 return array( 'mCompressed' );
 386         }
 387
 388         function __wakeup() {
 389                 // addItem() doesn't work if mItems is partially filled from mDiffs
 390                 $this->mFrozen = true;
 391                 $info = unserialize( gzinflate( $this->mCompressed ) );
 392                 unset( $this->mCompressed );
 393
 394                 if ( !$info ) {
 395                         // Empty object
 396                         return;
 397                 }
 398
 399                 if ( isset( $info['default'] ) ) {
 400                         $this->mDefaultKey = $info['default'];
 401                 }
 402                 $this->mItems[0] = $info['base'];
 403                 $this->mDiffs = $info['diffs'];
 404         }
 405 }