includes/HistoryBlob.php

   1 <?php
   2
   3 /**
   4  * Base class for general text storage via the "object" flag in old_flags, or
   5  * two-part external storage URLs. Used for represent efficient concatenated
   6  * storage, and migration-related pointer objects.
   7  */
   8 interface HistoryBlob
   9 {
  10         /**
  11          * Adds an item of text, returns a stub object which points to the item.
  12          * You must call setLocation() on the stub object before storing it to the
  13          * database
  14          *
  15          * @param $text string
  16          *
  17          * @return String: the key for getItem()
  18          */
  19         function addItem( $text );
  20
  21         /**
  22          * Get item by key, or false if the key is not present
  23          *
  24          * @param $key string
  25          *
  26          * @return String or false
  27          */
  28         function getItem( $key );
  29
  30         /**
  31          * Set the "default text"
  32          * This concept is an odd property of the current DB schema, whereby each text item has a revision
  33          * associated with it. The default text is the text of the associated revision. There may, however,
  34          * be other revisions in the same object.
  35          *
  36          * Default text is not required for two-part external storage URLs.
  37          *
  38          * @param $text string
  39          */
  40         function setText( $text );
  41
  42         /**
  43          * Get default text. This is called from Revision::getRevisionText()
  44          *
  45          * @return String
  46          */
  47         function getText();
  48 }
  49
  50 /**
  51  * Concatenated gzip (CGZ) storage
  52  * Improves compression ratio by concatenating like objects before gzipping
  53  */
  54 class ConcatenatedGzipHistoryBlob implements HistoryBlob
  55 {
  56         public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
  57         public $mSize = 0;
  58         public $mMaxSize = 10000000;
  59         public $mMaxCount = 100;
  60
  61         /** Constructor */
  62         public function __construct() {
  63                 if ( !function_exists( 'gzdeflate' ) ) {
  64                         throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
  65                 }
  66         }
  67
  68         /**
  69          * @param $text string
  70          * @return string
  71          */
  72         public function addItem( $text ) {
  73                 $this->uncompress();
  74                 $hash = md5( $text );
  75                 if ( !isset( $this->mItems[$hash] ) ) {
  76                         $this->mItems[$hash] = $text;
  77                         $this->mSize += strlen( $text );
  78                 }
  79                 return $hash;
  80         }
  81
  82         /**
  83          * @param $hash string
  84          * @return array|bool
  85          */
  86         public function getItem( $hash ) {
  87                 $this->uncompress();
  88                 if ( array_key_exists( $hash, $this->mItems ) ) {
  89                         return $this->mItems[$hash];
  90                 } else {
  91                         return false;
  92                 }
  93         }
  94
  95         /**
  96          * @param $text string
  97          */
  98         public function setText( $text ) {
  99                 $this->uncompress();
 100                 $this->mDefaultHash = $this->addItem( $text );
 101         }
 102
 103         /**
 104          * @return array|bool
 105          */
 106         public function getText() {
 107                 $this->uncompress();
 108                 return $this->getItem( $this->mDefaultHash );
 109         }
 110
 111         /**
 112          * Remove an item
 113          *
 114          * @param $hash string
 115          */
 116         public function removeItem( $hash ) {
 117                 $this->mSize -= strlen( $this->mItems[$hash] );
 118                 unset( $this->mItems[$hash] );
 119         }
 120
 121         /**
 122          * Compress the bulk data in the object
 123          */
 124         public function compress() {
 125                 if ( !$this->mCompressed  ) {
 126                         $this->mItems = gzdeflate( serialize( $this->mItems ) );
 127                         $this->mCompressed = true;
 128                 }
 129         }
 130
 131         /**
 132          * Uncompress bulk data
 133          */
 134         public function uncompress() {
 135                 if ( $this->mCompressed ) {
 136                         $this->mItems = unserialize( gzinflate( $this->mItems ) );
 137                         $this->mCompressed = false;
 138                 }
 139         }
 140
 141         /**
 142          * @return array
 143          */
 144         function __sleep() {
 145                 $this->compress();
 146                 return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
 147         }
 148
 149         function __wakeup() {
 150                 $this->uncompress();
 151         }
 152
 153         /**
 154          * Helper function for compression jobs
 155          * Returns true until the object is "full" and ready to be committed
 156          *
 157          * @return bool
 158          */
 159         public function isHappy() {
 160                 return $this->mSize < $this->mMaxSize
 161                         && count( $this->mItems ) < $this->mMaxCount;
 162         }
 163 }
 164
 165
 166 /**
 167  * Pointer object for an item within a CGZ blob stored in the text table.
 168  */
 169 class HistoryBlobStub {
 170         /**
 171          * One-step cache variable to hold base blobs; operations that
 172          * pull multiple revisions may often pull multiple times from
 173          * the same blob. By keeping the last-used one open, we avoid
 174          * redundant unserialization and decompression overhead.
 175          */
 176         protected static $blobCache = array();
 177
 178         var $mOldId, $mHash, $mRef;
 179
 180         /**
 181          * @param $hash Strng: the content hash of the text
 182          * @param $oldid Integer: the old_id for the CGZ object
 183          */
 184         function __construct( $hash = '', $oldid = 0 ) {
 185                 $this->mHash = $hash;
 186         }
 187
 188         /**
 189          * Sets the location (old_id) of the main object to which this object
 190          * points
 191          */
 192         function setLocation( $id ) {
 193                 $this->mOldId = $id;
 194         }
 195
 196         /**
 197          * Sets the location (old_id) of the referring object
 198          */
 199         function setReferrer( $id ) {
 200                 $this->mRef = $id;
 201         }
 202
 203         /**
 204          * Gets the location of the referring object
 205          */
 206         function getReferrer() {
 207                 return $this->mRef;
 208         }
 209
 210         /**
 211          * @return string
 212          */
 213         function getText() {
 214                 $fname = 'HistoryBlobStub::getText';
 215
 216                 if( isset( self::$blobCache[$this->mOldId] ) ) {
 217                         $obj = self::$blobCache[$this->mOldId];
 218                 } else {
 219                         $dbr = wfGetDB( DB_SLAVE );
 220                         $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId ) );
 221                         if( !$row ) {
 222                                 return false;
 223                         }
 224                         $flags = explode( ',', $row->old_flags );
 225                         if( in_array( 'external', $flags ) ) {
 226                                 $url=$row->old_text;
 227                                 $parts = explode( '://', $url, 2 );
 228                                 if ( !isset( $parts[1] ) || $parts[1] == '' ) {
 229                                         wfProfileOut( $fname );
 230                                         return false;
 231                                 }
 232                                 $row->old_text = ExternalStore::fetchFromUrl($url);
 233
 234                         }
 235                         if( !in_array( 'object', $flags ) ) {
 236                                 return false;
 237                         }
 238
 239                         if( in_array( 'gzip', $flags ) ) {
 240                                 // This shouldn't happen, but a bug in the compress script
 241                                 // may at times gzip-compress a HistoryBlob object row.
 242                                 $obj = unserialize( gzinflate( $row->old_text ) );
 243                         } else {
 244                                 $obj = unserialize( $row->old_text );
 245                         }
 246
 247                         if( !is_object( $obj ) ) {
 248                                 // Correct for old double-serialization bug.
 249                                 $obj = unserialize( $obj );
 250                         }
 251
 252                         // Save this item for reference; if pulling many
 253                         // items in a row we'll likely use it again.
 254                         $obj->uncompress();
 255                         self::$blobCache = array( $this->mOldId => $obj );
 256                 }
 257                 return $obj->getItem( $this->mHash );
 258         }
 259
 260         /**
 261          * Get the content hash
 262          *
 263          * @return string
 264          */
 265         function getHash() {
 266                 return $this->mHash;
 267         }
 268 }
 269
 270
 271 /**
 272  * To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the
 273  * leftover cur table as the backend. This avoids expensively copying hundreds
 274  * of megabytes of data during the conversion downtime.
 275  *
 276  * Serialized HistoryBlobCurStub objects will be inserted into the text table
 277  * on conversion if $wgFastSchemaUpgrades is set to true.
 278  */
 279 class HistoryBlobCurStub {
 280         var $mCurId;
 281
 282         /**
 283          * @param $curid Integer: the cur_id pointed to
 284          */
 285         function __construct( $curid = 0 ) {
 286                 $this->mCurId = $curid;
 287         }
 288
 289         /**
 290          * Sets the location (cur_id) of the main object to which this object
 291          * points
 292          *
 293          * @param $id int
 294          */
 295         function setLocation( $id ) {
 296                 $this->mCurId = $id;
 297         }
 298
 299         /**
 300          * @return string|false
 301          */
 302         function getText() {
 303                 $dbr = wfGetDB( DB_SLAVE );
 304                 $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId ) );
 305                 if( !$row ) {
 306                         return false;
 307                 }
 308                 return $row->cur_text;
 309         }
 310 }
 311
 312 /**
 313  * Diff-based history compression
 314  * Requires xdiff 1.5+ and zlib
 315  */
 316 class DiffHistoryBlob implements HistoryBlob {
 317         /** Uncompressed item cache */
 318         var $mItems = array();
 319
 320         /** Total uncompressed size */
 321         var $mSize = 0;
 322
 323         /**
 324          * Array of diffs. If a diff D from A to B is notated D = B - A, and Z is
 325          * an empty string:
 326          *
 327          *              { item[map[i]] - item[map[i-1]]   where i > 0
 328          *    diff[i] = {
 329          *              { item[map[i]] - Z                where i = 0
 330          */
 331         var $mDiffs;
 332
 333         /** The diff map, see above */
 334         var $mDiffMap;
 335
 336         /**
 337          * The key for getText()
 338          */
 339         var $mDefaultKey;
 340
 341         /**
 342          * Compressed storage
 343          */
 344         var $mCompressed;
 345
 346         /**
 347          * True if the object is locked against further writes
 348          */
 349         var $mFrozen = false;
 350
 351         /**
 352          * The maximum uncompressed size before the object becomes sad
 353          * Should be less than max_allowed_packet
 354          */
 355         var $mMaxSize = 10000000;
 356
 357         /**
 358          * The maximum number of text items before the object becomes sad
 359          */
 360         var $mMaxCount = 100;
 361
 362         /** Constants from xdiff.h */
 363         const XDL_BDOP_INS = 1;
 364         const XDL_BDOP_CPY = 2;
 365         const XDL_BDOP_INSB = 3;
 366
 367         function __construct() {
 368                 if ( !function_exists( 'gzdeflate' ) ) {
 369                         throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
 370                 }
 371         }
 372
 373         /**
 374          * @throws MWException
 375          * @param $text string
 376          * @return int
 377          */
 378         function addItem( $text ) {
 379                 if ( $this->mFrozen ) {
 380                         throw new MWException( __METHOD__.": Cannot add more items after sleep/wakeup" );
 381                 }
 382
 383                 $this->mItems[] = $text;
 384                 $this->mSize += strlen( $text );
 385                 $this->mDiffs = null; // later
 386                 return count( $this->mItems ) - 1;
 387         }
 388
 389         /**
 390          * @param $key string
 391          * @return string
 392          */
 393         function getItem( $key ) {
 394                 return $this->mItems[$key];
 395         }
 396
 397         /**
 398          * @param $text string
 399          */
 400         function setText( $text ) {
 401                 $this->mDefaultKey = $this->addItem( $text );
 402         }
 403
 404         /**
 405          * @return string
 406          */
 407         function getText() {
 408                 return $this->getItem( $this->mDefaultKey );
 409         }
 410
 411         /**
 412          * @throws MWException
 413          */
 414         function compress() {
 415                 if ( !function_exists( 'xdiff_string_rabdiff' ) ){
 416                         throw new MWException( "Need xdiff 1.5+ support to write DiffHistoryBlob\n" );
 417                 }
 418                 if ( isset( $this->mDiffs ) ) {
 419                         // Already compressed
 420                         return;
 421                 }
 422                 if ( !count( $this->mItems ) ) {
 423                         // Empty
 424                         return;
 425                 }
 426
 427                 // Create two diff sequences: one for main text and one for small text
 428                 $sequences = array(
 429                         'small' => array(
 430                                 'tail' => '',
 431                                 'diffs' => array(),
 432                                 'map' => array(),
 433                         ),
 434                         'main' => array(
 435                                 'tail' => '',
 436                                 'diffs' => array(),
 437                                 'map' => array(),
 438                         ),
 439                 );
 440                 $smallFactor = 0.5;
 441
 442                 for ( $i = 0; $i < count( $this->mItems ); $i++ ) {
 443                         $text = $this->mItems[$i];
 444                         if ( $i == 0 ) {
 445                                 $seqName = 'main';
 446                         } else {
 447                                 $mainTail = $sequences['main']['tail'];
 448                                 if ( strlen( $text ) < strlen( $mainTail ) * $smallFactor ) {
 449                                         $seqName = 'small';
 450                                 } else {
 451                                         $seqName = 'main';
 452                                 }
 453                         }
 454                         $seq =& $sequences[$seqName];
 455                         $tail = $seq['tail'];
 456                         $diff = $this->diff( $tail, $text );
 457                         $seq['diffs'][] = $diff;
 458                         $seq['map'][] = $i;
 459                         $seq['tail'] = $text;
 460                 }
 461                 unset( $seq ); // unlink dangerous alias
 462
 463                 // Knit the sequences together
 464                 $tail = '';
 465                 $this->mDiffs = array();
 466                 $this->mDiffMap = array();
 467                 foreach ( $sequences as $seq ) {
 468                         if ( !count( $seq['diffs'] ) ) {
 469                                 continue;
 470                         }
 471                         if ( $tail === '' ) {
 472                                 $this->mDiffs[] = $seq['diffs'][0];
 473                         } else {
 474                                 $head = $this->patch( '', $seq['diffs'][0] );
 475                                 $this->mDiffs[] = $this->diff( $tail, $head );
 476                         }
 477                         $this->mDiffMap[] = $seq['map'][0];
 478                         for ( $i = 1; $i < count( $seq['diffs'] ); $i++ ) {
 479                                 $this->mDiffs[] = $seq['diffs'][$i];
 480                                 $this->mDiffMap[] = $seq['map'][$i];
 481                         }
 482                         $tail = $seq['tail'];
 483                 }
 484         }
 485
 486         /**
 487          * @param $t1
 488          * @param $t2
 489          * @return string
 490          */
 491         function diff( $t1, $t2 ) {
 492                 # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
 493                 # "String is not zero-terminated"
 494                 wfSuppressWarnings();
 495                 $diff = xdiff_string_rabdiff( $t1, $t2 ) . '';
 496                 wfRestoreWarnings();
 497                 return $diff;
 498         }
 499
 500         /**
 501          * @param $base
 502          * @param $diff
 503          * @return bool|string
 504          */
 505         function patch( $base, $diff ) {
 506                 if ( function_exists( 'xdiff_string_bpatch' ) ) {
 507                         wfSuppressWarnings();
 508                         $text = xdiff_string_bpatch( $base, $diff ) . '';
 509                         wfRestoreWarnings();
 510                         return $text;
 511                 }
 512
 513                 # Pure PHP implementation
 514
 515                 $header = unpack( 'Vofp/Vcsize', substr( $diff, 0, 8 ) );
 516
 517                 # Check the checksum if mhash is available
 518                 if ( extension_loaded( 'mhash' ) ) {
 519                         $ofp = mhash( MHASH_ADLER32, $base );
 520                         if ( $ofp !== substr( $diff, 0, 4 ) ) {
 521                                 wfDebug( __METHOD__. ": incorrect base checksum\n" );
 522                                 return false;
 523                         }
 524                 }
 525                 if ( $header['csize'] != strlen( $base ) ) {
 526                         wfDebug( __METHOD__. ": incorrect base length\n" );
 527                         return false;
 528                 }
 529
 530                 $p = 8;
 531                 $out = '';
 532                 while ( $p < strlen( $diff ) ) {
 533                         $x = unpack( 'Cop', substr( $diff, $p, 1 ) );
 534                         $op = $x['op'];
 535                         ++$p;
 536                         switch ( $op ) {
 537                         case self::XDL_BDOP_INS:
 538                                 $x = unpack( 'Csize', substr( $diff, $p, 1 ) );
 539                                 $p++;
 540                                 $out .= substr( $diff, $p, $x['size'] );
 541                                 $p += $x['size'];
 542                                 break;
 543                         case self::XDL_BDOP_INSB:
 544                                 $x = unpack( 'Vcsize', substr( $diff, $p, 4 ) );
 545                                 $p += 4;
 546                                 $out .= substr( $diff, $p, $x['csize'] );
 547                                 $p += $x['csize'];
 548                                 break;
 549                         case self::XDL_BDOP_CPY:
 550                                 $x = unpack( 'Voff/Vcsize', substr( $diff, $p, 8 ) );
 551                                 $p += 8;
 552                                 $out .= substr( $base, $x['off'], $x['csize'] );
 553                                 break;
 554                         default:
 555                                 wfDebug( __METHOD__.": invalid op\n" );
 556                                 return false;
 557                         }
 558                 }
 559                 return $out;
 560         }
 561
 562         function uncompress() {
 563                 if ( !$this->mDiffs ) {
 564                         return;
 565                 }
 566                 $tail = '';
 567                 for ( $diffKey = 0; $diffKey < count( $this->mDiffs ); $diffKey++ ) {
 568                         $textKey = $this->mDiffMap[$diffKey];
 569                         $text = $this->patch( $tail, $this->mDiffs[$diffKey] );
 570                         $this->mItems[$textKey] = $text;
 571                         $tail = $text;
 572                 }
 573         }
 574
 575         /**
 576          * @return array
 577          */
 578         function __sleep() {
 579                 $this->compress();
 580                 if ( !count( $this->mItems ) ) {
 581                         // Empty object
 582                         $info = false;
 583                 } else {
 584                         // Take forward differences to improve the compression ratio for sequences
 585                         $map = '';
 586                         $prev = 0;
 587                         foreach ( $this->mDiffMap as $i ) {
 588                                 if ( $map !== '' ) {
 589                                         $map .= ',';
 590                                 }
 591                                 $map .= $i - $prev;
 592                                 $prev = $i;
 593                         }
 594                         $info = array(
 595                                 'diffs' => $this->mDiffs,
 596                                 'map' => $map
 597                         );
 598                 }
 599                 if ( isset( $this->mDefaultKey ) ) {
 600                         $info['default'] = $this->mDefaultKey;
 601                 }
 602                 $this->mCompressed = gzdeflate( serialize( $info ) );
 603                 return array( 'mCompressed' );
 604         }
 605
 606         function __wakeup() {
 607                 // addItem() doesn't work if mItems is partially filled from mDiffs
 608                 $this->mFrozen = true;
 609                 $info = unserialize( gzinflate( $this->mCompressed ) );
 610                 unset( $this->mCompressed );
 611
 612                 if ( !$info ) {
 613                         // Empty object
 614                         return;
 615                 }
 616
 617                 if ( isset( $info['default'] ) ) {
 618                         $this->mDefaultKey = $info['default'];
 619                 }
 620                 $this->mDiffs = $info['diffs'];
 621                 if ( isset( $info['base'] ) ) {
 622                         // Old format
 623                         $this->mDiffMap = range( 0, count( $this->mDiffs ) - 1 );
 624                         array_unshift( $this->mDiffs,
 625                                 pack( 'VVCV', 0, 0, self::XDL_BDOP_INSB, strlen( $info['base'] ) ) .
 626                                 $info['base'] );
 627                 } else {
 628                         // New format
 629                         $map = explode( ',', $info['map'] );
 630                         $cur = 0;
 631                         $this->mDiffMap = array();
 632                         foreach ( $map as $i ) {
 633                                 $cur += $i;
 634                                 $this->mDiffMap[] = $cur;
 635                         }
 636                 }
 637                 $this->uncompress();
 638         }
 639
 640         /**
 641          * Helper function for compression jobs
 642          * Returns true until the object is "full" and ready to be committed
 643          *
 644          * @return bool
 645          */
 646         function isHappy() {
 647                 return $this->mSize < $this->mMaxSize
 648                         && count( $this->mItems ) < $this->mMaxCount;
 649         }
 650
 651 }