includes/BagOStuff.php

   1 <?php
   2 #
   3 # Copyright (C) 2003-2004 Brion Vibber <brion@pobox.com>
   4 # http://www.mediawiki.org/
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with this program; if not, write to the Free Software Foundation, Inc.,
  18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19 # http://www.gnu.org/copyleft/gpl.html
  20 /**
  21  *
  22  */
  23
  24 /**
  25  * Simple generic object store
  26  *
  27  * interface is intended to be more or less compatible with
  28  * the PHP memcached client.
  29  *
  30  * backends for local hash array and SQL table included:
  31  * <code>
  32  *   $bag = new HashBagOStuff();
  33  *   $bag = new MysqlBagOStuff($tablename); # connect to db first
  34  * </code>
  35  *
  36  */
  37 class BagOStuff {
  38         var $debugmode;
  39
  40         function __construct() {
  41                 $this->set_debug( false );
  42         }
  43
  44         function set_debug($bool) {
  45                 $this->debugmode = $bool;
  46         }
  47
  48         /* *** THE GUTS OF THE OPERATION *** */
  49         /* Override these with functional things in subclasses */
  50
  51         function get($key) {
  52                 /* stub */
  53                 return false;
  54         }
  55
  56         function set($key, $value, $exptime=0) {
  57                 /* stub */
  58                 return false;
  59         }
  60
  61         function delete($key, $time=0) {
  62                 /* stub */
  63                 return false;
  64         }
  65
  66         function lock($key, $timeout = 0) {
  67                 /* stub */
  68                 return true;
  69         }
  70
  71         function unlock($key) {
  72                 /* stub */
  73                 return true;
  74         }
  75
  76         /* *** Emulated functions *** */
  77         /* Better performance can likely be got with custom written versions */
  78         function get_multi($keys) {
  79                 $out = array();
  80                 foreach($keys as $key)
  81                         $out[$key] = $this->get($key);
  82                 return $out;
  83         }
  84
  85         function set_multi($hash, $exptime=0) {
  86                 foreach($hash as $key => $value)
  87                         $this->set($key, $value, $exptime);
  88         }
  89
  90         function add($key, $value, $exptime=0) {
  91                 if( $this->get($key) == false ) {
  92                         $this->set($key, $value, $exptime);
  93                         return true;
  94                 }
  95         }
  96
  97         function add_multi($hash, $exptime=0) {
  98                 foreach($hash as $key => $value)
  99                         $this->add($key, $value, $exptime);
 100         }
 101
 102         function delete_multi($keys, $time=0) {
 103                 foreach($keys as $key)
 104                         $this->delete($key, $time);
 105         }
 106
 107         function replace($key, $value, $exptime=0) {
 108                 if( $this->get($key) !== false )
 109                         $this->set($key, $value, $exptime);
 110         }
 111
 112         function incr($key, $value=1) {
 113                 if ( !$this->lock($key) ) {
 114                         return false;
 115                 }
 116                 $value = intval($value);
 117                 if($value < 0) $value = 0;
 118
 119                 $n = false;
 120                 if( ($n = $this->get($key)) !== false ) {
 121                         $n += $value;
 122                         $this->set($key, $n); // exptime?
 123                 }
 124                 $this->unlock($key);
 125                 return $n;
 126         }
 127
 128         function decr($key, $value=1) {
 129                 if ( !$this->lock($key) ) {
 130                         return false;
 131                 }
 132                 $value = intval($value);
 133                 if($value < 0) $value = 0;
 134
 135                 $m = false;
 136                 if( ($n = $this->get($key)) !== false ) {
 137                         $m = $n - $value;
 138                         if($m < 0) $m = 0;
 139                         $this->set($key, $m); // exptime?
 140                 }
 141                 $this->unlock($key);
 142                 return $m;
 143         }
 144
 145         function _debug($text) {
 146                 if($this->debugmode)
 147                         wfDebug("BagOStuff debug: $text\n");
 148         }
 149
 150         /**
 151          * Convert an optionally relative time to an absolute time
 152          */
 153         static function convertExpiry( $exptime ) {
 154                 if(($exptime != 0) && ($exptime < 3600*24*30)) {
 155                         return time() + $exptime;
 156                 } else {
 157                         return $exptime;
 158                 }
 159         }
 160 }
 161
 162
 163 /**
 164  * Functional versions!
 165  * @todo document
 166  */
 167 class HashBagOStuff extends BagOStuff {
 168         /*
 169            This is a test of the interface, mainly. It stores
 170            things in an associative array, which is not going to
 171            persist between program runs.
 172         */
 173         var $bag;
 174
 175         function __construct() {
 176                 $this->bag = array();
 177         }
 178
 179         function _expire($key) {
 180                 $et = $this->bag[$key][1];
 181                 if(($et == 0) || ($et > time()))
 182                         return false;
 183                 $this->delete($key);
 184                 return true;
 185         }
 186
 187         function get($key) {
 188                 if(!$this->bag[$key])
 189                         return false;
 190                 if($this->_expire($key))
 191                         return false;
 192                 return $this->bag[$key][0];
 193         }
 194
 195         function set($key,$value,$exptime=0) {
 196                 $this->bag[$key] = array( $value, BagOStuff::convertExpiry( $exptime ) );
 197         }
 198
 199         function delete($key,$time=0) {
 200                 if(!$this->bag[$key])
 201                         return false;
 202                 unset($this->bag[$key]);
 203                 return true;
 204         }
 205 }
 206
 207 /*
 208 CREATE TABLE objectcache (
 209   keyname char(255) binary not null default '',
 210   value mediumblob,
 211   exptime datetime,
 212   unique key (keyname),
 213   key (exptime)
 214 );
 215 */
 216
 217 /**
 218  * @todo document
 219  * @abstract
 220  */
 221 abstract class SqlBagOStuff extends BagOStuff {
 222         var $table;
 223         var $lastexpireall = 0;
 224
 225         function __construct($tablename = 'objectcache') {
 226                 $this->table = $tablename;
 227         }
 228
 229         function get($key) {
 230                 /* expire old entries if any */
 231                 $this->garbageCollect();
 232
 233                 $res = $this->_query(
 234                         "SELECT value,exptime FROM $0 WHERE keyname='$1'", $key);
 235                 if(!$res) {
 236                         $this->_debug("get: ** error: " . $this->_dberror($res) . " **");
 237                         return false;
 238                 }
 239                 if($row=$this->_fetchobject($res)) {
 240                         $this->_debug("get: retrieved data; exp time is " . $row->exptime);
 241                         if ( $row->exptime != $this->_maxdatetime() &&
 242                           wfTimestamp( TS_UNIX, $row->exptime ) < time() )
 243                         {
 244                                 $this->_debug("get: key has expired, deleting");
 245                                 $this->delete($key);
 246                                 return false;
 247                         }
 248                         return $this->_unserialize($this->_blobdecode($row->value));
 249                 } else {
 250                         $this->_debug('get: no matching rows');
 251                 }
 252                 return false;
 253         }
 254
 255         function set($key,$value,$exptime=0) {
 256                 if ( wfReadOnly() ) {
 257                         return false;
 258                 }
 259                 $exptime = intval($exptime);
 260                 if($exptime < 0) $exptime = 0;
 261                 if($exptime == 0) {
 262                         $exp = $this->_maxdatetime();
 263                 } else {
 264                         if($exptime < 3.16e8) # ~10 years
 265                                 $exptime += time();
 266                         $exp = $this->_fromunixtime($exptime);
 267                 }
 268                 $this->delete( $key );
 269                 $this->_doinsert($this->getTableName(), array(
 270                                         'keyname' => $key,
 271                                         'value' => $this->_blobencode($this->_serialize($value)),
 272                                         'exptime' => $exp
 273                                 ));
 274                 return true; /* ? */
 275         }
 276
 277         function delete($key,$time=0) {
 278                 if ( wfReadOnly() ) {
 279                         return false;
 280                 }
 281                 $this->_query(
 282                         "DELETE FROM $0 WHERE keyname='$1'", $key );
 283                 return true; /* ? */
 284         }
 285
 286         function getTableName() {
 287                 return $this->table;
 288         }
 289
 290         function _query($sql) {
 291                 $reps = func_get_args();
 292                 $reps[0] = $this->getTableName();
 293                 // ewwww
 294                 for($i=0;$i<count($reps);$i++) {
 295                         $sql = str_replace(
 296                                 '$' . $i,
 297                                 $i > 0 ? $this->_strencode($reps[$i]) : $reps[$i],
 298                                 $sql);
 299                 }
 300                 $res = $this->_doquery($sql);
 301                 if($res == false) {
 302                         $this->_debug('query failed: ' . $this->_dberror($res));
 303                 }
 304                 return $res;
 305         }
 306
 307         function _strencode($str) {
 308                 /* Protect strings in SQL */
 309                 return str_replace( "'", "''", $str );
 310         }
 311         function _blobencode($str) {
 312                 return $str;
 313         }
 314         function _blobdecode($str) {
 315                 return $str;
 316         }
 317
 318         abstract function _doinsert($table, $vals);
 319         abstract function _doquery($sql);
 320
 321         function _freeresult($result) {
 322                 /* stub */
 323                 return false;
 324         }
 325
 326         function _dberror($result) {
 327                 /* stub */
 328                 return 'unknown error';
 329         }
 330
 331         abstract function _maxdatetime();
 332         abstract function _fromunixtime($ts);
 333
 334         function garbageCollect() {
 335                 /* Ignore 99% of requests */
 336                 if ( !mt_rand( 0, 100 ) ) {
 337                         $nowtime = time();
 338                         /* Avoid repeating the delete within a few seconds */
 339                         if ( $nowtime > ($this->lastexpireall + 1) ) {
 340                                 $this->lastexpireall = $nowtime;
 341                                 $this->expireall();
 342                         }
 343                 }
 344         }
 345
 346         function expireall() {
 347                 /* Remove any items that have expired */
 348                 if ( wfReadOnly() ) {
 349                         return false;
 350                 }
 351                 $now = $this->_fromunixtime( time() );
 352                 $this->_query( "DELETE FROM $0 WHERE exptime < '$now'" );
 353         }
 354
 355         function deleteall() {
 356                 /* Clear *all* items from cache table */
 357                 if ( wfReadOnly() ) {
 358                         return false;
 359                 }
 360                 $this->_query( "DELETE FROM $0" );
 361         }
 362
 363         /**
 364          * Serialize an object and, if possible, compress the representation.
 365          * On typical message and page data, this can provide a 3X decrease
 366          * in storage requirements.
 367          *
 368          * @param mixed $data
 369          * @return string
 370          */
 371         function _serialize( &$data ) {
 372                 $serial = serialize( $data );
 373                 if( function_exists( 'gzdeflate' ) ) {
 374                         return gzdeflate( $serial );
 375                 } else {
 376                         return $serial;
 377                 }
 378         }
 379
 380         /**
 381          * Unserialize and, if necessary, decompress an object.
 382          * @param string $serial
 383          * @return mixed
 384          */
 385         function _unserialize( $serial ) {
 386                 if( function_exists( 'gzinflate' ) ) {
 387                         $decomp = @gzinflate( $serial );
 388                         if( false !== $decomp ) {
 389                                 $serial = $decomp;
 390                         }
 391                 }
 392                 $ret = unserialize( $serial );
 393                 return $ret;
 394         }
 395 }
 396
 397 /**
 398  * @todo document
 399  */
 400 class MediaWikiBagOStuff extends SqlBagOStuff {
 401         var $tableInitialised = false;
 402
 403         function _doquery($sql) {
 404                 $dbw = wfGetDB( DB_MASTER );
 405                 return $dbw->query($sql, 'MediaWikiBagOStuff::_doquery');
 406         }
 407         function _doinsert($t, $v) {
 408                 $dbw = wfGetDB( DB_MASTER );
 409                 return $dbw->insert($t, $v, 'MediaWikiBagOStuff::_doinsert',
 410                         array( 'IGNORE' ) );
 411         }
 412         function _fetchobject($result) {
 413                 $dbw = wfGetDB( DB_MASTER );
 414                 return $dbw->fetchObject($result);
 415         }
 416         function _freeresult($result) {
 417                 $dbw = wfGetDB( DB_MASTER );
 418                 return $dbw->freeResult($result);
 419         }
 420         function _dberror($result) {
 421                 $dbw = wfGetDB( DB_MASTER );
 422                 return $dbw->lastError();
 423         }
 424         function _maxdatetime() {
 425                 if ( time() > 0x7fffffff ) {
 426                         return $this->_fromunixtime( 1<<62 );
 427                 } else {
 428                         return $this->_fromunixtime( 0x7fffffff );
 429                 }
 430         }
 431         function _fromunixtime($ts) {
 432                 $dbw = wfGetDB(DB_MASTER);
 433                 return $dbw->timestamp($ts);
 434         }
 435         function _strencode($s) {
 436                 $dbw = wfGetDB( DB_MASTER );
 437                 return $dbw->strencode($s);
 438         }
 439         function _blobencode($s) {
 440                 $dbw = wfGetDB( DB_MASTER );
 441                 return $dbw->encodeBlob($s);
 442         }
 443         function _blobdecode($s) {
 444                 $dbw = wfGetDB( DB_MASTER );
 445                 return $dbw->decodeBlob($s);
 446         }
 447         function getTableName() {
 448                 if ( !$this->tableInitialised ) {
 449                         $dbw = wfGetDB( DB_MASTER );
 450                         /* This is actually a hack, we should be able
 451                            to use Language classes here... or not */
 452                         if (!$dbw)
 453                                 throw new MWException("Could not connect to database");
 454                         $this->table = $dbw->tableName( $this->table );
 455                         $this->tableInitialised = true;
 456                 }
 457                 return $this->table;
 458         }
 459 }
 460
 461 /**
 462  * This is a wrapper for Turck MMCache's shared memory functions.
 463  *
 464  * You can store objects with mmcache_put() and mmcache_get(), but Turck seems
 465  * to use a weird custom serializer that randomly segfaults. So we wrap calls
 466  * with serialize()/unserialize().
 467  *
 468  * The thing I noticed about the Turck serialized data was that unlike ordinary
 469  * serialize(), it contained the names of methods, and judging by the amount of
 470  * binary data, perhaps even the bytecode of the methods themselves. It may be
 471  * that Turck's serializer is faster, so a possible future extension would be
 472  * to use it for arrays but not for objects.
 473  *
 474  */
 475 class TurckBagOStuff extends BagOStuff {
 476         function get($key) {
 477                 $val = mmcache_get( $key );
 478                 if ( is_string( $val ) ) {
 479                         $val = unserialize( $val );
 480                 }
 481                 return $val;
 482         }
 483
 484         function set($key, $value, $exptime=0) {
 485                 mmcache_put( $key, serialize( $value ), $exptime );
 486                 return true;
 487         }
 488
 489         function delete($key, $time=0) {
 490                 mmcache_rm( $key );
 491                 return true;
 492         }
 493
 494         function lock($key, $waitTimeout = 0 ) {
 495                 mmcache_lock( $key );
 496                 return true;
 497         }
 498
 499         function unlock($key) {
 500                 mmcache_unlock( $key );
 501                 return true;
 502         }
 503 }
 504
 505 /**
 506  * This is a wrapper for APC's shared memory functions
 507  *
 508  */
 509 class APCBagOStuff extends BagOStuff {
 510         function get($key) {
 511                 $val = apc_fetch($key);
 512                 if ( is_string( $val ) ) {
 513                         $val = unserialize( $val );
 514                 }
 515                 return $val;
 516         }
 517
 518         function set($key, $value, $exptime=0) {
 519                 apc_store($key, serialize($value), $exptime);
 520                 return true;
 521         }
 522
 523         function delete($key, $time=0) {
 524                 apc_delete($key);
 525                 return true;
 526         }
 527 }
 528
 529
 530 /**
 531  * This is a wrapper for eAccelerator's shared memory functions.
 532  *
 533  * This is basically identical to the Turck MMCache version,
 534  * mostly because eAccelerator is based on Turck MMCache.
 535  *
 536  */
 537 class eAccelBagOStuff extends BagOStuff {
 538         function get($key) {
 539                 $val = eaccelerator_get( $key );
 540                 if ( is_string( $val ) ) {
 541                         $val = unserialize( $val );
 542                 }
 543                 return $val;
 544         }
 545
 546         function set($key, $value, $exptime=0) {
 547                 eaccelerator_put( $key, serialize( $value ), $exptime );
 548                 return true;
 549         }
 550
 551         function delete($key, $time=0) {
 552                 eaccelerator_rm( $key );
 553                 return true;
 554         }
 555
 556         function lock($key, $waitTimeout = 0 ) {
 557                 eaccelerator_lock( $key );
 558                 return true;
 559         }
 560
 561         function unlock($key) {
 562                 eaccelerator_unlock( $key );
 563                 return true;
 564         }
 565 }
 566
 567 /**
 568  * Wrapper for XCache object caching functions; identical interface
 569  * to the APC wrapper
 570  */
 571 class XCacheBagOStuff extends BagOStuff {
 572
 573         /**
 574          * Get a value from the XCache object cache
 575          *
 576          * @param string $key Cache key
 577          * @return mixed
 578          */
 579         public function get( $key ) {
 580                 $val = xcache_get( $key );
 581                 if( is_string( $val ) )
 582                         $val = unserialize( $val );
 583                 return $val;
 584         }
 585
 586         /**
 587          * Store a value in the XCache object cache
 588          *
 589          * @param string $key Cache key
 590          * @param mixed $value Object to store
 591          * @param int $expire Expiration time
 592          * @return bool
 593          */
 594         public function set( $key, $value, $expire = 0 ) {
 595                 xcache_set( $key, serialize( $value ), $expire );
 596                 return true;
 597         }
 598
 599         /**
 600          * Remove a value from the XCache object cache
 601          *
 602          * @param string $key Cache key
 603          * @param int $time Not used in this implementation
 604          * @return bool
 605          */
 606         public function delete( $key, $time = 0 ) {
 607                 xcache_unset( $key );
 608                 return true;
 609         }
 610
 611 }
 612
 613 /**
 614  * @todo document
 615  */
 616 class DBABagOStuff extends BagOStuff {
 617         var $mHandler, $mFile, $mReader, $mWriter, $mDisabled;
 618
 619         function __construct( $handler = 'db3', $dir = false ) {
 620                 if ( $dir === false ) {
 621                         global $wgTmpDirectory;
 622                         $dir = $wgTmpDirectory;
 623                 }
 624                 $this->mFile = "$dir/mw-cache-" . wfWikiID();
 625                 $this->mFile .= '.db';
 626                 $this->mHandler = $handler;
 627         }
 628
 629         /**
 630          * Encode value and expiry for storage
 631          */
 632         function encode( $value, $expiry ) {
 633                 # Convert to absolute time
 634                 $expiry = BagOStuff::convertExpiry( $expiry );
 635                 return sprintf( '%010u', intval( $expiry ) ) . ' ' . serialize( $value );
 636         }
 637
 638         /**
 639          * @return list containing value first and expiry second
 640          */
 641         function decode( $blob ) {
 642                 if ( !is_string( $blob ) ) {
 643                         return array( null, 0 );
 644                 } else {
 645                         return array(
 646                                 unserialize( substr( $blob, 11 ) ),
 647                                 intval( substr( $blob, 0, 10 ) )
 648                         );
 649                 }
 650         }
 651
 652         function getReader() {
 653                 if ( file_exists( $this->mFile ) ) {
 654                         $handle = dba_open( $this->mFile, 'rl', $this->mHandler );
 655                 } else {
 656                         $handle = $this->getWriter();
 657                 }
 658                 if ( !$handle ) {
 659                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 660                 }
 661                 return $handle;
 662         }
 663
 664         function getWriter() {
 665                 $handle = dba_open( $this->mFile, 'cl', $this->mHandler );
 666                 if ( !$handle ) {
 667                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 668                 }
 669                 return $handle;
 670         }
 671
 672         function get( $key ) {
 673                 wfProfileIn( __METHOD__ );
 674                 wfDebug( __METHOD__."($key)\n" );
 675                 $handle = $this->getReader();
 676                 if ( !$handle ) {
 677                         return null;
 678                 }
 679                 $val = dba_fetch( $key, $handle );
 680                 list( $val, $expiry ) = $this->decode( $val );
 681                 # Must close ASAP because locks are held
 682                 dba_close( $handle );
 683
 684                 if ( !is_null( $val ) && $expiry && $expiry < time() ) {
 685                         # Key is expired, delete it
 686                         $handle = $this->getWriter();
 687                         dba_delete( $key, $handle );
 688                         dba_close( $handle );
 689                         wfDebug( __METHOD__.": $key expired\n" );
 690                         $val = null;
 691                 }
 692                 wfProfileOut( __METHOD__ );
 693                 return $val;
 694         }
 695
 696         function set( $key, $value, $exptime=0 ) {
 697                 wfProfileIn( __METHOD__ );
 698                 wfDebug( __METHOD__."($key)\n" );
 699                 $blob = $this->encode( $value, $exptime );
 700                 $handle = $this->getWriter();
 701                 if ( !$handle ) {
 702                         return false;
 703                 }
 704                 $ret = dba_replace( $key, $blob, $handle );
 705                 dba_close( $handle );
 706                 wfProfileOut( __METHOD__ );
 707                 return $ret;
 708         }
 709
 710         function delete( $key, $time = 0 ) {
 711                 wfProfileIn( __METHOD__ );
 712                 $handle = $this->getWriter();
 713                 if ( !$handle ) {
 714                         return false;
 715                 }
 716                 $ret = dba_delete( $key, $handle );
 717                 dba_close( $handle );
 718                 wfProfileOut( __METHOD__ );
 719                 return $ret;
 720         }
 721
 722         function add( $key, $value, $exptime = 0 ) {
 723                 wfProfileIn( __METHOD__ );
 724                 $blob = $this->encode( $value, $exptime );
 725                 $handle = $this->getWriter();
 726                 if ( !$handle ) {
 727                         return false;
 728                 }
 729                 $ret = dba_insert( $key, $blob, $handle );
 730                 # Insert failed, check to see if it failed due to an expired key
 731                 if ( !$ret ) {
 732                         list( $value, $expiry ) = $this->decode( dba_fetch( $key, $handle ) );
 733                         if ( $expiry < time() ) {
 734                                 # Yes expired, delete and try again
 735                                 dba_delete( $key, $handle );
 736                                 $ret = dba_insert( $key, $blob, $handle );
 737                                 # This time if it failed then it will be handled by the caller like any other race
 738                         }
 739                 }
 740
 741                 dba_close( $handle );
 742                 wfProfileOut( __METHOD__ );
 743                 return $ret;
 744         }
 745 }
 746
 747