tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php

   1 <?php
   2
   3 use Wikimedia\TestingAccessWrapper;
   4
   5 /**
   6  * @covers WANObjectCache::wrap
   7  * @covers WANObjectCache::unwrap
   8  * @covers WANObjectCache::worthRefreshExpiring
   9  * @covers WANObjectCache::worthRefreshPopular
  10  * @covers WANObjectCache::isValid
  11  * @covers WANObjectCache::getWarmupKeyMisses
  12  * @covers WANObjectCache::prefixCacheKeys
  13  * @covers WANObjectCache::getProcessCache
  14  * @covers WANObjectCache::getNonProcessCachedMultiKeys
  15  * @covers WANObjectCache::getRawKeysForWarmup
  16  * @covers WANObjectCache::getInterimValue
  17  * @covers WANObjectCache::setInterimValue
  18  */
  19 class WANObjectCacheTest extends PHPUnit\Framework\TestCase {
  20
  21         use MediaWikiCoversValidator;
  22         use PHPUnit4And6Compat;
  23
  24         /** @var WANObjectCache */
  25         private $cache;
  26         /** @var BagOStuff */
  27         private $internalCache;
  28
  29         protected function setUp() {
  30                 parent::setUp();
  31
  32                 $this->cache = new WANObjectCache( [
  33                         'cache' => new HashBagOStuff()
  34                 ] );
  35
  36                 $wanCache = TestingAccessWrapper::newFromObject( $this->cache );
  37                 /** @noinspection PhpUndefinedFieldInspection */
  38                 $this->internalCache = $wanCache->cache;
  39         }
  40
  41         /**
  42          * @dataProvider provideSetAndGet
  43          * @covers WANObjectCache::set()
  44          * @covers WANObjectCache::get()
  45          * @covers WANObjectCache::makeKey()
  46          * @param mixed $value
  47          * @param int $ttl
  48          */
  49         public function testSetAndGet( $value, $ttl ) {
  50                 $cache = $this->cache;
  51
  52                 $curTTL = null;
  53                 $asOf = null;
  54                 $key = $cache->makeKey( 'x', wfRandomString() );
  55
  56                 $cache->get( $key, $curTTL, [], $asOf );
  57                 $this->assertNull( $curTTL, "Current TTL is null" );
  58                 $this->assertNull( $asOf, "Current as-of-time is infinite" );
  59
  60                 $t = microtime( true );
  61
  62                 $cache->set( $key, $value, $cache::TTL_UNCACHEABLE );
  63                 $cache->get( $key, $curTTL, [], $asOf );
  64                 $this->assertNull( $curTTL, "Current TTL is null (TTL_UNCACHEABLE)" );
  65                 $this->assertNull( $asOf, "Current as-of-time is infinite (TTL_UNCACHEABLE)" );
  66
  67                 $cache->set( $key, $value, $ttl );
  68
  69                 $this->assertEquals( $value, $cache->get( $key, $curTTL, [], $asOf ) );
  70                 if ( is_infinite( $ttl ) || $ttl == 0 ) {
  71                         $this->assertTrue( is_infinite( $curTTL ), "Current TTL is infinite" );
  72                 } else {
  73                         $this->assertGreaterThan( 0, $curTTL, "Current TTL > 0" );
  74                         $this->assertLessThanOrEqual( $ttl, $curTTL, "Current TTL < nominal TTL" );
  75                 }
  76                 $this->assertGreaterThanOrEqual( $t - 1, $asOf, "As-of-time in range of set() time" );
  77                 $this->assertLessThanOrEqual( $t + 1, $asOf, "As-of-time in range of set() time" );
  78         }
  79
  80         public static function provideSetAndGet() {
  81                 return [
  82                         [ 14141, 3 ],
  83                         [ 3535.666, 3 ],
  84                         [ [], 3 ],
  85                         [ null, 3 ],
  86                         [ '0', 3 ],
  87                         [ (object)[ 'meow' ], 3 ],
  88                         [ INF, 3 ],
  89                         [ '', 3 ],
  90                         [ 'pizzacat', INF ],
  91                 ];
  92         }
  93
  94         /**
  95          * @covers WANObjectCache::get()
  96          * @covers WANObjectCache::makeGlobalKey()
  97          */
  98         public function testGetNotExists() {
  99                 $key = $this->cache->makeGlobalKey( 'y', wfRandomString(), 'p' );
 100                 $curTTL = null;
 101                 $value = $this->cache->get( $key, $curTTL );
 102
 103                 $this->assertFalse( $value, "Non-existing key has false value" );
 104                 $this->assertNull( $curTTL, "Non-existing key has null current TTL" );
 105         }
 106
 107         /**
 108          * @covers WANObjectCache::set()
 109          */
 110         public function testSetOver() {
 111                 $key = wfRandomString();
 112                 for ( $i = 0; $i < 3; ++$i ) {
 113                         $value = wfRandomString();
 114                         $this->cache->set( $key, $value, 3 );
 115
 116                         $this->assertEquals( $this->cache->get( $key ), $value );
 117                 }
 118         }
 119
 120         /**
 121          * @covers WANObjectCache::set()
 122          */
 123         public function testStaleSet() {
 124                 $key = wfRandomString();
 125                 $value = wfRandomString();
 126                 $this->cache->set( $key, $value, 3, [ 'since' => microtime( true ) - 30 ] );
 127
 128                 $this->assertFalse( $this->cache->get( $key ), "Stale set() value ignored" );
 129         }
 130
 131         public function testProcessCache() {
 132                 $mockWallClock = 1549343530.2053;
 133                 $this->cache->setMockTime( $mockWallClock );
 134
 135                 $hit = 0;
 136                 $callback = function () use ( &$hit ) {
 137                         ++$hit;
 138                         return 42;
 139                 };
 140                 $keys = [ wfRandomString(), wfRandomString(), wfRandomString() ];
 141                 $groups = [ 'thiscache:1', 'thatcache:1', 'somecache:1' ];
 142
 143                 foreach ( $keys as $i => $key ) {
 144                         $this->cache->getWithSetCallback(
 145                                 $key, 100, $callback, [ 'pcTTL' => 5, 'pcGroup' => $groups[$i] ] );
 146                 }
 147                 $this->assertEquals( 3, $hit );
 148
 149                 foreach ( $keys as $i => $key ) {
 150                         $this->cache->getWithSetCallback(
 151                                 $key, 100, $callback, [ 'pcTTL' => 5, 'pcGroup' => $groups[$i] ] );
 152                 }
 153                 $this->assertEquals( 3, $hit, "Values cached" );
 154
 155                 foreach ( $keys as $i => $key ) {
 156                         $this->cache->getWithSetCallback(
 157                                 "$key-2", 100, $callback, [ 'pcTTL' => 5, 'pcGroup' => $groups[$i] ] );
 158                 }
 159                 $this->assertEquals( 6, $hit );
 160
 161                 foreach ( $keys as $i => $key ) {
 162                         $this->cache->getWithSetCallback(
 163                                 "$key-2", 100, $callback, [ 'pcTTL' => 5, 'pcGroup' => $groups[$i] ] );
 164                 }
 165                 $this->assertEquals( 6, $hit, "New values cached" );
 166
 167                 foreach ( $keys as $i => $key ) {
 168                         // Should not evict from process cache
 169                         $this->cache->delete( $key );
 170                         $mockWallClock += 0.001; // cached values will be newer than tombstone
 171                         // Get into cache (specific process cache group)
 172                         $this->cache->getWithSetCallback(
 173                                 $key, 100, $callback, [ 'pcTTL' => 5, 'pcGroup' => $groups[$i] ] );
 174                 }
 175                 $this->assertEquals( 9, $hit, "Values evicted by delete()" );
 176
 177                 // Get into cache (default process cache group)
 178                 $key = reset( $keys );
 179                 $this->cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] );
 180                 $this->assertEquals( 9, $hit, "Value recently interim-cached" );
 181
 182                 $mockWallClock += 0.2; // interim key not brand new
 183                 $this->cache->clearProcessCache();
 184                 $this->cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] );
 185                 $this->assertEquals( 10, $hit, "Value calculated (interim key not recent and reset)" );
 186                 $this->cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] );
 187                 $this->assertEquals( 10, $hit, "Value process cached" );
 188
 189                 $mockWallClock += 0.2; // interim key not brand new
 190                 $outerCallback = function () use ( &$callback, $key ) {
 191                         $v = $this->cache->getWithSetCallback( $key, 100, $callback, [ 'pcTTL' => 5 ] );
 192
 193                         return 43 + $v;
 194                 };
 195                 // Outer key misses and refuses inner key process cache value
 196                 $this->cache->getWithSetCallback( "$key-miss-outer", 100, $outerCallback );
 197                 $this->assertEquals( 11, $hit, "Nested callback value process cache skipped" );
 198         }
 199
 200         /**
 201          * @dataProvider getWithSetCallback_provider
 202          * @covers WANObjectCache::getWithSetCallback()
 203          * @covers WANObjectCache::fetchOrRegenerate()
 204          * @param array $extOpts
 205          * @param bool $versioned
 206          */
 207         public function testGetWithSetCallback( array $extOpts, $versioned ) {
 208                 $cache = $this->cache;
 209
 210                 $key = wfRandomString();
 211                 $value = wfRandomString();
 212                 $cKey1 = wfRandomString();
 213                 $cKey2 = wfRandomString();
 214
 215                 $priorValue = null;
 216                 $priorAsOf = null;
 217                 $wasSet = 0;
 218                 $func = function ( $old, &$ttl, &$opts, $asOf )
 219                 use ( &$wasSet, &$priorValue, &$priorAsOf, $value ) {
 220                         ++$wasSet;
 221                         $priorValue = $old;
 222                         $priorAsOf = $asOf;
 223                         $ttl = 20; // override with another value
 224                         return $value;
 225                 };
 226
 227                 $mockWallClock = 1549343530.2053;
 228                 $priorTime = $mockWallClock; // reference time
 229                 $cache->setMockTime( $mockWallClock );
 230
 231                 $wasSet = 0;
 232                 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] + $extOpts );
 233                 $this->assertEquals( $value, $v, "Value returned" );
 234                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
 235                 $this->assertFalse( $priorValue, "No prior value" );
 236                 $this->assertNull( $priorAsOf, "No prior value" );
 237
 238                 $curTTL = null;
 239                 $cache->get( $key, $curTTL );
 240                 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' );
 241                 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' );
 242
 243                 $wasSet = 0;
 244                 $v = $cache->getWithSetCallback(
 245                         $key, 30, $func, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts );
 246                 $this->assertEquals( $value, $v, "Value returned" );
 247                 $this->assertEquals( 0, $wasSet, "Value not regenerated" );
 248
 249                 $mockWallClock += 1;
 250
 251                 $wasSet = 0;
 252                 $v = $cache->getWithSetCallback(
 253                         $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 254                 );
 255                 $this->assertEquals( $value, $v, "Value returned" );
 256                 $this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" );
 257                 $this->assertEquals( $value, $priorValue, "Has prior value" );
 258                 $this->assertInternalType( 'float', $priorAsOf, "Has prior value" );
 259                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 260                 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' );
 261                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 262                 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' );
 263
 264                 $mockWallClock += 0.2; // interim key is not brand new and check keys have past values
 265                 $priorTime = $mockWallClock; // reference time
 266                 $wasSet = 0;
 267                 $v = $cache->getWithSetCallback(
 268                         $key, 30, $func, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 269                 );
 270                 $this->assertEquals( $value, $v, "Value returned" );
 271                 $this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" );
 272                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 273                 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' );
 274                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 275                 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' );
 276
 277                 $curTTL = null;
 278                 $v = $cache->get( $key, $curTTL, [ $cKey1, $cKey2 ] );
 279                 $this->assertEquals( $value, $v, "Value returned" );
 280                 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" );
 281
 282                 $wasSet = 0;
 283                 $key = wfRandomString();
 284                 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts );
 285                 $this->assertEquals( $value, $v, "Value returned" );
 286                 $cache->delete( $key );
 287                 $v = $cache->getWithSetCallback( $key, 30, $func, [ 'pcTTL' => 5 ] + $extOpts );
 288                 $this->assertEquals( $value, $v, "Value still returned after deleted" );
 289                 $this->assertEquals( 1, $wasSet, "Value process cached while deleted" );
 290
 291                 $oldValReceived = -1;
 292                 $oldAsOfReceived = -1;
 293                 $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf )
 294                 use ( &$oldValReceived, &$oldAsOfReceived, &$wasSet ) {
 295                         ++$wasSet;
 296                         $oldValReceived = $oldVal;
 297                         $oldAsOfReceived = $oldAsOf;
 298
 299                         return 'xxx' . $wasSet;
 300                 };
 301
 302                 $mockWallClock = 1549343530.2053;
 303                 $priorTime = $mockWallClock; // reference time
 304
 305                 $wasSet = 0;
 306                 $key = wfRandomString();
 307                 $v = $cache->getWithSetCallback(
 308                         $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
 309                 $this->assertEquals( 'xxx1', $v, "Value returned" );
 310                 $this->assertEquals( false, $oldValReceived, "Callback got no stale value" );
 311                 $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" );
 312
 313                 $mockWallClock += 40;
 314                 $v = $cache->getWithSetCallback(
 315                         $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
 316                 $this->assertEquals( 'xxx2', $v, "Value still returned after expired" );
 317                 $this->assertEquals( 2, $wasSet, "Value recalculated while expired" );
 318                 $this->assertEquals( 'xxx1', $oldValReceived, "Callback got stale value" );
 319                 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got stale value" );
 320
 321                 $mockWallClock += 260;
 322                 $v = $cache->getWithSetCallback(
 323                         $key, 30, $checkFunc, [ 'staleTTL' => 50 ] + $extOpts );
 324                 $this->assertEquals( 'xxx3', $v, "Value still returned after expired" );
 325                 $this->assertEquals( 3, $wasSet, "Value recalculated while expired" );
 326                 $this->assertEquals( false, $oldValReceived, "Callback got no stale value" );
 327                 $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" );
 328
 329                 $mockWallClock = ( $priorTime - $cache::HOLDOFF_TTL - 1 );
 330                 $wasSet = 0;
 331                 $key = wfRandomString();
 332                 $checkKey = $cache->makeKey( 'template', 'X' );
 333                 $cache->touchCheckKey( $checkKey ); // init check key
 334                 $mockWallClock = $priorTime;
 335                 $v = $cache->getWithSetCallback(
 336                         $key,
 337                         $cache::TTL_INDEFINITE,
 338                         $checkFunc,
 339                         [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts
 340                 );
 341                 $this->assertEquals( 'xxx1', $v, "Value returned" );
 342                 $this->assertEquals( 1, $wasSet, "Value computed" );
 343                 $this->assertEquals( false, $oldValReceived, "Callback got no stale value" );
 344                 $this->assertEquals( null, $oldAsOfReceived, "Callback got no stale value" );
 345
 346                 $mockWallClock += $cache::TTL_HOUR; // some time passes
 347                 $v = $cache->getWithSetCallback(
 348                         $key,
 349                         $cache::TTL_INDEFINITE,
 350                         $checkFunc,
 351                         [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts
 352                 );
 353                 $this->assertEquals( 'xxx1', $v, "Cached value returned" );
 354                 $this->assertEquals( 1, $wasSet, "Cached value returned" );
 355
 356                 $cache->touchCheckKey( $checkKey ); // make key stale
 357                 $mockWallClock += 0.01; // ~1 week left of grace (barely stale to avoid refreshes)
 358
 359                 $v = $cache->getWithSetCallback(
 360                         $key,
 361                         $cache::TTL_INDEFINITE,
 362                         $checkFunc,
 363                         [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts
 364                 );
 365                 $this->assertEquals( 'xxx1', $v, "Value still returned after expired (in grace)" );
 366                 $this->assertEquals( 1, $wasSet, "Value still returned after expired (in grace)" );
 367
 368                 // Chance of refresh increase to unity as staleness approaches graceTTL
 369                 $mockWallClock += $cache::TTL_WEEK; // 8 days of being stale
 370                 $v = $cache->getWithSetCallback(
 371                         $key,
 372                         $cache::TTL_INDEFINITE,
 373                         $checkFunc,
 374                         [ 'graceTTL' => $cache::TTL_WEEK, 'checkKeys' => [ $checkKey ] ] + $extOpts
 375                 );
 376                 $this->assertEquals( 'xxx2', $v, "Value was recomputed (past grace)" );
 377                 $this->assertEquals( 2, $wasSet, "Value was recomputed (past grace)" );
 378                 $this->assertEquals( 'xxx1', $oldValReceived, "Callback got post-grace stale value" );
 379                 $this->assertNotEquals( null, $oldAsOfReceived, "Callback got post-grace stale value" );
 380         }
 381
 382         /**
 383          * @dataProvider getWithSetCallback_provider
 384          * @covers WANObjectCache::getWithSetCallback()
 385          * @covers WANObjectCache::fetchOrRegenerate()
 386          * @param array $extOpts
 387          * @param bool $versioned
 388          */
 389         function testGetWithSetcallback_touched( array $extOpts, $versioned ) {
 390                 $cache = $this->cache;
 391
 392                 $mockWallClock = 1549343530.2053;
 393                 $cache->setMockTime( $mockWallClock );
 394
 395                 $checkFunc = function ( $oldVal, &$ttl, array $setOpts, $oldAsOf )
 396                 use ( &$wasSet ) {
 397                         ++$wasSet;
 398
 399                         return 'xxx' . $wasSet;
 400                 };
 401
 402                 $key = wfRandomString();
 403                 $wasSet = 0;
 404                 $touched = null;
 405                 $touchedCallback = function () use ( &$touched ) {
 406                         return $touched;
 407                 };
 408                 $v = $cache->getWithSetCallback(
 409                         $key,
 410                         $cache::TTL_INDEFINITE,
 411                         $checkFunc,
 412                         [ 'touchedCallback' => $touchedCallback ] + $extOpts
 413                 );
 414                 $mockWallClock += 60;
 415                 $v = $cache->getWithSetCallback(
 416                         $key,
 417                         $cache::TTL_INDEFINITE,
 418                         $checkFunc,
 419                         [ 'touchedCallback' => $touchedCallback ] + $extOpts
 420                 );
 421                 $this->assertEquals( 'xxx1', $v, "Value was computed once" );
 422                 $this->assertEquals( 1, $wasSet, "Value was computed once" );
 423
 424                 $touched = $mockWallClock - 10;
 425                 $v = $cache->getWithSetCallback(
 426                         $key,
 427                         $cache::TTL_INDEFINITE,
 428                         $checkFunc,
 429                         [ 'touchedCallback' => $touchedCallback ] + $extOpts
 430                 );
 431                 $v = $cache->getWithSetCallback(
 432                         $key,
 433                         $cache::TTL_INDEFINITE,
 434                         $checkFunc,
 435                         [ 'touchedCallback' => $touchedCallback ] + $extOpts
 436                 );
 437                 $this->assertEquals( 'xxx2', $v, "Value was recomputed once" );
 438                 $this->assertEquals( 2, $wasSet, "Value was recomputed once" );
 439         }
 440
 441         public static function getWithSetCallback_provider() {
 442                 return [
 443                         [ [], false ],
 444                         [ [ 'version' => 1 ], true ]
 445                 ];
 446         }
 447
 448         public function testPreemtiveRefresh() {
 449                 $value = 'KatCafe';
 450                 $wasSet = 0;
 451                 $func = function ( $old, &$ttl, &$opts, $asOf ) use ( &$wasSet, &$value )
 452                 {
 453                         ++$wasSet;
 454                         return $value;
 455                 };
 456
 457                 $cache = new NearExpiringWANObjectCache( [ 'cache' => new HashBagOStuff() ] );
 458                 $mockWallClock = 1549343530.2053;
 459                 $cache->setMockTime( $mockWallClock );
 460
 461                 $wasSet = 0;
 462                 $key = wfRandomString();
 463                 $opts = [ 'lowTTL' => 30 ];
 464                 $v = $cache->getWithSetCallback( $key, 20, $func, $opts );
 465                 $this->assertEquals( $value, $v, "Value returned" );
 466                 $this->assertEquals( 1, $wasSet, "Value calculated" );
 467
 468                 $mockWallClock += 0.2; // interim key is not brand new
 469                 $v = $cache->getWithSetCallback( $key, 20, $func, $opts );
 470                 $this->assertEquals( 2, $wasSet, "Value re-calculated" );
 471
 472                 $wasSet = 0;
 473                 $key = wfRandomString();
 474                 $opts = [ 'lowTTL' => 1 ];
 475                 $v = $cache->getWithSetCallback( $key, 30, $func, $opts );
 476                 $this->assertEquals( $value, $v, "Value returned" );
 477                 $this->assertEquals( 1, $wasSet, "Value calculated" );
 478                 $v = $cache->getWithSetCallback( $key, 30, $func, $opts );
 479                 $this->assertEquals( 1, $wasSet, "Value cached" );
 480
 481                 $asycList = [];
 482                 $asyncHandler = function ( $callback ) use ( &$asycList ) {
 483                         $asycList[] = $callback;
 484                 };
 485                 $cache = new NearExpiringWANObjectCache( [
 486                         'cache'        => new HashBagOStuff(),
 487                         'asyncHandler' => $asyncHandler
 488                 ] );
 489
 490                 $mockWallClock = 1549343530.2053;
 491                 $priorTime = $mockWallClock; // reference time
 492                 $cache->setMockTime( $mockWallClock );
 493
 494                 $wasSet = 0;
 495                 $key = wfRandomString();
 496                 $opts = [ 'lowTTL' => 100 ];
 497                 $v = $cache->getWithSetCallback( $key, 300, $func, $opts );
 498                 $this->assertEquals( $value, $v, "Value returned" );
 499                 $this->assertEquals( 1, $wasSet, "Value calculated" );
 500                 $v = $cache->getWithSetCallback( $key, 300, $func, $opts );
 501                 $this->assertEquals( 1, $wasSet, "Cached value used" );
 502                 $this->assertEquals( $v, $value, "Value cached" );
 503
 504                 $mockWallClock += 250;
 505                 $v = $cache->getWithSetCallback( $key, 300, $func, $opts );
 506                 $this->assertEquals( $value, $v, "Value returned" );
 507                 $this->assertEquals( 1, $wasSet, "Stale value used" );
 508                 $this->assertEquals( 1, count( $asycList ), "Refresh deferred." );
 509                 $value = 'NewCatsInTown'; // change callback return value
 510                 $asycList[0](); // run the refresh callback
 511                 $asycList = [];
 512                 $this->assertEquals( 2, $wasSet, "Value calculated at later time" );
 513                 $this->assertEquals( 0, count( $asycList ), "No deferred refreshes added." );
 514                 $v = $cache->getWithSetCallback( $key, 300, $func, $opts );
 515                 $this->assertEquals( $value, $v, "New value stored" );
 516
 517                 $cache = new PopularityRefreshingWANObjectCache( [
 518                         'cache'   => new HashBagOStuff()
 519                 ] );
 520
 521                 $mockWallClock = $priorTime;
 522                 $cache->setMockTime( $mockWallClock );
 523
 524                 $wasSet = 0;
 525                 $key = wfRandomString();
 526                 $opts = [ 'hotTTR' => 900 ];
 527                 $v = $cache->getWithSetCallback( $key, 60, $func, $opts );
 528                 $this->assertEquals( $value, $v, "Value returned" );
 529                 $this->assertEquals( 1, $wasSet, "Value calculated" );
 530
 531                 $mockWallClock += 30;
 532
 533                 $v = $cache->getWithSetCallback( $key, 60, $func, $opts );
 534                 $this->assertEquals( 1, $wasSet, "Value cached" );
 535
 536                 $mockWallClock = $priorTime;
 537                 $wasSet = 0;
 538                 $key = wfRandomString();
 539                 $opts = [ 'hotTTR' => 10 ];
 540                 $v = $cache->getWithSetCallback( $key, 60, $func, $opts );
 541                 $this->assertEquals( $value, $v, "Value returned" );
 542                 $this->assertEquals( 1, $wasSet, "Value calculated" );
 543
 544                 $mockWallClock += 30;
 545
 546                 $v = $cache->getWithSetCallback( $key, 60, $func, $opts );
 547                 $this->assertEquals( $value, $v, "Value returned" );
 548                 $this->assertEquals( 2, $wasSet, "Value re-calculated" );
 549         }
 550
 551         /**
 552          * @dataProvider getMultiWithSetCallback_provider
 553          * @covers WANObjectCache::getMultiWithSetCallback
 554          * @covers WANObjectCache::makeMultiKeys
 555          * @covers WANObjectCache::getMulti
 556          * @param array $extOpts
 557          * @param bool $versioned
 558          */
 559         public function testGetMultiWithSetCallback( array $extOpts, $versioned ) {
 560                 $cache = $this->cache;
 561
 562                 $keyA = wfRandomString();
 563                 $keyB = wfRandomString();
 564                 $keyC = wfRandomString();
 565                 $cKey1 = wfRandomString();
 566                 $cKey2 = wfRandomString();
 567
 568                 $priorValue = null;
 569                 $priorAsOf = null;
 570                 $wasSet = 0;
 571                 $genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use (
 572                         &$wasSet, &$priorValue, &$priorAsOf
 573                 ) {
 574                         ++$wasSet;
 575                         $priorValue = $old;
 576                         $priorAsOf = $asOf;
 577                         $ttl = 20; // override with another value
 578                         return "@$id$";
 579                 };
 580
 581                 $mockWallClock = 1549343530.2053;
 582                 $priorTime = $mockWallClock; // reference time
 583                 $cache->setMockTime( $mockWallClock );
 584
 585                 $wasSet = 0;
 586                 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] );
 587                 $value = "@3353$";
 588                 $v = $cache->getMultiWithSetCallback(
 589                         $keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts );
 590                 $this->assertEquals( $value, $v[$keyA], "Value returned" );
 591                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
 592                 $this->assertFalse( $priorValue, "No prior value" );
 593                 $this->assertNull( $priorAsOf, "No prior value" );
 594
 595                 $curTTL = null;
 596                 $cache->get( $keyA, $curTTL );
 597                 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' );
 598                 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' );
 599
 600                 $wasSet = 0;
 601                 $value = "@efef$";
 602                 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
 603                 $v = $cache->getMultiWithSetCallback(
 604                         $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts );
 605                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 606                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
 607                 $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" );
 608
 609                 $v = $cache->getMultiWithSetCallback(
 610                         $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' => 5 ] + $extOpts );
 611                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 612                 $this->assertEquals( 1, $wasSet, "Value not regenerated" );
 613                 $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" );
 614
 615                 $mockWallClock += 1;
 616
 617                 $wasSet = 0;
 618                 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
 619                 $v = $cache->getMultiWithSetCallback(
 620                         $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 621                 );
 622                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 623                 $this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" );
 624                 $this->assertEquals( $value, $priorValue, "Has prior value" );
 625                 $this->assertInternalType( 'float', $priorAsOf, "Has prior value" );
 626                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 627                 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' );
 628                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 629                 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' );
 630
 631                 $mockWallClock += 0.01;
 632                 $priorTime = $mockWallClock;
 633                 $value = "@43636$";
 634                 $wasSet = 0;
 635                 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] );
 636                 $v = $cache->getMultiWithSetCallback(
 637                         $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 638                 );
 639                 $this->assertEquals( $value, $v[$keyC], "Value returned" );
 640                 $this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" );
 641                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 642                 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' );
 643                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 644                 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' );
 645
 646                 $curTTL = null;
 647                 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] );
 648                 $this->assertEquals( $value, $v, "Value returned" );
 649                 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" );
 650
 651                 $wasSet = 0;
 652                 $key = wfRandomString();
 653                 $keyedIds = new ArrayIterator( [ $key => 242424 ] );
 654                 $v = $cache->getMultiWithSetCallback(
 655                         $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
 656                 $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value returned" );
 657                 $cache->delete( $key );
 658                 $keyedIds = new ArrayIterator( [ $key => 242424 ] );
 659                 $v = $cache->getMultiWithSetCallback(
 660                         $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
 661                 $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" );
 662                 $this->assertEquals( 1, $wasSet, "Value process cached while deleted" );
 663
 664                 $calls = 0;
 665                 $ids = [ 1, 2, 3, 4, 5, 6 ];
 666                 $keyFunc = function ( $id, WANObjectCache $wanCache ) {
 667                         return $wanCache->makeKey( 'test', $id );
 668                 };
 669                 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc );
 670                 $genFunc = function ( $id, $oldValue, &$ttl, array &$setops ) use ( &$calls ) {
 671                         ++$calls;
 672
 673                         return "val-{$id}";
 674                 };
 675                 $values = $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc );
 676
 677                 $this->assertEquals(
 678                         [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ],
 679                         array_values( $values ),
 680                         "Correct values in correct order"
 681                 );
 682                 $this->assertEquals(
 683                         array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ),
 684                         array_keys( $values ),
 685                         "Correct keys in correct order"
 686                 );
 687                 $this->assertEquals( count( $ids ), $calls );
 688
 689                 $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc );
 690                 $this->assertEquals( count( $ids ), $calls, "Values cached" );
 691
 692                 // Mock the BagOStuff to assure only one getMulti() call given process caching
 693                 $localBag = $this->getMockBuilder( HashBagOStuff::class )
 694                         ->setMethods( [ 'getMulti' ] )->getMock();
 695                 $localBag->expects( $this->exactly( 1 ) )->method( 'getMulti' )->willReturn( [
 696                         WANObjectCache::VALUE_KEY_PREFIX . 'k1' => 'val-id1',
 697                         WANObjectCache::VALUE_KEY_PREFIX . 'k2' => 'val-id2'
 698                 ] );
 699                 $wanCache = new WANObjectCache( [ 'cache' => $localBag ] );
 700
 701                 // Warm the process cache
 702                 $keyedIds = new ArrayIterator( [ 'k1' => 'id1', 'k2' => 'id2' ] );
 703                 $this->assertEquals(
 704                         [ 'k1' => 'val-id1', 'k2' => 'val-id2' ],
 705                         $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] )
 706                 );
 707                 // Use the process cache
 708                 $this->assertEquals(
 709                         [ 'k1' => 'val-id1', 'k2' => 'val-id2' ],
 710                         $wanCache->getMultiWithSetCallback( $keyedIds, 10, $genFunc, [ 'pcTTL' => 5 ] )
 711                 );
 712         }
 713
 714         public static function getMultiWithSetCallback_provider() {
 715                 return [
 716                         [ [], false ],
 717                         [ [ 'version' => 1 ], true ]
 718                 ];
 719         }
 720
 721         /**
 722          * @dataProvider getMultiWithUnionSetCallback_provider
 723          * @covers WANObjectCache::getMultiWithUnionSetCallback()
 724          * @covers WANObjectCache::makeMultiKeys()
 725          * @param array $extOpts
 726          * @param bool $versioned
 727          */
 728         public function testGetMultiWithUnionSetCallback( array $extOpts, $versioned ) {
 729                 $cache = $this->cache;
 730
 731                 $keyA = wfRandomString();
 732                 $keyB = wfRandomString();
 733                 $keyC = wfRandomString();
 734                 $cKey1 = wfRandomString();
 735                 $cKey2 = wfRandomString();
 736
 737                 $wasSet = 0;
 738                 $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use (
 739                         &$wasSet, &$priorValue, &$priorAsOf
 740                 ) {
 741                         $newValues = [];
 742                         foreach ( $ids as $id ) {
 743                                 ++$wasSet;
 744                                 $newValues[$id] = "@$id$";
 745                                 $ttls[$id] = 20; // override with another value
 746                         }
 747
 748                         return $newValues;
 749                 };
 750
 751                 $mockWallClock = 1549343530.2053;
 752                 $priorTime = $mockWallClock; // reference time
 753                 $cache->setMockTime( $mockWallClock );
 754
 755                 $wasSet = 0;
 756                 $keyedIds = new ArrayIterator( [ $keyA => 3353 ] );
 757                 $value = "@3353$";
 758                 $v = $cache->getMultiWithUnionSetCallback(
 759                         $keyedIds, 30, $genFunc, $extOpts );
 760                 $this->assertEquals( $value, $v[$keyA], "Value returned" );
 761                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
 762
 763                 $curTTL = null;
 764                 $cache->get( $keyA, $curTTL );
 765                 $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between 19-20 (overriden)' );
 766                 $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL between 19-20 (overriden)' );
 767
 768                 $wasSet = 0;
 769                 $value = "@efef$";
 770                 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
 771                 $v = $cache->getMultiWithUnionSetCallback(
 772                         $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts );
 773                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 774                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
 775                 $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" );
 776
 777                 $v = $cache->getMultiWithUnionSetCallback(
 778                         $keyedIds, 30, $genFunc, [ 'lowTTL' => 0 ] + $extOpts );
 779                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 780                 $this->assertEquals( 1, $wasSet, "Value not regenerated" );
 781                 $this->assertEquals( 0, $cache->getWarmupKeyMisses(), "Keys warmed in warmup cache" );
 782
 783                 $mockWallClock += 1;
 784
 785                 $wasSet = 0;
 786                 $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
 787                 $v = $cache->getMultiWithUnionSetCallback(
 788                         $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 789                 );
 790                 $this->assertEquals( $value, $v[$keyB], "Value returned" );
 791                 $this->assertEquals( 1, $wasSet, "Value regenerated due to check keys" );
 792                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 793                 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys generated on miss' );
 794                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 795                 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys generated on miss' );
 796
 797                 $mockWallClock += 0.01;
 798                 $priorTime = $mockWallClock;
 799                 $value = "@43636$";
 800                 $wasSet = 0;
 801                 $keyedIds = new ArrayIterator( [ $keyC => 43636 ] );
 802                 $v = $cache->getMultiWithUnionSetCallback(
 803                         $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1, $cKey2 ] ] + $extOpts
 804                 );
 805                 $this->assertEquals( $value, $v[$keyC], "Value returned" );
 806                 $this->assertEquals( 1, $wasSet, "Value regenerated due to still-recent check keys" );
 807                 $t1 = $cache->getCheckKeyTime( $cKey1 );
 808                 $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did not change again' );
 809                 $t2 = $cache->getCheckKeyTime( $cKey2 );
 810                 $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did not change again' );
 811
 812                 $curTTL = null;
 813                 $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] );
 814                 $this->assertEquals( $value, $v, "Value returned" );
 815                 $this->assertLessThanOrEqual( 0, $curTTL, "Value has current TTL < 0 due to check keys" );
 816
 817                 $wasSet = 0;
 818                 $key = wfRandomString();
 819                 $keyedIds = new ArrayIterator( [ $key => 242424 ] );
 820                 $v = $cache->getMultiWithUnionSetCallback(
 821                         $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
 822                 $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value returned" );
 823                 $cache->delete( $key );
 824                 $keyedIds = new ArrayIterator( [ $key => 242424 ] );
 825                 $v = $cache->getMultiWithUnionSetCallback(
 826                         $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
 827                 $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value still returned after deleted" );
 828                 $this->assertEquals( 1, $wasSet, "Value process cached while deleted" );
 829
 830                 $calls = 0;
 831                 $ids = [ 1, 2, 3, 4, 5, 6 ];
 832                 $keyFunc = function ( $id, WANObjectCache $wanCache ) {
 833                         return $wanCache->makeKey( 'test', $id );
 834                 };
 835                 $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc );
 836                 $genFunc = function ( array $ids, array &$ttls, array &$setOpts ) use ( &$calls ) {
 837                         $newValues = [];
 838                         foreach ( $ids as $id ) {
 839                                 ++$calls;
 840                                 $newValues[$id] = "val-{$id}";
 841                         }
 842
 843                         return $newValues;
 844                 };
 845                 $values = $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc );
 846
 847                 $this->assertEquals(
 848                         [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6" ],
 849                         array_values( $values ),
 850                         "Correct values in correct order"
 851                 );
 852                 $this->assertEquals(
 853                         array_map( $keyFunc, $ids, array_fill( 0, count( $ids ), $this->cache ) ),
 854                         array_keys( $values ),
 855                         "Correct keys in correct order"
 856                 );
 857                 $this->assertEquals( count( $ids ), $calls );
 858
 859                 $cache->getMultiWithUnionSetCallback( $keyedIds, 10, $genFunc );
 860                 $this->assertEquals( count( $ids ), $calls, "Values cached" );
 861         }
 862
 863         public static function getMultiWithUnionSetCallback_provider() {
 864                 return [
 865                         [ [], false ],
 866                         [ [ 'version' => 1 ], true ]
 867                 ];
 868         }
 869
 870         /**
 871          * @covers WANObjectCache::getWithSetCallback()
 872          * @covers WANObjectCache::fetchOrRegenerate()
 873          */
 874         public function testLockTSE() {
 875                 $cache = $this->cache;
 876                 $key = wfRandomString();
 877                 $value = wfRandomString();
 878
 879                 $mockWallClock = 1549343530.2053;
 880                 $cache->setMockTime( $mockWallClock );
 881
 882                 $calls = 0;
 883                 $func = function () use ( &$calls, $value, $cache, $key ) {
 884                         ++$calls;
 885                         return $value;
 886                 };
 887
 888                 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'lockTSE' => 5 ] );
 889                 $this->assertEquals( $value, $ret );
 890                 $this->assertEquals( 1, $calls, 'Value was populated' );
 891
 892                 // Acquire the mutex to verify that getWithSetCallback uses lockTSE properly
 893                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
 894
 895                 $checkKeys = [ wfRandomString() ]; // new check keys => force misses
 896                 $ret = $cache->getWithSetCallback( $key, 30, $func,
 897                         [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] );
 898                 $this->assertEquals( $value, $ret, 'Old value used' );
 899                 $this->assertEquals( 1, $calls, 'Callback was not used' );
 900
 901                 $cache->delete( $key );
 902                 $mockWallClock += 0.001; // cached values will be newer than tombstone
 903                 $ret = $cache->getWithSetCallback( $key, 30, $func,
 904                         [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] );
 905                 $this->assertEquals( $value, $ret, 'Callback was used; interim saved' );
 906                 $this->assertEquals( 2, $calls, 'Callback was used; interim saved' );
 907
 908                 $ret = $cache->getWithSetCallback( $key, 30, $func,
 909                         [ 'lockTSE' => 5, 'checkKeys' => $checkKeys ] );
 910                 $this->assertEquals( $value, $ret, 'Callback was not used; used interim (mutex failed)' );
 911                 $this->assertEquals( 2, $calls, 'Callback was not used; used interim (mutex failed)' );
 912         }
 913
 914         /**
 915          * @covers WANObjectCache::getWithSetCallback()
 916          * @covers WANObjectCache::fetchOrRegenerate()
 917          * @covers WANObjectCache::set()
 918          */
 919         public function testLockTSESlow() {
 920                 $cache = $this->cache;
 921                 $key = wfRandomString();
 922                 $key2 = wfRandomString();
 923                 $value = wfRandomString();
 924
 925                 $mockWallClock = 1549343530.2053;
 926                 $cache->setMockTime( $mockWallClock );
 927
 928                 $calls = 0;
 929                 $func = function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value, &$mockWallClock ) {
 930                         ++$calls;
 931                         $setOpts['since'] = $mockWallClock - 10;
 932                         return $value;
 933                 };
 934
 935                 // Value should be given a low logical TTL due to snapshot lag
 936                 $curTTL = null;
 937                 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
 938                 $this->assertEquals( $value, $ret );
 939                 $this->assertEquals( $value, $cache->get( $key, $curTTL ), 'Value was populated' );
 940                 $this->assertEquals( 1, $curTTL, 'Value has reduced logical TTL', 0.01 );
 941                 $this->assertEquals( 1, $calls, 'Value was generated' );
 942
 943                 $mockWallClock += 2; // low logical TTL expired
 944
 945                 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
 946                 $this->assertEquals( $value, $ret );
 947                 $this->assertEquals( 2, $calls, 'Callback used (mutex acquired)' );
 948
 949                 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
 950                 $this->assertEquals( $value, $ret );
 951                 $this->assertEquals( 2, $calls, 'Callback was not used (interim value used)' );
 952
 953                 $mockWallClock += 2; // low logical TTL expired
 954                 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly
 955                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
 956
 957                 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
 958                 $this->assertEquals( $value, $ret );
 959                 $this->assertEquals( 2, $calls, 'Callback was not used (mutex not acquired)' );
 960
 961                 $mockWallClock += 301; // physical TTL expired
 962                 // Acquire a lock to verify that getWithSetCallback uses lockTSE properly
 963                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
 964
 965                 $ret = $cache->getWithSetCallback( $key, 300, $func, [ 'lockTSE' => 5 ] );
 966                 $this->assertEquals( $value, $ret );
 967                 $this->assertEquals( 3, $calls, 'Callback was used (mutex not acquired, not in cache)' );
 968
 969                 $calls = 0;
 970                 $func2 = function ( $oldValue, &$ttl, &$setOpts ) use ( &$calls, $value ) {
 971                         ++$calls;
 972                         $setOpts['lag'] = 15;
 973                         return $value;
 974                 };
 975
 976                 // Value should be given a low logical TTL due to replication lag
 977                 $curTTL = null;
 978                 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] );
 979                 $this->assertEquals( $value, $ret );
 980                 $this->assertEquals( $value, $cache->get( $key2, $curTTL ), 'Value was populated' );
 981                 $this->assertEquals( 30, $curTTL, 'Value has reduced logical TTL', 0.01 );
 982                 $this->assertEquals( 1, $calls, 'Value was generated' );
 983
 984                 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] );
 985                 $this->assertEquals( $value, $ret );
 986                 $this->assertEquals( 1, $calls, 'Callback was used (not expired)' );
 987
 988                 $mockWallClock += 31;
 989
 990                 $ret = $cache->getWithSetCallback( $key2, 300, $func2, [ 'lockTSE' => 5 ] );
 991                 $this->assertEquals( $value, $ret );
 992                 $this->assertEquals( 2, $calls, 'Callback was used (mutex acquired)' );
 993         }
 994
 995         /**
 996          * @covers WANObjectCache::getWithSetCallback()
 997          * @covers WANObjectCache::fetchOrRegenerate()
 998          */
 999         public function testBusyValue() {
1000                 $cache = $this->cache;
1001                 $key = wfRandomString();
1002                 $value = wfRandomString();
1003                 $busyValue = wfRandomString();
1004
1005                 $mockWallClock = 1549343530.2053;
1006                 $cache->setMockTime( $mockWallClock );
1007
1008                 $calls = 0;
1009                 $func = function () use ( &$calls, $value, $cache, $key ) {
1010                         ++$calls;
1011                         return $value;
1012                 };
1013
1014                 $ret = $cache->getWithSetCallback( $key, 30, $func, [ 'busyValue' => $busyValue ] );
1015                 $this->assertEquals( $value, $ret );
1016                 $this->assertEquals( 1, $calls, 'Value was populated' );
1017
1018                 $mockWallClock += 0.2; // interim keys not brand new
1019
1020                 // Acquire a lock to verify that getWithSetCallback uses busyValue properly
1021                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
1022
1023                 $checkKeys = [ wfRandomString() ]; // new check keys => force misses
1024                 $ret = $cache->getWithSetCallback( $key, 30, $func,
1025                         [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
1026                 $this->assertEquals( $value, $ret, 'Callback used' );
1027                 $this->assertEquals( 2, $calls, 'Callback used' );
1028
1029                 $ret = $cache->getWithSetCallback( $key, 30, $func,
1030                         [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
1031                 $this->assertEquals( $value, $ret, 'Old value used' );
1032                 $this->assertEquals( 2, $calls, 'Callback was not used' );
1033
1034                 $cache->delete( $key ); // no value at all anymore and still locked
1035                 $ret = $cache->getWithSetCallback( $key, 30, $func,
1036                         [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
1037                 $this->assertEquals( $busyValue, $ret, 'Callback was not used; used busy value' );
1038                 $this->assertEquals( 2, $calls, 'Callback was not used; used busy value' );
1039
1040                 $this->internalCache->delete( $cache::MUTEX_KEY_PREFIX . $key );
1041                 $mockWallClock += 0.001; // cached values will be newer than tombstone
1042                 $ret = $cache->getWithSetCallback( $key, 30, $func,
1043                         [ 'lockTSE' => 30, 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
1044                 $this->assertEquals( $value, $ret, 'Callback was used; saved interim' );
1045                 $this->assertEquals( 3, $calls, 'Callback was used; saved interim' );
1046
1047                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
1048                 $ret = $cache->getWithSetCallback( $key, 30, $func,
1049                         [ 'busyValue' => $busyValue, 'checkKeys' => $checkKeys ] );
1050                 $this->assertEquals( $value, $ret, 'Callback was not used; used interim' );
1051                 $this->assertEquals( 3, $calls, 'Callback was not used; used interim' );
1052         }
1053
1054         /**
1055          * @covers WANObjectCache::getMulti()
1056          */
1057         public function testGetMulti() {
1058                 $cache = $this->cache;
1059
1060                 $value1 = [ 'this' => 'is', 'a' => 'test' ];
1061                 $value2 = [ 'this' => 'is', 'another' => 'test' ];
1062
1063                 $key1 = wfRandomString();
1064                 $key2 = wfRandomString();
1065                 $key3 = wfRandomString();
1066
1067                 $mockWallClock = 1549343530.2053;
1068                 $priorTime = $mockWallClock; // reference time
1069                 $cache->setMockTime( $mockWallClock );
1070
1071                 $cache->set( $key1, $value1, 5 );
1072                 $cache->set( $key2, $value2, 10 );
1073
1074                 $curTTLs = [];
1075                 $this->assertSame(
1076                         [ $key1 => $value1, $key2 => $value2 ],
1077                         $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs ),
1078                         'Result array populated'
1079                 );
1080
1081                 $this->assertEquals( 2, count( $curTTLs ), "Two current TTLs in array" );
1082                 $this->assertGreaterThan( 0, $curTTLs[$key1], "Key 1 has current TTL > 0" );
1083                 $this->assertGreaterThan( 0, $curTTLs[$key2], "Key 2 has current TTL > 0" );
1084
1085                 $cKey1 = wfRandomString();
1086                 $cKey2 = wfRandomString();
1087
1088                 $mockWallClock += 1;
1089
1090                 $curTTLs = [];
1091                 $this->assertSame(
1092                         [ $key1 => $value1, $key2 => $value2 ],
1093                         $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ),
1094                         "Result array populated even with new check keys"
1095                 );
1096                 $t1 = $cache->getCheckKeyTime( $cKey1 );
1097                 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key 1 generated on miss' );
1098                 $t2 = $cache->getCheckKeyTime( $cKey2 );
1099                 $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check key 2 generated on miss' );
1100                 $this->assertEquals( 2, count( $curTTLs ), "Current TTLs array set" );
1101                 $this->assertLessThanOrEqual( 0, $curTTLs[$key1], 'Key 1 has current TTL <= 0' );
1102                 $this->assertLessThanOrEqual( 0, $curTTLs[$key2], 'Key 2 has current TTL <= 0' );
1103
1104                 $mockWallClock += 1;
1105
1106                 $curTTLs = [];
1107                 $this->assertEquals(
1108                         [ $key1 => $value1, $key2 => $value2 ],
1109                         $cache->getMulti( [ $key1, $key2, $key3 ], $curTTLs, [ $cKey1, $cKey2 ] ),
1110                         "Result array still populated even with new check keys"
1111                 );
1112                 $this->assertEquals( 2, count( $curTTLs ), "Current TTLs still array set" );
1113                 $this->assertLessThan( 0, $curTTLs[$key1], 'Key 1 has negative current TTL' );
1114                 $this->assertLessThan( 0, $curTTLs[$key2], 'Key 2 has negative current TTL' );
1115         }
1116
1117         /**
1118          * @covers WANObjectCache::getMulti()
1119          * @covers WANObjectCache::processCheckKeys()
1120          */
1121         public function testGetMultiCheckKeys() {
1122                 $cache = $this->cache;
1123
1124                 $checkAll = wfRandomString();
1125                 $check1 = wfRandomString();
1126                 $check2 = wfRandomString();
1127                 $check3 = wfRandomString();
1128                 $value1 = wfRandomString();
1129                 $value2 = wfRandomString();
1130
1131                 $mockWallClock = 1549343530.2053;
1132                 $cache->setMockTime( $mockWallClock );
1133
1134                 // Fake initial check key to be set in the past. Otherwise we'd have to sleep for
1135                 // several seconds during the test to assert the behaviour.
1136                 foreach ( [ $checkAll, $check1, $check2 ] as $checkKey ) {
1137                         $cache->touchCheckKey( $checkKey, WANObjectCache::HOLDOFF_NONE );
1138                 }
1139
1140                 $mockWallClock += 0.100;
1141
1142                 $cache->set( 'key1', $value1, 10 );
1143                 $cache->set( 'key2', $value2, 10 );
1144
1145                 $curTTLs = [];
1146                 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [
1147                         'key1' => $check1,
1148                         $checkAll,
1149                         'key2' => $check2,
1150                         'key3' => $check3,
1151                 ] );
1152                 $this->assertSame(
1153                         [ 'key1' => $value1, 'key2' => $value2 ],
1154                         $result,
1155                         'Initial values'
1156                 );
1157                 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key1'], 'Initial ttls' );
1158                 $this->assertLessThanOrEqual( 10.5, $curTTLs['key1'], 'Initial ttls' );
1159                 $this->assertGreaterThanOrEqual( 9.5, $curTTLs['key2'], 'Initial ttls' );
1160                 $this->assertLessThanOrEqual( 10.5, $curTTLs['key2'], 'Initial ttls' );
1161
1162                 $mockWallClock += 0.100;
1163                 $cache->touchCheckKey( $check1 );
1164
1165                 $curTTLs = [];
1166                 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [
1167                         'key1' => $check1,
1168                         $checkAll,
1169                         'key2' => $check2,
1170                         'key3' => $check3,
1171                 ] );
1172                 $this->assertSame(
1173                         [ 'key1' => $value1, 'key2' => $value2 ],
1174                         $result,
1175                         'key1 expired by check1, but value still provided'
1176                 );
1177                 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 TTL expired' );
1178                 $this->assertGreaterThan( 0, $curTTLs['key2'], 'key2 still valid' );
1179
1180                 $cache->touchCheckKey( $checkAll );
1181
1182                 $curTTLs = [];
1183                 $result = $cache->getMulti( [ 'key1', 'key2', 'key3' ], $curTTLs, [
1184                         'key1' => $check1,
1185                         $checkAll,
1186                         'key2' => $check2,
1187                         'key3' => $check3,
1188                 ] );
1189                 $this->assertEquals(
1190                         [ 'key1' => $value1, 'key2' => $value2 ],
1191                         $result,
1192                         'All keys expired by checkAll, but value still provided'
1193                 );
1194                 $this->assertLessThan( 0, $curTTLs['key1'], 'key1 expired by checkAll' );
1195                 $this->assertLessThan( 0, $curTTLs['key2'], 'key2 expired by checkAll' );
1196         }
1197
1198         /**
1199          * @covers WANObjectCache::get()
1200          * @covers WANObjectCache::processCheckKeys()
1201          */
1202         public function testCheckKeyInitHoldoff() {
1203                 $cache = $this->cache;
1204
1205                 for ( $i = 0; $i < 500; ++$i ) {
1206                         $key = wfRandomString();
1207                         $checkKey = wfRandomString();
1208                         // miss, set, hit
1209                         $cache->get( $key, $curTTL, [ $checkKey ] );
1210                         $cache->set( $key, 'val', 10 );
1211                         $curTTL = null;
1212                         $v = $cache->get( $key, $curTTL, [ $checkKey ] );
1213
1214                         $this->assertEquals( 'val', $v );
1215                         $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (miss/set/hit)" );
1216                 }
1217
1218                 for ( $i = 0; $i < 500; ++$i ) {
1219                         $key = wfRandomString();
1220                         $checkKey = wfRandomString();
1221                         // set, hit
1222                         $cache->set( $key, 'val', 10 );
1223                         $curTTL = null;
1224                         $v = $cache->get( $key, $curTTL, [ $checkKey ] );
1225
1226                         $this->assertEquals( 'val', $v );
1227                         $this->assertLessThan( 0, $curTTL, "Step $i: CTL < 0 (set/hit)" );
1228                 }
1229         }
1230
1231         /**
1232          * @covers WANObjectCache::delete
1233          * @covers WANObjectCache::relayDelete
1234          * @covers WANObjectCache::relayPurge
1235          */
1236         public function testDelete() {
1237                 $key = wfRandomString();
1238                 $value = wfRandomString();
1239                 $this->cache->set( $key, $value );
1240
1241                 $curTTL = null;
1242                 $v = $this->cache->get( $key, $curTTL );
1243                 $this->assertEquals( $value, $v, "Key was created with value" );
1244                 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" );
1245
1246                 $this->cache->delete( $key );
1247
1248                 $curTTL = null;
1249                 $v = $this->cache->get( $key, $curTTL );
1250                 $this->assertFalse( $v, "Deleted key has false value" );
1251                 $this->assertLessThan( 0, $curTTL, "Deleted key has current TTL < 0" );
1252
1253                 $this->cache->set( $key, $value . 'more' );
1254                 $v = $this->cache->get( $key, $curTTL );
1255                 $this->assertFalse( $v, "Deleted key is tombstoned and has false value" );
1256                 $this->assertLessThan( 0, $curTTL, "Deleted key is tombstoned and has current TTL < 0" );
1257
1258                 $this->cache->set( $key, $value );
1259                 $this->cache->delete( $key, WANObjectCache::HOLDOFF_NONE );
1260
1261                 $curTTL = null;
1262                 $v = $this->cache->get( $key, $curTTL );
1263                 $this->assertFalse( $v, "Deleted key has false value" );
1264                 $this->assertNull( $curTTL, "Deleted key has null current TTL" );
1265
1266                 $this->cache->set( $key, $value );
1267                 $v = $this->cache->get( $key, $curTTL );
1268                 $this->assertEquals( $value, $v, "Key was created with value" );
1269                 $this->assertGreaterThan( 0, $curTTL, "Existing key has current TTL > 0" );
1270         }
1271
1272         /**
1273          * @dataProvider getWithSetCallback_versions_provider
1274          * @covers WANObjectCache::getWithSetCallback()
1275          * @covers WANObjectCache::fetchOrRegenerate()
1276          * @param array $extOpts
1277          * @param bool $versioned
1278          */
1279         public function testGetWithSetCallback_versions( array $extOpts, $versioned ) {
1280                 $cache = $this->cache;
1281
1282                 $key = wfRandomString();
1283                 $valueV1 = wfRandomString();
1284                 $valueV2 = [ wfRandomString() ];
1285
1286                 $wasSet = 0;
1287                 $funcV1 = function () use ( &$wasSet, $valueV1 ) {
1288                         ++$wasSet;
1289
1290                         return $valueV1;
1291                 };
1292
1293                 $priorValue = false;
1294                 $priorAsOf = null;
1295                 $funcV2 = function ( $oldValue, &$ttl, $setOpts, $oldAsOf )
1296                 use ( &$wasSet, $valueV2, &$priorValue, &$priorAsOf ) {
1297                         $priorValue = $oldValue;
1298                         $priorAsOf = $oldAsOf;
1299                         ++$wasSet;
1300
1301                         return $valueV2; // new array format
1302                 };
1303
1304                 // Set the main key (version N if versioned)
1305                 $wasSet = 0;
1306                 $v = $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts );
1307                 $this->assertEquals( $valueV1, $v, "Value returned" );
1308                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
1309                 $cache->getWithSetCallback( $key, 30, $funcV1, $extOpts );
1310                 $this->assertEquals( 1, $wasSet, "Value not regenerated" );
1311                 $this->assertEquals( $valueV1, $v, "Value not regenerated" );
1312
1313                 if ( $versioned ) {
1314                         // Set the key for version N+1 format
1315                         $verOpts = [ 'version' => $extOpts['version'] + 1 ];
1316                 } else {
1317                         // Start versioning now with the unversioned key still there
1318                         $verOpts = [ 'version' => 1 ];
1319                 }
1320
1321                 // Value goes to secondary key since V1 already used $key
1322                 $wasSet = 0;
1323                 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts );
1324                 $this->assertEquals( $valueV2, $v, "Value returned" );
1325                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
1326                 $this->assertEquals( false, $priorValue, "Old value not given due to old format" );
1327                 $this->assertEquals( null, $priorAsOf, "Old value not given due to old format" );
1328
1329                 $wasSet = 0;
1330                 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts );
1331                 $this->assertEquals( $valueV2, $v, "Value not regenerated (secondary key)" );
1332                 $this->assertEquals( 0, $wasSet, "Value not regenerated (secondary key)" );
1333
1334                 // Clear out the older or unversioned key
1335                 $cache->delete( $key, 0 );
1336
1337                 // Set the key for next/first versioned format
1338                 $wasSet = 0;
1339                 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts );
1340                 $this->assertEquals( $valueV2, $v, "Value returned" );
1341                 $this->assertEquals( 1, $wasSet, "Value regenerated" );
1342
1343                 $v = $cache->getWithSetCallback( $key, 30, $funcV2, $verOpts + $extOpts );
1344                 $this->assertEquals( $valueV2, $v, "Value not regenerated (main key)" );
1345                 $this->assertEquals( 1, $wasSet, "Value not regenerated (main key)" );
1346         }
1347
1348         public static function getWithSetCallback_versions_provider() {
1349                 return [
1350                         [ [], false ],
1351                         [ [ 'version' => 1 ], true ]
1352                 ];
1353         }
1354
1355         /**
1356          * @covers WANObjectCache::useInterimHoldOffCaching
1357          * @covers WANObjectCache::getInterimValue
1358          */
1359         public function testInterimHoldOffCaching() {
1360                 $cache = $this->cache;
1361
1362                 $mockWallClock = 1549343530.2053;
1363                 $cache->setMockTime( $mockWallClock );
1364
1365                 $value = 'CRL-40-940';
1366                 $wasCalled = 0;
1367                 $func = function () use ( &$wasCalled, $value ) {
1368                         $wasCalled++;
1369
1370                         return $value;
1371                 };
1372
1373                 $cache->useInterimHoldOffCaching( true );
1374
1375                 $key = wfRandomString( 32 );
1376                 $v = $cache->getWithSetCallback( $key, 60, $func );
1377                 $v = $cache->getWithSetCallback( $key, 60, $func );
1378                 $this->assertEquals( 1, $wasCalled, 'Value cached' );
1379
1380                 $cache->delete( $key );
1381                 $mockWallClock += 0.001; // cached values will be newer than tombstone
1382                 $v = $cache->getWithSetCallback( $key, 60, $func );
1383                 $this->assertEquals( 2, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim
1384                 $v = $cache->getWithSetCallback( $key, 60, $func );
1385                 $this->assertEquals( 2, $wasCalled, 'Value interim cached' ); // reuses interim
1386
1387                 $mockWallClock += 0.2; // interim key not brand new
1388                 $v = $cache->getWithSetCallback( $key, 60, $func );
1389                 $this->assertEquals( 3, $wasCalled, 'Value regenerated (got mutex)' ); // sets interim
1390                 // Lock up the mutex so interim cache is used
1391                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
1392                 $v = $cache->getWithSetCallback( $key, 60, $func );
1393                 $this->assertEquals( 3, $wasCalled, 'Value interim cached (failed mutex)' );
1394                 $this->internalCache->delete( $cache::MUTEX_KEY_PREFIX . $key );
1395
1396                 $cache->useInterimHoldOffCaching( false );
1397
1398                 $wasCalled = 0;
1399                 $key = wfRandomString( 32 );
1400                 $v = $cache->getWithSetCallback( $key, 60, $func );
1401                 $v = $cache->getWithSetCallback( $key, 60, $func );
1402                 $this->assertEquals( 1, $wasCalled, 'Value cached' );
1403                 $cache->delete( $key );
1404                 $v = $cache->getWithSetCallback( $key, 60, $func );
1405                 $this->assertEquals( 2, $wasCalled, 'Value regenerated (got mutex)' );
1406                 $v = $cache->getWithSetCallback( $key, 60, $func );
1407                 $this->assertEquals( 3, $wasCalled, 'Value still regenerated (got mutex)' );
1408                 $v = $cache->getWithSetCallback( $key, 60, $func );
1409                 $this->assertEquals( 4, $wasCalled, 'Value still regenerated (got mutex)' );
1410                 // Lock up the mutex so interim cache is used
1411                 $this->internalCache->add( $cache::MUTEX_KEY_PREFIX . $key, 1, 0 );
1412                 $v = $cache->getWithSetCallback( $key, 60, $func );
1413                 $this->assertEquals( 5, $wasCalled, 'Value still regenerated (failed mutex)' );
1414         }
1415
1416         /**
1417          * @covers WANObjectCache::touchCheckKey
1418          * @covers WANObjectCache::resetCheckKey
1419          * @covers WANObjectCache::getCheckKeyTime
1420          * @covers WANObjectCache::getMultiCheckKeyTime
1421          * @covers WANObjectCache::makePurgeValue
1422          * @covers WANObjectCache::parsePurgeValue
1423          */
1424         public function testTouchKeys() {
1425                 $cache = $this->cache;
1426                 $key = wfRandomString();
1427
1428                 $mockWallClock = 1549343530.2053;
1429                 $priorTime = $mockWallClock; // reference time
1430                 $cache->setMockTime( $mockWallClock );
1431
1432                 $mockWallClock += 0.100;
1433                 $t0 = $cache->getCheckKeyTime( $key );
1434                 $this->assertGreaterThanOrEqual( $priorTime, $t0, 'Check key auto-created' );
1435
1436                 $priorTime = $mockWallClock;
1437                 $mockWallClock += 0.100;
1438                 $cache->touchCheckKey( $key );
1439                 $t1 = $cache->getCheckKeyTime( $key );
1440                 $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check key created' );
1441
1442                 $t2 = $cache->getCheckKeyTime( $key );
1443                 $this->assertEquals( $t1, $t2, 'Check key time did not change' );
1444
1445                 $mockWallClock += 0.100;
1446                 $cache->touchCheckKey( $key );
1447                 $t3 = $cache->getCheckKeyTime( $key );
1448                 $this->assertGreaterThan( $t2, $t3, 'Check key time increased' );
1449
1450                 $t4 = $cache->getCheckKeyTime( $key );
1451                 $this->assertEquals( $t3, $t4, 'Check key time did not change' );
1452
1453                 $mockWallClock += 0.100;
1454                 $cache->resetCheckKey( $key );
1455                 $t5 = $cache->getCheckKeyTime( $key );
1456                 $this->assertGreaterThan( $t4, $t5, 'Check key time increased' );
1457
1458                 $t6 = $cache->getCheckKeyTime( $key );
1459                 $this->assertEquals( $t5, $t6, 'Check key time did not change' );
1460         }
1461
1462         /**
1463          * @covers WANObjectCache::getMulti()
1464          */
1465         public function testGetWithSeveralCheckKeys() {
1466                 $key = wfRandomString();
1467                 $tKey1 = wfRandomString();
1468                 $tKey2 = wfRandomString();
1469                 $value = 'meow';
1470
1471                 $mockWallClock = 1549343530.2053;
1472                 $priorTime = $mockWallClock; // reference time
1473                 $this->cache->setMockTime( $mockWallClock );
1474
1475                 // Two check keys are newer (given hold-off) than $key, another is older
1476                 $this->internalCache->set(
1477                         WANObjectCache::TIME_KEY_PREFIX . $tKey2,
1478                         WANObjectCache::PURGE_VAL_PREFIX . ( $priorTime - 3 )
1479                 );
1480                 $this->internalCache->set(
1481                         WANObjectCache::TIME_KEY_PREFIX . $tKey2,
1482                         WANObjectCache::PURGE_VAL_PREFIX . ( $priorTime - 5 )
1483                 );
1484                 $this->internalCache->set(
1485                         WANObjectCache::TIME_KEY_PREFIX . $tKey1,
1486                         WANObjectCache::PURGE_VAL_PREFIX . ( $priorTime - 30 )
1487                 );
1488                 $this->cache->set( $key, $value, 30 );
1489
1490                 $curTTL = null;
1491                 $v = $this->cache->get( $key, $curTTL, [ $tKey1, $tKey2 ] );
1492                 $this->assertEquals( $value, $v, "Value matches" );
1493                 $this->assertLessThan( -4.9, $curTTL, "Correct CTL" );
1494                 $this->assertGreaterThan( -5.1, $curTTL, "Correct CTL" );
1495         }
1496
1497         /**
1498          * @covers WANObjectCache::reap()
1499          * @covers WANObjectCache::reapCheckKey()
1500          */
1501         public function testReap() {
1502                 $vKey1 = wfRandomString();
1503                 $vKey2 = wfRandomString();
1504                 $tKey1 = wfRandomString();
1505                 $tKey2 = wfRandomString();
1506                 $value = 'moo';
1507
1508                 $knownPurge = time() - 60;
1509                 $goodTime = microtime( true ) - 5;
1510                 $badTime = microtime( true ) - 300;
1511
1512                 $this->internalCache->set(
1513                         WANObjectCache::VALUE_KEY_PREFIX . $vKey1,
1514                         [
1515                                 WANObjectCache::FLD_FORMAT_VERSION => WANObjectCache::VERSION,
1516                                 WANObjectCache::FLD_VALUE => $value,
1517                                 WANObjectCache::FLD_TTL => 3600,
1518                                 WANObjectCache::FLD_TIME => $goodTime
1519                         ]
1520                 );
1521                 $this->internalCache->set(
1522                         WANObjectCache::VALUE_KEY_PREFIX . $vKey2,
1523                         [
1524                                 WANObjectCache::FLD_FORMAT_VERSION => WANObjectCache::VERSION,
1525                                 WANObjectCache::FLD_VALUE => $value,
1526                                 WANObjectCache::FLD_TTL => 3600,
1527                                 WANObjectCache::FLD_TIME => $badTime
1528                         ]
1529                 );
1530                 $this->internalCache->set(
1531                         WANObjectCache::TIME_KEY_PREFIX . $tKey1,
1532                         WANObjectCache::PURGE_VAL_PREFIX . $goodTime
1533                 );
1534                 $this->internalCache->set(
1535                         WANObjectCache::TIME_KEY_PREFIX . $tKey2,
1536                         WANObjectCache::PURGE_VAL_PREFIX . $badTime
1537                 );
1538
1539                 $this->assertEquals( $value, $this->cache->get( $vKey1 ) );
1540                 $this->assertEquals( $value, $this->cache->get( $vKey2 ) );
1541                 $this->cache->reap( $vKey1, $knownPurge, $bad1 );
1542                 $this->cache->reap( $vKey2, $knownPurge, $bad2 );
1543
1544                 $this->assertFalse( $bad1 );
1545                 $this->assertTrue( $bad2 );
1546
1547                 $this->cache->reapCheckKey( $tKey1, $knownPurge, $tBad1 );
1548                 $this->cache->reapCheckKey( $tKey2, $knownPurge, $tBad2 );
1549                 $this->assertFalse( $tBad1 );
1550                 $this->assertTrue( $tBad2 );
1551         }
1552
1553         /**
1554          * @covers WANObjectCache::reap()
1555          */
1556         public function testReap_fail() {
1557                 $backend = $this->getMockBuilder( EmptyBagOStuff::class )
1558                         ->setMethods( [ 'get', 'changeTTL' ] )->getMock();
1559                 $backend->expects( $this->once() )->method( 'get' )
1560                         ->willReturn( [
1561                                 WANObjectCache::FLD_FORMAT_VERSION => WANObjectCache::VERSION,
1562                                 WANObjectCache::FLD_VALUE => 'value',
1563                                 WANObjectCache::FLD_TTL => 3600,
1564                                 WANObjectCache::FLD_TIME => 300,
1565                         ] );
1566                 $backend->expects( $this->once() )->method( 'changeTTL' )
1567                         ->willReturn( false );
1568
1569                 $wanCache = new WANObjectCache( [
1570                         'cache' => $backend
1571                 ] );
1572
1573                 $isStale = null;
1574                 $ret = $wanCache->reap( 'key', 360, $isStale );
1575                 $this->assertTrue( $isStale, 'value was stale' );
1576                 $this->assertFalse( $ret, 'changeTTL failed' );
1577         }
1578
1579         /**
1580          * @covers WANObjectCache::set()
1581          */
1582         public function testSetWithLag() {
1583                 $value = 1;
1584
1585                 $key = wfRandomString();
1586                 $opts = [ 'lag' => 300, 'since' => microtime( true ) ];
1587                 $this->cache->set( $key, $value, 30, $opts );
1588                 $this->assertEquals( $value, $this->cache->get( $key ), "Rep-lagged value written." );
1589
1590                 $key = wfRandomString();
1591                 $opts = [ 'lag' => 0, 'since' => microtime( true ) - 300 ];
1592                 $this->cache->set( $key, $value, 30, $opts );
1593                 $this->assertEquals( false, $this->cache->get( $key ), "Trx-lagged value not written." );
1594
1595                 $key = wfRandomString();
1596                 $opts = [ 'lag' => 5, 'since' => microtime( true ) - 5 ];
1597                 $this->cache->set( $key, $value, 30, $opts );
1598                 $this->assertEquals( false, $this->cache->get( $key ), "Lagged value not written." );
1599         }
1600
1601         /**
1602          * @covers WANObjectCache::set()
1603          */
1604         public function testWritePending() {
1605                 $value = 1;
1606
1607                 $key = wfRandomString();
1608                 $opts = [ 'pending' => true ];
1609                 $this->cache->set( $key, $value, 30, $opts );
1610                 $this->assertEquals( false, $this->cache->get( $key ), "Pending value not written." );
1611         }
1612
1613         public function testMcRouterSupport() {
1614                 $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
1615                         ->setMethods( [ 'set', 'delete' ] )->getMock();
1616                 $localBag->expects( $this->never() )->method( 'set' );
1617                 $localBag->expects( $this->never() )->method( 'delete' );
1618                 $wanCache = new WANObjectCache( [
1619                         'cache' => $localBag,
1620                         'mcrouterAware' => true,
1621                         'region' => 'pmtpa',
1622                         'cluster' => 'mw-wan'
1623                 ] );
1624                 $valFunc = function () {
1625                         return 1;
1626                 };
1627
1628                 // None of these should use broadcasting commands (e.g. SET, DELETE)
1629                 $wanCache->get( 'x' );
1630                 $wanCache->get( 'x', $ctl, [ 'check1' ] );
1631                 $wanCache->getMulti( [ 'x', 'y' ] );
1632                 $wanCache->getMulti( [ 'x', 'y' ], $ctls, [ 'check2' ] );
1633                 $wanCache->getWithSetCallback( 'p', 30, $valFunc );
1634                 $wanCache->getCheckKeyTime( 'zzz' );
1635                 $wanCache->reap( 'x', time() - 300 );
1636                 $wanCache->reap( 'zzz', time() - 300 );
1637         }
1638
1639         public function testMcRouterSupportBroadcastDelete() {
1640                 $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
1641                         ->setMethods( [ 'set' ] )->getMock();
1642                 $wanCache = new WANObjectCache( [
1643                         'cache' => $localBag,
1644                         'mcrouterAware' => true,
1645                         'region' => 'pmtpa',
1646                         'cluster' => 'mw-wan'
1647                 ] );
1648
1649                 $localBag->expects( $this->once() )->method( 'set' )
1650                         ->with( "/*/mw-wan/" . $wanCache::VALUE_KEY_PREFIX . "test" );
1651
1652                 $wanCache->delete( 'test' );
1653         }
1654
1655         public function testMcRouterSupportBroadcastTouchCK() {
1656                 $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
1657                         ->setMethods( [ 'set' ] )->getMock();
1658                 $wanCache = new WANObjectCache( [
1659                         'cache' => $localBag,
1660                         'mcrouterAware' => true,
1661                         'region' => 'pmtpa',
1662                         'cluster' => 'mw-wan'
1663                 ] );
1664
1665                 $localBag->expects( $this->once() )->method( 'set' )
1666                         ->with( "/*/mw-wan/" . $wanCache::TIME_KEY_PREFIX . "test" );
1667
1668                 $wanCache->touchCheckKey( 'test' );
1669         }
1670
1671         public function testMcRouterSupportBroadcastResetCK() {
1672                 $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
1673                         ->setMethods( [ 'delete' ] )->getMock();
1674                 $wanCache = new WANObjectCache( [
1675                         'cache' => $localBag,
1676                         'mcrouterAware' => true,
1677                         'region' => 'pmtpa',
1678                         'cluster' => 'mw-wan'
1679                 ] );
1680
1681                 $localBag->expects( $this->once() )->method( 'delete' )
1682                         ->with( "/*/mw-wan/" . $wanCache::TIME_KEY_PREFIX . "test" );
1683
1684                 $wanCache->resetCheckKey( 'test' );
1685         }
1686
1687         public function testEpoch() {
1688                 $bag = new HashBagOStuff();
1689                 $cache = new WANObjectCache( [ 'cache' => $bag ] );
1690                 $key = $cache->makeGlobalKey( 'The whole of the Law' );
1691
1692                 $now = microtime( true );
1693                 $cache->setMockTime( $now );
1694
1695                 $cache->set( $key, 'Do what thou Wilt' );
1696                 $cache->touchCheckKey( $key );
1697
1698                 $then = $now;
1699                 $now += 30;
1700                 $this->assertEquals( 'Do what thou Wilt', $cache->get( $key ) );
1701                 $this->assertEquals( $then, $cache->getCheckKeyTime( $key ), 'Check key init', 0.01 );
1702
1703                 $cache = new WANObjectCache( [
1704                         'cache' => $bag,
1705                         'epoch' => $now - 3600
1706                 ] );
1707                 $cache->setMockTime( $now );
1708
1709                 $this->assertEquals( 'Do what thou Wilt', $cache->get( $key ) );
1710                 $this->assertEquals( $then, $cache->getCheckKeyTime( $key ), 'Check key kept', 0.01 );
1711
1712                 $now += 30;
1713                 $cache = new WANObjectCache( [
1714                         'cache' => $bag,
1715                         'epoch' => $now + 3600
1716                 ] );
1717                 $cache->setMockTime( $now );
1718
1719                 $this->assertFalse( $cache->get( $key ), 'Key rejected due to epoch' );
1720                 $this->assertEquals( $now, $cache->getCheckKeyTime( $key ), 'Check key reset', 0.01 );
1721         }
1722
1723         /**
1724          * @dataProvider provideAdaptiveTTL
1725          * @covers WANObjectCache::adaptiveTTL()
1726          * @param float|int $ago
1727          * @param int $maxTTL
1728          * @param int $minTTL
1729          * @param float $factor
1730          * @param int $adaptiveTTL
1731          */
1732         public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor, $adaptiveTTL ) {
1733                 $mtime = $ago ? time() - $ago : $ago;
1734                 $margin = 5;
1735                 $ttl = $this->cache->adaptiveTTL( $mtime, $maxTTL, $minTTL, $factor );
1736
1737                 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl );
1738                 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl );
1739
1740                 $ttl = $this->cache->adaptiveTTL( (string)$mtime, $maxTTL, $minTTL, $factor );
1741
1742                 $this->assertGreaterThanOrEqual( $adaptiveTTL - $margin, $ttl );
1743                 $this->assertLessThanOrEqual( $adaptiveTTL + $margin, $ttl );
1744         }
1745
1746         public static function provideAdaptiveTTL() {
1747                 return [
1748                         [ 3600, 900, 30, 0.2, 720 ],
1749                         [ 3600, 500, 30, 0.2, 500 ],
1750                         [ 3600, 86400, 800, 0.2, 800 ],
1751                         [ false, 86400, 800, 0.2, 800 ],
1752                         [ null, 86400, 800, 0.2, 800 ]
1753                 ];
1754         }
1755
1756         /**
1757          * @covers WANObjectCache::__construct
1758          * @covers WANObjectCache::newEmpty
1759          */
1760         public function testNewEmpty() {
1761                 $this->assertInstanceOf(
1762                         WANObjectCache::class,
1763                         WANObjectCache::newEmpty()
1764                 );
1765         }
1766
1767         /**
1768          * @covers WANObjectCache::setLogger
1769          */
1770         public function testSetLogger() {
1771                 $this->assertSame( null, $this->cache->setLogger( new Psr\Log\NullLogger ) );
1772         }
1773
1774         /**
1775          * @covers WANObjectCache::getQoS
1776          */
1777         public function testGetQoS() {
1778                 $backend = $this->getMockBuilder( HashBagOStuff::class )
1779                         ->setMethods( [ 'getQoS' ] )->getMock();
1780                 $backend->expects( $this->once() )->method( 'getQoS' )
1781                         ->willReturn( BagOStuff::QOS_UNKNOWN );
1782                 $wanCache = new WANObjectCache( [ 'cache' => $backend ] );
1783
1784                 $this->assertSame(
1785                         $wanCache::QOS_UNKNOWN,
1786                         $wanCache->getQoS( $wanCache::ATTR_EMULATION )
1787                 );
1788         }
1789
1790         /**
1791          * @covers WANObjectCache::makeKey
1792          */
1793         public function testMakeKey() {
1794                 $backend = $this->getMockBuilder( HashBagOStuff::class )
1795                         ->setMethods( [ 'makeKey' ] )->getMock();
1796                 $backend->expects( $this->once() )->method( 'makeKey' )
1797                         ->willReturn( 'special' );
1798
1799                 $wanCache = new WANObjectCache( [
1800                         'cache' => $backend
1801                 ] );
1802
1803                 $this->assertSame( 'special', $wanCache->makeKey( 'a', 'b' ) );
1804         }
1805
1806         /**
1807          * @covers WANObjectCache::makeGlobalKey
1808          */
1809         public function testMakeGlobalKey() {
1810                 $backend = $this->getMockBuilder( HashBagOStuff::class )
1811                         ->setMethods( [ 'makeGlobalKey' ] )->getMock();
1812                 $backend->expects( $this->once() )->method( 'makeGlobalKey' )
1813                         ->willReturn( 'special' );
1814
1815                 $wanCache = new WANObjectCache( [
1816                         'cache' => $backend
1817                 ] );
1818
1819                 $this->assertSame( 'special', $wanCache->makeGlobalKey( 'a', 'b' ) );
1820         }
1821
1822         public static function statsKeyProvider() {
1823                 return [
1824                         [ 'domain:page:5', 'page' ],
1825                         [ 'domain:main-key', 'main-key' ],
1826                         [ 'domain:page:history', 'page' ],
1827                         [ 'missingdomainkey', 'missingdomainkey' ]
1828                 ];
1829         }
1830
1831         /**
1832          * @dataProvider statsKeyProvider
1833          * @covers WANObjectCache::determineKeyClassForStats
1834          */
1835         public function testStatsKeyClass( $key, $class ) {
1836                 $wanCache = TestingAccessWrapper::newFromObject( new WANObjectCache( [
1837                         'cache' => new HashBagOStuff
1838                 ] ) );
1839
1840                 $this->assertEquals( $class, $wanCache->determineKeyClassForStats( $key ) );
1841         }
1842
1843         /**
1844          * @covers WANObjectCache::makeMultiKeys
1845          */
1846         public function testMakeMultiKeys() {
1847                 $cache = $this->cache;
1848
1849                 $ids = [ 1, 2, 3, 4, 4, 5, 6, 6, 7, 7 ];
1850                 $keyCallback = function ( $id, WANObjectCache $cache ) {
1851                         return $cache->makeKey( 'key', $id );
1852                 };
1853                 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback );
1854
1855                 $expected = [
1856                         "local:key:1" => 1,
1857                         "local:key:2" => 2,
1858                         "local:key:3" => 3,
1859                         "local:key:4" => 4,
1860                         "local:key:5" => 5,
1861                         "local:key:6" => 6,
1862                         "local:key:7" => 7
1863                 ];
1864                 $this->assertSame( $expected, iterator_to_array( $keyedIds ) );
1865
1866                 $ids = [ '1', '2', '3', '4', '4', '5', '6', '6', '7', '7' ];
1867                 $keyCallback = function ( $id, WANObjectCache $cache ) {
1868                         return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' );
1869                 };
1870                 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback );
1871
1872                 $expected = [
1873                         "global:key:1:a:1:b" => '1',
1874                         "global:key:2:a:2:b" => '2',
1875                         "global:key:3:a:3:b" => '3',
1876                         "global:key:4:a:4:b" => '4',
1877                         "global:key:5:a:5:b" => '5',
1878                         "global:key:6:a:6:b" => '6',
1879                         "global:key:7:a:7:b" => '7'
1880                 ];
1881                 $this->assertSame( $expected, iterator_to_array( $keyedIds ) );
1882         }
1883
1884         /**
1885          * @covers WANObjectCache::makeMultiKeys
1886          */
1887         public function testMakeMultiKeysIntString() {
1888                 $cache = $this->cache;
1889                 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7, '7' ];
1890                 $keyCallback = function ( $id, WANObjectCache $cache ) {
1891                         return $cache->makeGlobalKey( 'key', $id, 'a', $id, 'b' );
1892                 };
1893
1894                 $keyedIds = $cache->makeMultiKeys( $ids, $keyCallback );
1895
1896                 $expected = [
1897                         "global:key:1:a:1:b" => 1,
1898                         "global:key:2:a:2:b" => 2,
1899                         "global:key:3:a:3:b" => 3,
1900                         "global:key:4:a:4:b" => 4,
1901                         "global:key:5:a:5:b" => 5,
1902                         "global:key:6:a:6:b" => 6,
1903                         "global:key:7:a:7:b" => 7
1904                 ];
1905                 $this->assertSame( $expected, iterator_to_array( $keyedIds ) );
1906         }
1907
1908         /**
1909          * @covers WANObjectCache::makeMultiKeys
1910          * @expectedException UnexpectedValueException
1911          */
1912         public function testMakeMultiKeysCollision() {
1913                 $ids = [ 1, 2, 3, 4, '4', 5, 6, 6, 7 ];
1914
1915                 $this->cache->makeMultiKeys(
1916                         $ids,
1917                         function ( $id ) {
1918                                 return "keymod:" . $id % 3;
1919                         }
1920                 );
1921         }
1922
1923         /**
1924          * @covers WANObjectCache::multiRemap
1925          */
1926         public function testMultiRemap() {
1927                 $a = [ 'a', 'b', 'c' ];
1928                 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3 ];
1929
1930                 $this->assertEquals(
1931                         [ 'a' => 1, 'b' => 2, 'c' => 3 ],
1932                         $this->cache->multiRemap( $a, $res )
1933                 );
1934
1935                 $a = [ 'a', 'b', 'c', 'c', 'd' ];
1936                 $res = [ 'keyA' => 1, 'keyB' => 2, 'keyC' => 3, 'keyD' => 4 ];
1937
1938                 $this->assertEquals(
1939                         [ 'a' => 1, 'b' => 2, 'c' => 3, 'd' => 4 ],
1940                         $this->cache->multiRemap( $a, $res )
1941                 );
1942         }
1943
1944         /**
1945          * @covers WANObjectCache::hash256
1946          */
1947         public function testHash256() {
1948                 $bag = new HashBagOStuff();
1949                 $cache = new WANObjectCache( [ 'cache' => $bag, 'epoch' => 5 ] );
1950                 $this->assertEquals(
1951                         'f402bce76bfa1136adc705d8d5719911ce1fe61f0ad82ddf79a15f3c4de6ec4c',
1952                         $cache->hash256( 'x' )
1953                 );
1954
1955                 $cache = new WANObjectCache( [ 'cache' => $bag, 'epoch' => 50 ] );
1956                 $this->assertEquals(
1957                         'f79a126722f0a682c4c500509f1b61e836e56c4803f92edc89fc281da5caa54e',
1958                         $cache->hash256( 'x' )
1959                 );
1960
1961                 $cache = new WANObjectCache( [ 'cache' => $bag, 'secret' => 'garden' ] );
1962                 $this->assertEquals(
1963                         '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093',
1964                         $cache->hash256( 'x' )
1965                 );
1966
1967                 $cache = new WANObjectCache( [ 'cache' => $bag, 'secret' => 'garden', 'epoch' => 3 ] );
1968                 $this->assertEquals(
1969                         '48cd57016ffe29981a1114c45e5daef327d30fc6206cb73edc3cb94b4d8fe093',
1970                         $cache->hash256( 'x' )
1971                 );
1972         }
1973 }
1974
1975 class NearExpiringWANObjectCache extends WANObjectCache {
1976         const CLOCK_SKEW = 1;
1977
1978         protected function worthRefreshExpiring( $curTTL, $lowTTL ) {
1979                 return ( $curTTL > 0 && ( $curTTL + self::CLOCK_SKEW ) < $lowTTL );
1980         }
1981 }
1982
1983 class PopularityRefreshingWANObjectCache extends WANObjectCache {
1984         protected function worthRefreshPopular( $asOf, $ageNew, $timeTillRefresh, $now ) {
1985                 return ( ( $now - $asOf ) > $timeTillRefresh );
1986         }
1987 }