objectcache: use region prefixes for mcrouter-backed WAN cache

This allows for mcrouter to have proper cross-DC and intra-DC timeouts.

Change-Id: If48f740f435d266a2050839f34611c0c8f36b3a7

diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php
index 08424c9..d939819 100644 (file)
--- a/includes/libs/objectcache/WANObjectCache.php
+++ b/includes/libs/objectcache/WANObjectCache.php
@@ -47,17 +47,23 @@ use Psr\Log\NullLogger;
  * There are three supported ways to handle broadcasted operations:
  *   - a) Configure the 'purge' EventRelayer to point to a valid PubSub endpoint
  *         that has subscribed listeners on the cache servers applying the cache updates.
  * There are three supported ways to handle broadcasted operations:
  *   - a) Configure the 'purge' EventRelayer to point to a valid PubSub endpoint
  *         that has subscribed listeners on the cache servers applying the cache updates.

- *   - b) Ignore the 'purge' EventRelayer configuration (default is NullEventRelayer)
- *         and set up mcrouter as the underlying cache backend, using one of the memcached
- *         BagOStuff classes as 'cache'. Use OperationSelectorRoute in the mcrouter settings
- *         to configure 'set' and 'delete' operations to go to all DCs via AllAsyncRoute and
- *         configure other operations to go to the local DC via PoolRoute (for reference,
- *         see https://github.com/facebook/mcrouter/wiki/List-of-Route-Handles).
- *   - c) Ignore the 'purge' EventRelayer configuration (default is NullEventRelayer)
- *         and set up dynomite as cache middleware between the web servers and either
- *         memcached or redis. This will also broadcast all key setting operations, not just purges,
- *         which can be useful for cache warming. Writes are eventually consistent via the
- *         Dynamo replication model (see https://github.com/Netflix/dynomite).
+ *   - b) Ommit the 'purge' EventRelayer parameter and set up mcrouter as the underlying cache
+ *        backend, using a memcached BagOStuff class for the 'cache' parameter. The 'region'
+ *        and 'cluster' parameters must be provided and 'mcrouterAware' must be set to 'true'.
+ *        Configure mcrouter as follows:
+ *          - 1) Use Route Prefixing based on region (datacenter) and cache cluster.
+ *                See https://github.com/facebook/mcrouter/wiki/Routing-Prefix and
+ *                https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
+ *          - 2) To increase the consistency of delete() and touchCheckKey() during cache
+ *                server membership changes, you can use the OperationSelectorRoute to
+ *                configure 'set' and 'delete' operations to go to all servers in the cache
+ *                cluster, instead of just one server determined by hashing.
+ *                See https://github.com/facebook/mcrouter/wiki/List-of-Route-Handles
+ *   - c) Ommit the 'purge' EventRelayer parameter and set up dynomite as cache middleware
+ *         between the web servers and either memcached or redis. This will also broadcast all
+ *         key setting operations, not just purges, which can be useful for cache warming.
+ *         Writes are eventually consistent via the Dynamo replication model.
+ *         See https://github.com/Netflix/dynomite

  *
  * Broadcasted operations like delete() and touchCheckKey() are done asynchronously
  * in all datacenters this way, though the local one should likely be near immediate.
  *
  * Broadcasted operations like delete() and touchCheckKey() are done asynchronously
  * in all datacenters this way, though the local one should likely be near immediate.


@@ -87,6 +93,12 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
        protected $purgeChannel;
        /** @var EventRelayer Bus that handles purge broadcasts */
        protected $purgeRelayer;
        protected $purgeChannel;
        /** @var EventRelayer Bus that handles purge broadcasts */
        protected $purgeRelayer;

+       /** @bar bool Whether to use mcrouter key prefixing for routing */
+       protected $mcrouterAware;
+       /** @var string Physical region for mcrouter use */
+       protected $region;
+       /** @var string Cache cluster name for mcrouter use */
+       protected $cluster;

        /** @var LoggerInterface */
        protected $logger;
        /** @var StatsdDataFactoryInterface */
        /** @var LoggerInterface */
        protected $logger;
        /** @var StatsdDataFactoryInterface */


@@ -200,6 +212,16 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
         *       callback supplied by the getWithSetCallback() caller. The result will be saved
         *       as normal. The handler is expected to call the WAN cache callback at an opportune
         *       time (e.g. HTTP post-send), though generally within a few 100ms. [optional]
         *       callback supplied by the getWithSetCallback() caller. The result will be saved
         *       as normal. The handler is expected to call the WAN cache callback at an opportune
         *       time (e.g. HTTP post-send), though generally within a few 100ms. [optional]

+        *   - region: the current physical region. This is required when using mcrouter as the
+        *       backing store proxy. [optional]
+        *   - cluster: name of the cache cluster used by this WAN cache. The name must be the
+        *       same in all datacenters; the ("region","cluster") tuple is what distinguishes
+        *       the counterpart cache clusters among all the datacenter. The contents of
+        *       https://github.com/facebook/mcrouter/wiki/Config-Files give background on this.
+        *       This is required when using mcrouter as the backing store proxy. [optional]
+        *   - mcrouterAware: set as true if mcrouter is the backing store proxy and mcrouter
+        *       is configured to interpret /<region>/<cluster>/ key prefixes as routes. This
+        *       requires that "region" and "cluster" are both set above. [optional]

         */
        public function __construct( array $params ) {
                $this->cache = $params['cache'];
         */
        public function __construct( array $params ) {
                $this->cache = $params['cache'];


@@ -209,6 +231,10 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
                $this->purgeRelayer = isset( $params['relayers']['purge'] )
                        ? $params['relayers']['purge']
                        : new EventRelayerNull( [] );
                $this->purgeRelayer = isset( $params['relayers']['purge'] )
                        ? $params['relayers']['purge']
                        : new EventRelayerNull( [] );

+               $this->region = isset( $params['region'] ) ? $params['region'] : 'main';
+               $this->cluster = isset( $params['cluster'] ) ? $params['cluster'] : 'wan-main';
+               $this->mcrouterAware = !empty( $params['mcrouterAware'] );
+

                $this->setLogger( isset( $params['logger'] ) ? $params['logger'] : new NullLogger() );
                $this->stats = isset( $params['stats'] ) ? $params['stats'] : new NullStatsdDataFactory();
                $this->asyncHandler = isset( $params['asyncHandler'] ) ? $params['asyncHandler'] : null;
                $this->setLogger( isset( $params['logger'] ) ? $params['logger'] : new NullLogger() );
                $this->stats = isset( $params['stats'] ) ? $params['stats'] : new NullStatsdDataFactory();
                $this->asyncHandler = isset( $params['asyncHandler'] ) ? $params['asyncHandler'] : null;


@@ -1779,9 +1805,18 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
         * @return bool Success
         */
        protected function relayPurge( $key, $ttl, $holdoff ) {
         * @return bool Success
         */
        protected function relayPurge( $key, $ttl, $holdoff ) {

-               if ( $this->purgeRelayer instanceof EventRelayerNull ) {
+               if ( $this->mcrouterAware ) {
+                       // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
+                       // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
+                       $ok = $this->cache->set(
+                               "/*/{$this->cluster}/{$key}",
+                               $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
+                               $ttl
+                       );
+               } elseif ( $this->purgeRelayer instanceof EventRelayerNull ) {

                        // This handles the mcrouter and the single-DC case
                        // This handles the mcrouter and the single-DC case

-                       $ok = $this->cache->set( $key,
+                       $ok = $this->cache->set(
+                               $key,

                                $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
                                $ttl
                        );
                                $this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
                                $ttl
                        );


@@ -1810,8 +1845,12 @@ class WANObjectCache implements IExpiringStore, LoggerAwareInterface {
         * @return bool Success
         */
        protected function relayDelete( $key ) {
         * @return bool Success
         */
        protected function relayDelete( $key ) {

-               if ( $this->purgeRelayer instanceof EventRelayerNull ) {
-                       // This handles the mcrouter and the single-DC case
+               if ( $this->mcrouterAware ) {
+                       // See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
+                       // Wildcards select all matching routes, e.g. the WAN cluster on all DCs
+                       $ok = $this->cache->delete( "/*/{$this->cluster}/{$key}" );
+               } elseif ( $this->purgeRelayer instanceof EventRelayerNull ) {
+                       // Some other proxy handles broadcasting or there is only one datacenter

                        $ok = $this->cache->delete( $key );
                } else {
                        $event = $this->cache->modifySimpleRelayEvent( [
                        $ok = $this->cache->delete( $key );
                } else {
                        $event = $this->cache->modifySimpleRelayEvent( [

diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
index a2c3bdc..ca78f65 100644 (file)
--- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
+++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
@@ -1481,7 +1481,10 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase {
                $wanCache = new WANObjectCache( [
                        'cache' => $localBag,
                        'pool' => 'testcache-hash',
                $wanCache = new WANObjectCache( [
                        'cache' => $localBag,
                        'pool' => 'testcache-hash',

-                       'relayer' => new EventRelayerNull( [] )
+                       'relayer' => new EventRelayerNull( [] ),
+                       'mcrouterAware' => true,
+                       'region' => 'pmtpa',
+                       'cluster' => 'mw-wan'

                ] );
                $valFunc = function () {
                        return 1;
                ] );
                $valFunc = function () {
                        return 1;


@@ -1498,6 +1501,60 @@ class WANObjectCacheTest extends PHPUnit_Framework_TestCase {
                $wanCache->reap( 'zzz', time() - 300 );
        }
 
                $wanCache->reap( 'zzz', time() - 300 );
        }
 

+       public function testMcRouterSupportBroadcastDelete() {
+               $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
+                       ->setMethods( [ 'set' ] )->getMock();
+               $wanCache = new WANObjectCache( [
+                       'cache' => $localBag,
+                       'pool' => 'testcache-hash',
+                       'relayer' => new EventRelayerNull( [] ),
+                       'mcrouterAware' => true,
+                       'region' => 'pmtpa',
+                       'cluster' => 'mw-wan'
+               ] );
+
+               $localBag->expects( $this->once() )->method( 'set' )
+                       ->with( "/*/mw-wan/" . $wanCache::VALUE_KEY_PREFIX . "test" );
+
+               $wanCache->delete( 'test' );
+       }
+
+       public function testMcRouterSupportBroadcastTouchCK() {
+               $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
+                       ->setMethods( [ 'set' ] )->getMock();
+               $wanCache = new WANObjectCache( [
+                       'cache' => $localBag,
+                       'pool' => 'testcache-hash',
+                       'relayer' => new EventRelayerNull( [] ),
+                       'mcrouterAware' => true,
+                       'region' => 'pmtpa',
+                       'cluster' => 'mw-wan'
+               ] );
+
+               $localBag->expects( $this->once() )->method( 'set' )
+                       ->with( "/*/mw-wan/" . $wanCache::TIME_KEY_PREFIX . "test" );
+
+               $wanCache->touchCheckKey( 'test' );
+       }
+
+       public function testMcRouterSupportBroadcastResetCK() {
+               $localBag = $this->getMockBuilder( EmptyBagOStuff::class )
+                       ->setMethods( [ 'delete' ] )->getMock();
+               $wanCache = new WANObjectCache( [
+                       'cache' => $localBag,
+                       'pool' => 'testcache-hash',
+                       'relayer' => new EventRelayerNull( [] ),
+                       'mcrouterAware' => true,
+                       'region' => 'pmtpa',
+                       'cluster' => 'mw-wan'
+               ] );
+
+               $localBag->expects( $this->once() )->method( 'delete' )
+                       ->with( "/*/mw-wan/" . $wanCache::TIME_KEY_PREFIX . "test" );
+
+               $wanCache->resetCheckKey( 'test' );
+       }
+

        /**
         * @dataProvider provideAdaptiveTTL
         * @covers WANObjectCache::adaptiveTTL()
        /**
         * @dataProvider provideAdaptiveTTL
         * @covers WANObjectCache::adaptiveTTL()