[SiteStatsUpdate] Added support for memcached staging of stats updates.

Change-Id: I0bfdd6a4c9b8b09355bf008bd97456401f8b485b

diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 7084bc6..c210662 100644 (file)
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -1873,12 +1873,12 @@ $wgMaxSquidPurgeTitles = 400;
  * Routing configuration for HTCP multicast purging. Add elements here to
  * enable HTCP and determine which purges are sent where. If set to an empty
  * array, HTCP is disabled.
- * 
+ *
  * Each key in this array is a regular expression to match against the purged
  * URL, or an empty string to match all URLs. The purged URL is matched against
  * the regexes in the order specified, and the first rule whose regex matches
  * is used.
- * 
+ *
  * Example configuration to send purges for upload.wikimedia.org to one
  * multicast group and all other purges to another:
  * $wgHTCPMulticastRouting = array(
@@ -1891,7 +1891,7 @@ $wgMaxSquidPurgeTitles = 400;
  *                 'port' => 4827,
  *         ),
  * );
- * 
+ *
  * @see $wgHTCPMulticastTTL
  */
 $wgHTCPMulticastRouting = array();
@@ -1901,12 +1901,12 @@ $wgHTCPMulticastRouting = array();
  *
  * Note that MediaWiki uses the old non-RFC compliant HTCP format, which was
  * present in the earliest Squid implementations of the protocol.
- * 
+ *
  * This setting is DEPRECATED in favor of $wgHTCPMulticastRouting , and kept
  * for backwards compatibility only. If $wgHTCPMulticastRouting is set, this
  * setting is ignored. If $wgHTCPMulticastRouting is not set and this setting
  * is, it is used to populate $wgHTCPMulticastRouting.
- * 
+ *
  * @deprecated in favor of $wgHTCPMulticastRouting
  */
 $wgHTCPMulticastAddress = false;
@@ -4227,6 +4227,14 @@ $wgAggregateStatsID = false;
  */
 $wgDisableCounters = false;
 
+/**
+ * Set this to an integer to only do synchronous site_stats updates
+ * one every *this many* updates. The other requests go into pending
+ * delta values in $wgMemc. Make sure that $wgMemc is a global cache.
+ * If set to -1, updates *only* go to $wgMemc (useful for daemons).
+ */
+$wgSiteStatsAsyncFactor = false;
+
 /**
  * Parser test suite files to be run by parserTests.php when no specific
  * filename is passed to it.

diff --git a/includes/SiteStats.php b/includes/SiteStats.php
index b557ca2..2a1f25d 100644 (file)
--- a/includes/SiteStats.php
+++ b/includes/SiteStats.php
@@ -226,15 +226,15 @@ class SiteStatsUpdate implements DeferrableUpdate {
        protected $views = 0;
        protected $edits = 0;
        protected $pages = 0;
-       protected $goodPages = 0;
+       protected $articles = 0;
        protected $users = 0;
        protected $images = 0;
 
-       // @TODO: deprecate this
+       // @TODO: deprecate this constructor
        function __construct( $views, $edits, $good, $pages = 0, $users = 0 ) {
                $this->views = $views;
                $this->edits = $edits;
-               $this->goodPages = $good;
+               $this->articles = $good;
                $this->pages = $pages;
                $this->users = $users;
        }
@@ -246,7 +246,7 @@ class SiteStatsUpdate implements DeferrableUpdate {
        public static function factory( array $deltas ) {
                $update = new self( 0, 0, 0 );
 
-               $fields = array( 'views', 'edits', 'pages', 'goodPages', 'users', 'images' );
+               $fields = array( 'views', 'edits', 'pages', 'articles', 'users', 'images' );
                foreach ( $fields as $field ) {
                        if ( isset( $deltas[$field] ) && $deltas[$field] ) {
                                $update->$field = $deltas[$field];
@@ -256,43 +256,56 @@ class SiteStatsUpdate implements DeferrableUpdate {
                return $update;
        }
 
-       /**
-        * @param $sql
-        * @param $field
-        * @param $delta
-        */
-       function appendUpdate( &$sql, $field, $delta ) {
-               if ( $delta ) {
-                       if ( $sql ) {
-                               $sql .= ',';
-                       }
-                       if ( $delta < 0 ) {
-                               $sql .= "$field=$field-1";
-                       } else {
-                               $sql .= "$field=$field+1";
-                       }
-               }
-       }
-
        public function doUpdate() {
-               $dbw = wfGetDB( DB_MASTER );
+               global $wgSiteStatsAsyncFactor;
+
+               $rate = $wgSiteStatsAsyncFactor; // convenience
+               // If set to do so, only do actual DB updates 1 every $rate times.
+               // The other times, just update "pending delta" values in memcached.
+               if ( $rate && ( $rate < 0 || mt_rand( 0, $rate - 1 ) != 0 ) ) {
+                       $this->doUpdatePendingDeltas();
+               } else {
+                       $dbw = wfGetDB( DB_MASTER );
+                       // Need a separate transaction because this a global lock
+                       $dbw->begin( __METHOD__ );
 
-               $updates = '';
+                       $lockKey = wfMemcKey( 'site_stats' ); // prepend wiki ID
+                       if ( $rate ) {
+                               // Lock the table so we don't have double DB/memcached updates
+                               if ( !$dbw->lock( $lockKey, __METHOD__, 1 ) ) {
+                                       $dbw->commit( __METHOD__ );
+                                       $this->doUpdatePendingDeltas();
+                                       return;
+                               }
+                               $pd = $this->getPendingDeltas();
+                               // Piggy-back the async deltas onto those of this stats update....
+                               $this->views    += ( $pd['ss_total_views']['+'] - $pd['ss_total_views']['-'] );
+                               $this->edits    += ( $pd['ss_total_edits']['+'] - $pd['ss_total_edits']['-'] );
+                               $this->articles += ( $pd['ss_good_articles']['+'] - $pd['ss_good_articles']['-'] );
+                               $this->pages    += ( $pd['ss_total_pages']['+'] - $pd['ss_total_pages']['-'] );
+                               $this->users    += ( $pd['ss_users']['+'] - $pd['ss_users']['-'] );
+                               $this->images   += ( $pd['ss_images']['+'] - $pd['ss_images']['-'] );
+                       }
 
-               $this->appendUpdate( $updates, 'ss_total_views', $this->views );
-               $this->appendUpdate( $updates, 'ss_total_edits', $this->edits );
-               $this->appendUpdate( $updates, 'ss_good_articles', $this->goodPages );
-               $this->appendUpdate( $updates, 'ss_total_pages', $this->pages );
-               $this->appendUpdate( $updates, 'ss_users', $this->users );
-               $this->appendUpdate( $updates, 'ss_images', $this->images );
+                       // Build up an SQL query of deltas and apply them...
+                       $updates = '';
+                       $this->appendUpdate( $updates, 'ss_total_views', $this->views );
+                       $this->appendUpdate( $updates, 'ss_total_edits', $this->edits );
+                       $this->appendUpdate( $updates, 'ss_good_articles', $this->articles );
+                       $this->appendUpdate( $updates, 'ss_total_pages', $this->pages );
+                       $this->appendUpdate( $updates, 'ss_users', $this->users );
+                       $this->appendUpdate( $updates, 'ss_images', $this->images );
+                       if ( $updates != '' ) {
+                               $dbw->update( 'site_stats', array( $updates ), array(), __METHOD__ );
+                       }
 
-               if ( $updates ) {
-                       $site_stats = $dbw->tableName( 'site_stats' );
-                       $sql = "UPDATE $site_stats SET $updates";
+                       if ( $rate ) {
+                               // Decrement the async deltas now that we applied them
+                               $this->removePendingDeltas( $pd );
+                               // Commit the updates and unlock the table
+                               $dbw->unlock( $lockKey, __METHOD__ );
+                       }
 
-                       # Need a separate transaction because this a global lock
-                       $dbw->begin( __METHOD__ );
-                       $dbw->query( $sql, __METHOD__ );
                        $dbw->commit( __METHOD__ );
                }
        }
@@ -325,6 +338,102 @@ class SiteStatsUpdate implements DeferrableUpdate {
                );
                return $activeUsers;
        }
+
+       protected function doUpdatePendingDeltas() {
+               $this->adjustPending( 'ss_total_views', $this->views );
+               $this->adjustPending( 'ss_total_edits', $this->edits );
+               $this->adjustPending( 'ss_good_articles', $this->articles );
+               $this->adjustPending( 'ss_total_pages', $this->pages );
+               $this->adjustPending( 'ss_users', $this->users );
+               $this->adjustPending( 'ss_images', $this->images );
+       }
+
+       /**
+        * @param $sql string
+        * @param $field string
+        * @param $delta integer
+        */
+       protected function appendUpdate( &$sql, $field, $delta ) {
+               if ( $delta ) {
+                       if ( $sql ) {
+                               $sql .= ',';
+                       }
+                       if ( $delta < 0 ) {
+                               $sql .= "$field=$field-" . abs( $delta );
+                       } else {
+                               $sql .= "$field=$field+" . abs( $delta );
+                       }
+               }
+       }
+
+       /**
+        * @param $type string
+        * @param $sign string ('+' or '-')
+        * @return void
+        */
+       private function getTypeCacheKey( $type, $sign ) {
+               return wfMemcKey( 'sitestatsupdate', 'pendingdelta', $type, $sign );
+       }
+
+       /**
+        * Adjust the pending deltas for a stat type.
+        * Each stat type has two pending counters, one for increments and decrements
+        * @param $type string
+        * @param $delta integer Delta (positive or negative)
+        * @return void
+        */
+       protected function adjustPending( $type, $delta ) {
+               global $wgMemc;
+
+               if ( $delta < 0 ) { // decrement
+                       $key = $this->getTypeCacheKey( $type, '-' );
+               } else { // increment
+                       $key = $this->getTypeCacheKey( $type, '+' );
+               }
+
+               $magnitude = abs( $delta );
+               if ( !$wgMemc->incr( $key, $magnitude ) ) { // not there?
+                       if ( !$wgMemc->add( $key, $magnitude ) ) { // race?
+                               $wgMemc->incr( $key, $magnitude );
+                       }
+               }
+       }
+
+       /**
+        * Get pending delta counters for each stat type
+        * @return Array Positive and negative deltas for each type
+        * @return void
+        */
+       protected function getPendingDeltas() {
+               global $wgMemc;
+
+               $pending = array();
+               foreach ( array( 'ss_total_views', 'ss_total_edits',
+                       'ss_good_articles', 'ss_total_pages', 'ss_users', 'ss_images' ) as $type )
+               {
+                       // Get pending increments and pending decrements
+                       $pending[$type]['+'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '+' ) );
+                       $pending[$type]['-'] = (int)$wgMemc->get( $this->getTypeCacheKey( $type, '-' ) );
+               }
+
+               return $pending;
+       }
+
+       /**
+        * Reduce pending delta counters after updates have been applied
+        * @param Array Result of getPendingDeltas(), used for DB update
+        * @return void
+        */
+       protected function removePendingDeltas( array $pd ) {
+               global $wgMemc;
+
+               foreach ( $pd as $type => $deltas ) {
+                       foreach ( $deltas as $sign => $magnitude ) {
+                               // Lower the pending counter now that we applied these changes
+                               $wgMemc->decr( $this->getTypeCacheKey( $type, $sign ), $magnitude );
+                       }
+               }
+       }
 }
 
 /**