includes/deferred/SiteStatsUpdate.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20 use MediaWiki\MediaWikiServices;
  21 use Wikimedia\Assert\Assert;
  22 use Wikimedia\Rdbms\IDatabase;
  23
  24 /**
  25  * Class for handling updates to the site_stats table
  26  */
  27 class SiteStatsUpdate implements DeferrableUpdate, MergeableUpdate {
  28         /** @var BagOStuff */
  29         protected $stash;
  30         /** @var int */
  31         protected $edits = 0;
  32         /** @var int */
  33         protected $pages = 0;
  34         /** @var int */
  35         protected $articles = 0;
  36         /** @var int */
  37         protected $users = 0;
  38         /** @var int */
  39         protected $images = 0;
  40
  41         private static $counters = [ 'edits', 'pages', 'articles', 'users', 'images' ];
  42
  43         // @todo deprecate this constructor
  44         function __construct( $views, $edits, $good, $pages = 0, $users = 0 ) {
  45                 $this->edits = $edits;
  46                 $this->articles = $good;
  47                 $this->pages = $pages;
  48                 $this->users = $users;
  49
  50                 $this->stash = MediaWikiServices::getInstance()->getMainObjectStash();
  51         }
  52
  53         public function merge( MergeableUpdate $update ) {
  54                 /** @var SiteStatsUpdate $update */
  55                 Assert::parameterType( __CLASS__, $update, '$update' );
  56
  57                 foreach ( self::$counters as $field ) {
  58                         $this->$field += $update->$field;
  59                 }
  60         }
  61
  62         /**
  63          * @param array $deltas
  64          * @return SiteStatsUpdate
  65          */
  66         public static function factory( array $deltas ) {
  67                 $update = new self( 0, 0, 0 );
  68
  69                 foreach ( $deltas as $name => $unused ) {
  70                         if ( !in_array( $name, self::$counters ) ) { // T187585
  71                                 throw new UnexpectedValueException( __METHOD__ . ": no field called '$name'" );
  72                         }
  73                 }
  74
  75                 foreach ( self::$counters as $field ) {
  76                         if ( isset( $deltas[$field] ) && $deltas[$field] ) {
  77                                 $update->$field = $deltas[$field];
  78                         }
  79                 }
  80
  81                 return $update;
  82         }
  83
  84         public function doUpdate() {
  85                 $this->doUpdateContextStats();
  86
  87                 $rate = MediaWikiServices::getInstance()->getMainConfig()->get( 'SiteStatsAsyncFactor' );
  88                 // If set to do so, only do actual DB updates 1 every $rate times.
  89                 // The other times, just update "pending delta" values in memcached.
  90                 if ( $rate && ( $rate < 0 || mt_rand( 0, $rate - 1 ) != 0 ) ) {
  91                         $this->doUpdatePendingDeltas();
  92                 } else {
  93                         // Need a separate transaction because this a global lock
  94                         DeferredUpdates::addCallableUpdate( [ $this, 'tryDBUpdateInternal' ] );
  95                 }
  96         }
  97
  98         /**
  99          * Do not call this outside of SiteStatsUpdate
 100          */
 101         public function tryDBUpdateInternal() {
 102                 $services = MediaWikiServices::getInstance();
 103                 $config = $services->getMainConfig();
 104
 105                 $dbw = $services->getDBLoadBalancer()->getConnection( DB_MASTER );
 106                 $lockKey = $dbw->getDomainID() . ':site_stats'; // prepend wiki ID
 107                 $pd = [];
 108                 if ( $config->get( 'SiteStatsAsyncFactor' ) ) {
 109                         // Lock the table so we don't have double DB/memcached updates
 110                         if ( !$dbw->lock( $lockKey, __METHOD__, 0 ) ) {
 111                                 $this->doUpdatePendingDeltas();
 112
 113                                 return;
 114                         }
 115                         $pd = $this->getPendingDeltas();
 116                         // Piggy-back the async deltas onto those of this stats update....
 117                         $this->edits += ( $pd['ss_total_edits']['+'] - $pd['ss_total_edits']['-'] );
 118                         $this->articles += ( $pd['ss_good_articles']['+'] - $pd['ss_good_articles']['-'] );
 119                         $this->pages += ( $pd['ss_total_pages']['+'] - $pd['ss_total_pages']['-'] );
 120                         $this->users += ( $pd['ss_users']['+'] - $pd['ss_users']['-'] );
 121                         $this->images += ( $pd['ss_images']['+'] - $pd['ss_images']['-'] );
 122                 }
 123
 124                 // Build up an SQL query of deltas and apply them...
 125                 $updates = '';
 126                 $this->appendUpdate( $updates, 'ss_total_edits', $this->edits );
 127                 $this->appendUpdate( $updates, 'ss_good_articles', $this->articles );
 128                 $this->appendUpdate( $updates, 'ss_total_pages', $this->pages );
 129                 $this->appendUpdate( $updates, 'ss_users', $this->users );
 130                 $this->appendUpdate( $updates, 'ss_images', $this->images );
 131                 if ( $updates != '' ) {
 132                         $dbw->update( 'site_stats', [ $updates ], [], __METHOD__ );
 133                 }
 134
 135                 if ( $config->get( 'SiteStatsAsyncFactor' ) ) {
 136                         // Decrement the async deltas now that we applied them
 137                         $this->removePendingDeltas( $pd );
 138                         // Commit the updates and unlock the table
 139                         $dbw->unlock( $lockKey, __METHOD__ );
 140                 }
 141
 142                 // Invalid cache used by parser functions
 143                 SiteStats::unload();
 144         }
 145
 146         /**
 147          * @param IDatabase $dbw
 148          * @return bool|mixed
 149          */
 150         public static function cacheUpdate( IDatabase $dbw ) {
 151                 $services = MediaWikiServices::getInstance();
 152                 $config = $services->getMainConfig();
 153
 154                 $dbr = $services->getDBLoadBalancer()->getConnection( DB_REPLICA, 'vslow' );
 155                 # Get non-bot users than did some recent action other than making accounts.
 156                 # If account creation is included, the number gets inflated ~20+ fold on enwiki.
 157                 $rcQuery = RecentChange::getQueryInfo();
 158                 $activeUsers = $dbr->selectField(
 159                         $rcQuery['tables'],
 160                         'COUNT( DISTINCT ' . $rcQuery['fields']['rc_user_text'] . ' )',
 161                         [
 162                                 'rc_type != ' . $dbr->addQuotes( RC_EXTERNAL ), // Exclude external (Wikidata)
 163                                 ActorMigration::newMigration()->isNotAnon( $rcQuery['fields']['rc_user'] ),
 164                                 'rc_bot' => 0,
 165                                 'rc_log_type != ' . $dbr->addQuotes( 'newusers' ) . ' OR rc_log_type IS NULL',
 166                                 'rc_timestamp >= ' . $dbr->addQuotes(
 167                                         $dbr->timestamp( time() - $config->get( 'ActiveUserDays' ) * 24 * 3600 ) ),
 168                         ],
 169                         __METHOD__,
 170                         [],
 171                         $rcQuery['joins']
 172                 );
 173                 $dbw->update(
 174                         'site_stats',
 175                         [ 'ss_active_users' => intval( $activeUsers ) ],
 176                         [ 'ss_row_id' => 1 ],
 177                         __METHOD__
 178                 );
 179
 180                 // Invalid cache used by parser functions
 181                 SiteStats::unload();
 182
 183                 return $activeUsers;
 184         }
 185
 186         protected function doUpdateContextStats() {
 187                 $stats = MediaWikiServices::getInstance()->getStatsdDataFactory();
 188                 foreach ( [ 'edits', 'articles', 'pages', 'users', 'images' ] as $type ) {
 189                         $delta = $this->$type;
 190                         if ( $delta !== 0 ) {
 191                                 $stats->updateCount( "site.$type", $delta );
 192                         }
 193                 }
 194         }
 195
 196         protected function doUpdatePendingDeltas() {
 197                 $this->adjustPending( 'ss_total_edits', $this->edits );
 198                 $this->adjustPending( 'ss_good_articles', $this->articles );
 199                 $this->adjustPending( 'ss_total_pages', $this->pages );
 200                 $this->adjustPending( 'ss_users', $this->users );
 201                 $this->adjustPending( 'ss_images', $this->images );
 202         }
 203
 204         /**
 205          * @param string &$sql
 206          * @param string $field
 207          * @param int $delta
 208          */
 209         protected function appendUpdate( &$sql, $field, $delta ) {
 210                 if ( $delta ) {
 211                         if ( $sql ) {
 212                                 $sql .= ',';
 213                         }
 214                         if ( $delta < 0 ) {
 215                                 $sql .= "$field=$field-" . abs( $delta );
 216                         } else {
 217                                 $sql .= "$field=$field+" . abs( $delta );
 218                         }
 219                 }
 220         }
 221
 222         /**
 223          * @param BagOStuff $stash
 224          * @param string $type
 225          * @param string $sign ('+' or '-')
 226          * @return string
 227          */
 228         private function getTypeCacheKey( BagOStuff $stash, $type, $sign ) {
 229                 return $stash->makeKey( 'sitestatsupdate', 'pendingdelta', $type, $sign );
 230         }
 231
 232         /**
 233          * Adjust the pending deltas for a stat type.
 234          * Each stat type has two pending counters, one for increments and decrements
 235          * @param string $type
 236          * @param int $delta Delta (positive or negative)
 237          */
 238         protected function adjustPending( $type, $delta ) {
 239                 if ( $delta < 0 ) { // decrement
 240                         $key = $this->getTypeCacheKey( $this->stash, $type, '-' );
 241                 } else { // increment
 242                         $key = $this->getTypeCacheKey( $this->stash, $type, '+' );
 243                 }
 244
 245                 $magnitude = abs( $delta );
 246                 $this->stash->incrWithInit( $key, 0, $magnitude, $magnitude );
 247         }
 248
 249         /**
 250          * Get pending delta counters for each stat type
 251          * @return array Positive and negative deltas for each type
 252          */
 253         protected function getPendingDeltas() {
 254                 $pending = [];
 255                 foreach ( [ 'ss_total_edits',
 256                         'ss_good_articles', 'ss_total_pages', 'ss_users', 'ss_images' ] as $type
 257                 ) {
 258                         // Get pending increments and pending decrements
 259                         $flg = BagOStuff::READ_LATEST;
 260                         $pending[$type]['+'] = (int)$this->stash->get(
 261                                 $this->getTypeCacheKey( $this->stash, $type, '+' ),
 262                                 $flg
 263                         );
 264                         $pending[$type]['-'] = (int)$this->stash->get(
 265                                 $this->getTypeCacheKey( $this->stash, $type, '-' ),
 266                                 $flg
 267                         );
 268                 }
 269
 270                 return $pending;
 271         }
 272
 273         /**
 274          * Reduce pending delta counters after updates have been applied
 275          * @param array $pd Result of getPendingDeltas(), used for DB update
 276          */
 277         protected function removePendingDeltas( array $pd ) {
 278                 foreach ( $pd as $type => $deltas ) {
 279                         foreach ( $deltas as $sign => $magnitude ) {
 280                                 // Lower the pending counter now that we applied these changes
 281                                 $key = $this->getTypeCacheKey( $this->stash, $type, $sign );
 282                                 $this->stash->decr( $key, $magnitude );
 283                         }
 284                 }
 285         }
 286 }