includes/jobqueue/jobs/RecentChangesUpdateJob.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @author Aaron Schulz
  20  * @ingroup JobQueue
  21  */
  22 use MediaWiki\MediaWikiServices;
  23 use Wikimedia\Rdbms\DBReplicationWaitError;
  24
  25 /**
  26  * Job for pruning recent changes
  27  *
  28  * @ingroup JobQueue
  29  * @since 1.25
  30  */
  31 class RecentChangesUpdateJob extends Job {
  32         function __construct( Title $title, array $params ) {
  33                 parent::__construct( 'recentChangesUpdate', $title, $params );
  34
  35                 if ( !isset( $params['type'] ) ) {
  36                         throw new Exception( "Missing 'type' parameter." );
  37                 }
  38
  39                 $this->removeDuplicates = true;
  40         }
  41
  42         /**
  43          * @return RecentChangesUpdateJob
  44          */
  45         final public static function newPurgeJob() {
  46                 return new self(
  47                         SpecialPage::getTitleFor( 'Recentchanges' ), [ 'type' => 'purge' ]
  48                 );
  49         }
  50
  51         /**
  52          * @return RecentChangesUpdateJob
  53          * @since 1.26
  54          */
  55         final public static function newCacheUpdateJob() {
  56                 return new self(
  57                         SpecialPage::getTitleFor( 'Recentchanges' ), [ 'type' => 'cacheUpdate' ]
  58                 );
  59         }
  60
  61         public function run() {
  62                 if ( $this->params['type'] === 'purge' ) {
  63                         $this->purgeExpiredRows();
  64                 } elseif ( $this->params['type'] === 'cacheUpdate' ) {
  65                         $this->updateActiveUsers();
  66                 } else {
  67                         throw new InvalidArgumentException(
  68                                 "Invalid 'type' parameter '{$this->params['type']}'." );
  69                 }
  70
  71                 return true;
  72         }
  73
  74         protected function purgeExpiredRows() {
  75                 global $wgRCMaxAge, $wgUpdateRowsPerQuery;
  76
  77                 $lockKey = wfWikiID() . ':recentchanges-prune';
  78
  79                 $dbw = wfGetDB( DB_MASTER );
  80                 if ( !$dbw->lockIsFree( $lockKey, __METHOD__ )
  81                         || !$dbw->lock( $lockKey, __METHOD__, 1 )
  82                 ) {
  83                         return; // already in progress
  84                 }
  85
  86                 $factory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  87                 $ticket = $factory->getEmptyTransactionTicket( __METHOD__ );
  88                 $cutoff = $dbw->timestamp( time() - $wgRCMaxAge );
  89                 do {
  90                         $rcIds = $dbw->selectFieldValues( 'recentchanges',
  91                                 'rc_id',
  92                                 [ 'rc_timestamp < ' . $dbw->addQuotes( $cutoff ) ],
  93                                 __METHOD__,
  94                                 [ 'LIMIT' => $wgUpdateRowsPerQuery ]
  95                         );
  96                         if ( $rcIds ) {
  97                                 $dbw->delete( 'recentchanges', [ 'rc_id' => $rcIds ], __METHOD__ );
  98                                 // There might be more, so try waiting for replica DBs
  99                                 try {
 100                                         $factory->commitAndWaitForReplication(
 101                                                 __METHOD__, $ticket, [ 'timeout' => 3 ]
 102                                         );
 103                                 } catch ( DBReplicationWaitError $e ) {
 104                                         // Another job will continue anyway
 105                                         break;
 106                                 }
 107                         }
 108                 } while ( $rcIds );
 109
 110                 $dbw->unlock( $lockKey, __METHOD__ );
 111         }
 112
 113         protected function updateActiveUsers() {
 114                 global $wgActiveUserDays;
 115
 116                 // Users that made edits at least this many days ago are "active"
 117                 $days = $wgActiveUserDays;
 118                 // Pull in the full window of active users in this update
 119                 $window = $wgActiveUserDays * 86400;
 120
 121                 $dbw = wfGetDB( DB_MASTER );
 122                 // JobRunner uses DBO_TRX, but doesn't call begin/commit itself;
 123                 // onTransactionIdle() will run immediately since there is no trx.
 124                 $dbw->onTransactionIdle(
 125                         function () use ( $dbw, $days, $window ) {
 126                                 $factory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
 127                                 $ticket = $factory->getEmptyTransactionTicket( __METHOD__ );
 128                                 // Avoid disconnect/ping() cycle that makes locks fall off
 129                                 $dbw->setSessionOptions( [ 'connTimeout' => 900 ] );
 130
 131                                 $lockKey = wfWikiID() . '-activeusers';
 132                                 if ( !$dbw->lockIsFree( $lockKey, __METHOD__ ) || !$dbw->lock( $lockKey, __METHOD__, 1 ) ) {
 133                                         // Exclusive update (avoids duplicate entries)… it's usually fine to just drop out here,
 134                                         // if the Job is already running.
 135                                         return;
 136                                 }
 137
 138                                 $nowUnix = time();
 139                                 // Get the last-updated timestamp for the cache
 140                                 $cTime = $dbw->selectField( 'querycache_info',
 141                                         'qci_timestamp',
 142                                         [ 'qci_type' => 'activeusers' ]
 143                                 );
 144                                 $cTimeUnix = $cTime ? wfTimestamp( TS_UNIX, $cTime ) : 1;
 145
 146                                 // Pick the date range to fetch from. This is normally from the last
 147                                 // update to till the present time, but has a limited window for sanity.
 148                                 // If the window is limited, multiple runs are need to fully populate it.
 149                                 $sTimestamp = max( $cTimeUnix, $nowUnix - $days * 86400 );
 150                                 $eTimestamp = min( $sTimestamp + $window, $nowUnix );
 151
 152                                 // Get all the users active since the last update
 153                                 $res = $dbw->select(
 154                                         [ 'recentchanges' ],
 155                                         [ 'rc_user_text', 'lastedittime' => 'MAX(rc_timestamp)' ],
 156                                         [
 157                                                 'rc_user > 0', // actual accounts
 158                                                 'rc_type != ' . $dbw->addQuotes( RC_EXTERNAL ), // no wikidata
 159                                                 'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes( 'newusers' ),
 160                                                 'rc_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $sTimestamp ) ),
 161                                                 'rc_timestamp <= ' . $dbw->addQuotes( $dbw->timestamp( $eTimestamp ) )
 162                                         ],
 163                                         __METHOD__,
 164                                         [
 165                                                 'GROUP BY' => [ 'rc_user_text' ],
 166                                                 'ORDER BY' => 'NULL' // avoid filesort
 167                                         ]
 168                                 );
 169                                 $names = [];
 170                                 foreach ( $res as $row ) {
 171                                         $names[$row->rc_user_text] = $row->lastedittime;
 172                                 }
 173
 174                                 // Find which of the recently active users are already accounted for
 175                                 if ( count( $names ) ) {
 176                                         $res = $dbw->select( 'querycachetwo',
 177                                                 [ 'user_name' => 'qcc_title' ],
 178                                                 [
 179                                                         'qcc_type' => 'activeusers',
 180                                                         'qcc_namespace' => NS_USER,
 181                                                         'qcc_title' => array_keys( $names ),
 182                                                         'qcc_value >= ' . $dbw->addQuotes( $nowUnix - $days * 86400 ), // TS_UNIX
 183                                                  ],
 184                                                 __METHOD__
 185                                         );
 186                                         // Note: In order for this to be actually consistent, we would need
 187                                         // to update these rows with the new lastedittime.
 188                                         foreach ( $res as $row ) {
 189                                                 unset( $names[$row->user_name] );
 190                                         }
 191                                 }
 192
 193                                 // Insert the users that need to be added to the list
 194                                 if ( count( $names ) ) {
 195                                         $newRows = [];
 196                                         foreach ( $names as $name => $lastEditTime ) {
 197                                                 $newRows[] = [
 198                                                         'qcc_type' => 'activeusers',
 199                                                         'qcc_namespace' => NS_USER,
 200                                                         'qcc_title' => $name,
 201                                                         'qcc_value' => wfTimestamp( TS_UNIX, $lastEditTime ),
 202                                                         'qcc_namespacetwo' => 0, // unused
 203                                                         'qcc_titletwo' => '' // unused
 204                                                 ];
 205                                         }
 206                                         foreach ( array_chunk( $newRows, 500 ) as $rowBatch ) {
 207                                                 $dbw->insert( 'querycachetwo', $rowBatch, __METHOD__ );
 208                                                 $factory->commitAndWaitForReplication( __METHOD__, $ticket );
 209                                         }
 210                                 }
 211
 212                                 // If a transaction was already started, it might have an old
 213                                 // snapshot, so kludge the timestamp range back as needed.
 214                                 $asOfTimestamp = min( $eTimestamp, (int)$dbw->trxTimestamp() );
 215
 216                                 // Touch the data freshness timestamp
 217                                 $dbw->replace( 'querycache_info',
 218                                         [ 'qci_type' ],
 219                                         [ 'qci_type' => 'activeusers',
 220                                                 'qci_timestamp' => $dbw->timestamp( $asOfTimestamp ) ], // not always $now
 221                                         __METHOD__
 222                                 );
 223
 224                                 $dbw->unlock( $lockKey, __METHOD__ );
 225
 226                                 // Rotate out users that have not edited in too long (according to old data set)
 227                                 $dbw->delete( 'querycachetwo',
 228                                         [
 229                                                 'qcc_type' => 'activeusers',
 230                                                 'qcc_value < ' . $dbw->addQuotes( $nowUnix - $days * 86400 ) // TS_UNIX
 231                                         ],
 232                                         __METHOD__
 233                                 );
 234                         },
 235                         __METHOD__
 236                 );
 237         }
 238 }