includes/jobqueue/jobs/RecentChangesUpdateJob.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup JobQueue
  20  */
  21 use MediaWiki\MediaWikiServices;
  22
  23 /**
  24  * Job for pruning recent changes
  25  *
  26  * @ingroup JobQueue
  27  * @since 1.25
  28  */
  29 class RecentChangesUpdateJob extends Job {
  30         function __construct( Title $title, array $params ) {
  31                 parent::__construct( 'recentChangesUpdate', $title, $params );
  32
  33                 if ( !isset( $params['type'] ) ) {
  34                         throw new Exception( "Missing 'type' parameter." );
  35                 }
  36
  37                 $this->executionFlags |= self::JOB_NO_EXPLICIT_TRX_ROUND;
  38                 $this->removeDuplicates = true;
  39         }
  40
  41         /**
  42          * @return RecentChangesUpdateJob
  43          */
  44         final public static function newPurgeJob() {
  45                 return new self(
  46                         SpecialPage::getTitleFor( 'Recentchanges' ), [ 'type' => 'purge' ]
  47                 );
  48         }
  49
  50         /**
  51          * @return RecentChangesUpdateJob
  52          * @since 1.26
  53          */
  54         final public static function newCacheUpdateJob() {
  55                 return new self(
  56                         SpecialPage::getTitleFor( 'Recentchanges' ), [ 'type' => 'cacheUpdate' ]
  57                 );
  58         }
  59
  60         public function run() {
  61                 if ( $this->params['type'] === 'purge' ) {
  62                         $this->purgeExpiredRows();
  63                 } elseif ( $this->params['type'] === 'cacheUpdate' ) {
  64                         $this->updateActiveUsers();
  65                 } else {
  66                         throw new InvalidArgumentException(
  67                                 "Invalid 'type' parameter '{$this->params['type']}'." );
  68                 }
  69
  70                 return true;
  71         }
  72
  73         protected function purgeExpiredRows() {
  74                 global $wgRCMaxAge, $wgUpdateRowsPerQuery;
  75
  76                 $lockKey = wfWikiID() . ':recentchanges-prune';
  77
  78                 $dbw = wfGetDB( DB_MASTER );
  79                 if ( !$dbw->lock( $lockKey, __METHOD__, 0 ) ) {
  80                         // already in progress
  81                         return;
  82                 }
  83
  84                 $factory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  85                 $ticket = $factory->getEmptyTransactionTicket( __METHOD__ );
  86                 $cutoff = $dbw->timestamp( time() - $wgRCMaxAge );
  87                 $rcQuery = RecentChange::getQueryInfo();
  88                 do {
  89                         $rcIds = [];
  90                         $rows = [];
  91                         $res = $dbw->select(
  92                                 $rcQuery['tables'],
  93                                 $rcQuery['fields'],
  94                                 [ 'rc_timestamp < ' . $dbw->addQuotes( $cutoff ) ],
  95                                 __METHOD__,
  96                                 [ 'LIMIT' => $wgUpdateRowsPerQuery ],
  97                                 $rcQuery['joins']
  98                         );
  99                         foreach ( $res as $row ) {
 100                                 $rcIds[] = $row->rc_id;
 101                                 $rows[] = $row;
 102                         }
 103                         if ( $rcIds ) {
 104                                 $dbw->delete( 'recentchanges', [ 'rc_id' => $rcIds ], __METHOD__ );
 105                                 Hooks::run( 'RecentChangesPurgeRows', [ $rows ] );
 106                                 // There might be more, so try waiting for replica DBs
 107                                 if ( !$factory->commitAndWaitForReplication(
 108                                         __METHOD__, $ticket, [ 'timeout' => 3 ]
 109                                 ) ) {
 110                                         // Another job will continue anyway
 111                                         break;
 112                                 }
 113                         }
 114                 } while ( $rcIds );
 115
 116                 $dbw->unlock( $lockKey, __METHOD__ );
 117         }
 118
 119         protected function updateActiveUsers() {
 120                 global $wgActiveUserDays;
 121
 122                 // Users that made edits at least this many days ago are "active"
 123                 $days = $wgActiveUserDays;
 124                 // Pull in the full window of active users in this update
 125                 $window = $wgActiveUserDays * 86400;
 126
 127                 $dbw = wfGetDB( DB_MASTER );
 128                 $factory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
 129                 $ticket = $factory->getEmptyTransactionTicket( __METHOD__ );
 130
 131                 $lockKey = wfWikiID() . '-activeusers';
 132                 if ( !$dbw->lock( $lockKey, __METHOD__, 0 ) ) {
 133                         // Exclusive update (avoids duplicate entries)… it's usually fine to just
 134                         // drop out here, if the Job is already running.
 135                         return;
 136                 }
 137
 138                 // Long-running queries expected
 139                 $dbw->setSessionOptions( [ 'connTimeout' => 900 ] );
 140
 141                 $nowUnix = time();
 142                 // Get the last-updated timestamp for the cache
 143                 $cTime = $dbw->selectField( 'querycache_info',
 144                         'qci_timestamp',
 145                         [ 'qci_type' => 'activeusers' ]
 146                 );
 147                 $cTimeUnix = $cTime ? wfTimestamp( TS_UNIX, $cTime ) : 1;
 148
 149                 // Pick the date range to fetch from. This is normally from the last
 150                 // update to till the present time, but has a limited window for sanity.
 151                 // If the window is limited, multiple runs are need to fully populate it.
 152                 $sTimestamp = max( $cTimeUnix, $nowUnix - $days * 86400 );
 153                 $eTimestamp = min( $sTimestamp + $window, $nowUnix );
 154
 155                 // Get all the users active since the last update
 156                 $actorQuery = ActorMigration::newMigration()->getJoin( 'rc_user' );
 157                 $res = $dbw->select(
 158                         [ 'recentchanges' ] + $actorQuery['tables'],
 159                         [
 160                                 'rc_user_text' => $actorQuery['fields']['rc_user_text'],
 161                                 'lastedittime' => 'MAX(rc_timestamp)'
 162                         ],
 163                         [
 164                                 $actorQuery['fields']['rc_user'] . ' > 0', // actual accounts
 165                                 'rc_type != ' . $dbw->addQuotes( RC_EXTERNAL ), // no wikidata
 166                                 'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes( 'newusers' ),
 167                                 'rc_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $sTimestamp ) ),
 168                                 'rc_timestamp <= ' . $dbw->addQuotes( $dbw->timestamp( $eTimestamp ) )
 169                         ],
 170                         __METHOD__,
 171                         [
 172                                 'GROUP BY' => [ 'rc_user_text' ],
 173                                 'ORDER BY' => 'NULL' // avoid filesort
 174                         ],
 175                         $actorQuery['joins']
 176                 );
 177                 $names = [];
 178                 foreach ( $res as $row ) {
 179                         $names[$row->rc_user_text] = $row->lastedittime;
 180                 }
 181
 182                 // Find which of the recently active users are already accounted for
 183                 if ( count( $names ) ) {
 184                         $res = $dbw->select( 'querycachetwo',
 185                                 [ 'user_name' => 'qcc_title' ],
 186                                 [
 187                                         'qcc_type' => 'activeusers',
 188                                         'qcc_namespace' => NS_USER,
 189                                         'qcc_title' => array_keys( $names ),
 190                                         'qcc_value >= ' . $dbw->addQuotes( $nowUnix - $days * 86400 ), // TS_UNIX
 191                                  ],
 192                                 __METHOD__
 193                         );
 194                         // Note: In order for this to be actually consistent, we would need
 195                         // to update these rows with the new lastedittime.
 196                         foreach ( $res as $row ) {
 197                                 unset( $names[$row->user_name] );
 198                         }
 199                 }
 200
 201                 // Insert the users that need to be added to the list
 202                 if ( count( $names ) ) {
 203                         $newRows = [];
 204                         foreach ( $names as $name => $lastEditTime ) {
 205                                 $newRows[] = [
 206                                         'qcc_type' => 'activeusers',
 207                                         'qcc_namespace' => NS_USER,
 208                                         'qcc_title' => $name,
 209                                         'qcc_value' => wfTimestamp( TS_UNIX, $lastEditTime ),
 210                                         'qcc_namespacetwo' => 0, // unused
 211                                         'qcc_titletwo' => '' // unused
 212                                 ];
 213                         }
 214                         foreach ( array_chunk( $newRows, 500 ) as $rowBatch ) {
 215                                 $dbw->insert( 'querycachetwo', $rowBatch, __METHOD__ );
 216                                 $factory->commitAndWaitForReplication( __METHOD__, $ticket );
 217                         }
 218                 }
 219
 220                 // If a transaction was already started, it might have an old
 221                 // snapshot, so kludge the timestamp range back as needed.
 222                 $asOfTimestamp = min( $eTimestamp, (int)$dbw->trxTimestamp() );
 223
 224                 // Touch the data freshness timestamp
 225                 $dbw->replace( 'querycache_info',
 226                         [ 'qci_type' ],
 227                         [ 'qci_type' => 'activeusers',
 228                                 'qci_timestamp' => $dbw->timestamp( $asOfTimestamp ) ], // not always $now
 229                         __METHOD__
 230                 );
 231
 232                 $dbw->unlock( $lockKey, __METHOD__ );
 233
 234                 // Rotate out users that have not edited in too long (according to old data set)
 235                 $dbw->delete( 'querycachetwo',
 236                         [
 237                                 'qcc_type' => 'activeusers',
 238                                 'qcc_value < ' . $dbw->addQuotes( $nowUnix - $days * 86400 ) // TS_UNIX
 239                         ],
 240                         __METHOD__
 241                 );
 242         }
 243 }