includes/jobqueue/jobs/RecentChangesUpdateJob.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @author Aaron Schulz
  20  * @ingroup JobQueue
  21  */
  22
  23 /**
  24  * Job for pruning recent changes
  25  *
  26  * @ingroup JobQueue
  27  * @since 1.25
  28  */
  29 class RecentChangesUpdateJob extends Job {
  30         function __construct( Title $title, array $params ) {
  31                 parent::__construct( 'recentChangesUpdate', $title, $params );
  32
  33                 if ( !isset( $params['type'] ) ) {
  34                         throw new Exception( "Missing 'type' parameter." );
  35                 }
  36
  37                 $this->removeDuplicates = true;
  38         }
  39
  40         /**
  41          * @return RecentChangesUpdateJob
  42          */
  43         final public static function newPurgeJob() {
  44                 return new self(
  45                         SpecialPage::getTitleFor( 'Recentchanges' ), array( 'type' => 'purge' )
  46                 );
  47         }
  48
  49         /**
  50          * @return RecentChangesUpdateJob
  51          * @since 1.26
  52          */
  53         final public static function newCacheUpdateJob() {
  54                 return new self(
  55                         SpecialPage::getTitleFor( 'Recentchanges' ), array( 'type' => 'cacheUpdate' )
  56                 );
  57         }
  58
  59         public function run() {
  60                 if ( $this->params['type'] === 'purge' ) {
  61                         $this->purgeExpiredRows();
  62                 } elseif ( $this->params['type'] === 'cacheUpdate' ) {
  63                         $this->updateActiveUsers();
  64                 } else {
  65                         throw new InvalidArgumentException(
  66                                 "Invalid 'type' parameter '{$this->params['type']}'." );
  67                 }
  68
  69                 return true;
  70         }
  71
  72         protected function purgeExpiredRows() {
  73                 global $wgRCMaxAge;
  74
  75                 $lockKey = wfWikiID() . ':recentchanges-prune';
  76
  77                 $dbw = wfGetDB( DB_MASTER );
  78                 if ( !$dbw->lockIsFree( $lockKey, __METHOD__ )
  79                         || !$dbw->lock( $lockKey, __METHOD__, 1 )
  80                 ) {
  81                         return; // already in progress
  82                 }
  83
  84                 $batchSize = 100; // avoid slave lag
  85                 $cutoff = $dbw->timestamp( time() - $wgRCMaxAge );
  86                 do {
  87                         $rcIds = $dbw->selectFieldValues( 'recentchanges',
  88                                 'rc_id',
  89                                 array( 'rc_timestamp < ' . $dbw->addQuotes( $cutoff ) ),
  90                                 __METHOD__,
  91                                 array( 'LIMIT' => $batchSize )
  92                         );
  93                         if ( $rcIds ) {
  94                                 $dbw->delete( 'recentchanges', array( 'rc_id' => $rcIds ), __METHOD__ );
  95                         }
  96                         // Commit in chunks to avoid slave lag
  97                         $dbw->commit( __METHOD__, 'flush' );
  98
  99                         if ( count( $rcIds ) === $batchSize ) {
 100                                 // There might be more, so try waiting for slaves
 101                                 if ( !wfWaitForSlaves( null, false, false, /* $timeout = */ 3 ) ) {
 102                                         // Another job will continue anyway
 103                                         break;
 104                                 }
 105                         }
 106                 } while ( $rcIds );
 107
 108                 $dbw->unlock( $lockKey, __METHOD__ );
 109         }
 110
 111         protected function updateActiveUsers() {
 112                 global $wgActiveUserDays;
 113
 114                 // Users that made edits at least this many days ago are "active"
 115                 $days = $wgActiveUserDays;
 116                 // Pull in the full window of active users in this update
 117                 $window = $wgActiveUserDays * 86400;
 118
 119                 $dbw = wfGetDB( DB_MASTER );
 120                 // JobRunner uses DBO_TRX, but doesn't call begin/commit itself;
 121                 // onTransactionIdle() will run immediately since there is no trx.
 122                 $dbw->onTransactionIdle( function() use ( $dbw, $days, $window ) {
 123                         // Avoid disconnect/ping() cycle that makes locks fall off
 124                         $dbw->setSessionOptions( array( 'connTimeout' => 900 ) );
 125
 126                         $lockKey = wfWikiID() . '-activeusers';
 127                         if ( !$dbw->lock( $lockKey, __METHOD__, 1 ) ) {
 128                                 return false; // exclusive update (avoids duplicate entries)
 129                         }
 130
 131                         $nowUnix = time();
 132                         // Get the last-updated timestamp for the cache
 133                         $cTime = $dbw->selectField( 'querycache_info',
 134                                 'qci_timestamp',
 135                                 array( 'qci_type' => 'activeusers' )
 136                         );
 137                         $cTimeUnix = $cTime ? wfTimestamp( TS_UNIX, $cTime ) : 1;
 138
 139                         // Pick the date range to fetch from. This is normally from the last
 140                         // update to till the present time, but has a limited window for sanity.
 141                         // If the window is limited, multiple runs are need to fully populate it.
 142                         $sTimestamp = max( $cTimeUnix, $nowUnix - $days * 86400 );
 143                         $eTimestamp = min( $sTimestamp + $window, $nowUnix );
 144
 145                         // Get all the users active since the last update
 146                         $res = $dbw->select(
 147                                 array( 'recentchanges' ),
 148                                 array( 'rc_user_text', 'lastedittime' => 'MAX(rc_timestamp)' ),
 149                                 array(
 150                                         'rc_user > 0', // actual accounts
 151                                         'rc_type != ' . $dbw->addQuotes( RC_EXTERNAL ), // no wikidata
 152                                         'rc_log_type IS NULL OR rc_log_type != ' . $dbw->addQuotes( 'newusers' ),
 153                                         'rc_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $sTimestamp ) ),
 154                                         'rc_timestamp <= ' . $dbw->addQuotes( $dbw->timestamp( $eTimestamp ) )
 155                                 ),
 156                                 __METHOD__,
 157                                 array(
 158                                         'GROUP BY' => array( 'rc_user_text' ),
 159                                         'ORDER BY' => 'NULL' // avoid filesort
 160                                 )
 161                         );
 162                         $names = array();
 163                         foreach ( $res as $row ) {
 164                                 $names[$row->rc_user_text] = $row->lastedittime;
 165                         }
 166
 167                         // Rotate out users that have not edited in too long (according to old data set)
 168                         $dbw->delete( 'querycachetwo',
 169                                 array(
 170                                         'qcc_type' => 'activeusers',
 171                                         'qcc_value < ' . $dbw->addQuotes( $nowUnix - $days * 86400 ) // TS_UNIX
 172                                 ),
 173                                 __METHOD__
 174                         );
 175
 176                         // Find which of the recently active users are already accounted for
 177                         if ( count( $names ) ) {
 178                                 $res = $dbw->select( 'querycachetwo',
 179                                         array( 'user_name' => 'qcc_title' ),
 180                                         array(
 181                                                 'qcc_type' => 'activeusers',
 182                                                 'qcc_namespace' => NS_USER,
 183                                                 'qcc_title' => array_keys( $names ) ),
 184                                         __METHOD__
 185                                 );
 186                                 foreach ( $res as $row ) {
 187                                         unset( $names[$row->user_name] );
 188                                 }
 189                         }
 190
 191                         // Insert the users that need to be added to the list
 192                         if ( count( $names ) ) {
 193                                 $newRows = array();
 194                                 foreach ( $names as $name => $lastEditTime ) {
 195                                         $newRows[] = array(
 196                                                 'qcc_type' => 'activeusers',
 197                                                 'qcc_namespace' => NS_USER,
 198                                                 'qcc_title' => $name,
 199                                                 'qcc_value' => wfTimestamp( TS_UNIX, $lastEditTime ),
 200                                                 'qcc_namespacetwo' => 0, // unused
 201                                                 'qcc_titletwo' => '' // unused
 202                                         );
 203                                 }
 204                                 foreach ( array_chunk( $newRows, 500 ) as $rowBatch ) {
 205                                         $dbw->insert( 'querycachetwo', $rowBatch, __METHOD__ );
 206                                         wfWaitForSlaves();
 207                                 }
 208                         }
 209
 210                         // If a transaction was already started, it might have an old
 211                         // snapshot, so kludge the timestamp range back as needed.
 212                         $asOfTimestamp = min( $eTimestamp, (int)$dbw->trxTimestamp() );
 213
 214                         // Touch the data freshness timestamp
 215                         $dbw->replace( 'querycache_info',
 216                                 array( 'qci_type' ),
 217                                 array( 'qci_type' => 'activeusers',
 218                                         'qci_timestamp' => $dbw->timestamp( $asOfTimestamp ) ), // not always $now
 219                                 __METHOD__
 220                         );
 221
 222                         $dbw->unlock( $lockKey, __METHOD__ );
 223                 } );
 224         }
 225 }