includes/jobqueue/jobs/CategoryMembershipChangeJob.php

   1 <?php
   2 /**
   3  * Updater for link tracking tables after a page edit.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22 use MediaWiki\MediaWikiServices;
  23 use Wikimedia\Rdbms\LBFactory;
  24
  25 /**
  26  * Job to add recent change entries mentioning category membership changes
  27  *
  28  * Parameters include:
  29  *   - pageId : page ID
  30  *   - revTimestamp : timestamp of the triggering revision
  31  *
  32  * Category changes will be mentioned for revisions at/after the timestamp for this page
  33  *
  34  * @since 1.27
  35  */
  36 class CategoryMembershipChangeJob extends Job {
  37         /** @var int|null */
  38         private $ticket;
  39
  40         const ENQUEUE_FUDGE_SEC = 60;
  41
  42         public function __construct( Title $title, array $params ) {
  43                 parent::__construct( 'categoryMembershipChange', $title, $params );
  44                 // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
  45                 // older revision job gets inserted while the newer revision job is de-duplicated.
  46                 $this->removeDuplicates = true;
  47         }
  48
  49         public function run() {
  50                 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  51                 $lb = $lbFactory->getMainLB();
  52                 $dbw = $lb->getConnection( DB_MASTER );
  53
  54                 $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
  55
  56                 $page = WikiPage::newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
  57                 if ( !$page ) {
  58                         $this->setLastError( "Could not find page #{$this->params['pageId']}" );
  59                         return false; // deleted?
  60                 }
  61
  62                 // Cut down on the time spent in safeWaitForMasterPos() in the critical section
  63                 $dbr = $lb->getConnection( DB_REPLICA, [ 'recentchanges' ] );
  64                 if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
  65                         $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
  66                         return false;
  67                 }
  68
  69                 // Use a named lock so that jobs for this page see each others' changes
  70                 $lockKey = "CategoryMembershipUpdates:{$page->getId()}";
  71                 $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
  72                 if ( !$scopedLock ) {
  73                         $this->setLastError( "Could not acquire lock '$lockKey'" );
  74                         return false;
  75                 }
  76
  77                 // Wait till replica DB is caught up so that jobs for this page see each others' changes
  78                 if ( !$lb->safeWaitForMasterPos( $dbr ) ) {
  79                         $this->setLastError( "Timed out while waiting for replica DB to catch up" );
  80                         return false;
  81                 }
  82                 // Clear any stale REPEATABLE-READ snapshot
  83                 $dbr->flushSnapshot( __METHOD__ );
  84
  85                 $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
  86                 // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
  87                 // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
  88                 $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
  89
  90                 // Get the newest revision that has a SRC_CATEGORIZE row...
  91                 $row = $dbr->selectRow(
  92                         [ 'revision', 'recentchanges' ],
  93                         [ 'rev_timestamp', 'rev_id' ],
  94                         [
  95                                 'rev_page' => $page->getId(),
  96                                 'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) )
  97                         ],
  98                         __METHOD__,
  99                         [ 'ORDER BY' => 'rev_timestamp DESC, rev_id DESC' ],
 100                         [
 101                                 'recentchanges' => [
 102                                         'INNER JOIN',
 103                                         [
 104                                                 'rc_this_oldid = rev_id',
 105                                                 'rc_source' => RecentChange::SRC_CATEGORIZE,
 106                                                 // Allow rc_cur_id or rc_timestamp index usage
 107                                                 'rc_cur_id = rev_page',
 108                                                 'rc_timestamp >= rev_timestamp'
 109                                         ]
 110                                 ]
 111                         ]
 112                 );
 113                 // Only consider revisions newer than any such revision
 114                 if ( $row ) {
 115                         $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
 116                         $lastRevId = (int)$row->rev_id;
 117                 } else {
 118                         $lastRevId = 0;
 119                 }
 120
 121                 // Find revisions to this page made around and after this revision which lack category
 122                 // notifications in recent changes. This lets jobs pick up were the last one left off.
 123                 $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
 124                 $revQuery = Revision::getQueryInfo();
 125                 $res = $dbr->select(
 126                         $revQuery['tables'],
 127                         $revQuery['fields'],
 128                         [
 129                                 'rev_page' => $page->getId(),
 130                                 "rev_timestamp > $encCutoff" .
 131                                         " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
 132                         ],
 133                         __METHOD__,
 134                         [ 'ORDER BY' => 'rev_timestamp ASC, rev_id ASC' ],
 135                         $revQuery['joins']
 136                 );
 137
 138                 // Apply all category updates in revision timestamp order
 139                 foreach ( $res as $row ) {
 140                         $this->notifyUpdatesForRevision( $lbFactory, $page, Revision::newFromRow( $row ) );
 141                 }
 142
 143                 return true;
 144         }
 145
 146         /**
 147          * @param LBFactory $lbFactory
 148          * @param WikiPage $page
 149          * @param Revision $newRev
 150          * @throws MWException
 151          */
 152         protected function notifyUpdatesForRevision(
 153                 LBFactory $lbFactory, WikiPage $page, Revision $newRev
 154         ) {
 155                 $config = RequestContext::getMain()->getConfig();
 156                 $title = $page->getTitle();
 157
 158                 // Get the new revision
 159                 if ( !$newRev->getContent() ) {
 160                         return; // deleted?
 161                 }
 162
 163                 // Get the prior revision (the same for null edits)
 164                 if ( $newRev->getParentId() ) {
 165                         $oldRev = Revision::newFromId( $newRev->getParentId(), Revision::READ_LATEST );
 166                         if ( !$oldRev->getContent() ) {
 167                                 return; // deleted?
 168                         }
 169                 } else {
 170                         $oldRev = null;
 171                 }
 172
 173                 // Parse the new revision and get the categories
 174                 $categoryChanges = $this->getExplicitCategoriesChanges( $title, $newRev, $oldRev );
 175                 list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
 176                 if ( !$categoryInserts && !$categoryDeletes ) {
 177                         return; // nothing to do
 178                 }
 179
 180                 $catMembChange = new CategoryMembershipChange( $title, $newRev );
 181                 $catMembChange->checkTemplateLinks();
 182
 183                 $batchSize = $config->get( 'UpdateRowsPerQuery' );
 184                 $insertCount = 0;
 185
 186                 foreach ( $categoryInserts as $categoryName ) {
 187                         $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
 188                         $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
 189                         if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
 190                                 $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
 191                         }
 192                 }
 193
 194                 foreach ( $categoryDeletes as $categoryName ) {
 195                         $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
 196                         $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
 197                         if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
 198                                 $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
 199                         }
 200                 }
 201         }
 202
 203         private function getExplicitCategoriesChanges(
 204                 Title $title, Revision $newRev, Revision $oldRev = null
 205         ) {
 206                 // Inject the same timestamp for both revision parses to avoid seeing category changes
 207                 // due to time-based parser functions. Inject the same page title for the parses too.
 208                 // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
 209                 $parseTimestamp = $newRev->getTimestamp();
 210                 // Parse the old rev and get the categories. Do not use link tables as that
 211                 // assumes these updates are perfectly FIFO and that link tables are always
 212                 // up to date, neither of which are true.
 213                 $oldCategories = $oldRev
 214                         ? $this->getCategoriesAtRev( $title, $oldRev, $parseTimestamp )
 215                         : [];
 216                 // Parse the new revision and get the categories
 217                 $newCategories = $this->getCategoriesAtRev( $title, $newRev, $parseTimestamp );
 218
 219                 $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
 220                 $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
 221
 222                 return [ $categoryInserts, $categoryDeletes ];
 223         }
 224
 225         /**
 226          * @param Title $title
 227          * @param Revision $rev
 228          * @param string $parseTimestamp TS_MW
 229          *
 230          * @return string[] category names
 231          */
 232         private function getCategoriesAtRev( Title $title, Revision $rev, $parseTimestamp ) {
 233                 $content = $rev->getContent();
 234                 $options = $content->getContentHandler()->makeParserOptions( 'canonical' );
 235                 $options->setTimestamp( $parseTimestamp );
 236                 // This could possibly use the parser cache if it checked the revision ID,
 237                 // but that's more complicated than it's worth.
 238                 $output = $content->getParserOutput( $title, $rev->getId(), $options );
 239
 240                 // array keys will cast numeric category names to ints
 241                 // so we need to cast them back to strings to avoid breaking things!
 242                 return array_map( 'strval', array_keys( $output->getCategories() ) );
 243         }
 244
 245         public function getDeduplicationInfo() {
 246                 $info = parent::getDeduplicationInfo();
 247                 unset( $info['params']['revTimestamp'] ); // first job wins
 248
 249                 return $info;
 250         }
 251 }