includes/jobqueue/jobs/CategoryMembershipChangeJob.php

   1 <?php
   2 /**
   3  * Updater for link tracking tables after a page edit.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22 use MediaWiki\MediaWikiServices;
  23 use Wikimedia\Rdbms\LBFactory;
  24
  25 /**
  26  * Job to add recent change entries mentioning category membership changes
  27  *
  28  * This allows users to easily scan categories for recent page membership changes
  29  *
  30  * Parameters include:
  31  *   - pageId : page ID
  32  *   - revTimestamp : timestamp of the triggering revision
  33  *
  34  * Category changes will be mentioned for revisions at/after the timestamp for this page
  35  *
  36  * @since 1.27
  37  */
  38 class CategoryMembershipChangeJob extends Job {
  39         /** @var int|null */
  40         private $ticket;
  41
  42         const ENQUEUE_FUDGE_SEC = 60;
  43
  44         /**
  45          * @var ParserCache
  46          */
  47         private $parserCache;
  48
  49         /**
  50          * @param Title $title The title of the page for which to update category membership.
  51          * @param string $revisionTimestamp The timestamp of the new revision that triggered the job.
  52          * @return JobSpecification
  53          */
  54         public static function newSpec( Title $title, $revisionTimestamp ) {
  55                 return new JobSpecification(
  56                         'categoryMembershipChange',
  57                         [
  58                                 'pageId' => $title->getArticleID(),
  59                                 'revTimestamp' => $revisionTimestamp,
  60                         ],
  61                         [],
  62                         $title
  63                 );
  64         }
  65
  66         /**
  67          * Constructor for use by the Job Queue infrastructure.
  68          * @note Don't call this when queueing a new instance, use newSpec() instead.
  69          * @param ParserCache $parserCache Cache outputs of PHP parser.
  70          * @param Title $title Title of the categorized page.
  71          * @param array $params Such latest revision instance of the categorized page.
  72          */
  73         public function __construct( ParserCache $parserCache, Title $title, array $params ) {
  74                 parent::__construct( 'categoryMembershipChange', $title, $params );
  75                 // Only need one job per page. Note that ENQUEUE_FUDGE_SEC handles races where an
  76                 // older revision job gets inserted while the newer revision job is de-duplicated.
  77                 $this->removeDuplicates = true;
  78                 $this->parserCache = $parserCache;
  79         }
  80
  81         public function run() {
  82                 $lbFactory = MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
  83                 $lb = $lbFactory->getMainLB();
  84                 $dbw = $lb->getConnectionRef( DB_MASTER );
  85
  86                 $this->ticket = $lbFactory->getEmptyTransactionTicket( __METHOD__ );
  87
  88                 $page = WikiPage::newFromID( $this->params['pageId'], WikiPage::READ_LATEST );
  89                 if ( !$page ) {
  90                         $this->setLastError( "Could not find page #{$this->params['pageId']}" );
  91                         return false; // deleted?
  92                 }
  93
  94                 // Cut down on the time spent in waitForMasterPos() in the critical section
  95                 $dbr = $lb->getConnectionRef( DB_REPLICA, [ 'recentchanges' ] );
  96                 if ( !$lb->waitForMasterPos( $dbr ) ) {
  97                         $this->setLastError( "Timed out while pre-waiting for replica DB to catch up" );
  98                         return false;
  99                 }
 100
 101                 // Use a named lock so that jobs for this page see each others' changes
 102                 $lockKey = "{$dbw->getDomainID()}:CategoryMembershipChange:{$page->getId()}"; // per-wiki
 103                 $scopedLock = $dbw->getScopedLockAndFlush( $lockKey, __METHOD__, 3 );
 104                 if ( !$scopedLock ) {
 105                         $this->setLastError( "Could not acquire lock '$lockKey'" );
 106                         return false;
 107                 }
 108
 109                 // Wait till replica DB is caught up so that jobs for this page see each others' changes
 110                 if ( !$lb->waitForMasterPos( $dbr ) ) {
 111                         $this->setLastError( "Timed out while waiting for replica DB to catch up" );
 112                         return false;
 113                 }
 114                 // Clear any stale REPEATABLE-READ snapshot
 115                 $dbr->flushSnapshot( __METHOD__ );
 116
 117                 $cutoffUnix = wfTimestamp( TS_UNIX, $this->params['revTimestamp'] );
 118                 // Using ENQUEUE_FUDGE_SEC handles jobs inserted out of revision order due to the delay
 119                 // between COMMIT and actual enqueueing of the CategoryMembershipChangeJob job.
 120                 $cutoffUnix -= self::ENQUEUE_FUDGE_SEC;
 121
 122                 // Get the newest page revision that has a SRC_CATEGORIZE row.
 123                 // Assume that category changes before it were already handled.
 124                 $row = $dbr->selectRow(
 125                         'revision',
 126                         [ 'rev_timestamp', 'rev_id' ],
 127                         [
 128                                 'rev_page' => $page->getId(),
 129                                 'rev_timestamp >= ' . $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) ),
 130                                 'EXISTS (' . $dbr->selectSQLText(
 131                                         'recentchanges',
 132                                         '1',
 133                                         [
 134                                                 'rc_this_oldid = rev_id',
 135                                                 'rc_source' => RecentChange::SRC_CATEGORIZE,
 136                                                 // Allow rc_cur_id or rc_timestamp index usage
 137                                                 'rc_cur_id = rev_page',
 138                                                 'rc_timestamp = rev_timestamp'
 139                                         ]
 140                                 ) . ')'
 141                         ],
 142                         __METHOD__,
 143                         [ 'ORDER BY' => 'rev_timestamp DESC, rev_id DESC' ]
 144                 );
 145                 // Only consider revisions newer than any such revision
 146                 if ( $row ) {
 147                         $cutoffUnix = wfTimestamp( TS_UNIX, $row->rev_timestamp );
 148                         $lastRevId = (int)$row->rev_id;
 149                 } else {
 150                         $lastRevId = 0;
 151                 }
 152
 153                 // Find revisions to this page made around and after this revision which lack category
 154                 // notifications in recent changes. This lets jobs pick up were the last one left off.
 155                 $encCutoff = $dbr->addQuotes( $dbr->timestamp( $cutoffUnix ) );
 156                 $revQuery = Revision::getQueryInfo();
 157                 $res = $dbr->select(
 158                         $revQuery['tables'],
 159                         $revQuery['fields'],
 160                         [
 161                                 'rev_page' => $page->getId(),
 162                                 "rev_timestamp > $encCutoff" .
 163                                         " OR (rev_timestamp = $encCutoff AND rev_id > $lastRevId)"
 164                         ],
 165                         __METHOD__,
 166                         [ 'ORDER BY' => 'rev_timestamp ASC, rev_id ASC' ],
 167                         $revQuery['joins']
 168                 );
 169
 170                 // Apply all category updates in revision timestamp order
 171                 foreach ( $res as $row ) {
 172                         $this->notifyUpdatesForRevision( $lbFactory, $page, Revision::newFromRow( $row ) );
 173                 }
 174
 175                 return true;
 176         }
 177
 178         /**
 179          * @param LBFactory $lbFactory
 180          * @param WikiPage $page
 181          * @param Revision $newRev
 182          * @throws MWException
 183          */
 184         protected function notifyUpdatesForRevision(
 185                 LBFactory $lbFactory, WikiPage $page, Revision $newRev
 186         ) {
 187                 $config = RequestContext::getMain()->getConfig();
 188                 $title = $page->getTitle();
 189
 190                 // Get the new revision
 191                 if ( !$newRev->getContent() ) {
 192                         return; // deleted?
 193                 }
 194
 195                 // Get the prior revision (the same for null edits)
 196                 if ( $newRev->getParentId() ) {
 197                         $oldRev = Revision::newFromId( $newRev->getParentId(), Revision::READ_LATEST );
 198                         if ( !$oldRev || !$oldRev->getContent() ) {
 199                                 return; // deleted?
 200                         }
 201                 } else {
 202                         $oldRev = null;
 203                 }
 204
 205                 // Parse the new revision and get the categories
 206                 $categoryChanges = $this->getExplicitCategoriesChanges( $page, $newRev, $oldRev );
 207                 list( $categoryInserts, $categoryDeletes ) = $categoryChanges;
 208                 if ( !$categoryInserts && !$categoryDeletes ) {
 209                         return; // nothing to do
 210                 }
 211
 212                 $catMembChange = new CategoryMembershipChange( $title, $newRev );
 213                 $catMembChange->checkTemplateLinks();
 214
 215                 $batchSize = $config->get( 'UpdateRowsPerQuery' );
 216                 $insertCount = 0;
 217
 218                 foreach ( $categoryInserts as $categoryName ) {
 219                         $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
 220                         $catMembChange->triggerCategoryAddedNotification( $categoryTitle );
 221                         if ( $insertCount++ && ( $insertCount % $batchSize ) == 0 ) {
 222                                 $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
 223                         }
 224                 }
 225
 226                 foreach ( $categoryDeletes as $categoryName ) {
 227                         $categoryTitle = Title::makeTitle( NS_CATEGORY, $categoryName );
 228                         $catMembChange->triggerCategoryRemovedNotification( $categoryTitle );
 229                         if ( $insertCount++ && ( $insertCount++ % $batchSize ) == 0 ) {
 230                                 $lbFactory->commitAndWaitForReplication( __METHOD__, $this->ticket );
 231                         }
 232                 }
 233         }
 234
 235         private function getExplicitCategoriesChanges(
 236                 WikiPage $page, Revision $newRev, Revision $oldRev = null
 237         ) {
 238                 // Inject the same timestamp for both revision parses to avoid seeing category changes
 239                 // due to time-based parser functions. Inject the same page title for the parses too.
 240                 // Note that REPEATABLE-READ makes template/file pages appear unchanged between parses.
 241                 $parseTimestamp = $newRev->getTimestamp();
 242                 // Parse the old rev and get the categories. Do not use link tables as that
 243                 // assumes these updates are perfectly FIFO and that link tables are always
 244                 // up to date, neither of which are true.
 245                 $oldCategories = $oldRev
 246                         ? $this->getCategoriesAtRev( $page, $oldRev, $parseTimestamp )
 247                         : [];
 248                 // Parse the new revision and get the categories
 249                 $newCategories = $this->getCategoriesAtRev( $page, $newRev, $parseTimestamp );
 250
 251                 $categoryInserts = array_values( array_diff( $newCategories, $oldCategories ) );
 252                 $categoryDeletes = array_values( array_diff( $oldCategories, $newCategories ) );
 253
 254                 return [ $categoryInserts, $categoryDeletes ];
 255         }
 256
 257         /**
 258          * @param WikiPage $page
 259          * @param Revision $rev
 260          * @param string $parseTimestamp TS_MW
 261          *
 262          * @return string[] category names
 263          */
 264         private function getCategoriesAtRev( WikiPage $page, Revision $rev, $parseTimestamp ) {
 265                 $renderer = MediaWikiServices::getInstance()->getRevisionRenderer();
 266                 $options = $page->makeParserOptions( 'canonical' );
 267                 $options->setTimestamp( $parseTimestamp );
 268
 269                 $output = $rev->isCurrent() ? $this->parserCache->get( $page, $options ) : null;
 270
 271                 if ( !$output || $output->getCacheRevisionId() !== $rev->getId() ) {
 272                         $output = $renderer->getRenderedRevision( $rev->getRevisionRecord(), $options )
 273                                 ->getRevisionParserOutput();
 274                 }
 275
 276                 // array keys will cast numeric category names to ints
 277                 // so we need to cast them back to strings to avoid breaking things!
 278                 return array_map( 'strval', array_keys( $output->getCategories() ) );
 279         }
 280
 281         public function getDeduplicationInfo() {
 282                 $info = parent::getDeduplicationInfo();
 283                 unset( $info['params']['revTimestamp'] ); // first job wins
 284
 285                 return $info;
 286         }
 287 }