includes/LinksUpdate.php

   1 <?php
   2 /**
   3  * See docs/deferred.txt
   4  *
   5  * @todo document (e.g. one-sentence top-level class description).
   6  */
   7 class LinksUpdate {
   8
   9         /**@{{
  10          * @private
  11          */
  12         var $mId,            //!< Page ID of the article linked from
  13                 $mTitle,         //!< Title object of the article linked from
  14                 $mLinks,         //!< Map of title strings to IDs for the links in the document
  15                 $mImages,        //!< DB keys of the images used, in the array key only
  16                 $mTemplates,     //!< Map of title strings to IDs for the template references, including broken ones
  17                 $mExternals,     //!< URLs of external links, array key only
  18                 $mCategories,    //!< Map of category names to sort keys
  19                 $mInterlangs,    //!< Map of language codes to titles
  20                 $mDb,            //!< Database connection reference
  21                 $mOptions,       //!< SELECT options to be used (array)
  22                 $mRecursive;     //!< Whether to queue jobs for recursive updates
  23         /**@}}*/
  24
  25         /**
  26          * Constructor
  27          *
  28          * @param Title $title Title of the page we're updating
  29          * @param ParserOutput $parserOutput Output from a full parse of this page
  30          * @param bool $recursive Queue jobs for recursive updates?
  31          */
  32         function LinksUpdate( $title, $parserOutput, $recursive = true ) {
  33                 global $wgAntiLockFlags;
  34
  35                 if ( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) {
  36                         $this->mOptions = array();
  37                 } else {
  38                         $this->mOptions = array( 'FOR UPDATE' );
  39                 }
  40                 $this->mDb = wfGetDB( DB_MASTER );
  41
  42                 if ( !is_object( $title ) ) {
  43                         throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " .
  44                                 "Please see Article::editUpdates() for an invocation example.\n" );
  45                 }
  46                 $this->mTitle = $title;
  47                 $this->mId = $title->getArticleID();
  48
  49                 $this->mLinks = $parserOutput->getLinks();
  50                 $this->mImages = $parserOutput->getImages();
  51                 $this->mTemplates = $parserOutput->getTemplates();
  52                 $this->mExternals = $parserOutput->getExternalLinks();
  53                 $this->mCategories = $parserOutput->getCategories();
  54
  55                 # Convert the format of the interlanguage links
  56                 # I didn't want to change it in the ParserOutput, because that array is passed all
  57                 # the way back to the skin, so either a skin API break would be required, or an
  58                 # inefficient back-conversion.
  59                 $ill = $parserOutput->getLanguageLinks();
  60                 $this->mInterlangs = array();
  61                 foreach ( $ill as $link ) {
  62                         list( $key, $title ) = explode( ':', $link, 2 );
  63                         $this->mInterlangs[$key] = $title;
  64                 }
  65
  66                 $this->mRecursive = $recursive;
  67
  68                 wfRunHooks( 'LinksUpdateConstructed', array( &$this ) );
  69         }
  70
  71         /**
  72          * Update link tables with outgoing links from an updated article
  73          */
  74         function doUpdate() {
  75                 global $wgUseDumbLinkUpdate;
  76
  77                 wfRunHooks( 'LinksUpdate', array( &$this ) );
  78                 if ( $wgUseDumbLinkUpdate ) {
  79                         $this->doDumbUpdate();
  80                 } else {
  81                         $this->doIncrementalUpdate();
  82                 }
  83                 wfRunHooks( 'LinksUpdateComplete', array( &$this ) );
  84
  85         }
  86
  87         function doIncrementalUpdate() {
  88                 $fname = 'LinksUpdate::doIncrementalUpdate';
  89                 wfProfileIn( $fname );
  90
  91                 # Page links
  92                 $existing = $this->getExistingLinks();
  93                 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
  94                         $this->getLinkInsertions( $existing ) );
  95
  96                 # Image links
  97                 $existing = $this->getExistingImages();
  98                 $this->incrTableUpdate( 'imagelinks', 'il', $this->getImageDeletions( $existing ),
  99                         $this->getImageInsertions( $existing ) );
 100
 101                 # Invalidate all image description pages which had links added or removed
 102                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
 103                 $this->invalidateImageDescriptions( $imageUpdates );
 104
 105                 # External links
 106                 $existing = $this->getExistingExternals();
 107                 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
 108                 $this->getExternalInsertions( $existing ) );
 109
 110                 # Language links
 111                 $existing = $this->getExistingInterlangs();
 112                 $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ),
 113                         $this->getInterlangInsertions( $existing ) );
 114
 115                 # Template links
 116                 $existing = $this->getExistingTemplates();
 117                 $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
 118                         $this->getTemplateInsertions( $existing ) );
 119
 120                 # Category links
 121                 $existing = $this->getExistingCategories();
 122                 $this->incrTableUpdate( 'categorylinks', 'cl', $this->getCategoryDeletions( $existing ),
 123                         $this->getCategoryInsertions( $existing ) );
 124
 125                 # Invalidate all categories which were added, deleted or changed (set symmetric difference)
 126                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 127                 $this->invalidateCategories( $categoryUpdates );
 128
 129                 # Refresh links of all pages including this page
 130                 # This will be in a separate transaction
 131                 if ( $this->mRecursive ) {
 132                         $this->queueRecursiveJobs();
 133                 }
 134
 135                 wfProfileOut( $fname );
 136         }
 137
 138         /**
 139          * Link update which clears the previous entries and inserts new ones
 140          * May be slower or faster depending on level of lock contention and write speed of DB
 141          * Also useful where link table corruption needs to be repaired, e.g. in refreshLinks.php
 142          */
 143         function doDumbUpdate() {
 144                 $fname = 'LinksUpdate::doDumbUpdate';
 145                 wfProfileIn( $fname );
 146
 147                 # Refresh category pages and image description pages
 148                 $existing = $this->getExistingCategories();
 149                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 150                 $existing = $this->getExistingImages();
 151                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
 152
 153                 $this->dumbTableUpdate( 'pagelinks',     $this->getLinkInsertions(),     'pl_from' );
 154                 $this->dumbTableUpdate( 'imagelinks',    $this->getImageInsertions(),    'il_from' );
 155                 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
 156                 $this->dumbTableUpdate( 'templatelinks', $this->getTemplateInsertions(), 'tl_from' );
 157                 $this->dumbTableUpdate( 'externallinks', $this->getExternalInsertions(), 'el_from' );
 158                 $this->dumbTableUpdate( 'langlinks',     $this->getInterlangInsertions(), 'll_from' );
 159
 160                 # Update the cache of all the category pages and image description pages which were changed
 161                 $this->invalidateCategories( $categoryUpdates );
 162                 $this->invalidateImageDescriptions( $imageUpdates );
 163
 164                 # Refresh links of all pages including this page
 165                 # This will be in a separate transaction
 166                 if ( $this->mRecursive ) {
 167                         $this->queueRecursiveJobs();
 168                 }
 169
 170                 wfProfileOut( $fname );
 171         }
 172
 173         function queueRecursiveJobs() {
 174                 wfProfileIn( __METHOD__ );
 175
 176                 $batchSize = 100;
 177                 $dbr = wfGetDB( DB_SLAVE );
 178                 $res = $dbr->select( array( 'templatelinks', 'page' ),
 179                         array( 'page_namespace', 'page_title' ),
 180                         array(
 181                                 'page_id=tl_from',
 182                                 'tl_namespace' => $this->mTitle->getNamespace(),
 183                                 'tl_title' => $this->mTitle->getDBkey()
 184                         ), __METHOD__
 185                 );
 186
 187                 $done = false;
 188                 while ( !$done ) {
 189                         $jobs = array();
 190                         for ( $i = 0; $i < $batchSize; $i++ ) {
 191                                 $row = $dbr->fetchObject( $res );
 192                                 if ( !$row ) {
 193                                         $done = true;
 194                                         break;
 195                                 }
 196                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 197                                 $jobs[] = new RefreshLinksJob( $title, '' );
 198                         }
 199                         Job::batchInsert( $jobs );
 200                 }
 201                 $dbr->freeResult( $res );
 202                 wfProfileOut( __METHOD__ );
 203         }
 204
 205         /**
 206          * Invalidate the cache of a list of pages from a single namespace
 207          *
 208          * @param integer $namespace
 209          * @param array $dbkeys
 210          */
 211         function invalidatePages( $namespace, $dbkeys ) {
 212                 $fname = 'LinksUpdate::invalidatePages';
 213
 214                 if ( !count( $dbkeys ) ) {
 215                         return;
 216                 }
 217
 218                 /**
 219                  * Determine which pages need to be updated
 220                  * This is necessary to prevent the job queue from smashing the DB with
 221                  * large numbers of concurrent invalidations of the same page
 222                  */
 223                 $now = $this->mDb->timestamp();
 224                 $ids = array();
 225                 $res = $this->mDb->select( 'page', array( 'page_id' ),
 226                         array(
 227                                 'page_namespace' => $namespace,
 228                                 'page_title IN (' . $this->mDb->makeList( $dbkeys ) . ')',
 229                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 230                         ), $fname
 231                 );
 232                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 233                         $ids[] = $row->page_id;
 234                 }
 235                 if ( !count( $ids ) ) {
 236                         return;
 237                 }
 238
 239                 /**
 240                  * Do the update
 241                  * We still need the page_touched condition, in case the row has changed since
 242                  * the non-locking select above.
 243                  */
 244                 $this->mDb->update( 'page', array( 'page_touched' => $now ),
 245                         array(
 246                                 'page_id IN (' . $this->mDb->makeList( $ids ) . ')',
 247                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 248                         ), $fname
 249                 );
 250         }
 251
 252         function invalidateCategories( $cats ) {
 253                 $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) );
 254         }
 255
 256         function invalidateImageDescriptions( $images ) {
 257                 $this->invalidatePages( NS_IMAGE, array_keys( $images ) );
 258         }
 259
 260         function dumbTableUpdate( $table, $insertions, $fromField ) {
 261                 $fname = 'LinksUpdate::dumbTableUpdate';
 262                 $this->mDb->delete( $table, array( $fromField => $this->mId ), $fname );
 263                 if ( count( $insertions ) ) {
 264                         # The link array was constructed without FOR UPDATE, so there may be collisions
 265                         # This may cause minor link table inconsistencies, which is better than
 266                         # crippling the site with lock contention.
 267                         $this->mDb->insert( $table, $insertions, $fname, array( 'IGNORE' ) );
 268                 }
 269         }
 270
 271         /**
 272          * Make a WHERE clause from a 2-d NS/dbkey array
 273          *
 274          * @param array $arr 2-d array indexed by namespace and DB key
 275          * @param string $prefix Field name prefix, without the underscore
 276          */
 277         function makeWhereFrom2d( &$arr, $prefix ) {
 278                 $lb = new LinkBatch;
 279                 $lb->setArray( $arr );
 280                 return $lb->constructSet( $prefix, $this->mDb );
 281         }
 282
 283         /**
 284          * Update a table by doing a delete query then an insert query
 285          * @private
 286          */
 287         function incrTableUpdate( $table, $prefix, $deletions, $insertions ) {
 288                 $fname = 'LinksUpdate::incrTableUpdate';
 289                 $where = array( "{$prefix}_from" => $this->mId );
 290                 if ( $table == 'pagelinks' || $table == 'templatelinks' ) {
 291                         $clause = $this->makeWhereFrom2d( $deletions, $prefix );
 292                         if ( $clause ) {
 293                                 $where[] = $clause;
 294                         } else {
 295                                 $where = false;
 296                         }
 297                 } else {
 298                         if ( $table == 'langlinks' ) {
 299                                 $toField = 'll_lang';
 300                         } else {
 301                                 $toField = $prefix . '_to';
 302                         }
 303                         if ( count( $deletions ) ) {
 304                                 $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')';
 305                         } else {
 306                                 $where = false;
 307                         }
 308                 }
 309                 if ( $where ) {
 310                         $this->mDb->delete( $table, $where, $fname );
 311                 }
 312                 if ( count( $insertions ) ) {
 313                         $this->mDb->insert( $table, $insertions, $fname, 'IGNORE' );
 314                 }
 315         }
 316
 317
 318         /**
 319          * Get an array of pagelinks insertions for passing to the DB
 320          * Skips the titles specified by the 2-D array $existing
 321          * @private
 322          */
 323         function getLinkInsertions( $existing = array() ) {
 324                 $arr = array();
 325                 foreach( $this->mLinks as $ns => $dbkeys ) {
 326                         # array_diff_key() was introduced in PHP 5.1, there is a compatibility function
 327                         # in GlobalFunctions.php
 328                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 329                         foreach ( $diffs as $dbk => $id ) {
 330                                 $arr[] = array(
 331                                         'pl_from'      => $this->mId,
 332                                         'pl_namespace' => $ns,
 333                                         'pl_title'     => $dbk
 334                                 );
 335                         }
 336                 }
 337                 return $arr;
 338         }
 339
 340         /**
 341          * Get an array of template insertions. Like getLinkInsertions()
 342          * @private
 343          */
 344         function getTemplateInsertions( $existing = array() ) {
 345                 $arr = array();
 346                 foreach( $this->mTemplates as $ns => $dbkeys ) {
 347                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 348                         foreach ( $diffs as $dbk => $id ) {
 349                                 $arr[] = array(
 350                                         'tl_from'      => $this->mId,
 351                                         'tl_namespace' => $ns,
 352                                         'tl_title'     => $dbk
 353                                 );
 354                         }
 355                 }
 356                 return $arr;
 357         }
 358
 359         /**
 360          * Get an array of image insertions
 361          * Skips the names specified in $existing
 362          * @private
 363          */
 364         function getImageInsertions( $existing = array() ) {
 365                 $arr = array();
 366                 $diffs = array_diff_key( $this->mImages, $existing );
 367                 foreach( $diffs as $iname => $dummy ) {
 368                         $arr[] = array(
 369                                 'il_from' => $this->mId,
 370                                 'il_to'   => $iname
 371                         );
 372                 }
 373                 return $arr;
 374         }
 375
 376         /**
 377          * Get an array of externallinks insertions. Skips the names specified in $existing
 378          * @private
 379          */
 380         function getExternalInsertions( $existing = array() ) {
 381                 $arr = array();
 382                 $diffs = array_diff_key( $this->mExternals, $existing );
 383                 foreach( $diffs as $url => $dummy ) {
 384                         $arr[] = array(
 385                                 'el_from'   => $this->mId,
 386                                 'el_to'     => $url,
 387                                 'el_index'  => wfMakeUrlIndex( $url ),
 388                         );
 389                 }
 390                 return $arr;
 391         }
 392
 393         /**
 394          * Get an array of category insertions
 395          * @param array $existing Array mapping existing category names to sort keys. If both
 396          * match a link in $this, the link will be omitted from the output
 397          * @private
 398          */
 399         function getCategoryInsertions( $existing = array() ) {
 400                 $diffs = array_diff_assoc( $this->mCategories, $existing );
 401                 $arr = array();
 402                 foreach ( $diffs as $name => $sortkey ) {
 403                         $arr[] = array(
 404                                 'cl_from'    => $this->mId,
 405                                 'cl_to'      => $name,
 406                                 'cl_sortkey' => $sortkey,
 407                                 'cl_timestamp' => $this->mDb->timestamp()
 408                         );
 409                 }
 410                 return $arr;
 411         }
 412
 413         /**
 414          * Get an array of interlanguage link insertions
 415          * @param array $existing Array mapping existing language codes to titles
 416          * @private
 417          */
 418         function getInterlangInsertions( $existing = array() ) {
 419             $diffs = array_diff_assoc( $this->mInterlangs, $existing );
 420             $arr = array();
 421             foreach( $diffs as $lang => $title ) {
 422                 $arr[] = array(
 423                     'll_from'  => $this->mId,
 424                     'll_lang'  => $lang,
 425                     'll_title' => $title
 426                 );
 427             }
 428             return $arr;
 429         }
 430
 431         /**
 432          * Given an array of existing links, returns those links which are not in $this
 433          * and thus should be deleted.
 434          * @private
 435          */
 436         function getLinkDeletions( $existing ) {
 437                 $del = array();
 438                 foreach ( $existing as $ns => $dbkeys ) {
 439                         if ( isset( $this->mLinks[$ns] ) ) {
 440                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] );
 441                         } else {
 442                                 $del[$ns] = $existing[$ns];
 443                         }
 444                 }
 445                 return $del;
 446         }
 447
 448         /**
 449          * Given an array of existing templates, returns those templates which are not in $this
 450          * and thus should be deleted.
 451          * @private
 452          */
 453         function getTemplateDeletions( $existing ) {
 454                 $del = array();
 455                 foreach ( $existing as $ns => $dbkeys ) {
 456                         if ( isset( $this->mTemplates[$ns] ) ) {
 457                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] );
 458                         } else {
 459                                 $del[$ns] = $existing[$ns];
 460                         }
 461                 }
 462                 return $del;
 463         }
 464
 465         /**
 466          * Given an array of existing images, returns those images which are not in $this
 467          * and thus should be deleted.
 468          * @private
 469          */
 470         function getImageDeletions( $existing ) {
 471                 return array_diff_key( $existing, $this->mImages );
 472         }
 473
 474         /**
 475          * Given an array of existing external links, returns those links which are not
 476          * in $this and thus should be deleted.
 477          * @private
 478          */
 479         function getExternalDeletions( $existing ) {
 480                 return array_diff_key( $existing, $this->mExternals );
 481         }
 482
 483         /**
 484          * Given an array of existing categories, returns those categories which are not in $this
 485          * and thus should be deleted.
 486          * @private
 487          */
 488         function getCategoryDeletions( $existing ) {
 489                 return array_diff_assoc( $existing, $this->mCategories );
 490         }
 491
 492         /**
 493          * Given an array of existing interlanguage links, returns those links which are not
 494          * in $this and thus should be deleted.
 495          * @private
 496          */
 497         function getInterlangDeletions( $existing ) {
 498             return array_diff_assoc( $existing, $this->mInterlangs );
 499         }
 500
 501         /**
 502          * Get an array of existing links, as a 2-D array
 503          * @private
 504          */
 505         function getExistingLinks() {
 506                 $fname = 'LinksUpdate::getExistingLinks';
 507                 $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ),
 508                         array( 'pl_from' => $this->mId ), $fname, $this->mOptions );
 509                 $arr = array();
 510                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 511                         if ( !isset( $arr[$row->pl_namespace] ) ) {
 512                                 $arr[$row->pl_namespace] = array();
 513                         }
 514                         $arr[$row->pl_namespace][$row->pl_title] = 1;
 515                 }
 516                 $this->mDb->freeResult( $res );
 517                 return $arr;
 518         }
 519
 520         /**
 521          * Get an array of existing templates, as a 2-D array
 522          * @private
 523          */
 524         function getExistingTemplates() {
 525                 $fname = 'LinksUpdate::getExistingTemplates';
 526                 $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ),
 527                         array( 'tl_from' => $this->mId ), $fname, $this->mOptions );
 528                 $arr = array();
 529                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 530                         if ( !isset( $arr[$row->tl_namespace] ) ) {
 531                                 $arr[$row->tl_namespace] = array();
 532                         }
 533                         $arr[$row->tl_namespace][$row->tl_title] = 1;
 534                 }
 535                 $this->mDb->freeResult( $res );
 536                 return $arr;
 537         }
 538
 539         /**
 540          * Get an array of existing images, image names in the keys
 541          * @private
 542          */
 543         function getExistingImages() {
 544                 $fname = 'LinksUpdate::getExistingImages';
 545                 $res = $this->mDb->select( 'imagelinks', array( 'il_to' ),
 546                         array( 'il_from' => $this->mId ), $fname, $this->mOptions );
 547                 $arr = array();
 548                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 549                         $arr[$row->il_to] = 1;
 550                 }
 551                 $this->mDb->freeResult( $res );
 552                 return $arr;
 553         }
 554
 555         /**
 556          * Get an array of existing external links, URLs in the keys
 557          * @private
 558          */
 559         function getExistingExternals() {
 560                 $fname = 'LinksUpdate::getExistingExternals';
 561                 $res = $this->mDb->select( 'externallinks', array( 'el_to' ),
 562                         array( 'el_from' => $this->mId ), $fname, $this->mOptions );
 563                 $arr = array();
 564                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 565                         $arr[$row->el_to] = 1;
 566                 }
 567                 $this->mDb->freeResult( $res );
 568                 return $arr;
 569         }
 570
 571         /**
 572          * Get an array of existing categories, with the name in the key and sort key in the value.
 573          * @private
 574          */
 575         function getExistingCategories() {
 576                 $fname = 'LinksUpdate::getExistingCategories';
 577                 $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey' ),
 578                         array( 'cl_from' => $this->mId ), $fname, $this->mOptions );
 579                 $arr = array();
 580                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 581                         $arr[$row->cl_to] = $row->cl_sortkey;
 582                 }
 583                 $this->mDb->freeResult( $res );
 584                 return $arr;
 585         }
 586
 587         /**
 588          * Get an array of existing interlanguage links, with the language code in the key and the
 589          * title in the value.
 590          * @private
 591          */
 592         function getExistingInterlangs() {
 593                 $fname = 'LinksUpdate::getExistingInterlangs';
 594                 $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ),
 595                         array( 'll_from' => $this->mId ), $fname, $this->mOptions );
 596                 $arr = array();
 597                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 598                         $arr[$row->ll_lang] = $row->ll_title;
 599                 }
 600                 return $arr;
 601         }
 602
 603         /**
 604          * Return the title object of the page being updated
 605          */
 606         function getTitle() {
 607                 return $this->mTitle;
 608         }
 609 }
 610