includes/LinksUpdate.php

   1 <?php
   2 /**
   3  * See deferred.txt
   4  * @package MediaWiki
   5  */
   6
   7 /**
   8  * @todo document
   9  * @package MediaWiki
  10  */
  11 class LinksUpdate {
  12
  13         /**@{{
  14          * @private
  15          */
  16         var $mId,            //!< Page ID of the article linked from
  17                 $mTitle,         //!< Title object of the article linked from
  18                 $mLinks,         //!< Map of title strings to IDs for the links in the document
  19                 $mImages,        //!< DB keys of the images used, in the array key only
  20                 $mTemplates,     //!< Map of title strings to IDs for the template references, including broken ones
  21                 $mExternals,     //!< URLs of external links, array key only
  22                 $mCategories,    //!< Map of category names to sort keys
  23                 $mInterlangs,    //!< Map of language codes to titles
  24                 $mDb,            //!< Database connection reference
  25                 $mOptions,       //!< SELECT options to be used (array)
  26                 $mRecursive;     //!< Whether to queue jobs for recursive updates
  27         /**@}}*/
  28
  29         /**
  30          * Constructor
  31          * Initialize private variables
  32          * @param $title Integer: FIXME
  33          * @param $parserOutput FIXME
  34          * @param $recursive Boolean: FIXME, default 'true'.
  35          */
  36         function LinksUpdate( $title, $parserOutput, $recursive = true ) {
  37                 global $wgAntiLockFlags;
  38
  39                 if ( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) {
  40                         $this->mOptions = array();
  41                 } else {
  42                         $this->mOptions = array( 'FOR UPDATE' );
  43                 }
  44                 $this->mDb =& wfGetDB( DB_MASTER );
  45
  46                 if ( !is_object( $title ) ) {
  47                         throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " .
  48                                 "Please see Article::editUpdates() for an invocation example.\n" );
  49                 }
  50                 $this->mTitle = $title;
  51                 $this->mId = $title->getArticleID();
  52
  53                 $this->mLinks = $parserOutput->getLinks();
  54                 $this->mImages = $parserOutput->getImages();
  55                 $this->mTemplates = $parserOutput->getTemplates();
  56                 $this->mExternals = $parserOutput->getExternalLinks();
  57                 $this->mCategories = $parserOutput->getCategories();
  58
  59                 # Convert the format of the interlanguage links
  60                 # I didn't want to change it in the ParserOutput, because that array is passed all
  61                 # the way back to the skin, so either a skin API break would be required, or an
  62                 # inefficient back-conversion.
  63                 $ill = $parserOutput->getLanguageLinks();
  64                 $this->mInterlangs = array();
  65                 foreach ( $ill as $link ) {
  66                         list( $key, $title ) = explode( ':', $link, 2 );
  67                         $this->mInterlangs[$key] = $title;
  68                 }
  69
  70                 $this->mRecursive = $recursive;
  71         }
  72
  73         /**
  74          * Update link tables with outgoing links from an updated article
  75          */
  76         function doUpdate() {
  77                 global $wgUseDumbLinkUpdate;
  78                 if ( $wgUseDumbLinkUpdate ) {
  79                         $this->doDumbUpdate();
  80                 } else {
  81                         $this->doIncrementalUpdate();
  82                 }
  83         }
  84
  85         function doIncrementalUpdate() {
  86                 $fname = 'LinksUpdate::doIncrementalUpdate';
  87                 wfProfileIn( $fname );
  88
  89                 # Page links
  90                 $existing = $this->getExistingLinks();
  91                 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
  92                         $this->getLinkInsertions( $existing ) );
  93
  94                 # Image links
  95                 $existing = $this->getExistingImages();
  96                 $this->incrTableUpdate( 'imagelinks', 'il', $this->getImageDeletions( $existing ),
  97                         $this->getImageInsertions( $existing ) );
  98
  99                 # Invalidate all image description pages which had links added or removed
 100                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
 101                 $this->invalidateImageDescriptions( $imageUpdates );
 102
 103                 # External links
 104                 $existing = $this->getExistingExternals();
 105                 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
 106                 $this->getExternalInsertions( $existing ) );
 107
 108                 # Language links
 109                 $existing = $this->getExistingInterlangs();
 110                 $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ),
 111                         $this->getInterlangInsertions( $existing ) );
 112
 113                 # Template links
 114                 $existing = $this->getExistingTemplates();
 115                 $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
 116                         $this->getTemplateInsertions( $existing ) );
 117
 118                 # Category links
 119                 $existing = $this->getExistingCategories();
 120                 $this->incrTableUpdate( 'categorylinks', 'cl', $this->getCategoryDeletions( $existing ),
 121                         $this->getCategoryInsertions( $existing ) );
 122
 123                 # Invalidate all categories which were added, deleted or changed (set symmetric difference)
 124                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 125                 $this->invalidateCategories( $categoryUpdates );
 126
 127                 # Refresh links of all pages including this page
 128                 # This will be in a separate transaction
 129                 if ( $this->mRecursive ) {
 130                         $this->queueRecursiveJobs();
 131                 }
 132
 133                 wfProfileOut( $fname );
 134         }
 135
 136         /**
 137          * Link update which clears the previous entries and inserts new ones
 138          * May be slower or faster depending on level of lock contention and write speed of DB
 139          * Also useful where link table corruption needs to be repaired, e.g. in refreshLinks.php
 140          */
 141         function doDumbUpdate() {
 142                 $fname = 'LinksUpdate::doDumbUpdate';
 143                 wfProfileIn( $fname );
 144
 145                 # Refresh category pages and image description pages
 146                 $existing = $this->getExistingCategories();
 147                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 148                 $existing = $this->getExistingImages();
 149                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
 150
 151                 $this->dumbTableUpdate( 'pagelinks',     $this->getLinkInsertions(),     'pl_from' );
 152                 $this->dumbTableUpdate( 'imagelinks',    $this->getImageInsertions(),    'il_from' );
 153                 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
 154                 $this->dumbTableUpdate( 'templatelinks', $this->getTemplateInsertions(), 'tl_from' );
 155                 $this->dumbTableUpdate( 'externallinks', $this->getExternalInsertions(), 'el_from' );
 156                 $this->dumbTableUpdate( 'langlinks',     $this->getInterlangInsertions(), 'll_from' );
 157
 158                 # Update the cache of all the category pages and image description pages which were changed
 159                 $this->invalidateCategories( $categoryUpdates );
 160                 $this->invalidateImageDescriptions( $imageUpdates );
 161
 162                 # Refresh links of all pages including this page
 163                 # This will be in a separate transaction
 164                 if ( $this->mRecursive ) {
 165                         $this->queueRecursiveJobs();
 166                 }
 167
 168                 wfProfileOut( $fname );
 169         }
 170
 171         function queueRecursiveJobs() {
 172                 wfProfileIn( __METHOD__ );
 173
 174                 $batchSize = 100;
 175                 $dbr =& wfGetDB( DB_SLAVE );
 176                 $res = $dbr->select( array( 'templatelinks', 'page' ),
 177                         array( 'page_namespace', 'page_title' ),
 178                         array(
 179                                 'page_id=tl_from',
 180                                 'tl_namespace' => $this->mTitle->getNamespace(),
 181                                 'tl_title' => $this->mTitle->getDBkey()
 182                         ), __METHOD__
 183                 );
 184
 185                 $done = false;
 186                 while ( !$done ) {
 187                         $jobs = array();
 188                         for ( $i = 0; $i < $batchSize; $i++ ) {
 189                                 $row = $dbr->fetchObject( $res );
 190                                 if ( !$row ) {
 191                                         $done = true;
 192                                         break;
 193                                 }
 194                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 195                                 $jobs[] = Job::factory( 'refreshLinks', $title );
 196                         }
 197                         Job::batchInsert( $jobs );
 198                 }
 199                 $dbr->freeResult( $res );
 200                 wfProfileOut( __METHOD__ );
 201         }
 202
 203         /**
 204          * Invalidate the cache of a list of pages from a single namespace
 205          *
 206          * @param integer $namespace
 207          * @param array $dbkeys
 208          */
 209         function invalidatePages( $namespace, $dbkeys ) {
 210                 $fname = 'LinksUpdate::invalidatePages';
 211
 212                 if ( !count( $dbkeys ) ) {
 213                         return;
 214                 }
 215
 216                 /**
 217                  * Determine which pages need to be updated
 218                  * This is necessary to prevent the job queue from smashing the DB with
 219                  * large numbers of concurrent invalidations of the same page
 220                  */
 221                 $now = $this->mDb->timestamp();
 222                 $ids = array();
 223                 $res = $this->mDb->select( 'page', array( 'page_id' ),
 224                         array(
 225                                 'page_namespace' => $namespace,
 226                                 'page_title IN (' . $this->mDb->makeList( $dbkeys ) . ')',
 227                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 228                         ), $fname
 229                 );
 230                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 231                         $ids[] = $row->page_id;
 232                 }
 233                 if ( !count( $ids ) ) {
 234                         return;
 235                 }
 236
 237                 /**
 238                  * Do the update
 239                  * We still need the page_touched condition, in case the row has changed since
 240                  * the non-locking select above.
 241                  */
 242                 $this->mDb->update( 'page', array( 'page_touched' => $now ),
 243                         array(
 244                                 'page_id IN (' . $this->mDb->makeList( $ids ) . ')',
 245                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 246                         ), $fname
 247                 );
 248         }
 249
 250         function invalidateCategories( $cats ) {
 251                 $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) );
 252         }
 253
 254         function invalidateImageDescriptions( $images ) {
 255                 $this->invalidatePages( NS_IMAGE, array_keys( $images ) );
 256         }
 257
 258         function dumbTableUpdate( $table, $insertions, $fromField ) {
 259                 $fname = 'LinksUpdate::dumbTableUpdate';
 260                 $this->mDb->delete( $table, array( $fromField => $this->mId ), $fname );
 261                 if ( count( $insertions ) ) {
 262                         # The link array was constructed without FOR UPDATE, so there may be collisions
 263                         # This may cause minor link table inconsistencies, which is better than
 264                         # crippling the site with lock contention.
 265                         $this->mDb->insert( $table, $insertions, $fname, array( 'IGNORE' ) );
 266                 }
 267         }
 268
 269         /**
 270          * Make a WHERE clause from a 2-d NS/dbkey array
 271          *
 272          * @param array $arr 2-d array indexed by namespace and DB key
 273          * @param string $prefix Field name prefix, without the underscore
 274          */
 275         function makeWhereFrom2d( &$arr, $prefix ) {
 276                 $lb = new LinkBatch;
 277                 $lb->setArray( $arr );
 278                 return $lb->constructSet( $prefix, $this->mDb );
 279         }
 280
 281         /**
 282          * Update a table by doing a delete query then an insert query
 283          * @private
 284          */
 285         function incrTableUpdate( $table, $prefix, $deletions, $insertions ) {
 286                 $fname = 'LinksUpdate::incrTableUpdate';
 287                 $where = array( "{$prefix}_from" => $this->mId );
 288                 if ( $table == 'pagelinks' || $table == 'templatelinks' ) {
 289                         $clause = $this->makeWhereFrom2d( $deletions, $prefix );
 290                         if ( $clause ) {
 291                                 $where[] = $clause;
 292                         } else {
 293                                 $where = false;
 294                         }
 295                 } else {
 296                         if ( $table == 'langlinks' ) {
 297                                 $toField = 'll_lang';
 298                         } else {
 299                                 $toField = $prefix . '_to';
 300                         }
 301                         if ( count( $deletions ) ) {
 302                                 $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')';
 303                         } else {
 304                                 $where = false;
 305                         }
 306                 }
 307                 if ( $where ) {
 308                         $this->mDb->delete( $table, $where, $fname );
 309                 }
 310                 if ( count( $insertions ) ) {
 311                         $this->mDb->insert( $table, $insertions, $fname, 'IGNORE' );
 312                 }
 313         }
 314
 315
 316         /**
 317          * Get an array of pagelinks insertions for passing to the DB
 318          * Skips the titles specified by the 2-D array $existing
 319          * @private
 320          */
 321         function getLinkInsertions( $existing = array() ) {
 322                 $arr = array();
 323                 foreach( $this->mLinks as $ns => $dbkeys ) {
 324                         # array_diff_key() was introduced in PHP 5.1, there is a compatibility function
 325                         # in GlobalFunctions.php
 326                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 327                         foreach ( $diffs as $dbk => $id ) {
 328                                 $arr[] = array(
 329                                         'pl_from'      => $this->mId,
 330                                         'pl_namespace' => $ns,
 331                                         'pl_title'     => $dbk
 332                                 );
 333                         }
 334                 }
 335                 return $arr;
 336         }
 337
 338         /**
 339          * Get an array of template insertions. Like getLinkInsertions()
 340          * @private
 341          */
 342         function getTemplateInsertions( $existing = array() ) {
 343                 $arr = array();
 344                 foreach( $this->mTemplates as $ns => $dbkeys ) {
 345                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 346                         foreach ( $diffs as $dbk => $id ) {
 347                                 $arr[] = array(
 348                                         'tl_from'      => $this->mId,
 349                                         'tl_namespace' => $ns,
 350                                         'tl_title'     => $dbk
 351                                 );
 352                         }
 353                 }
 354                 return $arr;
 355         }
 356
 357         /**
 358          * Get an array of image insertions
 359          * Skips the names specified in $existing
 360          * @private
 361          */
 362         function getImageInsertions( $existing = array() ) {
 363                 $arr = array();
 364                 $diffs = array_diff_key( $this->mImages, $existing );
 365                 foreach( $diffs as $iname => $dummy ) {
 366                         $arr[] = array(
 367                                 'il_from' => $this->mId,
 368                                 'il_to'   => $iname
 369                         );
 370                 }
 371                 return $arr;
 372         }
 373
 374         /**
 375          * Get an array of externallinks insertions. Skips the names specified in $existing
 376          * @private
 377          */
 378         function getExternalInsertions( $existing = array() ) {
 379                 $arr = array();
 380                 $diffs = array_diff_key( $this->mExternals, $existing );
 381                 foreach( $diffs as $url => $dummy ) {
 382                         $arr[] = array(
 383                                 'el_from'   => $this->mId,
 384                                 'el_to'     => $url,
 385                                 'el_index'  => wfMakeUrlIndex( $url ),
 386                         );
 387                 }
 388                 return $arr;
 389         }
 390
 391         /**
 392          * Get an array of category insertions
 393          * @param array $existing Array mapping existing category names to sort keys. If both
 394          * match a link in $this, the link will be omitted from the output
 395          * @private
 396          */
 397         function getCategoryInsertions( $existing = array() ) {
 398                 $diffs = array_diff_assoc( $this->mCategories, $existing );
 399                 $arr = array();
 400                 foreach ( $diffs as $name => $sortkey ) {
 401                         $arr[] = array(
 402                                 'cl_from'    => $this->mId,
 403                                 'cl_to'      => $name,
 404                                 'cl_sortkey' => $sortkey,
 405                                 'cl_timestamp' => $this->mDb->timestamp()
 406                         );
 407                 }
 408                 return $arr;
 409         }
 410
 411         /**
 412          * Get an array of interlanguage link insertions
 413          * @param array $existing Array mapping existing language codes to titles
 414          * @private
 415          */
 416         function getInterlangInsertions( $existing = array() ) {
 417             $diffs = array_diff_assoc( $this->mInterlangs, $existing );
 418             $arr = array();
 419             foreach( $diffs as $lang => $title ) {
 420                 $arr[] = array(
 421                     'll_from'  => $this->mId,
 422                     'll_lang'  => $lang,
 423                     'll_title' => $title
 424                 );
 425             }
 426             return $arr;
 427         }
 428
 429         /**
 430          * Given an array of existing links, returns those links which are not in $this
 431          * and thus should be deleted.
 432          * @private
 433          */
 434         function getLinkDeletions( $existing ) {
 435                 $del = array();
 436                 foreach ( $existing as $ns => $dbkeys ) {
 437                         if ( isset( $this->mLinks[$ns] ) ) {
 438                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] );
 439                         } else {
 440                                 $del[$ns] = $existing[$ns];
 441                         }
 442                 }
 443                 return $del;
 444         }
 445
 446         /**
 447          * Given an array of existing templates, returns those templates which are not in $this
 448          * and thus should be deleted.
 449          * @private
 450          */
 451         function getTemplateDeletions( $existing ) {
 452                 $del = array();
 453                 foreach ( $existing as $ns => $dbkeys ) {
 454                         if ( isset( $this->mTemplates[$ns] ) ) {
 455                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] );
 456                         } else {
 457                                 $del[$ns] = $existing[$ns];
 458                         }
 459                 }
 460                 return $del;
 461         }
 462
 463         /**
 464          * Given an array of existing images, returns those images which are not in $this
 465          * and thus should be deleted.
 466          * @private
 467          */
 468         function getImageDeletions( $existing ) {
 469                 return array_diff_key( $existing, $this->mImages );
 470         }
 471
 472         /**
 473          * Given an array of existing external links, returns those links which are not
 474          * in $this and thus should be deleted.
 475          * @private
 476          */
 477         function getExternalDeletions( $existing ) {
 478                 return array_diff_key( $existing, $this->mExternals );
 479         }
 480
 481         /**
 482          * Given an array of existing categories, returns those categories which are not in $this
 483          * and thus should be deleted.
 484          * @private
 485          */
 486         function getCategoryDeletions( $existing ) {
 487                 return array_diff_assoc( $existing, $this->mCategories );
 488         }
 489
 490         /**
 491          * Given an array of existing interlanguage links, returns those links which are not
 492          * in $this and thus should be deleted.
 493          * @private
 494          */
 495         function getInterlangDeletions( $existing ) {
 496             return array_diff_assoc( $existing, $this->mInterlangs );
 497         }
 498
 499         /**
 500          * Get an array of existing links, as a 2-D array
 501          * @private
 502          */
 503         function getExistingLinks() {
 504                 $fname = 'LinksUpdate::getExistingLinks';
 505                 $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ),
 506                         array( 'pl_from' => $this->mId ), $fname, $this->mOptions );
 507                 $arr = array();
 508                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 509                         if ( !isset( $arr[$row->pl_namespace] ) ) {
 510                                 $arr[$row->pl_namespace] = array();
 511                         }
 512                         $arr[$row->pl_namespace][$row->pl_title] = 1;
 513                 }
 514                 $this->mDb->freeResult( $res );
 515                 return $arr;
 516         }
 517
 518         /**
 519          * Get an array of existing templates, as a 2-D array
 520          * @private
 521          */
 522         function getExistingTemplates() {
 523                 $fname = 'LinksUpdate::getExistingTemplates';
 524                 $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ),
 525                         array( 'tl_from' => $this->mId ), $fname, $this->mOptions );
 526                 $arr = array();
 527                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 528                         if ( !isset( $arr[$row->tl_namespace] ) ) {
 529                                 $arr[$row->tl_namespace] = array();
 530                         }
 531                         $arr[$row->tl_namespace][$row->tl_title] = 1;
 532                 }
 533                 $this->mDb->freeResult( $res );
 534                 return $arr;
 535         }
 536
 537         /**
 538          * Get an array of existing images, image names in the keys
 539          * @private
 540          */
 541         function getExistingImages() {
 542                 $fname = 'LinksUpdate::getExistingImages';
 543                 $res = $this->mDb->select( 'imagelinks', array( 'il_to' ),
 544                         array( 'il_from' => $this->mId ), $fname, $this->mOptions );
 545                 $arr = array();
 546                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 547                         $arr[$row->il_to] = 1;
 548                 }
 549                 $this->mDb->freeResult( $res );
 550                 return $arr;
 551         }
 552
 553         /**
 554          * Get an array of existing external links, URLs in the keys
 555          * @private
 556          */
 557         function getExistingExternals() {
 558                 $fname = 'LinksUpdate::getExistingExternals';
 559                 $res = $this->mDb->select( 'externallinks', array( 'el_to' ),
 560                         array( 'el_from' => $this->mId ), $fname, $this->mOptions );
 561                 $arr = array();
 562                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 563                         $arr[$row->el_to] = 1;
 564                 }
 565                 $this->mDb->freeResult( $res );
 566                 return $arr;
 567         }
 568
 569         /**
 570          * Get an array of existing categories, with the name in the key and sort key in the value.
 571          * @private
 572          */
 573         function getExistingCategories() {
 574                 $fname = 'LinksUpdate::getExistingCategories';
 575                 $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey' ),
 576                         array( 'cl_from' => $this->mId ), $fname, $this->mOptions );
 577                 $arr = array();
 578                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 579                         $arr[$row->cl_to] = $row->cl_sortkey;
 580                 }
 581                 $this->mDb->freeResult( $res );
 582                 return $arr;
 583         }
 584
 585         /**
 586          * Get an array of existing interlanguage links, with the language code in the key and the
 587          * title in the value.
 588          * @private
 589          */
 590         function getExistingInterlangs() {
 591                 $fname = 'LinksUpdate::getExistingInterlangs';
 592                 $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ),
 593                         array( 'll_from' => $this->mId ), $fname, $this->mOptions );
 594                 $arr = array();
 595                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 596                         $arr[$row->ll_lang] = $row->ll_title;
 597                 }
 598                 return $arr;
 599         }
 600 }
 601 ?>