includes/LinksUpdate.php

   1 <?php
   2 /**
   3  * See docs/deferred.txt
   4  *
   5  * @todo document (e.g. one-sentence top-level class description).
   6  */
   7 class LinksUpdate {
   8
   9         /**@{{
  10          * @private
  11          */
  12         var     $mId,            //!< Page ID of the article linked from
  13                 $mTitle,         //!< Title object of the article linked from
  14                 $mLinks,         //!< Map of title strings to IDs for the links in the document
  15                 $mImages,        //!< DB keys of the images used, in the array key only
  16                 $mTemplates,     //!< Map of title strings to IDs for the template references, including broken ones
  17                 $mExternals,     //!< URLs of external links, array key only
  18                 $mCategories,    //!< Map of category names to sort keys
  19                 $mInterlangs,    //!< Map of language codes to titles
  20                 $mDb,            //!< Database connection reference
  21                 $mOptions,       //!< SELECT options to be used (array)
  22                 $mRecursive;     //!< Whether to queue jobs for recursive updates
  23         /**@}}*/
  24
  25         /**
  26          * Constructor
  27          * Initialize private variables
  28          * @param $title Integer: FIXME
  29          * @param $parserOutput FIXME
  30          * @param $recursive Boolean: FIXME, default 'true'.
  31          */
  32         function LinksUpdate( $title, $parserOutput, $recursive = true ) {
  33                 global $wgAntiLockFlags;
  34
  35                 if ( $wgAntiLockFlags & ALF_NO_LINK_LOCK ) {
  36                         $this->mOptions = array();
  37                 } else {
  38                         $this->mOptions = array( 'FOR UPDATE' );
  39                 }
  40                 $this->mDb = wfGetDB( DB_MASTER );
  41
  42                 if ( !is_object( $title ) ) {
  43                         throw new MWException( "The calling convention to LinksUpdate::LinksUpdate() has changed. " .
  44                                 "Please see Article::editUpdates() for an invocation example.\n" );
  45                 }
  46                 $this->mTitle = $title;
  47                 $this->mId = $title->getArticleID();
  48
  49                 $this->mLinks = $parserOutput->getLinks();
  50                 $this->mImages = $parserOutput->getImages();
  51                 $this->mTemplates = $parserOutput->getTemplates();
  52                 $this->mExternals = $parserOutput->getExternalLinks();
  53                 $this->mCategories = $parserOutput->getCategories();
  54
  55                 # Convert the format of the interlanguage links
  56                 # I didn't want to change it in the ParserOutput, because that array is passed all
  57                 # the way back to the skin, so either a skin API break would be required, or an
  58                 # inefficient back-conversion.
  59                 $ill = $parserOutput->getLanguageLinks();
  60                 $this->mInterlangs = array();
  61                 foreach ( $ill as $link ) {
  62                         list( $key, $title ) = explode( ':', $link, 2 );
  63                         $this->mInterlangs[$key] = $title;
  64                 }
  65
  66                 $this->mRecursive = $recursive;
  67         }
  68
  69         /**
  70          * Update link tables with outgoing links from an updated article
  71          */
  72         function doUpdate() {
  73                 global $wgUseDumbLinkUpdate;
  74                 if ( $wgUseDumbLinkUpdate ) {
  75                         $this->doDumbUpdate();
  76                 } else {
  77                         $this->doIncrementalUpdate();
  78                 }
  79         }
  80
  81         function doIncrementalUpdate() {
  82                 $fname = 'LinksUpdate::doIncrementalUpdate';
  83                 wfProfileIn( $fname );
  84
  85                 # Page links
  86                 $existing = $this->getExistingLinks();
  87                 $this->incrTableUpdate( 'pagelinks', 'pl', $this->getLinkDeletions( $existing ),
  88                         $this->getLinkInsertions( $existing ) );
  89
  90                 # Image links
  91                 $existing = $this->getExistingImages();
  92                 $this->incrTableUpdate( 'imagelinks', 'il', $this->getImageDeletions( $existing ),
  93                         $this->getImageInsertions( $existing ) );
  94
  95                 # Invalidate all image description pages which had links added or removed
  96                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
  97                 $this->invalidateImageDescriptions( $imageUpdates );
  98
  99                 # External links
 100                 $existing = $this->getExistingExternals();
 101                 $this->incrTableUpdate( 'externallinks', 'el', $this->getExternalDeletions( $existing ),
 102                 $this->getExternalInsertions( $existing ) );
 103
 104                 # Language links
 105                 $existing = $this->getExistingInterlangs();
 106                 $this->incrTableUpdate( 'langlinks', 'll', $this->getInterlangDeletions( $existing ),
 107                         $this->getInterlangInsertions( $existing ) );
 108
 109                 # Template links
 110                 $existing = $this->getExistingTemplates();
 111                 $this->incrTableUpdate( 'templatelinks', 'tl', $this->getTemplateDeletions( $existing ),
 112                         $this->getTemplateInsertions( $existing ) );
 113
 114                 # Category links
 115                 $existing = $this->getExistingCategories();
 116                 $this->incrTableUpdate( 'categorylinks', 'cl', $this->getCategoryDeletions( $existing ),
 117                         $this->getCategoryInsertions( $existing ) );
 118
 119                 # Invalidate all categories which were added, deleted or changed (set symmetric difference)
 120                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 121                 $this->invalidateCategories( $categoryUpdates );
 122
 123                 # Refresh links of all pages including this page
 124                 # This will be in a separate transaction
 125                 if ( $this->mRecursive ) {
 126                         $this->queueRecursiveJobs();
 127                 }
 128
 129                 wfProfileOut( $fname );
 130         }
 131
 132         /**
 133          * Link update which clears the previous entries and inserts new ones
 134          * May be slower or faster depending on level of lock contention and write speed of DB
 135          * Also useful where link table corruption needs to be repaired, e.g. in refreshLinks.php
 136          */
 137         function doDumbUpdate() {
 138                 $fname = 'LinksUpdate::doDumbUpdate';
 139                 wfProfileIn( $fname );
 140
 141                 # Refresh category pages and image description pages
 142                 $existing = $this->getExistingCategories();
 143                 $categoryUpdates = array_diff_assoc( $existing, $this->mCategories ) + array_diff_assoc( $this->mCategories, $existing );
 144                 $existing = $this->getExistingImages();
 145                 $imageUpdates = array_diff_key( $existing, $this->mImages ) + array_diff_key( $this->mImages, $existing );
 146
 147                 $this->dumbTableUpdate( 'pagelinks',     $this->getLinkInsertions(),     'pl_from' );
 148                 $this->dumbTableUpdate( 'imagelinks',    $this->getImageInsertions(),    'il_from' );
 149                 $this->dumbTableUpdate( 'categorylinks', $this->getCategoryInsertions(), 'cl_from' );
 150                 $this->dumbTableUpdate( 'templatelinks', $this->getTemplateInsertions(), 'tl_from' );
 151                 $this->dumbTableUpdate( 'externallinks', $this->getExternalInsertions(), 'el_from' );
 152                 $this->dumbTableUpdate( 'langlinks',     $this->getInterlangInsertions(), 'll_from' );
 153
 154                 # Update the cache of all the category pages and image description pages which were changed
 155                 $this->invalidateCategories( $categoryUpdates );
 156                 $this->invalidateImageDescriptions( $imageUpdates );
 157
 158                 # Refresh links of all pages including this page
 159                 # This will be in a separate transaction
 160                 if ( $this->mRecursive ) {
 161                         $this->queueRecursiveJobs();
 162                 }
 163
 164                 wfProfileOut( $fname );
 165         }
 166
 167         function queueRecursiveJobs() {
 168                 wfProfileIn( __METHOD__ );
 169
 170                 $batchSize = 100;
 171                 $dbr = wfGetDB( DB_SLAVE );
 172                 $res = $dbr->select( array( 'templatelinks', 'page' ),
 173                         array( 'page_namespace', 'page_title' ),
 174                         array(
 175                                 'page_id=tl_from',
 176                                 'tl_namespace' => $this->mTitle->getNamespace(),
 177                                 'tl_title' => $this->mTitle->getDBkey()
 178                         ), __METHOD__
 179                 );
 180
 181                 $done = false;
 182                 while ( !$done ) {
 183                         $jobs = array();
 184                         for ( $i = 0; $i < $batchSize; $i++ ) {
 185                                 $row = $dbr->fetchObject( $res );
 186                                 if ( !$row ) {
 187                                         $done = true;
 188                                         break;
 189                                 }
 190                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 191                                 $jobs[] = Job::factory( 'refreshLinks', $title );
 192                         }
 193                         Job::batchInsert( $jobs );
 194                 }
 195                 $dbr->freeResult( $res );
 196                 wfProfileOut( __METHOD__ );
 197         }
 198
 199         /**
 200          * Invalidate the cache of a list of pages from a single namespace
 201          *
 202          * @param integer $namespace
 203          * @param array $dbkeys
 204          */
 205         function invalidatePages( $namespace, $dbkeys ) {
 206                 $fname = 'LinksUpdate::invalidatePages';
 207
 208                 if ( !count( $dbkeys ) ) {
 209                         return;
 210                 }
 211
 212                 /**
 213                  * Determine which pages need to be updated
 214                  * This is necessary to prevent the job queue from smashing the DB with
 215                  * large numbers of concurrent invalidations of the same page
 216                  */
 217                 $now = $this->mDb->timestamp();
 218                 $ids = array();
 219                 $res = $this->mDb->select( 'page', array( 'page_id' ),
 220                         array(
 221                                 'page_namespace' => $namespace,
 222                                 'page_title IN (' . $this->mDb->makeList( $dbkeys ) . ')',
 223                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 224                         ), $fname
 225                 );
 226                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 227                         $ids[] = $row->page_id;
 228                 }
 229                 if ( !count( $ids ) ) {
 230                         return;
 231                 }
 232
 233                 /**
 234                  * Do the update
 235                  * We still need the page_touched condition, in case the row has changed since
 236                  * the non-locking select above.
 237                  */
 238                 $this->mDb->update( 'page', array( 'page_touched' => $now ),
 239                         array(
 240                                 'page_id IN (' . $this->mDb->makeList( $ids ) . ')',
 241                                 'page_touched < ' . $this->mDb->addQuotes( $now )
 242                         ), $fname
 243                 );
 244         }
 245
 246         function invalidateCategories( $cats ) {
 247                 $this->invalidatePages( NS_CATEGORY, array_keys( $cats ) );
 248         }
 249
 250         function invalidateImageDescriptions( $images ) {
 251                 $this->invalidatePages( NS_IMAGE, array_keys( $images ) );
 252         }
 253
 254         function dumbTableUpdate( $table, $insertions, $fromField ) {
 255                 $fname = 'LinksUpdate::dumbTableUpdate';
 256                 $this->mDb->delete( $table, array( $fromField => $this->mId ), $fname );
 257                 if ( count( $insertions ) ) {
 258                         # The link array was constructed without FOR UPDATE, so there may be collisions
 259                         # This may cause minor link table inconsistencies, which is better than
 260                         # crippling the site with lock contention.
 261                         $this->mDb->insert( $table, $insertions, $fname, array( 'IGNORE' ) );
 262                 }
 263         }
 264
 265         /**
 266          * Make a WHERE clause from a 2-d NS/dbkey array
 267          *
 268          * @param array $arr 2-d array indexed by namespace and DB key
 269          * @param string $prefix Field name prefix, without the underscore
 270          */
 271         function makeWhereFrom2d( &$arr, $prefix ) {
 272                 $lb = new LinkBatch;
 273                 $lb->setArray( $arr );
 274                 return $lb->constructSet( $prefix, $this->mDb );
 275         }
 276
 277         /**
 278          * Update a table by doing a delete query then an insert query
 279          * @private
 280          */
 281         function incrTableUpdate( $table, $prefix, $deletions, $insertions ) {
 282                 $fname = 'LinksUpdate::incrTableUpdate';
 283                 $where = array( "{$prefix}_from" => $this->mId );
 284                 if ( $table == 'pagelinks' || $table == 'templatelinks' ) {
 285                         $clause = $this->makeWhereFrom2d( $deletions, $prefix );
 286                         if ( $clause ) {
 287                                 $where[] = $clause;
 288                         } else {
 289                                 $where = false;
 290                         }
 291                 } else {
 292                         if ( $table == 'langlinks' ) {
 293                                 $toField = 'll_lang';
 294                         } else {
 295                                 $toField = $prefix . '_to';
 296                         }
 297                         if ( count( $deletions ) ) {
 298                                 $where[] = "$toField IN (" . $this->mDb->makeList( array_keys( $deletions ) ) . ')';
 299                         } else {
 300                                 $where = false;
 301                         }
 302                 }
 303                 if ( $where ) {
 304                         $this->mDb->delete( $table, $where, $fname );
 305                 }
 306                 if ( count( $insertions ) ) {
 307                         $this->mDb->insert( $table, $insertions, $fname, 'IGNORE' );
 308                 }
 309         }
 310
 311
 312         /**
 313          * Get an array of pagelinks insertions for passing to the DB
 314          * Skips the titles specified by the 2-D array $existing
 315          * @private
 316          */
 317         function getLinkInsertions( $existing = array() ) {
 318                 $arr = array();
 319                 foreach( $this->mLinks as $ns => $dbkeys ) {
 320                         # array_diff_key() was introduced in PHP 5.1, there is a compatibility function
 321                         # in GlobalFunctions.php
 322                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 323                         foreach ( $diffs as $dbk => $id ) {
 324                                 $arr[] = array(
 325                                         'pl_from'      => $this->mId,
 326                                         'pl_namespace' => $ns,
 327                                         'pl_title'     => $dbk
 328                                 );
 329                         }
 330                 }
 331                 return $arr;
 332         }
 333
 334         /**
 335          * Get an array of template insertions. Like getLinkInsertions()
 336          * @private
 337          */
 338         function getTemplateInsertions( $existing = array() ) {
 339                 $arr = array();
 340                 foreach( $this->mTemplates as $ns => $dbkeys ) {
 341                         $diffs = isset( $existing[$ns] ) ? array_diff_key( $dbkeys, $existing[$ns] ) : $dbkeys;
 342                         foreach ( $diffs as $dbk => $id ) {
 343                                 $arr[] = array(
 344                                         'tl_from'      => $this->mId,
 345                                         'tl_namespace' => $ns,
 346                                         'tl_title'     => $dbk
 347                                 );
 348                         }
 349                 }
 350                 return $arr;
 351         }
 352
 353         /**
 354          * Get an array of image insertions
 355          * Skips the names specified in $existing
 356          * @private
 357          */
 358         function getImageInsertions( $existing = array() ) {
 359                 $arr = array();
 360                 $diffs = array_diff_key( $this->mImages, $existing );
 361                 foreach( $diffs as $iname => $dummy ) {
 362                         $arr[] = array(
 363                                 'il_from' => $this->mId,
 364                                 'il_to'   => $iname
 365                         );
 366                 }
 367                 return $arr;
 368         }
 369
 370         /**
 371          * Get an array of externallinks insertions. Skips the names specified in $existing
 372          * @private
 373          */
 374         function getExternalInsertions( $existing = array() ) {
 375                 $arr = array();
 376                 $diffs = array_diff_key( $this->mExternals, $existing );
 377                 foreach( $diffs as $url => $dummy ) {
 378                         $arr[] = array(
 379                                 'el_from'   => $this->mId,
 380                                 'el_to'     => $url,
 381                                 'el_index'  => wfMakeUrlIndex( $url ),
 382                         );
 383                 }
 384                 return $arr;
 385         }
 386
 387         /**
 388          * Get an array of category insertions
 389          * @param array $existing Array mapping existing category names to sort keys. If both
 390          * match a link in $this, the link will be omitted from the output
 391          * @private
 392          */
 393         function getCategoryInsertions( $existing = array() ) {
 394                 $diffs = array_diff_assoc( $this->mCategories, $existing );
 395                 $arr = array();
 396                 foreach ( $diffs as $name => $sortkey ) {
 397                         $arr[] = array(
 398                                 'cl_from'    => $this->mId,
 399                                 'cl_to'      => $name,
 400                                 'cl_sortkey' => $sortkey,
 401                                 'cl_timestamp' => $this->mDb->timestamp()
 402                         );
 403                 }
 404                 return $arr;
 405         }
 406
 407         /**
 408          * Get an array of interlanguage link insertions
 409          * @param array $existing Array mapping existing language codes to titles
 410          * @private
 411          */
 412         function getInterlangInsertions( $existing = array() ) {
 413             $diffs = array_diff_assoc( $this->mInterlangs, $existing );
 414             $arr = array();
 415             foreach( $diffs as $lang => $title ) {
 416                 $arr[] = array(
 417                     'll_from'  => $this->mId,
 418                     'll_lang'  => $lang,
 419                     'll_title' => $title
 420                 );
 421             }
 422             return $arr;
 423         }
 424
 425         /**
 426          * Given an array of existing links, returns those links which are not in $this
 427          * and thus should be deleted.
 428          * @private
 429          */
 430         function getLinkDeletions( $existing ) {
 431                 $del = array();
 432                 foreach ( $existing as $ns => $dbkeys ) {
 433                         if ( isset( $this->mLinks[$ns] ) ) {
 434                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mLinks[$ns] );
 435                         } else {
 436                                 $del[$ns] = $existing[$ns];
 437                         }
 438                 }
 439                 return $del;
 440         }
 441
 442         /**
 443          * Given an array of existing templates, returns those templates which are not in $this
 444          * and thus should be deleted.
 445          * @private
 446          */
 447         function getTemplateDeletions( $existing ) {
 448                 $del = array();
 449                 foreach ( $existing as $ns => $dbkeys ) {
 450                         if ( isset( $this->mTemplates[$ns] ) ) {
 451                                 $del[$ns] = array_diff_key( $existing[$ns], $this->mTemplates[$ns] );
 452                         } else {
 453                                 $del[$ns] = $existing[$ns];
 454                         }
 455                 }
 456                 return $del;
 457         }
 458
 459         /**
 460          * Given an array of existing images, returns those images which are not in $this
 461          * and thus should be deleted.
 462          * @private
 463          */
 464         function getImageDeletions( $existing ) {
 465                 return array_diff_key( $existing, $this->mImages );
 466         }
 467
 468         /**
 469          * Given an array of existing external links, returns those links which are not
 470          * in $this and thus should be deleted.
 471          * @private
 472          */
 473         function getExternalDeletions( $existing ) {
 474                 return array_diff_key( $existing, $this->mExternals );
 475         }
 476
 477         /**
 478          * Given an array of existing categories, returns those categories which are not in $this
 479          * and thus should be deleted.
 480          * @private
 481          */
 482         function getCategoryDeletions( $existing ) {
 483                 return array_diff_assoc( $existing, $this->mCategories );
 484         }
 485
 486         /**
 487          * Given an array of existing interlanguage links, returns those links which are not
 488          * in $this and thus should be deleted.
 489          * @private
 490          */
 491         function getInterlangDeletions( $existing ) {
 492             return array_diff_assoc( $existing, $this->mInterlangs );
 493         }
 494
 495         /**
 496          * Get an array of existing links, as a 2-D array
 497          * @private
 498          */
 499         function getExistingLinks() {
 500                 $fname = 'LinksUpdate::getExistingLinks';
 501                 $res = $this->mDb->select( 'pagelinks', array( 'pl_namespace', 'pl_title' ),
 502                         array( 'pl_from' => $this->mId ), $fname, $this->mOptions );
 503                 $arr = array();
 504                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 505                         if ( !isset( $arr[$row->pl_namespace] ) ) {
 506                                 $arr[$row->pl_namespace] = array();
 507                         }
 508                         $arr[$row->pl_namespace][$row->pl_title] = 1;
 509                 }
 510                 $this->mDb->freeResult( $res );
 511                 return $arr;
 512         }
 513
 514         /**
 515          * Get an array of existing templates, as a 2-D array
 516          * @private
 517          */
 518         function getExistingTemplates() {
 519                 $fname = 'LinksUpdate::getExistingTemplates';
 520                 $res = $this->mDb->select( 'templatelinks', array( 'tl_namespace', 'tl_title' ),
 521                         array( 'tl_from' => $this->mId ), $fname, $this->mOptions );
 522                 $arr = array();
 523                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 524                         if ( !isset( $arr[$row->tl_namespace] ) ) {
 525                                 $arr[$row->tl_namespace] = array();
 526                         }
 527                         $arr[$row->tl_namespace][$row->tl_title] = 1;
 528                 }
 529                 $this->mDb->freeResult( $res );
 530                 return $arr;
 531         }
 532
 533         /**
 534          * Get an array of existing images, image names in the keys
 535          * @private
 536          */
 537         function getExistingImages() {
 538                 $fname = 'LinksUpdate::getExistingImages';
 539                 $res = $this->mDb->select( 'imagelinks', array( 'il_to' ),
 540                         array( 'il_from' => $this->mId ), $fname, $this->mOptions );
 541                 $arr = array();
 542                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 543                         $arr[$row->il_to] = 1;
 544                 }
 545                 $this->mDb->freeResult( $res );
 546                 return $arr;
 547         }
 548
 549         /**
 550          * Get an array of existing external links, URLs in the keys
 551          * @private
 552          */
 553         function getExistingExternals() {
 554                 $fname = 'LinksUpdate::getExistingExternals';
 555                 $res = $this->mDb->select( 'externallinks', array( 'el_to' ),
 556                         array( 'el_from' => $this->mId ), $fname, $this->mOptions );
 557                 $arr = array();
 558                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 559                         $arr[$row->el_to] = 1;
 560                 }
 561                 $this->mDb->freeResult( $res );
 562                 return $arr;
 563         }
 564
 565         /**
 566          * Get an array of existing categories, with the name in the key and sort key in the value.
 567          * @private
 568          */
 569         function getExistingCategories() {
 570                 $fname = 'LinksUpdate::getExistingCategories';
 571                 $res = $this->mDb->select( 'categorylinks', array( 'cl_to', 'cl_sortkey' ),
 572                         array( 'cl_from' => $this->mId ), $fname, $this->mOptions );
 573                 $arr = array();
 574                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 575                         $arr[$row->cl_to] = $row->cl_sortkey;
 576                 }
 577                 $this->mDb->freeResult( $res );
 578                 return $arr;
 579         }
 580
 581         /**
 582          * Get an array of existing interlanguage links, with the language code in the key and the
 583          * title in the value.
 584          * @private
 585          */
 586         function getExistingInterlangs() {
 587                 $fname = 'LinksUpdate::getExistingInterlangs';
 588                 $res = $this->mDb->select( 'langlinks', array( 'll_lang', 'll_title' ),
 589                         array( 'll_from' => $this->mId ), $fname, $this->mOptions );
 590                 $arr = array();
 591                 while ( $row = $this->mDb->fetchObject( $res ) ) {
 592                         $arr[$row->ll_lang] = $row->ll_title;
 593                 }
 594                 return $arr;
 595         }
 596 }
 597 ?>