Merge "LinkHolderArray: Use LinkBatch for generating WHERE in query"
[lhc/web/wiklou.git] / includes / parser / LinkHolderArray.php
1 <?php
2 /**
3 * Holder of replacement pairs for wiki links
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup Parser
22 */
23
24 /**
25 * @ingroup Parser
26 */
27 class LinkHolderArray {
28 public $internals = [];
29 public $interwikis = [];
30 public $size = 0;
31
32 /**
33 * @var Parser
34 */
35 public $parent;
36 protected $tempIdOffset;
37
38 /**
39 * @param Parser $parent
40 */
41 public function __construct( $parent ) {
42 $this->parent = $parent;
43 }
44
45 /**
46 * Reduce memory usage to reduce the impact of circular references
47 */
48 public function __destruct() {
49 foreach ( $this as $name => $value ) {
50 unset( $this->$name );
51 }
52 }
53
54 /**
55 * Don't serialize the parent object, it is big, and not needed when it is
56 * a parameter to mergeForeign(), which is the only application of
57 * serializing at present.
58 *
59 * Compact the titles, only serialize the text form.
60 * @return array
61 */
62 public function __sleep() {
63 foreach ( $this->internals as &$nsLinks ) {
64 foreach ( $nsLinks as &$entry ) {
65 unset( $entry['title'] );
66 }
67 }
68 unset( $nsLinks );
69 unset( $entry );
70
71 foreach ( $this->interwikis as &$entry ) {
72 unset( $entry['title'] );
73 }
74 unset( $entry );
75
76 return [ 'internals', 'interwikis', 'size' ];
77 }
78
79 /**
80 * Recreate the Title objects
81 */
82 public function __wakeup() {
83 foreach ( $this->internals as &$nsLinks ) {
84 foreach ( $nsLinks as &$entry ) {
85 $entry['title'] = Title::newFromText( $entry['pdbk'] );
86 }
87 }
88 unset( $nsLinks );
89 unset( $entry );
90
91 foreach ( $this->interwikis as &$entry ) {
92 $entry['title'] = Title::newFromText( $entry['pdbk'] );
93 }
94 unset( $entry );
95 }
96
97 /**
98 * Merge another LinkHolderArray into this one
99 * @param LinkHolderArray $other
100 */
101 public function merge( $other ) {
102 foreach ( $other->internals as $ns => $entries ) {
103 $this->size += count( $entries );
104 if ( !isset( $this->internals[$ns] ) ) {
105 $this->internals[$ns] = $entries;
106 } else {
107 $this->internals[$ns] += $entries;
108 }
109 }
110 $this->interwikis += $other->interwikis;
111 }
112
113 /**
114 * Merge a LinkHolderArray from another parser instance into this one. The
115 * keys will not be preserved. Any text which went with the old
116 * LinkHolderArray and needs to work with the new one should be passed in
117 * the $texts array. The strings in this array will have their link holders
118 * converted for use in the destination link holder. The resulting array of
119 * strings will be returned.
120 *
121 * @param LinkHolderArray $other
122 * @param array $texts Array of strings
123 * @return array
124 */
125 public function mergeForeign( $other, $texts ) {
126 $this->tempIdOffset = $idOffset = $this->parent->nextLinkID();
127 $maxId = 0;
128
129 # Renumber internal links
130 foreach ( $other->internals as $ns => $nsLinks ) {
131 foreach ( $nsLinks as $key => $entry ) {
132 $newKey = $idOffset + $key;
133 $this->internals[$ns][$newKey] = $entry;
134 $maxId = $newKey > $maxId ? $newKey : $maxId;
135 }
136 }
137 $texts = preg_replace_callback( '/(<!--LINK \d+:)(\d+)(-->)/',
138 [ $this, 'mergeForeignCallback' ], $texts );
139
140 # Renumber interwiki links
141 foreach ( $other->interwikis as $key => $entry ) {
142 $newKey = $idOffset + $key;
143 $this->interwikis[$newKey] = $entry;
144 $maxId = $newKey > $maxId ? $newKey : $maxId;
145 }
146 $texts = preg_replace_callback( '/(<!--IWLINK )(\d+)(-->)/',
147 [ $this, 'mergeForeignCallback' ], $texts );
148
149 # Set the parent link ID to be beyond the highest used ID
150 $this->parent->setLinkID( $maxId + 1 );
151 $this->tempIdOffset = null;
152 return $texts;
153 }
154
155 /**
156 * @param array $m
157 * @return string
158 */
159 protected function mergeForeignCallback( $m ) {
160 return $m[1] . ( $m[2] + $this->tempIdOffset ) . $m[3];
161 }
162
163 /**
164 * Get a subset of the current LinkHolderArray which is sufficient to
165 * interpret the given text.
166 * @param string $text
167 * @return LinkHolderArray
168 */
169 public function getSubArray( $text ) {
170 $sub = new LinkHolderArray( $this->parent );
171
172 # Internal links
173 $pos = 0;
174 while ( $pos < strlen( $text ) ) {
175 if ( !preg_match( '/<!--LINK (\d+):(\d+)-->/',
176 $text, $m, PREG_OFFSET_CAPTURE, $pos )
177 ) {
178 break;
179 }
180 $ns = $m[1][0];
181 $key = $m[2][0];
182 $sub->internals[$ns][$key] = $this->internals[$ns][$key];
183 $pos = $m[0][1] + strlen( $m[0][0] );
184 }
185
186 # Interwiki links
187 $pos = 0;
188 while ( $pos < strlen( $text ) ) {
189 if ( !preg_match( '/<!--IWLINK (\d+)-->/', $text, $m, PREG_OFFSET_CAPTURE, $pos ) ) {
190 break;
191 }
192 $key = $m[1][0];
193 $sub->interwikis[$key] = $this->interwikis[$key];
194 $pos = $m[0][1] + strlen( $m[0][0] );
195 }
196 return $sub;
197 }
198
199 /**
200 * Returns true if the memory requirements of this object are getting large
201 * @return bool
202 */
203 public function isBig() {
204 global $wgLinkHolderBatchSize;
205 return $this->size > $wgLinkHolderBatchSize;
206 }
207
208 /**
209 * Clear all stored link holders.
210 * Make sure you don't have any text left using these link holders, before you call this
211 */
212 public function clear() {
213 $this->internals = [];
214 $this->interwikis = [];
215 $this->size = 0;
216 }
217
218 /**
219 * Make a link placeholder. The text returned can be later resolved to a real link with
220 * replaceLinkHolders(). This is done for two reasons: firstly to avoid further
221 * parsing of interwiki links, and secondly to allow all existence checks and
222 * article length checks (for stub links) to be bundled into a single query.
223 *
224 * @param Title $nt
225 * @param string $text
226 * @param array $query [optional]
227 * @param string $trail [optional]
228 * @param string $prefix [optional]
229 * @return string
230 */
231 public function makeHolder( $nt, $text = '', $query = [], $trail = '', $prefix = '' ) {
232 if ( !is_object( $nt ) ) {
233 # Fail gracefully
234 $retVal = "<!-- ERROR -->{$prefix}{$text}{$trail}";
235 } else {
236 # Separate the link trail from the rest of the link
237 list( $inside, $trail ) = Linker::splitTrail( $trail );
238
239 $entry = [
240 'title' => $nt,
241 'text' => $prefix . $text . $inside,
242 'pdbk' => $nt->getPrefixedDBkey(),
243 ];
244 if ( $query !== [] ) {
245 $entry['query'] = $query;
246 }
247
248 if ( $nt->isExternal() ) {
249 // Use a globally unique ID to keep the objects mergable
250 $key = $this->parent->nextLinkID();
251 $this->interwikis[$key] = $entry;
252 $retVal = "<!--IWLINK $key-->{$trail}";
253 } else {
254 $key = $this->parent->nextLinkID();
255 $ns = $nt->getNamespace();
256 $this->internals[$ns][$key] = $entry;
257 $retVal = "<!--LINK $ns:$key-->{$trail}";
258 }
259 $this->size++;
260 }
261 return $retVal;
262 }
263
264 /**
265 * Replace <!--LINK--> link placeholders with actual links, in the buffer
266 *
267 * @param string $text
268 */
269 public function replace( &$text ) {
270
271 $this->replaceInternal( $text );
272 $this->replaceInterwiki( $text );
273
274 }
275
276 /**
277 * Replace internal links
278 * @param string $text
279 */
280 protected function replaceInternal( &$text ) {
281 if ( !$this->internals ) {
282 return;
283 }
284
285 global $wgContLang, $wgContentHandlerUseDB, $wgPageLanguageUseDB;
286
287 $colours = [];
288 $linkCache = LinkCache::singleton();
289 $output = $this->parent->getOutput();
290
291 $dbr = wfGetDB( DB_SLAVE );
292 $threshold = $this->parent->getOptions()->getStubThreshold();
293
294 # Sort by namespace
295 ksort( $this->internals );
296
297 $linkcolour_ids = [];
298
299 # Generate query
300 $lb = new LinkBatch();
301 $lb->setCaller( __METHOD__ );
302
303 foreach ( $this->internals as $ns => $entries ) {
304 foreach ( $entries as $entry ) {
305 /** @var Title $title */
306 $title = $entry['title'];
307 $pdbk = $entry['pdbk'];
308
309 # Skip invalid entries.
310 # Result will be ugly, but prevents crash.
311 if ( is_null( $title ) ) {
312 continue;
313 }
314
315 # Check if it's a static known link, e.g. interwiki
316 if ( $title->isAlwaysKnown() ) {
317 $colours[$pdbk] = '';
318 } elseif ( $ns == NS_SPECIAL ) {
319 $colours[$pdbk] = 'new';
320 } else {
321 $id = $linkCache->getGoodLinkID( $pdbk );
322 if ( $id != 0 ) {
323 $colours[$pdbk] = Linker::getLinkColour( $title, $threshold );
324 $output->addLink( $title, $id );
325 $linkcolour_ids[$id] = $pdbk;
326 } elseif ( $linkCache->isBadLink( $pdbk ) ) {
327 $colours[$pdbk] = 'new';
328 } else {
329 # Not in the link cache, add it to the query
330 $lb->addObj( $title );
331 }
332 }
333 }
334 }
335 if ( !$lb->isEmpty() ) {
336 $fields = [ 'page_id', 'page_namespace', 'page_title',
337 'page_is_redirect', 'page_len', 'page_latest' ];
338
339 if ( $wgContentHandlerUseDB ) {
340 $fields[] = 'page_content_model';
341 }
342 if ( $wgPageLanguageUseDB ) {
343 $fields[] = 'page_lang';
344 }
345
346 $res = $dbr->select(
347 'page',
348 $fields,
349 $lb->constructSet( 'page', $dbr ),
350 __METHOD__
351 );
352
353 # Fetch data and form into an associative array
354 # non-existent = broken
355 foreach ( $res as $s ) {
356 $title = Title::makeTitle( $s->page_namespace, $s->page_title );
357 $pdbk = $title->getPrefixedDBkey();
358 $linkCache->addGoodLinkObjFromRow( $title, $s );
359 $output->addLink( $title, $s->page_id );
360 # @todo FIXME: Convoluted data flow
361 # The redirect status and length is passed to getLinkColour via the LinkCache
362 # Use formal parameters instead
363 $colours[$pdbk] = Linker::getLinkColour( $title, $threshold );
364 // add id to the extension todolist
365 $linkcolour_ids[$s->page_id] = $pdbk;
366 }
367 unset( $res );
368 }
369 if ( count( $linkcolour_ids ) ) {
370 // pass an array of page_ids to an extension
371 Hooks::run( 'GetLinkColours', [ $linkcolour_ids, &$colours ] );
372 }
373
374 # Do a second query for different language variants of links and categories
375 if ( $wgContLang->hasVariants() ) {
376 $this->doVariants( $colours );
377 }
378
379 # Construct search and replace arrays
380 $replacePairs = [];
381 foreach ( $this->internals as $ns => $entries ) {
382 foreach ( $entries as $index => $entry ) {
383 $pdbk = $entry['pdbk'];
384 $title = $entry['title'];
385 $query = isset( $entry['query'] ) ? $entry['query'] : [];
386 $key = "$ns:$index";
387 $searchkey = "<!--LINK $key-->";
388 $displayText = $entry['text'];
389 if ( isset( $entry['selflink'] ) ) {
390 $replacePairs[$searchkey] = Linker::makeSelfLinkObj( $title, $displayText, $query );
391 continue;
392 }
393 if ( $displayText === '' ) {
394 $displayText = null;
395 }
396 if ( !isset( $colours[$pdbk] ) ) {
397 $colours[$pdbk] = 'new';
398 }
399 $attribs = [];
400 if ( $colours[$pdbk] == 'new' ) {
401 $linkCache->addBadLinkObj( $title );
402 $output->addLink( $title, 0 );
403 $type = [ 'broken' ];
404 } else {
405 if ( $colours[$pdbk] != '' ) {
406 $attribs['class'] = $colours[$pdbk];
407 }
408 $type = [ 'known', 'noclasses' ];
409 }
410 $replacePairs[$searchkey] = Linker::link( $title, $displayText,
411 $attribs, $query, $type );
412 }
413 }
414 $replacer = new HashtableReplacer( $replacePairs, 1 );
415
416 # Do the thing
417 $text = preg_replace_callback(
418 '/(<!--LINK .*?-->)/',
419 $replacer->cb(),
420 $text
421 );
422
423 }
424
425 /**
426 * Replace interwiki links
427 * @param string $text
428 */
429 protected function replaceInterwiki( &$text ) {
430 if ( empty( $this->interwikis ) ) {
431 return;
432 }
433
434 # Make interwiki link HTML
435 $output = $this->parent->getOutput();
436 $replacePairs = [];
437 $options = [
438 'stubThreshold' => $this->parent->getOptions()->getStubThreshold(),
439 ];
440 foreach ( $this->interwikis as $key => $link ) {
441 $replacePairs[$key] = Linker::link( $link['title'], $link['text'], [], [], $options );
442 $output->addInterwikiLink( $link['title'] );
443 }
444 $replacer = new HashtableReplacer( $replacePairs, 1 );
445
446 $text = preg_replace_callback(
447 '/<!--IWLINK (.*?)-->/',
448 $replacer->cb(),
449 $text );
450 }
451
452 /**
453 * Modify $this->internals and $colours according to language variant linking rules
454 * @param array $colours
455 */
456 protected function doVariants( &$colours ) {
457 global $wgContLang, $wgContentHandlerUseDB, $wgPageLanguageUseDB;
458 $linkBatch = new LinkBatch();
459 $variantMap = []; // maps $pdbkey_Variant => $keys (of link holders)
460 $output = $this->parent->getOutput();
461 $linkCache = LinkCache::singleton();
462 $threshold = $this->parent->getOptions()->getStubThreshold();
463 $titlesToBeConverted = '';
464 $titlesAttrs = [];
465
466 // Concatenate titles to a single string, thus we only need auto convert the
467 // single string to all variants. This would improve parser's performance
468 // significantly.
469 foreach ( $this->internals as $ns => $entries ) {
470 if ( $ns == NS_SPECIAL ) {
471 continue;
472 }
473 foreach ( $entries as $index => $entry ) {
474 $pdbk = $entry['pdbk'];
475 // we only deal with new links (in its first query)
476 if ( !isset( $colours[$pdbk] ) || $colours[$pdbk] === 'new' ) {
477 $titlesAttrs[] = [ $index, $entry['title'] ];
478 // separate titles with \0 because it would never appears
479 // in a valid title
480 $titlesToBeConverted .= $entry['title']->getText() . "\0";
481 }
482 }
483 }
484
485 // Now do the conversion and explode string to text of titles
486 $titlesAllVariants = $wgContLang->autoConvertToAllVariants( rtrim( $titlesToBeConverted, "\0" ) );
487 $allVariantsName = array_keys( $titlesAllVariants );
488 foreach ( $titlesAllVariants as &$titlesVariant ) {
489 $titlesVariant = explode( "\0", $titlesVariant );
490 }
491
492 // Then add variants of links to link batch
493 $parentTitle = $this->parent->getTitle();
494 foreach ( $titlesAttrs as $i => $attrs ) {
495 /** @var Title $title */
496 list( $index, $title ) = $attrs;
497 $ns = $title->getNamespace();
498 $text = $title->getText();
499
500 foreach ( $allVariantsName as $variantName ) {
501 $textVariant = $titlesAllVariants[$variantName][$i];
502 if ( $textVariant === $text ) {
503 continue;
504 }
505
506 $variantTitle = Title::makeTitle( $ns, $textVariant );
507 if ( is_null( $variantTitle ) ) {
508 continue;
509 }
510
511 // Self-link checking for mixed/different variant titles. At this point, we
512 // already know the exact title does not exist, so the link cannot be to a
513 // variant of the current title that exists as a separate page.
514 if ( $variantTitle->equals( $parentTitle ) && !$title->hasFragment() ) {
515 $this->internals[$ns][$index]['selflink'] = true;
516 continue 2;
517 }
518
519 $linkBatch->addObj( $variantTitle );
520 $variantMap[$variantTitle->getPrefixedDBkey()][] = "$ns:$index";
521 }
522 }
523
524 // process categories, check if a category exists in some variant
525 $categoryMap = []; // maps $category_variant => $category (dbkeys)
526 $varCategories = []; // category replacements oldDBkey => newDBkey
527 foreach ( $output->getCategoryLinks() as $category ) {
528 $categoryTitle = Title::makeTitleSafe( NS_CATEGORY, $category );
529 $linkBatch->addObj( $categoryTitle );
530 $variants = $wgContLang->autoConvertToAllVariants( $category );
531 foreach ( $variants as $variant ) {
532 if ( $variant !== $category ) {
533 $variantTitle = Title::makeTitleSafe( NS_CATEGORY, $variant );
534 if ( is_null( $variantTitle ) ) {
535 continue;
536 }
537 $linkBatch->addObj( $variantTitle );
538 $categoryMap[$variant] = [ $category, $categoryTitle ];
539 }
540 }
541 }
542
543 if ( !$linkBatch->isEmpty() ) {
544 // construct query
545 $dbr = wfGetDB( DB_SLAVE );
546 $fields = [ 'page_id', 'page_namespace', 'page_title',
547 'page_is_redirect', 'page_len', 'page_latest' ];
548
549 if ( $wgContentHandlerUseDB ) {
550 $fields[] = 'page_content_model';
551 }
552 if ( $wgPageLanguageUseDB ) {
553 $fields[] = 'page_lang';
554 }
555
556 $varRes = $dbr->select( 'page',
557 $fields,
558 $linkBatch->constructSet( 'page', $dbr ),
559 __METHOD__
560 );
561
562 $linkcolour_ids = [];
563
564 // for each found variants, figure out link holders and replace
565 foreach ( $varRes as $s ) {
566 $variantTitle = Title::makeTitle( $s->page_namespace, $s->page_title );
567 $varPdbk = $variantTitle->getPrefixedDBkey();
568 $vardbk = $variantTitle->getDBkey();
569
570 $holderKeys = [];
571 if ( isset( $variantMap[$varPdbk] ) ) {
572 $holderKeys = $variantMap[$varPdbk];
573 $linkCache->addGoodLinkObjFromRow( $variantTitle, $s );
574 $output->addLink( $variantTitle, $s->page_id );
575 }
576
577 // loop over link holders
578 foreach ( $holderKeys as $key ) {
579 list( $ns, $index ) = explode( ':', $key, 2 );
580 $entry =& $this->internals[$ns][$index];
581 $pdbk = $entry['pdbk'];
582
583 if ( !isset( $colours[$pdbk] ) || $colours[$pdbk] === 'new' ) {
584 // found link in some of the variants, replace the link holder data
585 $entry['title'] = $variantTitle;
586 $entry['pdbk'] = $varPdbk;
587
588 // set pdbk and colour
589 # @todo FIXME: Convoluted data flow
590 # The redirect status and length is passed to getLinkColour via the LinkCache
591 # Use formal parameters instead
592 $colours[$varPdbk] = Linker::getLinkColour( $variantTitle, $threshold );
593 $linkcolour_ids[$s->page_id] = $pdbk;
594 }
595 }
596
597 // check if the object is a variant of a category
598 if ( isset( $categoryMap[$vardbk] ) ) {
599 list( $oldkey, $oldtitle ) = $categoryMap[$vardbk];
600 if ( !isset( $varCategories[$oldkey] ) && !$oldtitle->exists() ) {
601 $varCategories[$oldkey] = $vardbk;
602 }
603 }
604 }
605 Hooks::run( 'GetLinkColours', [ $linkcolour_ids, &$colours ] );
606
607 // rebuild the categories in original order (if there are replacements)
608 if ( count( $varCategories ) > 0 ) {
609 $newCats = [];
610 $originalCats = $output->getCategories();
611 foreach ( $originalCats as $cat => $sortkey ) {
612 // make the replacement
613 if ( array_key_exists( $cat, $varCategories ) ) {
614 $newCats[$varCategories[$cat]] = $sortkey;
615 } else {
616 $newCats[$cat] = $sortkey;
617 }
618 }
619 $output->setCategoryLinks( $newCats );
620 }
621 }
622 }
623
624 /**
625 * Replace <!--LINK--> link placeholders with plain text of links
626 * (not HTML-formatted).
627 *
628 * @param string $text
629 * @return string
630 */
631 public function replaceText( $text ) {
632
633 $text = preg_replace_callback(
634 '/<!--(LINK|IWLINK) (.*?)-->/',
635 [ &$this, 'replaceTextCallback' ],
636 $text );
637
638 return $text;
639 }
640
641 /**
642 * Callback for replaceText()
643 *
644 * @param array $matches
645 * @return string
646 * @private
647 */
648 public function replaceTextCallback( $matches ) {
649 $type = $matches[1];
650 $key = $matches[2];
651 if ( $type == 'LINK' ) {
652 list( $ns, $index ) = explode( ':', $key, 2 );
653 if ( isset( $this->internals[$ns][$index]['text'] ) ) {
654 return $this->internals[$ns][$index]['text'];
655 }
656 } elseif ( $type == 'IWLINK' ) {
657 if ( isset( $this->interwikis[$key]['text'] ) ) {
658 return $this->interwikis[$key]['text'];
659 }
660 }
661 return $matches[0];
662 }
663 }