Merge "Exclude redirects from Special:Fewestrevisions"
[lhc/web/wiklou.git] / maintenance / namespaceDupes.php
1 <?php
2 /**
3 * Check for articles to fix after adding/deleting namespaces
4 *
5 * Copyright © 2005-2007 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
22 *
23 * @file
24 * @ingroup Maintenance
25 */
26
27 require_once __DIR__ . '/Maintenance.php';
28
29 use MediaWiki\Linker\LinkTarget;
30 use MediaWiki\MediaWikiServices;
31 use MediaWiki\Storage\RevisionRecord;
32 use Wikimedia\Rdbms\IResultWrapper;
33 use Wikimedia\Rdbms\IMaintainableDatabase;
34
35 /**
36 * Maintenance script that checks for articles to fix after
37 * adding/deleting namespaces.
38 *
39 * @ingroup Maintenance
40 */
41 class NamespaceDupes extends Maintenance {
42
43 /**
44 * @var IMaintainableDatabase
45 */
46 protected $db;
47
48 private $resolvablePages = 0;
49 private $totalPages = 0;
50
51 private $resolvableLinks = 0;
52 private $totalLinks = 0;
53
54 public function __construct() {
55 parent::__construct();
56 $this->addDescription( 'Find and fix pages affected by namespace addition/removal' );
57 $this->addOption( 'fix', 'Attempt to automatically fix errors' );
58 $this->addOption( 'merge', "Instead of renaming conflicts, do a history merge with " .
59 "the correct title" );
60 $this->addOption( 'add-suffix', "Dupes will be renamed with correct namespace with " .
61 "<text> appended after the article name", false, true );
62 $this->addOption( 'add-prefix', "Dupes will be renamed with correct namespace with " .
63 "<text> prepended before the article name", false, true );
64 $this->addOption( 'source-pseudo-namespace', "Move all pages with the given source " .
65 "prefix (with an implied colon following it). If --dest-namespace is not specified, " .
66 "the colon will be replaced with a hyphen.",
67 false, true );
68 $this->addOption( 'dest-namespace', "In combination with --source-pseudo-namespace, " .
69 "specify the namespace ID of the destination.", false, true );
70 $this->addOption( 'move-talk', "If this is specified, pages in the Talk namespace that " .
71 "begin with a conflicting prefix will be renamed, for example " .
72 "Talk:File:Foo -> File_Talk:Foo" );
73 }
74
75 public function execute() {
76 $this->db = $this->getDB( DB_MASTER );
77
78 $options = [
79 'fix' => $this->hasOption( 'fix' ),
80 'merge' => $this->hasOption( 'merge' ),
81 'add-suffix' => $this->getOption( 'add-suffix', '' ),
82 'add-prefix' => $this->getOption( 'add-prefix', '' ),
83 'move-talk' => $this->hasOption( 'move-talk' ),
84 'source-pseudo-namespace' => $this->getOption( 'source-pseudo-namespace', '' ),
85 'dest-namespace' => intval( $this->getOption( 'dest-namespace', 0 ) ) ];
86
87 if ( $options['source-pseudo-namespace'] !== '' ) {
88 $retval = $this->checkPrefix( $options );
89 } else {
90 $retval = $this->checkAll( $options );
91 }
92
93 if ( $retval ) {
94 $this->output( "\nLooks good!\n" );
95 } else {
96 $this->output( "\nOh noeees\n" );
97 }
98 }
99
100 /**
101 * Check all namespaces
102 *
103 * @param array $options Associative array of validated command-line options
104 *
105 * @return bool
106 */
107 private function checkAll( $options ) {
108 global $wgNamespaceAliases, $wgCapitalLinks;
109
110 $contLang = MediaWikiServices::getInstance()->getContentLanguage();
111 $spaces = [];
112
113 // List interwikis first, so they'll be overridden
114 // by any conflicting local namespaces.
115 foreach ( $this->getInterwikiList() as $prefix ) {
116 $name = $contLang->ucfirst( $prefix );
117 $spaces[$name] = 0;
118 }
119
120 // Now pull in all canonical and alias namespaces...
121 foreach (
122 MediaWikiServices::getInstance()->getNamespaceInfo()->getCanonicalNamespaces()
123 as $ns => $name
124 ) {
125 // This includes $wgExtraNamespaces
126 if ( $name !== '' ) {
127 $spaces[$name] = $ns;
128 }
129 }
130 foreach ( $contLang->getNamespaces() as $ns => $name ) {
131 if ( $name !== '' ) {
132 $spaces[$name] = $ns;
133 }
134 }
135 foreach ( $wgNamespaceAliases as $name => $ns ) {
136 $spaces[$name] = $ns;
137 }
138 foreach ( $contLang->getNamespaceAliases() as $name => $ns ) {
139 $spaces[$name] = $ns;
140 }
141
142 // We'll need to check for lowercase keys as well,
143 // since we're doing case-sensitive searches in the db.
144 foreach ( $spaces as $name => $ns ) {
145 $moreNames = [];
146 $moreNames[] = $contLang->uc( $name );
147 $moreNames[] = $contLang->ucfirst( $contLang->lc( $name ) );
148 $moreNames[] = $contLang->ucwords( $name );
149 $moreNames[] = $contLang->ucwords( $contLang->lc( $name ) );
150 $moreNames[] = $contLang->ucwordbreaks( $name );
151 $moreNames[] = $contLang->ucwordbreaks( $contLang->lc( $name ) );
152 if ( !$wgCapitalLinks ) {
153 foreach ( $moreNames as $altName ) {
154 $moreNames[] = $contLang->lcfirst( $altName );
155 }
156 $moreNames[] = $contLang->lcfirst( $name );
157 }
158 foreach ( array_unique( $moreNames ) as $altName ) {
159 if ( $altName !== $name ) {
160 $spaces[$altName] = $ns;
161 }
162 }
163 }
164
165 // Sort by namespace index, and if there are two with the same index,
166 // break the tie by sorting by name
167 $origSpaces = $spaces;
168 uksort( $spaces, function ( $a, $b ) use ( $origSpaces ) {
169 return $origSpaces[$a] <=> $origSpaces[$b]
170 ?: $a <=> $b;
171 } );
172
173 $ok = true;
174 foreach ( $spaces as $name => $ns ) {
175 $ok = $this->checkNamespace( $ns, $name, $options ) && $ok;
176 }
177
178 $this->output( "{$this->totalPages} pages to fix, " .
179 "{$this->resolvablePages} were resolvable.\n\n" );
180
181 foreach ( $spaces as $name => $ns ) {
182 if ( $ns != 0 ) {
183 /* Fix up link destinations for non-interwiki links only.
184 *
185 * For example if a page has [[Foo:Bar]] and then a Foo namespace
186 * is introduced, pagelinks needs to be updated to have
187 * page_namespace = NS_FOO.
188 *
189 * If instead an interwiki prefix was introduced called "Foo",
190 * the link should instead be moved to the iwlinks table. If a new
191 * language is introduced called "Foo", or if there is a pagelink
192 * [[fr:Bar]] when interlanguage magic links are turned on, the
193 * link would have to be moved to the langlinks table. Let's put
194 * those cases in the too-hard basket for now. The consequences are
195 * not especially severe.
196 * @fixme Handle interwiki links, and pagelinks to Category:, File:
197 * which probably need reparsing.
198 */
199
200 $this->checkLinkTable( 'pagelinks', 'pl', $ns, $name, $options );
201 $this->checkLinkTable( 'templatelinks', 'tl', $ns, $name, $options );
202
203 // The redirect table has interwiki links randomly mixed in, we
204 // need to filter those out. For example [[w:Foo:Bar]] would
205 // have rd_interwiki=w and rd_namespace=0, which would match the
206 // query for a conflicting namespace "Foo" if filtering wasn't done.
207 $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options,
208 [ 'rd_interwiki' => null ] );
209 $this->checkLinkTable( 'redirect', 'rd', $ns, $name, $options,
210 [ 'rd_interwiki' => '' ] );
211 }
212 }
213
214 $this->output( "{$this->totalLinks} links to fix, " .
215 "{$this->resolvableLinks} were resolvable.\n" );
216
217 return $ok;
218 }
219
220 /**
221 * Get the interwiki list
222 *
223 * @return array
224 */
225 private function getInterwikiList() {
226 $result = MediaWikiServices::getInstance()->getInterwikiLookup()->getAllPrefixes();
227 $prefixes = [];
228 foreach ( $result as $row ) {
229 $prefixes[] = $row['iw_prefix'];
230 }
231
232 return $prefixes;
233 }
234
235 /**
236 * Check a given prefix and try to move it into the given destination namespace
237 *
238 * @param int $ns Destination namespace id
239 * @param string $name
240 * @param array $options Associative array of validated command-line options
241 * @return bool
242 */
243 private function checkNamespace( $ns, $name, $options ) {
244 $targets = $this->getTargetList( $ns, $name, $options );
245 $count = $targets->numRows();
246 $this->totalPages += $count;
247 if ( $count == 0 ) {
248 return true;
249 }
250
251 $dryRunNote = $options['fix'] ? '' : ' DRY RUN ONLY';
252
253 $ok = true;
254 foreach ( $targets as $row ) {
255 // Find the new title and determine the action to take
256
257 $newTitle = $this->getDestinationTitle( $ns, $name,
258 $row->page_namespace, $row->page_title, $options );
259 $logStatus = false;
260 if ( !$newTitle ) {
261 $logStatus = 'invalid title';
262 $action = 'abort';
263 } elseif ( $newTitle->exists() ) {
264 if ( $options['merge'] ) {
265 if ( $this->canMerge( $row->page_id, $newTitle, $logStatus ) ) {
266 $action = 'merge';
267 } else {
268 $action = 'abort';
269 }
270 } elseif ( $options['add-prefix'] == '' && $options['add-suffix'] == '' ) {
271 $action = 'abort';
272 $logStatus = 'dest title exists and --add-prefix not specified';
273 } else {
274 $newTitle = $this->getAlternateTitle( $newTitle, $options );
275 if ( !$newTitle ) {
276 $action = 'abort';
277 $logStatus = 'alternate title is invalid';
278 } elseif ( $newTitle->exists() ) {
279 $action = 'abort';
280 $logStatus = 'title conflict';
281 } else {
282 $action = 'move';
283 $logStatus = 'alternate';
284 }
285 }
286 } else {
287 $action = 'move';
288 $logStatus = 'no conflict';
289 }
290
291 // Take the action or log a dry run message
292
293 $logTitle = "id={$row->page_id} ns={$row->page_namespace} dbk={$row->page_title}";
294 $pageOK = true;
295
296 switch ( $action ) {
297 case 'abort':
298 $this->output( "$logTitle *** $logStatus\n" );
299 $pageOK = false;
300 break;
301 case 'move':
302 $this->output( "$logTitle -> " .
303 $newTitle->getPrefixedDBkey() . " ($logStatus)$dryRunNote\n" );
304
305 if ( $options['fix'] ) {
306 $pageOK = $this->movePage( $row->page_id, $newTitle );
307 }
308 break;
309 case 'merge':
310 $this->output( "$logTitle => " .
311 $newTitle->getPrefixedDBkey() . " (merge)$dryRunNote\n" );
312
313 if ( $options['fix'] ) {
314 $pageOK = $this->mergePage( $row, $newTitle );
315 }
316 break;
317 }
318
319 if ( $pageOK ) {
320 $this->resolvablePages++;
321 } else {
322 $ok = false;
323 }
324 }
325
326 return $ok;
327 }
328
329 /**
330 * Check and repair the destination fields in a link table
331 * @param string $table The link table name
332 * @param string $fieldPrefix The field prefix in the link table
333 * @param int $ns Destination namespace id
334 * @param string $name
335 * @param array $options Associative array of validated command-line options
336 * @param array $extraConds Extra conditions for the SQL query
337 */
338 private function checkLinkTable( $table, $fieldPrefix, $ns, $name, $options,
339 $extraConds = []
340 ) {
341 $batchConds = [];
342 $fromField = "{$fieldPrefix}_from";
343 $namespaceField = "{$fieldPrefix}_namespace";
344 $titleField = "{$fieldPrefix}_title";
345 $batchSize = 500;
346 while ( true ) {
347 $res = $this->db->select(
348 $table,
349 [ $fromField, $namespaceField, $titleField ],
350 array_merge( $batchConds, $extraConds, [
351 $namespaceField => 0,
352 $titleField . $this->db->buildLike( "$name:", $this->db->anyString() )
353 ] ),
354 __METHOD__,
355 [
356 'ORDER BY' => [ $titleField, $fromField ],
357 'LIMIT' => $batchSize
358 ]
359 );
360
361 if ( $res->numRows() == 0 ) {
362 break;
363 }
364 foreach ( $res as $row ) {
365 $logTitle = "from={$row->$fromField} ns={$row->$namespaceField} " .
366 "dbk={$row->$titleField}";
367 $destTitle = $this->getDestinationTitle( $ns, $name,
368 $row->$namespaceField, $row->$titleField, $options );
369 $this->totalLinks++;
370 if ( !$destTitle ) {
371 $this->output( "$table $logTitle *** INVALID\n" );
372 continue;
373 }
374 $this->resolvableLinks++;
375 if ( !$options['fix'] ) {
376 $this->output( "$table $logTitle -> " .
377 $destTitle->getPrefixedDBkey() . " DRY RUN\n" );
378 continue;
379 }
380
381 $this->db->update( $table,
382 // SET
383 [
384 $namespaceField => $destTitle->getNamespace(),
385 $titleField => $destTitle->getDBkey()
386 ],
387 // WHERE
388 [
389 $namespaceField => 0,
390 $titleField => $row->$titleField,
391 $fromField => $row->$fromField
392 ],
393 __METHOD__,
394 [ 'IGNORE' ]
395 );
396 $this->output( "$table $logTitle -> " .
397 $destTitle->getPrefixedDBkey() . "\n" );
398 }
399 $encLastTitle = $this->db->addQuotes( $row->$titleField );
400 $encLastFrom = $this->db->addQuotes( $row->$fromField );
401
402 $batchConds = [
403 "$titleField > $encLastTitle " .
404 "OR ($titleField = $encLastTitle AND $fromField > $encLastFrom)" ];
405
406 wfWaitForSlaves();
407 }
408 }
409
410 /**
411 * Move the given pseudo-namespace, either replacing the colon with a hyphen
412 * (useful for pseudo-namespaces that conflict with interwiki links) or move
413 * them to another namespace if specified.
414 * @param array $options Associative array of validated command-line options
415 * @return bool
416 */
417 private function checkPrefix( $options ) {
418 $prefix = $options['source-pseudo-namespace'];
419 $ns = $options['dest-namespace'];
420 $this->output( "Checking prefix \"$prefix\" vs namespace $ns\n" );
421
422 return $this->checkNamespace( $ns, $prefix, $options );
423 }
424
425 /**
426 * Find pages in main and talk namespaces that have a prefix of the new
427 * namespace so we know titles that will need migrating
428 *
429 * @param int $ns Destination namespace id
430 * @param string $name Prefix that is being made a namespace
431 * @param array $options Associative array of validated command-line options
432 *
433 * @return IResultWrapper
434 */
435 private function getTargetList( $ns, $name, $options ) {
436 if (
437 $options['move-talk'] &&
438 MediaWikiServices::getInstance()->getNamespaceInfo()->isSubject( $ns )
439 ) {
440 $checkNamespaces = [ NS_MAIN, NS_TALK ];
441 } else {
442 $checkNamespaces = NS_MAIN;
443 }
444
445 return $this->db->select( 'page',
446 [
447 'page_id',
448 'page_title',
449 'page_namespace',
450 ],
451 [
452 'page_namespace' => $checkNamespaces,
453 'page_title' . $this->db->buildLike( "$name:", $this->db->anyString() ),
454 ],
455 __METHOD__
456 );
457 }
458
459 /**
460 * Get the preferred destination title for a given target page.
461 * @param int $ns The destination namespace ID
462 * @param string $name The conflicting prefix
463 * @param int $sourceNs The source namespace
464 * @param int $sourceDbk The source DB key (i.e. page_title)
465 * @param array $options Associative array of validated command-line options
466 * @return Title|false
467 */
468 private function getDestinationTitle( $ns, $name, $sourceNs, $sourceDbk, $options ) {
469 $dbk = substr( $sourceDbk, strlen( "$name:" ) );
470 if ( $ns == 0 ) {
471 // An interwiki; try an alternate encoding with '-' for ':'
472 $dbk = "$name-" . $dbk;
473 }
474 $destNS = $ns;
475 $nsInfo = MediaWikiServices::getInstance()->getNamespaceInfo();
476 if ( $sourceNs == NS_TALK && $nsInfo->isSubject( $ns ) ) {
477 // This is an associated talk page moved with the --move-talk feature.
478 $destNS = $nsInfo->getTalk( $destNS );
479 }
480 $newTitle = Title::makeTitleSafe( $destNS, $dbk );
481 if ( !$newTitle || !$newTitle->canExist() ) {
482 return false;
483 }
484 return $newTitle;
485 }
486
487 /**
488 * Get an alternative title to move a page to. This is used if the
489 * preferred destination title already exists.
490 *
491 * @param LinkTarget $linkTarget
492 * @param array $options Associative array of validated command-line options
493 * @return Title|bool
494 */
495 private function getAlternateTitle( LinkTarget $linkTarget, $options ) {
496 $prefix = $options['add-prefix'];
497 $suffix = $options['add-suffix'];
498 if ( $prefix == '' && $suffix == '' ) {
499 return false;
500 }
501 while ( true ) {
502 $dbk = $prefix . $linkTarget->getDBkey() . $suffix;
503 $title = Title::makeTitleSafe( $linkTarget->getNamespace(), $dbk );
504 if ( !$title ) {
505 return false;
506 }
507 if ( !$title->exists() ) {
508 return $title;
509 }
510 }
511 }
512
513 /**
514 * Move a page
515 *
516 * @param integer $id The page_id
517 * @param LinkTarget $newLinkTarget The new title link target
518 * @return bool
519 */
520 private function movePage( $id, LinkTarget $newLinkTarget ) {
521 $this->db->update( 'page',
522 [
523 "page_namespace" => $newLinkTarget->getNamespace(),
524 "page_title" => $newLinkTarget->getDBkey(),
525 ],
526 [
527 "page_id" => $id,
528 ],
529 __METHOD__ );
530
531 // Update *_from_namespace in links tables
532 $fromNamespaceTables = [
533 [ 'pagelinks', 'pl' ],
534 [ 'templatelinks', 'tl' ],
535 [ 'imagelinks', 'il' ] ];
536 foreach ( $fromNamespaceTables as $tableInfo ) {
537 list( $table, $fieldPrefix ) = $tableInfo;
538 $this->db->update( $table,
539 // SET
540 [ "{$fieldPrefix}_from_namespace" => $newLinkTarget->getNamespace() ],
541 // WHERE
542 [ "{$fieldPrefix}_from" => $id ],
543 __METHOD__ );
544 }
545
546 return true;
547 }
548
549 /**
550 * Determine if we can merge a page.
551 * We check if an inaccessible revision would become the latest and
552 * deny the merge if so -- it's theoretically possible to update the
553 * latest revision, but opens a can of worms -- search engine updates,
554 * recentchanges review, etc.
555 *
556 * @param integer $id The page_id
557 * @param LinkTarget $linkTarget The new link target
558 * @param string $logStatus This is set to the log status message on failure
559 * @return bool
560 */
561 private function canMerge( $id, LinkTarget $linkTarget, &$logStatus ) {
562 $latestDest = Revision::newFromTitle(
563 $linkTarget, 0, RevisionRecord::READ_LATEST
564 );
565 $latestSource = Revision::newFromPageId(
566 $id, 0, RevisionRecord::READ_LATEST
567 );
568 if ( $latestSource->getTimestamp() > $latestDest->getTimestamp() ) {
569 $logStatus = 'cannot merge since source is later';
570 return false;
571 } else {
572 return true;
573 }
574 }
575
576 /**
577 * Merge page histories
578 *
579 * @param stdClass $row Page row
580 * @param Title $newTitle The new title
581 * @return bool
582 */
583 private function mergePage( $row, Title $newTitle ) {
584 $id = $row->page_id;
585
586 // Construct the WikiPage object we will need later, while the
587 // page_id still exists. Note that this cannot use makeTitleSafe(),
588 // we are deliberately constructing an invalid title.
589 $sourceTitle = Title::makeTitle( $row->page_namespace, $row->page_title );
590 $sourceTitle->resetArticleID( $id );
591 $wikiPage = new WikiPage( $sourceTitle );
592 $wikiPage->loadPageData( 'fromdbmaster' );
593
594 $destId = $newTitle->getArticleID();
595 $this->beginTransaction( $this->db, __METHOD__ );
596 $this->db->update( 'revision',
597 // SET
598 [ 'rev_page' => $destId ],
599 // WHERE
600 [ 'rev_page' => $id ],
601 __METHOD__ );
602
603 $this->db->delete( 'page', [ 'page_id' => $id ], __METHOD__ );
604
605 $this->commitTransaction( $this->db, __METHOD__ );
606
607 /* Call LinksDeletionUpdate to delete outgoing links from the old title,
608 * and update category counts.
609 *
610 * Calling external code with a fake broken Title is a fairly dubious
611 * idea. It's necessary because it's quite a lot of code to duplicate,
612 * but that also makes it fragile since it would be easy for someone to
613 * accidentally introduce an assumption of title validity to the code we
614 * are calling.
615 */
616 DeferredUpdates::addUpdate( new LinksDeletionUpdate( $wikiPage ) );
617 DeferredUpdates::doUpdates();
618
619 return true;
620 }
621 }
622
623 $maintClass = NamespaceDupes::class;
624 require_once RUN_MAINTENANCE_IF_MAIN;