includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use Mediawiki\MediaWikiServices;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         $page = $request->getText( 'pages' );
 102                         $this->curonly = $request->getCheck( 'curonly' );
 103                         $rawOffset = $request->getVal( 'offset' );
 104
 105                         if ( $rawOffset ) {
 106                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 107                         } else {
 108                                 $offset = null;
 109                         }
 110
 111                         $maxHistory = $config->get( 'ExportMaxHistory' );
 112                         $limit = $request->getInt( 'limit' );
 113                         $dir = $request->getVal( 'dir' );
 114                         $history = [
 115                                 'dir' => 'asc',
 116                                 'offset' => false,
 117                                 'limit' => $maxHistory,
 118                         ];
 119                         $historyCheck = $request->getCheck( 'history' );
 120
 121                         if ( $this->curonly ) {
 122                                 $history = WikiExporter::CURRENT;
 123                         } elseif ( !$historyCheck ) {
 124                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 125                                         $history['limit'] = $limit;
 126                                 }
 127
 128                                 if ( !is_null( $offset ) ) {
 129                                         $history['offset'] = $offset;
 130                                 }
 131
 132                                 if ( strtolower( $dir ) == 'desc' ) {
 133                                         $history['dir'] = 'desc';
 134                                 }
 135                         }
 136
 137                         if ( $page != '' ) {
 138                                 $this->doExport = true;
 139                         }
 140                 } else {
 141                         // Default to current-only for GET requests.
 142                         $page = $request->getText( 'pages', $par );
 143                         $historyCheck = $request->getCheck( 'history' );
 144
 145                         if ( $historyCheck ) {
 146                                 $history = WikiExporter::FULL;
 147                         } else {
 148                                 $history = WikiExporter::CURRENT;
 149                         }
 150
 151                         if ( $page != '' ) {
 152                                 $this->doExport = true;
 153                         }
 154                 }
 155
 156                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 157                         // Override
 158                         $history = WikiExporter::CURRENT;
 159                 }
 160
 161                 $list_authors = $request->getCheck( 'listauthors' );
 162                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 163                         $list_authors = false;
 164                 }
 165
 166                 if ( $this->doExport ) {
 167                         $this->getOutput()->disable();
 168
 169                         // Cancel output buffering and gzipping if set
 170                         // This should provide safer streaming for pages with history
 171                         wfResetOutputBuffers();
 172                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 173                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 174
 175                         if ( $request->getCheck( 'wpDownload' ) ) {
 176                                 // Provide a sane filename suggestion
 177                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 178                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 179                         }
 180
 181                         $this->doExport( $page, $history, $list_authors, $exportall );
 182
 183                         return;
 184                 }
 185
 186                 $out = $this->getOutput();
 187                 $out->addWikiMsg( 'exporttext' );
 188
 189                 if ( $page == '' ) {
 190                         $categoryName = $request->getText( 'catname' );
 191                 } else {
 192                         $categoryName = '';
 193                 }
 194
 195                 $formDescriptor = [
 196                         'catname' => [
 197                                 'type' => 'textwithbutton',
 198                                 'name' => 'catname',
 199                                 'horizontal-label' => true,
 200                                 'label-message' => 'export-addcattext',
 201                                 'default' => $categoryName,
 202                                 'size' => 40,
 203                                 'buttontype' => 'submit',
 204                                 'buttonname' => 'addcat',
 205                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 206                                 'hide-if' => [ '===', 'exportall', '1' ],
 207                         ],
 208                 ];
 209                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 210                         $formDescriptor += [
 211                                 'nsindex' => [
 212                                         'type' => 'namespaceselectwithbutton',
 213                                         'default' => $nsindex,
 214                                         'label-message' => 'export-addnstext',
 215                                         'horizontal-label' => true,
 216                                         'name' => 'nsindex',
 217                                         'id' => 'namespace',
 218                                         'cssclass' => 'namespaceselector',
 219                                         'buttontype' => 'submit',
 220                                         'buttonname' => 'addns',
 221                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 222                                         'hide-if' => [ '===', 'exportall', '1' ],
 223                                 ],
 224                         ];
 225                 }
 226
 227                 if ( $config->get( 'ExportAllowAll' ) ) {
 228                         $formDescriptor += [
 229                                 'exportall' => [
 230                                         'type' => 'check',
 231                                         'label-message' => 'exportall',
 232                                         'name' => 'exportall',
 233                                         'id' => 'exportall',
 234                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 235                                 ],
 236                         ];
 237                 }
 238
 239                 $formDescriptor += [
 240                         'textarea' => [
 241                                 'class' => 'HTMLTextAreaField',
 242                                 'name' => 'pages',
 243                                 'label-message' => 'export-manual',
 244                                 'nodata' => true,
 245                                 'rows' => 10,
 246                                 'default' => $page,
 247                                 'hide-if' => [ '===', 'exportall', '1' ],
 248                         ],
 249                 ];
 250
 251                 if ( $config->get( 'ExportAllowHistory' ) ) {
 252                         $formDescriptor += [
 253                                 'curonly' => [
 254                                         'type' => 'check',
 255                                         'label-message' => 'exportcuronly',
 256                                         'name' => 'curonly',
 257                                         'id' => 'curonly',
 258                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 259                                 ],
 260                         ];
 261                 } else {
 262                         $out->addWikiMsg( 'exportnohistory' );
 263                 }
 264
 265                 $formDescriptor += [
 266                         'templates' => [
 267                                 'type' => 'check',
 268                                 'label-message' => 'export-templates',
 269                                 'name' => 'templates',
 270                                 'id' => 'wpExportTemplates',
 271                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 272                         ],
 273                 ];
 274
 275                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 276                         $formDescriptor += [
 277                                 'pagelink-depth' => [
 278                                         'type' => 'text',
 279                                         'name' => 'pagelink-depth',
 280                                         'id' => 'pagelink-depth',
 281                                         'label-message' => 'export-pagelinks',
 282                                         'default' => '0',
 283                                         'size' => 20,
 284                                 ],
 285                         ];
 286                 }
 287
 288                 $formDescriptor += [
 289                         'wpDownload' => [
 290                                 'type' => 'check',
 291                                 'name' =>'wpDownload',
 292                                 'id' => 'wpDownload',
 293                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 294                                 'label-message' => 'export-download',
 295                         ],
 296                 ];
 297
 298                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 299                         $formDescriptor += [
 300                                 'listauthors' => [
 301                                         'type' => 'check',
 302                                         'label-message' => 'exportlistauthors',
 303                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 304                                         'name' => 'listauthors',
 305                                         'id' => 'listauthors',
 306                                 ],
 307                         ];
 308                 }
 309
 310                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 311                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 312                 $htmlForm->prepareForm()->displayForm( false );
 313                 $this->addHelpLink( 'Help:Export' );
 314         }
 315
 316         /**
 317          * @return bool
 318          */
 319         private function userCanOverrideExportDepth() {
 320                 return $this->getUser()->isAllowed( 'override-export-depth' );
 321         }
 322
 323         /**
 324          * Do the actual page exporting
 325          *
 326          * @param string $page User input on what page(s) to export
 327          * @param int $history One of the WikiExporter history export constants
 328          * @param bool $list_authors Whether to add distinct author list (when
 329          *   not returning full history)
 330          * @param bool $exportall Whether to export everything
 331          */
 332         private function doExport( $page, $history, $list_authors, $exportall ) {
 333
 334                 // If we are grabbing everything, enable full history and ignore the rest
 335                 if ( $exportall ) {
 336                         $history = WikiExporter::FULL;
 337                 } else {
 338                         $pageSet = []; // Inverted index of all pages to look up
 339
 340                         // Split up and normalize input
 341                         foreach ( explode( "\n", $page ) as $pageName ) {
 342                                 $pageName = trim( $pageName );
 343                                 $title = Title::newFromText( $pageName );
 344                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 345                                         // Only record each page once!
 346                                         $pageSet[$title->getPrefixedText()] = true;
 347                                 }
 348                         }
 349
 350                         // Set of original pages to pass on to further manipulation...
 351                         $inputPages = array_keys( $pageSet );
 352
 353                         // Look up any linked pages if asked...
 354                         if ( $this->templates ) {
 355                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 356                         }
 357                         $linkDepth = $this->pageLinkDepth;
 358                         if ( $linkDepth ) {
 359                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 360                         }
 361
 362                         $pages = array_keys( $pageSet );
 363
 364                         // Normalize titles to the same format and remove dupes, see T19374
 365                         foreach ( $pages as $k => $v ) {
 366                                 $pages[$k] = str_replace( " ", "_", $v );
 367                         }
 368
 369                         $pages = array_unique( $pages );
 370                 }
 371
 372                 /* Ok, let's get to it... */
 373                 if ( $history == WikiExporter::CURRENT ) {
 374                         $lb = false;
 375                         $db = wfGetDB( DB_REPLICA );
 376                         $buffer = WikiExporter::BUFFER;
 377                 } else {
 378                         // Use an unbuffered query; histories may be very long!
 379                         $lb = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->newMainLB();
 380                         $db = $lb->getConnection( DB_REPLICA );
 381                         $buffer = WikiExporter::STREAM;
 382
 383                         // This might take a while... :D
 384                         MediaWiki\suppressWarnings();
 385                         set_time_limit( 0 );
 386                         MediaWiki\restoreWarnings();
 387                 }
 388
 389                 $exporter = new WikiExporter( $db, $history, $buffer );
 390                 $exporter->list_authors = $list_authors;
 391                 $exporter->openStream();
 392
 393                 if ( $exportall ) {
 394                         $exporter->allPages();
 395                 } else {
 396                         foreach ( $pages as $page ) {
 397                                 # T10824: Only export pages the user can read
 398                                 $title = Title::newFromText( $page );
 399                                 if ( is_null( $title ) ) {
 400                                         // @todo Perhaps output an <error> tag or something.
 401                                         continue;
 402                                 }
 403
 404                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 405                                         // @todo Perhaps output an <error> tag or something.
 406                                         continue;
 407                                 }
 408
 409                                 $exporter->pageByTitle( $title );
 410                         }
 411                 }
 412
 413                 $exporter->closeStream();
 414
 415                 if ( $lb ) {
 416                         $lb->closeAll();
 417                 }
 418         }
 419
 420         /**
 421          * @param Title $title
 422          * @return array
 423          */
 424         private function getPagesFromCategory( $title ) {
 425                 global $wgContLang;
 426
 427                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 428
 429                 $name = $title->getDBkey();
 430
 431                 $dbr = wfGetDB( DB_REPLICA );
 432                 $res = $dbr->select(
 433                         [ 'page', 'categorylinks' ],
 434                         [ 'page_namespace', 'page_title' ],
 435                         [ 'cl_from=page_id', 'cl_to' => $name ],
 436                         __METHOD__,
 437                         [ 'LIMIT' => $maxPages ]
 438                 );
 439
 440                 $pages = [];
 441
 442                 foreach ( $res as $row ) {
 443                         $n = $row->page_title;
 444                         if ( $row->page_namespace ) {
 445                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 446                                 $n = $ns . ':' . $n;
 447                         }
 448
 449                         $pages[] = $n;
 450                 }
 451
 452                 return $pages;
 453         }
 454
 455         /**
 456          * @param int $nsindex
 457          * @return array
 458          */
 459         private function getPagesFromNamespace( $nsindex ) {
 460                 global $wgContLang;
 461
 462                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 463
 464                 $dbr = wfGetDB( DB_REPLICA );
 465                 $res = $dbr->select(
 466                         'page',
 467                         [ 'page_namespace', 'page_title' ],
 468                         [ 'page_namespace' => $nsindex ],
 469                         __METHOD__,
 470                         [ 'LIMIT' => $maxPages ]
 471                 );
 472
 473                 $pages = [];
 474
 475                 foreach ( $res as $row ) {
 476                         $n = $row->page_title;
 477
 478                         if ( $row->page_namespace ) {
 479                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 480                                 $n = $ns . ':' . $n;
 481                         }
 482
 483                         $pages[] = $n;
 484                 }
 485
 486                 return $pages;
 487         }
 488
 489         /**
 490          * Expand a list of pages to include templates used in those pages.
 491          * @param array $inputPages List of titles to look up
 492          * @param array $pageSet Associative array indexed by titles for output
 493          * @return array Associative array index by titles
 494          */
 495         private function getTemplates( $inputPages, $pageSet ) {
 496                 return $this->getLinks( $inputPages, $pageSet,
 497                         'templatelinks',
 498                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 499                         [ 'page_id=tl_from' ]
 500                 );
 501         }
 502
 503         /**
 504          * Validate link depth setting, if available.
 505          * @param int $depth
 506          * @return int
 507          */
 508         private function validateLinkDepth( $depth ) {
 509                 if ( $depth < 0 ) {
 510                         return 0;
 511                 }
 512
 513                 if ( !$this->userCanOverrideExportDepth() ) {
 514                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 515                         if ( $depth > $maxLinkDepth ) {
 516                                 return $maxLinkDepth;
 517                         }
 518                 }
 519
 520                 /*
 521                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 522                  * crazy-big export from being done by someone setting the depth
 523                  * number too high. In other words, last resort safety net.
 524                  */
 525
 526                 return intval( min( $depth, 5 ) );
 527         }
 528
 529         /**
 530          * Expand a list of pages to include pages linked to from that page.
 531          * @param array $inputPages
 532          * @param array $pageSet
 533          * @param int $depth
 534          * @return array
 535          */
 536         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 537                 // @codingStandardsIgnoreStart Squiz.WhiteSpace.SemicolonSpacing.Incorrect
 538                 for ( ; $depth > 0; --$depth ) {
 539                         // @codingStandardsIgnoreEnd
 540                         $pageSet = $this->getLinks(
 541                                 $inputPages, $pageSet, 'pagelinks',
 542                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 543                                 [ 'page_id=pl_from' ]
 544                         );
 545                         $inputPages = array_keys( $pageSet );
 546                 }
 547
 548                 return $pageSet;
 549         }
 550
 551         /**
 552          * Expand a list of pages to include items used in those pages.
 553          * @param array $inputPages Array of page titles
 554          * @param array $pageSet
 555          * @param string $table
 556          * @param array $fields Array of field names
 557          * @param array $join
 558          * @return array
 559          */
 560         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 561                 $dbr = wfGetDB( DB_REPLICA );
 562
 563                 foreach ( $inputPages as $page ) {
 564                         $title = Title::newFromText( $page );
 565
 566                         if ( $title ) {
 567                                 $pageSet[$title->getPrefixedText()] = true;
 568                                 /// @todo FIXME: May or may not be more efficient to batch these
 569                                 ///        by namespace when given multiple input pages.
 570                                 $result = $dbr->select(
 571                                         [ 'page', $table ],
 572                                         $fields,
 573                                         array_merge(
 574                                                 $join,
 575                                                 [
 576                                                         'page_namespace' => $title->getNamespace(),
 577                                                         'page_title' => $title->getDBkey()
 578                                                 ]
 579                                         ),
 580                                         __METHOD__
 581                                 );
 582
 583                                 foreach ( $result as $row ) {
 584                                         $template = Title::makeTitle( $row->namespace, $row->title );
 585                                         $pageSet[$template->getPrefixedText()] = true;
 586                                 }
 587                         }
 588                 }
 589
 590                 return $pageSet;
 591         }
 592
 593         protected function getGroupName() {
 594                 return 'pagetools';
 595         }
 596 }