includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use MediaWiki\MediaWikiServices;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         $page = $request->getText( 'pages' );
 102                         $this->curonly = $request->getCheck( 'curonly' );
 103                         $rawOffset = $request->getVal( 'offset' );
 104
 105                         if ( $rawOffset ) {
 106                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 107                         } else {
 108                                 $offset = null;
 109                         }
 110
 111                         $maxHistory = $config->get( 'ExportMaxHistory' );
 112                         $limit = $request->getInt( 'limit' );
 113                         $dir = $request->getVal( 'dir' );
 114                         $history = [
 115                                 'dir' => 'asc',
 116                                 'offset' => false,
 117                                 'limit' => $maxHistory,
 118                         ];
 119                         $historyCheck = $request->getCheck( 'history' );
 120
 121                         if ( $this->curonly ) {
 122                                 $history = WikiExporter::CURRENT;
 123                         } elseif ( !$historyCheck ) {
 124                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 125                                         $history['limit'] = $limit;
 126                                 }
 127
 128                                 if ( !is_null( $offset ) ) {
 129                                         $history['offset'] = $offset;
 130                                 }
 131
 132                                 if ( strtolower( $dir ) == 'desc' ) {
 133                                         $history['dir'] = 'desc';
 134                                 }
 135                         }
 136
 137                         if ( $page != '' ) {
 138                                 $this->doExport = true;
 139                         }
 140                 } else {
 141                         // Default to current-only for GET requests.
 142                         $page = $request->getText( 'pages', $par );
 143                         $historyCheck = $request->getCheck( 'history' );
 144
 145                         if ( $historyCheck ) {
 146                                 $history = WikiExporter::FULL;
 147                         } else {
 148                                 $history = WikiExporter::CURRENT;
 149                         }
 150
 151                         if ( $page != '' ) {
 152                                 $this->doExport = true;
 153                         }
 154                 }
 155
 156                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 157                         // Override
 158                         $history = WikiExporter::CURRENT;
 159                 }
 160
 161                 $list_authors = $request->getCheck( 'listauthors' );
 162                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 163                         $list_authors = false;
 164                 }
 165
 166                 if ( $this->doExport ) {
 167                         $this->getOutput()->disable();
 168
 169                         // Cancel output buffering and gzipping if set
 170                         // This should provide safer streaming for pages with history
 171                         wfResetOutputBuffers();
 172                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 173                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 174
 175                         if ( $request->getCheck( 'wpDownload' ) ) {
 176                                 // Provide a sane filename suggestion
 177                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 178                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 179                         }
 180
 181                         $this->doExport( $page, $history, $list_authors, $exportall );
 182
 183                         return;
 184                 }
 185
 186                 $out = $this->getOutput();
 187                 $out->addWikiMsg( 'exporttext' );
 188
 189                 if ( $page == '' ) {
 190                         $categoryName = $request->getText( 'catname' );
 191                 } else {
 192                         $categoryName = '';
 193                 }
 194
 195                 $formDescriptor = [
 196                         'catname' => [
 197                                 'type' => 'textwithbutton',
 198                                 'name' => 'catname',
 199                                 'horizontal-label' => true,
 200                                 'label-message' => 'export-addcattext',
 201                                 'default' => $categoryName,
 202                                 'size' => 40,
 203                                 'buttontype' => 'submit',
 204                                 'buttonname' => 'addcat',
 205                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 206                                 'hide-if' => [ '===', 'exportall', '1' ],
 207                         ],
 208                 ];
 209                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 210                         $formDescriptor += [
 211                                 'nsindex' => [
 212                                         'type' => 'namespaceselectwithbutton',
 213                                         'default' => $nsindex,
 214                                         'label-message' => 'export-addnstext',
 215                                         'horizontal-label' => true,
 216                                         'name' => 'nsindex',
 217                                         'id' => 'namespace',
 218                                         'cssclass' => 'namespaceselector',
 219                                         'buttontype' => 'submit',
 220                                         'buttonname' => 'addns',
 221                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 222                                         'hide-if' => [ '===', 'exportall', '1' ],
 223                                 ],
 224                         ];
 225                 }
 226
 227                 if ( $config->get( 'ExportAllowAll' ) ) {
 228                         $formDescriptor += [
 229                                 'exportall' => [
 230                                         'type' => 'check',
 231                                         'label-message' => 'exportall',
 232                                         'name' => 'exportall',
 233                                         'id' => 'exportall',
 234                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 235                                 ],
 236                         ];
 237                 }
 238
 239                 $formDescriptor += [
 240                         'textarea' => [
 241                                 'class' => HTMLTextAreaField::class,
 242                                 'name' => 'pages',
 243                                 'label-message' => 'export-manual',
 244                                 'nodata' => true,
 245                                 'rows' => 10,
 246                                 'default' => $page,
 247                                 'hide-if' => [ '===', 'exportall', '1' ],
 248                         ],
 249                 ];
 250
 251                 if ( $config->get( 'ExportAllowHistory' ) ) {
 252                         $formDescriptor += [
 253                                 'curonly' => [
 254                                         'type' => 'check',
 255                                         'label-message' => 'exportcuronly',
 256                                         'name' => 'curonly',
 257                                         'id' => 'curonly',
 258                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 259                                 ],
 260                         ];
 261                 } else {
 262                         $out->addWikiMsg( 'exportnohistory' );
 263                 }
 264
 265                 $formDescriptor += [
 266                         'templates' => [
 267                                 'type' => 'check',
 268                                 'label-message' => 'export-templates',
 269                                 'name' => 'templates',
 270                                 'id' => 'wpExportTemplates',
 271                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 272                         ],
 273                 ];
 274
 275                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 276                         $formDescriptor += [
 277                                 'pagelink-depth' => [
 278                                         'type' => 'text',
 279                                         'name' => 'pagelink-depth',
 280                                         'id' => 'pagelink-depth',
 281                                         'label-message' => 'export-pagelinks',
 282                                         'default' => '0',
 283                                         'size' => 20,
 284                                 ],
 285                         ];
 286                 }
 287
 288                 $formDescriptor += [
 289                         'wpDownload' => [
 290                                 'type' => 'check',
 291                                 'name' => 'wpDownload',
 292                                 'id' => 'wpDownload',
 293                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 294                                 'label-message' => 'export-download',
 295                         ],
 296                 ];
 297
 298                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 299                         $formDescriptor += [
 300                                 'listauthors' => [
 301                                         'type' => 'check',
 302                                         'label-message' => 'exportlistauthors',
 303                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 304                                         'name' => 'listauthors',
 305                                         'id' => 'listauthors',
 306                                 ],
 307                         ];
 308                 }
 309
 310                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 311                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 312                 $htmlForm->prepareForm()->displayForm( false );
 313                 $this->addHelpLink( 'Help:Export' );
 314         }
 315
 316         /**
 317          * @return bool
 318          */
 319         private function userCanOverrideExportDepth() {
 320                 return $this->getUser()->isAllowed( 'override-export-depth' );
 321         }
 322
 323         /**
 324          * Do the actual page exporting
 325          *
 326          * @param string $page User input on what page(s) to export
 327          * @param int $history One of the WikiExporter history export constants
 328          * @param bool $list_authors Whether to add distinct author list (when
 329          *   not returning full history)
 330          * @param bool $exportall Whether to export everything
 331          */
 332         private function doExport( $page, $history, $list_authors, $exportall ) {
 333                 // If we are grabbing everything, enable full history and ignore the rest
 334                 if ( $exportall ) {
 335                         $history = WikiExporter::FULL;
 336                 } else {
 337                         $pageSet = []; // Inverted index of all pages to look up
 338
 339                         // Split up and normalize input
 340                         foreach ( explode( "\n", $page ) as $pageName ) {
 341                                 $pageName = trim( $pageName );
 342                                 $title = Title::newFromText( $pageName );
 343                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 344                                         // Only record each page once!
 345                                         $pageSet[$title->getPrefixedText()] = true;
 346                                 }
 347                         }
 348
 349                         // Set of original pages to pass on to further manipulation...
 350                         $inputPages = array_keys( $pageSet );
 351
 352                         // Look up any linked pages if asked...
 353                         if ( $this->templates ) {
 354                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 355                         }
 356                         $linkDepth = $this->pageLinkDepth;
 357                         if ( $linkDepth ) {
 358                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 359                         }
 360
 361                         $pages = array_keys( $pageSet );
 362
 363                         // Normalize titles to the same format and remove dupes, see T19374
 364                         foreach ( $pages as $k => $v ) {
 365                                 $pages[$k] = str_replace( " ", "_", $v );
 366                         }
 367
 368                         $pages = array_unique( $pages );
 369                 }
 370
 371                 /* Ok, let's get to it... */
 372                 if ( $history == WikiExporter::CURRENT ) {
 373                         $lb = false;
 374                         $db = wfGetDB( DB_REPLICA );
 375                         $buffer = WikiExporter::BUFFER;
 376                 } else {
 377                         // Use an unbuffered query; histories may be very long!
 378                         $lb = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->newMainLB();
 379                         $db = $lb->getConnection( DB_REPLICA );
 380                         $buffer = WikiExporter::STREAM;
 381
 382                         // This might take a while... :D
 383                         Wikimedia\suppressWarnings();
 384                         set_time_limit( 0 );
 385                         Wikimedia\restoreWarnings();
 386                 }
 387
 388                 $exporter = new WikiExporter( $db, $history, $buffer );
 389                 $exporter->list_authors = $list_authors;
 390                 $exporter->openStream();
 391
 392                 if ( $exportall ) {
 393                         $exporter->allPages();
 394                 } else {
 395                         foreach ( $pages as $page ) {
 396                                 # T10824: Only export pages the user can read
 397                                 $title = Title::newFromText( $page );
 398                                 if ( is_null( $title ) ) {
 399                                         // @todo Perhaps output an <error> tag or something.
 400                                         continue;
 401                                 }
 402
 403                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 404                                         // @todo Perhaps output an <error> tag or something.
 405                                         continue;
 406                                 }
 407
 408                                 $exporter->pageByTitle( $title );
 409                         }
 410                 }
 411
 412                 $exporter->closeStream();
 413
 414                 if ( $lb ) {
 415                         $lb->closeAll();
 416                 }
 417         }
 418
 419         /**
 420          * @param Title $title
 421          * @return string[]
 422          */
 423         private function getPagesFromCategory( $title ) {
 424                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 425
 426                 $name = $title->getDBkey();
 427
 428                 $dbr = wfGetDB( DB_REPLICA );
 429                 $res = $dbr->select(
 430                         [ 'page', 'categorylinks' ],
 431                         [ 'page_namespace', 'page_title' ],
 432                         [ 'cl_from=page_id', 'cl_to' => $name ],
 433                         __METHOD__,
 434                         [ 'LIMIT' => $maxPages ]
 435                 );
 436
 437                 $pages = [];
 438
 439                 foreach ( $res as $row ) {
 440                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 441                 }
 442
 443                 return $pages;
 444         }
 445
 446         /**
 447          * @param int $nsindex
 448          * @return string[]
 449          */
 450         private function getPagesFromNamespace( $nsindex ) {
 451                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 452
 453                 $dbr = wfGetDB( DB_REPLICA );
 454                 $res = $dbr->select(
 455                         'page',
 456                         [ 'page_namespace', 'page_title' ],
 457                         [ 'page_namespace' => $nsindex ],
 458                         __METHOD__,
 459                         [ 'LIMIT' => $maxPages ]
 460                 );
 461
 462                 $pages = [];
 463
 464                 foreach ( $res as $row ) {
 465                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 466                 }
 467
 468                 return $pages;
 469         }
 470
 471         /**
 472          * Expand a list of pages to include templates used in those pages.
 473          * @param array $inputPages List of titles to look up
 474          * @param array $pageSet Associative array indexed by titles for output
 475          * @return array Associative array index by titles
 476          */
 477         private function getTemplates( $inputPages, $pageSet ) {
 478                 return $this->getLinks( $inputPages, $pageSet,
 479                         'templatelinks',
 480                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 481                         [ 'page_id=tl_from' ]
 482                 );
 483         }
 484
 485         /**
 486          * Validate link depth setting, if available.
 487          * @param int $depth
 488          * @return int
 489          */
 490         private function validateLinkDepth( $depth ) {
 491                 if ( $depth < 0 ) {
 492                         return 0;
 493                 }
 494
 495                 if ( !$this->userCanOverrideExportDepth() ) {
 496                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 497                         if ( $depth > $maxLinkDepth ) {
 498                                 return $maxLinkDepth;
 499                         }
 500                 }
 501
 502                 /*
 503                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 504                  * crazy-big export from being done by someone setting the depth
 505                  * number too high. In other words, last resort safety net.
 506                  */
 507
 508                 return intval( min( $depth, 5 ) );
 509         }
 510
 511         /**
 512          * Expand a list of pages to include pages linked to from that page.
 513          * @param array $inputPages
 514          * @param array $pageSet
 515          * @param int $depth
 516          * @return array
 517          */
 518         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 519                 for ( ; $depth > 0; --$depth ) {
 520                         $pageSet = $this->getLinks(
 521                                 $inputPages, $pageSet, 'pagelinks',
 522                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 523                                 [ 'page_id=pl_from' ]
 524                         );
 525                         $inputPages = array_keys( $pageSet );
 526                 }
 527
 528                 return $pageSet;
 529         }
 530
 531         /**
 532          * Expand a list of pages to include items used in those pages.
 533          * @param array $inputPages Array of page titles
 534          * @param array $pageSet
 535          * @param string $table
 536          * @param array $fields Array of field names
 537          * @param array $join
 538          * @return array
 539          */
 540         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 541                 $dbr = wfGetDB( DB_REPLICA );
 542
 543                 foreach ( $inputPages as $page ) {
 544                         $title = Title::newFromText( $page );
 545
 546                         if ( $title ) {
 547                                 $pageSet[$title->getPrefixedText()] = true;
 548                                 /// @todo FIXME: May or may not be more efficient to batch these
 549                                 ///        by namespace when given multiple input pages.
 550                                 $result = $dbr->select(
 551                                         [ 'page', $table ],
 552                                         $fields,
 553                                         array_merge(
 554                                                 $join,
 555                                                 [
 556                                                         'page_namespace' => $title->getNamespace(),
 557                                                         'page_title' => $title->getDBkey()
 558                                                 ]
 559                                         ),
 560                                         __METHOD__
 561                                 );
 562
 563                                 foreach ( $result as $row ) {
 564                                         $template = Title::makeTitle( $row->namespace, $row->title );
 565                                         $pageSet[$template->getPrefixedText()] = true;
 566                                 }
 567                         }
 568                 }
 569
 570                 return $pageSet;
 571         }
 572
 573         protected function getGroupName() {
 574                 return 'pagetools';
 575         }
 576 }