includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use MediaWiki\Logger\LoggerFactory;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         // Log to see if certain parameters are actually used.
 102                         // If not, we could deprecate them and do some cleanup, here and in WikiExporter.
 103                         LoggerFactory::getInstance( 'export' )->debug(
 104                                 'Special:Export POST, dir: [{dir}], offset: [{offset}], limit: [{limit}]', [
 105                                 'dir' => $request->getRawVal( 'dir' ),
 106                                 'offset' => $request->getRawVal( 'offset' ),
 107                                 'limit' => $request->getRawVal( 'limit' ),
 108                         ] );
 109
 110                         $page = $request->getText( 'pages' );
 111                         $this->curonly = $request->getCheck( 'curonly' );
 112                         $rawOffset = $request->getVal( 'offset' );
 113
 114                         if ( $rawOffset ) {
 115                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 116                         } else {
 117                                 $offset = null;
 118                         }
 119
 120                         $maxHistory = $config->get( 'ExportMaxHistory' );
 121                         $limit = $request->getInt( 'limit' );
 122                         $dir = $request->getVal( 'dir' );
 123                         $history = [
 124                                 'dir' => 'asc',
 125                                 'offset' => false,
 126                                 'limit' => $maxHistory,
 127                         ];
 128                         $historyCheck = $request->getCheck( 'history' );
 129
 130                         if ( $this->curonly ) {
 131                                 $history = WikiExporter::CURRENT;
 132                         } elseif ( !$historyCheck ) {
 133                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 134                                         $history['limit'] = $limit;
 135                                 }
 136
 137                                 if ( !is_null( $offset ) ) {
 138                                         $history['offset'] = $offset;
 139                                 }
 140
 141                                 if ( strtolower( $dir ) == 'desc' ) {
 142                                         $history['dir'] = 'desc';
 143                                 }
 144                         }
 145
 146                         if ( $page != '' ) {
 147                                 $this->doExport = true;
 148                         }
 149                 } else {
 150                         // Default to current-only for GET requests.
 151                         $page = $request->getText( 'pages', $par );
 152                         $historyCheck = $request->getCheck( 'history' );
 153
 154                         if ( $historyCheck ) {
 155                                 $history = WikiExporter::FULL;
 156                         } else {
 157                                 $history = WikiExporter::CURRENT;
 158                         }
 159
 160                         if ( $page != '' ) {
 161                                 $this->doExport = true;
 162                         }
 163                 }
 164
 165                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 166                         // Override
 167                         $history = WikiExporter::CURRENT;
 168                 }
 169
 170                 $list_authors = $request->getCheck( 'listauthors' );
 171                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 172                         $list_authors = false;
 173                 }
 174
 175                 if ( $this->doExport ) {
 176                         $this->getOutput()->disable();
 177
 178                         // Cancel output buffering and gzipping if set
 179                         // This should provide safer streaming for pages with history
 180                         wfResetOutputBuffers();
 181                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 182                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 183
 184                         if ( $request->getCheck( 'wpDownload' ) ) {
 185                                 // Provide a sane filename suggestion
 186                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 187                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 188                         }
 189
 190                         $this->doExport( $page, $history, $list_authors, $exportall );
 191
 192                         return;
 193                 }
 194
 195                 $out = $this->getOutput();
 196                 $out->addWikiMsg( 'exporttext' );
 197
 198                 if ( $page == '' ) {
 199                         $categoryName = $request->getText( 'catname' );
 200                 } else {
 201                         $categoryName = '';
 202                 }
 203
 204                 $formDescriptor = [
 205                         'catname' => [
 206                                 'type' => 'textwithbutton',
 207                                 'name' => 'catname',
 208                                 'horizontal-label' => true,
 209                                 'label-message' => 'export-addcattext',
 210                                 'default' => $categoryName,
 211                                 'size' => 40,
 212                                 'buttontype' => 'submit',
 213                                 'buttonname' => 'addcat',
 214                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 215                                 'hide-if' => [ '===', 'exportall', '1' ],
 216                         ],
 217                 ];
 218                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 219                         $formDescriptor += [
 220                                 'nsindex' => [
 221                                         'type' => 'namespaceselectwithbutton',
 222                                         'default' => $nsindex,
 223                                         'label-message' => 'export-addnstext',
 224                                         'horizontal-label' => true,
 225                                         'name' => 'nsindex',
 226                                         'id' => 'namespace',
 227                                         'cssclass' => 'namespaceselector',
 228                                         'buttontype' => 'submit',
 229                                         'buttonname' => 'addns',
 230                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 231                                         'hide-if' => [ '===', 'exportall', '1' ],
 232                                 ],
 233                         ];
 234                 }
 235
 236                 if ( $config->get( 'ExportAllowAll' ) ) {
 237                         $formDescriptor += [
 238                                 'exportall' => [
 239                                         'type' => 'check',
 240                                         'label-message' => 'exportall',
 241                                         'name' => 'exportall',
 242                                         'id' => 'exportall',
 243                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 244                                 ],
 245                         ];
 246                 }
 247
 248                 $formDescriptor += [
 249                         'textarea' => [
 250                                 'class' => HTMLTextAreaField::class,
 251                                 'name' => 'pages',
 252                                 'label-message' => 'export-manual',
 253                                 'nodata' => true,
 254                                 'rows' => 10,
 255                                 'default' => $page,
 256                                 'hide-if' => [ '===', 'exportall', '1' ],
 257                         ],
 258                 ];
 259
 260                 if ( $config->get( 'ExportAllowHistory' ) ) {
 261                         $formDescriptor += [
 262                                 'curonly' => [
 263                                         'type' => 'check',
 264                                         'label-message' => 'exportcuronly',
 265                                         'name' => 'curonly',
 266                                         'id' => 'curonly',
 267                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 268                                 ],
 269                         ];
 270                 } else {
 271                         $out->addWikiMsg( 'exportnohistory' );
 272                 }
 273
 274                 $formDescriptor += [
 275                         'templates' => [
 276                                 'type' => 'check',
 277                                 'label-message' => 'export-templates',
 278                                 'name' => 'templates',
 279                                 'id' => 'wpExportTemplates',
 280                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 281                         ],
 282                 ];
 283
 284                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 285                         $formDescriptor += [
 286                                 'pagelink-depth' => [
 287                                         'type' => 'text',
 288                                         'name' => 'pagelink-depth',
 289                                         'id' => 'pagelink-depth',
 290                                         'label-message' => 'export-pagelinks',
 291                                         'default' => '0',
 292                                         'size' => 20,
 293                                 ],
 294                         ];
 295                 }
 296
 297                 $formDescriptor += [
 298                         'wpDownload' => [
 299                                 'type' => 'check',
 300                                 'name' => 'wpDownload',
 301                                 'id' => 'wpDownload',
 302                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 303                                 'label-message' => 'export-download',
 304                         ],
 305                 ];
 306
 307                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 308                         $formDescriptor += [
 309                                 'listauthors' => [
 310                                         'type' => 'check',
 311                                         'label-message' => 'exportlistauthors',
 312                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 313                                         'name' => 'listauthors',
 314                                         'id' => 'listauthors',
 315                                 ],
 316                         ];
 317                 }
 318
 319                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 320                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 321                 $htmlForm->prepareForm()->displayForm( false );
 322                 $this->addHelpLink( 'Help:Export' );
 323         }
 324
 325         /**
 326          * @return bool
 327          */
 328         private function userCanOverrideExportDepth() {
 329                 return $this->getUser()->isAllowed( 'override-export-depth' );
 330         }
 331
 332         /**
 333          * Do the actual page exporting
 334          *
 335          * @param string $page User input on what page(s) to export
 336          * @param int $history One of the WikiExporter history export constants
 337          * @param bool $list_authors Whether to add distinct author list (when
 338          *   not returning full history)
 339          * @param bool $exportall Whether to export everything
 340          */
 341         private function doExport( $page, $history, $list_authors, $exportall ) {
 342                 // If we are grabbing everything, enable full history and ignore the rest
 343                 if ( $exportall ) {
 344                         $history = WikiExporter::FULL;
 345                 } else {
 346                         $pageSet = []; // Inverted index of all pages to look up
 347
 348                         // Split up and normalize input
 349                         foreach ( explode( "\n", $page ) as $pageName ) {
 350                                 $pageName = trim( $pageName );
 351                                 $title = Title::newFromText( $pageName );
 352                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 353                                         // Only record each page once!
 354                                         $pageSet[$title->getPrefixedText()] = true;
 355                                 }
 356                         }
 357
 358                         // Set of original pages to pass on to further manipulation...
 359                         $inputPages = array_keys( $pageSet );
 360
 361                         // Look up any linked pages if asked...
 362                         if ( $this->templates ) {
 363                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 364                         }
 365                         $linkDepth = $this->pageLinkDepth;
 366                         if ( $linkDepth ) {
 367                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 368                         }
 369
 370                         $pages = array_keys( $pageSet );
 371
 372                         // Normalize titles to the same format and remove dupes, see T19374
 373                         foreach ( $pages as $k => $v ) {
 374                                 $pages[$k] = str_replace( " ", "_", $v );
 375                         }
 376
 377                         $pages = array_unique( $pages );
 378                 }
 379
 380                 /* Ok, let's get to it... */
 381                 $db = wfGetDB( DB_REPLICA );
 382
 383                 $exporter = new WikiExporter( $db, $history );
 384                 $exporter->list_authors = $list_authors;
 385                 $exporter->openStream();
 386
 387                 if ( $exportall ) {
 388                         $exporter->allPages();
 389                 } else {
 390                         foreach ( $pages as $page ) {
 391                                 # T10824: Only export pages the user can read
 392                                 $title = Title::newFromText( $page );
 393                                 if ( is_null( $title ) ) {
 394                                         // @todo Perhaps output an <error> tag or something.
 395                                         continue;
 396                                 }
 397
 398                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 399                                         // @todo Perhaps output an <error> tag or something.
 400                                         continue;
 401                                 }
 402
 403                                 $exporter->pageByTitle( $title );
 404                         }
 405                 }
 406
 407                 $exporter->closeStream();
 408         }
 409
 410         /**
 411          * @param Title $title
 412          * @return string[]
 413          */
 414         private function getPagesFromCategory( $title ) {
 415                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 416
 417                 $name = $title->getDBkey();
 418
 419                 $dbr = wfGetDB( DB_REPLICA );
 420                 $res = $dbr->select(
 421                         [ 'page', 'categorylinks' ],
 422                         [ 'page_namespace', 'page_title' ],
 423                         [ 'cl_from=page_id', 'cl_to' => $name ],
 424                         __METHOD__,
 425                         [ 'LIMIT' => $maxPages ]
 426                 );
 427
 428                 $pages = [];
 429
 430                 foreach ( $res as $row ) {
 431                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 432                 }
 433
 434                 return $pages;
 435         }
 436
 437         /**
 438          * @param int $nsindex
 439          * @return string[]
 440          */
 441         private function getPagesFromNamespace( $nsindex ) {
 442                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 443
 444                 $dbr = wfGetDB( DB_REPLICA );
 445                 $res = $dbr->select(
 446                         'page',
 447                         [ 'page_namespace', 'page_title' ],
 448                         [ 'page_namespace' => $nsindex ],
 449                         __METHOD__,
 450                         [ 'LIMIT' => $maxPages ]
 451                 );
 452
 453                 $pages = [];
 454
 455                 foreach ( $res as $row ) {
 456                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 457                 }
 458
 459                 return $pages;
 460         }
 461
 462         /**
 463          * Expand a list of pages to include templates used in those pages.
 464          * @param array $inputPages List of titles to look up
 465          * @param array $pageSet Associative array indexed by titles for output
 466          * @return array Associative array index by titles
 467          */
 468         private function getTemplates( $inputPages, $pageSet ) {
 469                 return $this->getLinks( $inputPages, $pageSet,
 470                         'templatelinks',
 471                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 472                         [ 'page_id=tl_from' ]
 473                 );
 474         }
 475
 476         /**
 477          * Validate link depth setting, if available.
 478          * @param int $depth
 479          * @return int
 480          */
 481         private function validateLinkDepth( $depth ) {
 482                 if ( $depth < 0 ) {
 483                         return 0;
 484                 }
 485
 486                 if ( !$this->userCanOverrideExportDepth() ) {
 487                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 488                         if ( $depth > $maxLinkDepth ) {
 489                                 return $maxLinkDepth;
 490                         }
 491                 }
 492
 493                 /*
 494                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 495                  * crazy-big export from being done by someone setting the depth
 496                  * number too high. In other words, last resort safety net.
 497                  */
 498
 499                 return intval( min( $depth, 5 ) );
 500         }
 501
 502         /**
 503          * Expand a list of pages to include pages linked to from that page.
 504          * @param array $inputPages
 505          * @param array $pageSet
 506          * @param int $depth
 507          * @return array
 508          */
 509         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 510                 for ( ; $depth > 0; --$depth ) {
 511                         $pageSet = $this->getLinks(
 512                                 $inputPages, $pageSet, 'pagelinks',
 513                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 514                                 [ 'page_id=pl_from' ]
 515                         );
 516                         $inputPages = array_keys( $pageSet );
 517                 }
 518
 519                 return $pageSet;
 520         }
 521
 522         /**
 523          * Expand a list of pages to include items used in those pages.
 524          * @param array $inputPages Array of page titles
 525          * @param array $pageSet
 526          * @param string $table
 527          * @param array $fields Array of field names
 528          * @param array $join
 529          * @return array
 530          */
 531         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 532                 $dbr = wfGetDB( DB_REPLICA );
 533
 534                 foreach ( $inputPages as $page ) {
 535                         $title = Title::newFromText( $page );
 536
 537                         if ( $title ) {
 538                                 $pageSet[$title->getPrefixedText()] = true;
 539                                 /// @todo FIXME: May or may not be more efficient to batch these
 540                                 ///        by namespace when given multiple input pages.
 541                                 $result = $dbr->select(
 542                                         [ 'page', $table ],
 543                                         $fields,
 544                                         array_merge(
 545                                                 $join,
 546                                                 [
 547                                                         'page_namespace' => $title->getNamespace(),
 548                                                         'page_title' => $title->getDBkey()
 549                                                 ]
 550                                         ),
 551                                         __METHOD__
 552                                 );
 553
 554                                 foreach ( $result as $row ) {
 555                                         $template = Title::makeTitle( $row->namespace, $row->title );
 556                                         $pageSet[$template->getPrefixedText()] = true;
 557                                 }
 558                         }
 559                 }
 560
 561                 return $pageSet;
 562         }
 563
 564         protected function getGroupName() {
 565                 return 'pagetools';
 566         }
 567 }