includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use MediaWiki\Logger\LoggerFactory;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         // Log to see if certain parameters are actually used.
 102                         // If not, we could deprecate them and do some cleanup, here and in WikiExporter.
 103                         LoggerFactory::getInstance( 'export' )->debug(
 104                                 'Special:Export POST, dir: [{dir}], offset: [{offset}], limit: [{limit}]', [
 105                                 'dir' => $request->getRawVal( 'dir' ),
 106                                 'offset' => $request->getRawVal( 'offset' ),
 107                                 'limit' => $request->getRawVal( 'limit' ),
 108                         ] );
 109
 110                         $page = $request->getText( 'pages' );
 111                         $this->curonly = $request->getCheck( 'curonly' );
 112                         $rawOffset = $request->getVal( 'offset' );
 113
 114                         if ( $rawOffset ) {
 115                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 116                         } else {
 117                                 $offset = null;
 118                         }
 119
 120                         $maxHistory = $config->get( 'ExportMaxHistory' );
 121                         $limit = $request->getInt( 'limit' );
 122                         $dir = $request->getVal( 'dir' );
 123                         $history = [
 124                                 'dir' => 'asc',
 125                                 'offset' => false,
 126                                 'limit' => $maxHistory,
 127                         ];
 128                         $historyCheck = $request->getCheck( 'history' );
 129
 130                         if ( $this->curonly ) {
 131                                 $history = WikiExporter::CURRENT;
 132                         } elseif ( !$historyCheck ) {
 133                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 134                                         $history['limit'] = $limit;
 135                                 }
 136
 137                                 if ( !is_null( $offset ) ) {
 138                                         $history['offset'] = $offset;
 139                                 }
 140
 141                                 if ( strtolower( $dir ) == 'desc' ) {
 142                                         $history['dir'] = 'desc';
 143                                 }
 144                         }
 145
 146                         if ( $page != '' ) {
 147                                 $this->doExport = true;
 148                         }
 149                 } else {
 150                         // Default to current-only for GET requests.
 151                         $page = $request->getText( 'pages', $par );
 152                         $historyCheck = $request->getCheck( 'history' );
 153
 154                         if ( $historyCheck ) {
 155                                 $history = WikiExporter::FULL;
 156                         } else {
 157                                 $history = WikiExporter::CURRENT;
 158                         }
 159
 160                         if ( $page != '' ) {
 161                                 $this->doExport = true;
 162                         }
 163                 }
 164
 165                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 166                         // Override
 167                         $history = WikiExporter::CURRENT;
 168                 }
 169
 170                 $list_authors = $request->getCheck( 'listauthors' );
 171                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 172                         $list_authors = false;
 173                 }
 174
 175                 if ( $this->doExport ) {
 176                         $this->getOutput()->disable();
 177
 178                         // Cancel output buffering and gzipping if set
 179                         // This should provide safer streaming for pages with history
 180                         wfResetOutputBuffers();
 181                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 182                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 183
 184                         if ( $request->getCheck( 'wpDownload' ) ) {
 185                                 // Provide a sane filename suggestion
 186                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 187                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 188                         }
 189
 190                         $this->doExport( $page, $history, $list_authors, $exportall );
 191
 192                         return;
 193                 }
 194
 195                 $out = $this->getOutput();
 196                 $out->addWikiMsg( 'exporttext' );
 197
 198                 if ( $page == '' ) {
 199                         $categoryName = $request->getText( 'catname' );
 200                 } else {
 201                         $categoryName = '';
 202                 }
 203
 204                 $formDescriptor = [
 205                         'catname' => [
 206                                 'type' => 'textwithbutton',
 207                                 'name' => 'catname',
 208                                 'horizontal-label' => true,
 209                                 'label-message' => 'export-addcattext',
 210                                 'default' => $categoryName,
 211                                 'size' => 40,
 212                                 'buttontype' => 'submit',
 213                                 'buttonname' => 'addcat',
 214                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 215                                 'hide-if' => [ '===', 'exportall', '1' ],
 216                         ],
 217                 ];
 218                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 219                         $formDescriptor += [
 220                                 'nsindex' => [
 221                                         'type' => 'namespaceselectwithbutton',
 222                                         'default' => $nsindex,
 223                                         'label-message' => 'export-addnstext',
 224                                         'horizontal-label' => true,
 225                                         'name' => 'nsindex',
 226                                         'id' => 'namespace',
 227                                         'cssclass' => 'namespaceselector',
 228                                         'buttontype' => 'submit',
 229                                         'buttonname' => 'addns',
 230                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 231                                         'hide-if' => [ '===', 'exportall', '1' ],
 232                                 ],
 233                         ];
 234                 }
 235
 236                 if ( $config->get( 'ExportAllowAll' ) ) {
 237                         $formDescriptor += [
 238                                 'exportall' => [
 239                                         'type' => 'check',
 240                                         'label-message' => 'exportall',
 241                                         'name' => 'exportall',
 242                                         'id' => 'exportall',
 243                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 244                                 ],
 245                         ];
 246                 }
 247
 248                 $formDescriptor += [
 249                         'textarea' => [
 250                                 'class' => HTMLTextAreaField::class,
 251                                 'name' => 'pages',
 252                                 'label-message' => 'export-manual',
 253                                 'nodata' => true,
 254                                 'rows' => 10,
 255                                 'default' => $page,
 256                                 'hide-if' => [ '===', 'exportall', '1' ],
 257                         ],
 258                 ];
 259
 260                 if ( $config->get( 'ExportAllowHistory' ) ) {
 261                         $formDescriptor += [
 262                                 'curonly' => [
 263                                         'type' => 'check',
 264                                         'label-message' => 'exportcuronly',
 265                                         'name' => 'curonly',
 266                                         'id' => 'curonly',
 267                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 268                                 ],
 269                         ];
 270                 } else {
 271                         $out->addWikiMsg( 'exportnohistory' );
 272                 }
 273
 274                 $formDescriptor += [
 275                         'templates' => [
 276                                 'type' => 'check',
 277                                 'label-message' => 'export-templates',
 278                                 'name' => 'templates',
 279                                 'id' => 'wpExportTemplates',
 280                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 281                         ],
 282                 ];
 283
 284                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 285                         $formDescriptor += [
 286                                 'pagelink-depth' => [
 287                                         'type' => 'text',
 288                                         'name' => 'pagelink-depth',
 289                                         'id' => 'pagelink-depth',
 290                                         'label-message' => 'export-pagelinks',
 291                                         'default' => '0',
 292                                         'size' => 20,
 293                                 ],
 294                         ];
 295                 }
 296
 297                 $formDescriptor += [
 298                         'wpDownload' => [
 299                                 'type' => 'check',
 300                                 'name' => 'wpDownload',
 301                                 'id' => 'wpDownload',
 302                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 303                                 'label-message' => 'export-download',
 304                         ],
 305                 ];
 306
 307                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 308                         $formDescriptor += [
 309                                 'listauthors' => [
 310                                         'type' => 'check',
 311                                         'label-message' => 'exportlistauthors',
 312                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 313                                         'name' => 'listauthors',
 314                                         'id' => 'listauthors',
 315                                 ],
 316                         ];
 317                 }
 318
 319                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 320                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 321                 $htmlForm->prepareForm()->displayForm( false );
 322                 $this->addHelpLink( 'Help:Export' );
 323         }
 324
 325         /**
 326          * @return bool
 327          */
 328         private function userCanOverrideExportDepth() {
 329                 return $this->getUser()->isAllowed( 'override-export-depth' );
 330         }
 331
 332         /**
 333          * Do the actual page exporting
 334          *
 335          * @param string $page User input on what page(s) to export
 336          * @param int $history One of the WikiExporter history export constants
 337          * @param bool $list_authors Whether to add distinct author list (when
 338          *   not returning full history)
 339          * @param bool $exportall Whether to export everything
 340          */
 341         private function doExport( $page, $history, $list_authors, $exportall ) {
 342                 // If we are grabbing everything, enable full history and ignore the rest
 343                 if ( $exportall ) {
 344                         $history = WikiExporter::FULL;
 345                 } else {
 346                         $pageSet = []; // Inverted index of all pages to look up
 347
 348                         // Split up and normalize input
 349                         foreach ( explode( "\n", $page ) as $pageName ) {
 350                                 $pageName = trim( $pageName );
 351                                 $title = Title::newFromText( $pageName );
 352                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 353                                         // Only record each page once!
 354                                         $pageSet[$title->getPrefixedText()] = true;
 355                                 }
 356                         }
 357
 358                         // Set of original pages to pass on to further manipulation...
 359                         $inputPages = array_keys( $pageSet );
 360
 361                         // Look up any linked pages if asked...
 362                         if ( $this->templates ) {
 363                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 364                         }
 365                         $linkDepth = $this->pageLinkDepth;
 366                         if ( $linkDepth ) {
 367                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 368                         }
 369
 370                         $pages = array_keys( $pageSet );
 371
 372                         // Normalize titles to the same format and remove dupes, see T19374
 373                         foreach ( $pages as $k => $v ) {
 374                                 $pages[$k] = str_replace( " ", "_", $v );
 375                         }
 376
 377                         $pages = array_unique( $pages );
 378                 }
 379
 380                 /* Ok, let's get to it... */
 381                 $lb = false;
 382                 $db = wfGetDB( DB_REPLICA );
 383
 384                 $exporter = new WikiExporter( $db, $history );
 385                 $exporter->list_authors = $list_authors;
 386                 $exporter->openStream();
 387
 388                 if ( $exportall ) {
 389                         $exporter->allPages();
 390                 } else {
 391                         foreach ( $pages as $page ) {
 392                                 # T10824: Only export pages the user can read
 393                                 $title = Title::newFromText( $page );
 394                                 if ( is_null( $title ) ) {
 395                                         // @todo Perhaps output an <error> tag or something.
 396                                         continue;
 397                                 }
 398
 399                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 400                                         // @todo Perhaps output an <error> tag or something.
 401                                         continue;
 402                                 }
 403
 404                                 $exporter->pageByTitle( $title );
 405                         }
 406                 }
 407
 408                 $exporter->closeStream();
 409
 410                 if ( $lb ) {
 411                         $lb->closeAll();
 412                 }
 413         }
 414
 415         /**
 416          * @param Title $title
 417          * @return string[]
 418          */
 419         private function getPagesFromCategory( $title ) {
 420                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 421
 422                 $name = $title->getDBkey();
 423
 424                 $dbr = wfGetDB( DB_REPLICA );
 425                 $res = $dbr->select(
 426                         [ 'page', 'categorylinks' ],
 427                         [ 'page_namespace', 'page_title' ],
 428                         [ 'cl_from=page_id', 'cl_to' => $name ],
 429                         __METHOD__,
 430                         [ 'LIMIT' => $maxPages ]
 431                 );
 432
 433                 $pages = [];
 434
 435                 foreach ( $res as $row ) {
 436                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 437                 }
 438
 439                 return $pages;
 440         }
 441
 442         /**
 443          * @param int $nsindex
 444          * @return string[]
 445          */
 446         private function getPagesFromNamespace( $nsindex ) {
 447                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 448
 449                 $dbr = wfGetDB( DB_REPLICA );
 450                 $res = $dbr->select(
 451                         'page',
 452                         [ 'page_namespace', 'page_title' ],
 453                         [ 'page_namespace' => $nsindex ],
 454                         __METHOD__,
 455                         [ 'LIMIT' => $maxPages ]
 456                 );
 457
 458                 $pages = [];
 459
 460                 foreach ( $res as $row ) {
 461                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 462                 }
 463
 464                 return $pages;
 465         }
 466
 467         /**
 468          * Expand a list of pages to include templates used in those pages.
 469          * @param array $inputPages List of titles to look up
 470          * @param array $pageSet Associative array indexed by titles for output
 471          * @return array Associative array index by titles
 472          */
 473         private function getTemplates( $inputPages, $pageSet ) {
 474                 return $this->getLinks( $inputPages, $pageSet,
 475                         'templatelinks',
 476                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 477                         [ 'page_id=tl_from' ]
 478                 );
 479         }
 480
 481         /**
 482          * Validate link depth setting, if available.
 483          * @param int $depth
 484          * @return int
 485          */
 486         private function validateLinkDepth( $depth ) {
 487                 if ( $depth < 0 ) {
 488                         return 0;
 489                 }
 490
 491                 if ( !$this->userCanOverrideExportDepth() ) {
 492                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 493                         if ( $depth > $maxLinkDepth ) {
 494                                 return $maxLinkDepth;
 495                         }
 496                 }
 497
 498                 /*
 499                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 500                  * crazy-big export from being done by someone setting the depth
 501                  * number too high. In other words, last resort safety net.
 502                  */
 503
 504                 return intval( min( $depth, 5 ) );
 505         }
 506
 507         /**
 508          * Expand a list of pages to include pages linked to from that page.
 509          * @param array $inputPages
 510          * @param array $pageSet
 511          * @param int $depth
 512          * @return array
 513          */
 514         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 515                 for ( ; $depth > 0; --$depth ) {
 516                         $pageSet = $this->getLinks(
 517                                 $inputPages, $pageSet, 'pagelinks',
 518                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 519                                 [ 'page_id=pl_from' ]
 520                         );
 521                         $inputPages = array_keys( $pageSet );
 522                 }
 523
 524                 return $pageSet;
 525         }
 526
 527         /**
 528          * Expand a list of pages to include items used in those pages.
 529          * @param array $inputPages Array of page titles
 530          * @param array $pageSet
 531          * @param string $table
 532          * @param array $fields Array of field names
 533          * @param array $join
 534          * @return array
 535          */
 536         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 537                 $dbr = wfGetDB( DB_REPLICA );
 538
 539                 foreach ( $inputPages as $page ) {
 540                         $title = Title::newFromText( $page );
 541
 542                         if ( $title ) {
 543                                 $pageSet[$title->getPrefixedText()] = true;
 544                                 /// @todo FIXME: May or may not be more efficient to batch these
 545                                 ///        by namespace when given multiple input pages.
 546                                 $result = $dbr->select(
 547                                         [ 'page', $table ],
 548                                         $fields,
 549                                         array_merge(
 550                                                 $join,
 551                                                 [
 552                                                         'page_namespace' => $title->getNamespace(),
 553                                                         'page_title' => $title->getDBkey()
 554                                                 ]
 555                                         ),
 556                                         __METHOD__
 557                                 );
 558
 559                                 foreach ( $result as $row ) {
 560                                         $template = Title::makeTitle( $row->namespace, $row->title );
 561                                         $pageSet[$template->getPrefixedText()] = true;
 562                                 }
 563                         }
 564                 }
 565
 566                 return $pageSet;
 567         }
 568
 569         protected function getGroupName() {
 570                 return 'pagetools';
 571         }
 572 }