includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use MediaWiki\MediaWikiServices;
  27 use MediaWiki\Logger\LoggerFactory;
  28
  29 /**
  30  * A special page that allows users to export pages in a XML file
  31  *
  32  * @ingroup SpecialPage
  33  */
  34 class SpecialExport extends SpecialPage {
  35         private $curonly, $doExport, $pageLinkDepth, $templates;
  36
  37         public function __construct() {
  38                 parent::__construct( 'Export' );
  39         }
  40
  41         public function execute( $par ) {
  42                 $this->setHeaders();
  43                 $this->outputHeader();
  44                 $config = $this->getConfig();
  45
  46                 // Set some variables
  47                 $this->curonly = true;
  48                 $this->doExport = false;
  49                 $request = $this->getRequest();
  50                 $this->templates = $request->getCheck( 'templates' );
  51                 $this->pageLinkDepth = $this->validateLinkDepth(
  52                         $request->getIntOrNull( 'pagelink-depth' )
  53                 );
  54                 $nsindex = '';
  55                 $exportall = false;
  56
  57                 if ( $request->getCheck( 'addcat' ) ) {
  58                         $page = $request->getText( 'pages' );
  59                         $catname = $request->getText( 'catname' );
  60
  61                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  62                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  63                                 if ( $t ) {
  64                                         /**
  65                                          * @todo FIXME: This can lead to hitting memory limit for very large
  66                                          * categories. Ideally we would do the lookup synchronously
  67                                          * during the export in a single query.
  68                                          */
  69                                         $catpages = $this->getPagesFromCategory( $t );
  70                                         if ( $catpages ) {
  71                                                 if ( $page !== '' ) {
  72                                                         $page .= "\n";
  73                                                 }
  74                                                 $page .= implode( "\n", $catpages );
  75                                         }
  76                                 }
  77                         }
  78                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  79                         $page = $request->getText( 'pages' );
  80                         $nsindex = $request->getText( 'nsindex', '' );
  81
  82                         if ( strval( $nsindex ) !== '' ) {
  83                                 /**
  84                                  * Same implementation as above, so same @todo
  85                                  */
  86                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  87                                 if ( $nspages ) {
  88                                         $page .= "\n" . implode( "\n", $nspages );
  89                                 }
  90                         }
  91                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  92                         $this->doExport = true;
  93                         $exportall = true;
  94
  95                         /* Although $page and $history are not used later on, we
  96                         nevertheless set them to avoid that PHP notices about using
  97                         undefined variables foul up our XML output (see call to
  98                         doExport(...) further down) */
  99                         $page = '';
 100                         $history = '';
 101                 } elseif ( $request->wasPosted() && $par == '' ) {
 102                         // Log to see if certain parameters are actually used.
 103                         // If not, we could deprecate them and do some cleanup, here and in WikiExporter.
 104                         LoggerFactory::getInstance( 'export' )->debug(
 105                                 'Special:Export POST, dir: [{dir}], offset: [{offset}], limit: [{limit}]', [
 106                                 'dir' => $request->getRawVal( 'dir' ),
 107                                 'offset' => $request->getRawVal( 'offset' ),
 108                                 'limit' => $request->getRawVal( 'limit' ),
 109                         ] );
 110
 111                         $page = $request->getText( 'pages' );
 112                         $this->curonly = $request->getCheck( 'curonly' );
 113                         $rawOffset = $request->getVal( 'offset' );
 114
 115                         if ( $rawOffset ) {
 116                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 117                         } else {
 118                                 $offset = null;
 119                         }
 120
 121                         $maxHistory = $config->get( 'ExportMaxHistory' );
 122                         $limit = $request->getInt( 'limit' );
 123                         $dir = $request->getVal( 'dir' );
 124                         $history = [
 125                                 'dir' => 'asc',
 126                                 'offset' => false,
 127                                 'limit' => $maxHistory,
 128                         ];
 129                         $historyCheck = $request->getCheck( 'history' );
 130
 131                         if ( $this->curonly ) {
 132                                 $history = WikiExporter::CURRENT;
 133                         } elseif ( !$historyCheck ) {
 134                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 135                                         $history['limit'] = $limit;
 136                                 }
 137
 138                                 if ( !is_null( $offset ) ) {
 139                                         $history['offset'] = $offset;
 140                                 }
 141
 142                                 if ( strtolower( $dir ) == 'desc' ) {
 143                                         $history['dir'] = 'desc';
 144                                 }
 145                         }
 146
 147                         if ( $page != '' ) {
 148                                 $this->doExport = true;
 149                         }
 150                 } else {
 151                         // Default to current-only for GET requests.
 152                         $page = $request->getText( 'pages', $par );
 153                         $historyCheck = $request->getCheck( 'history' );
 154
 155                         if ( $historyCheck ) {
 156                                 $history = WikiExporter::FULL;
 157                         } else {
 158                                 $history = WikiExporter::CURRENT;
 159                         }
 160
 161                         if ( $page != '' ) {
 162                                 $this->doExport = true;
 163                         }
 164                 }
 165
 166                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 167                         // Override
 168                         $history = WikiExporter::CURRENT;
 169                 }
 170
 171                 $list_authors = $request->getCheck( 'listauthors' );
 172                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 173                         $list_authors = false;
 174                 }
 175
 176                 if ( $this->doExport ) {
 177                         $this->getOutput()->disable();
 178
 179                         // Cancel output buffering and gzipping if set
 180                         // This should provide safer streaming for pages with history
 181                         wfResetOutputBuffers();
 182                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 183                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 184
 185                         if ( $request->getCheck( 'wpDownload' ) ) {
 186                                 // Provide a sane filename suggestion
 187                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 188                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 189                         }
 190
 191                         $this->doExport( $page, $history, $list_authors, $exportall );
 192
 193                         return;
 194                 }
 195
 196                 $out = $this->getOutput();
 197                 $out->addWikiMsg( 'exporttext' );
 198
 199                 if ( $page == '' ) {
 200                         $categoryName = $request->getText( 'catname' );
 201                 } else {
 202                         $categoryName = '';
 203                 }
 204
 205                 $formDescriptor = [
 206                         'catname' => [
 207                                 'type' => 'textwithbutton',
 208                                 'name' => 'catname',
 209                                 'horizontal-label' => true,
 210                                 'label-message' => 'export-addcattext',
 211                                 'default' => $categoryName,
 212                                 'size' => 40,
 213                                 'buttontype' => 'submit',
 214                                 'buttonname' => 'addcat',
 215                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 216                                 'hide-if' => [ '===', 'exportall', '1' ],
 217                         ],
 218                 ];
 219                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 220                         $formDescriptor += [
 221                                 'nsindex' => [
 222                                         'type' => 'namespaceselectwithbutton',
 223                                         'default' => $nsindex,
 224                                         'label-message' => 'export-addnstext',
 225                                         'horizontal-label' => true,
 226                                         'name' => 'nsindex',
 227                                         'id' => 'namespace',
 228                                         'cssclass' => 'namespaceselector',
 229                                         'buttontype' => 'submit',
 230                                         'buttonname' => 'addns',
 231                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 232                                         'hide-if' => [ '===', 'exportall', '1' ],
 233                                 ],
 234                         ];
 235                 }
 236
 237                 if ( $config->get( 'ExportAllowAll' ) ) {
 238                         $formDescriptor += [
 239                                 'exportall' => [
 240                                         'type' => 'check',
 241                                         'label-message' => 'exportall',
 242                                         'name' => 'exportall',
 243                                         'id' => 'exportall',
 244                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 245                                 ],
 246                         ];
 247                 }
 248
 249                 $formDescriptor += [
 250                         'textarea' => [
 251                                 'class' => HTMLTextAreaField::class,
 252                                 'name' => 'pages',
 253                                 'label-message' => 'export-manual',
 254                                 'nodata' => true,
 255                                 'rows' => 10,
 256                                 'default' => $page,
 257                                 'hide-if' => [ '===', 'exportall', '1' ],
 258                         ],
 259                 ];
 260
 261                 if ( $config->get( 'ExportAllowHistory' ) ) {
 262                         $formDescriptor += [
 263                                 'curonly' => [
 264                                         'type' => 'check',
 265                                         'label-message' => 'exportcuronly',
 266                                         'name' => 'curonly',
 267                                         'id' => 'curonly',
 268                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 269                                 ],
 270                         ];
 271                 } else {
 272                         $out->addWikiMsg( 'exportnohistory' );
 273                 }
 274
 275                 $formDescriptor += [
 276                         'templates' => [
 277                                 'type' => 'check',
 278                                 'label-message' => 'export-templates',
 279                                 'name' => 'templates',
 280                                 'id' => 'wpExportTemplates',
 281                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 282                         ],
 283                 ];
 284
 285                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 286                         $formDescriptor += [
 287                                 'pagelink-depth' => [
 288                                         'type' => 'text',
 289                                         'name' => 'pagelink-depth',
 290                                         'id' => 'pagelink-depth',
 291                                         'label-message' => 'export-pagelinks',
 292                                         'default' => '0',
 293                                         'size' => 20,
 294                                 ],
 295                         ];
 296                 }
 297
 298                 $formDescriptor += [
 299                         'wpDownload' => [
 300                                 'type' => 'check',
 301                                 'name' => 'wpDownload',
 302                                 'id' => 'wpDownload',
 303                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 304                                 'label-message' => 'export-download',
 305                         ],
 306                 ];
 307
 308                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 309                         $formDescriptor += [
 310                                 'listauthors' => [
 311                                         'type' => 'check',
 312                                         'label-message' => 'exportlistauthors',
 313                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 314                                         'name' => 'listauthors',
 315                                         'id' => 'listauthors',
 316                                 ],
 317                         ];
 318                 }
 319
 320                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 321                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 322                 $htmlForm->prepareForm()->displayForm( false );
 323                 $this->addHelpLink( 'Help:Export' );
 324         }
 325
 326         /**
 327          * @return bool
 328          */
 329         private function userCanOverrideExportDepth() {
 330                 return $this->getUser()->isAllowed( 'override-export-depth' );
 331         }
 332
 333         /**
 334          * Do the actual page exporting
 335          *
 336          * @param string $page User input on what page(s) to export
 337          * @param int $history One of the WikiExporter history export constants
 338          * @param bool $list_authors Whether to add distinct author list (when
 339          *   not returning full history)
 340          * @param bool $exportall Whether to export everything
 341          */
 342         private function doExport( $page, $history, $list_authors, $exportall ) {
 343                 // If we are grabbing everything, enable full history and ignore the rest
 344                 if ( $exportall ) {
 345                         $history = WikiExporter::FULL;
 346                 } else {
 347                         $pageSet = []; // Inverted index of all pages to look up
 348
 349                         // Split up and normalize input
 350                         foreach ( explode( "\n", $page ) as $pageName ) {
 351                                 $pageName = trim( $pageName );
 352                                 $title = Title::newFromText( $pageName );
 353                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 354                                         // Only record each page once!
 355                                         $pageSet[$title->getPrefixedText()] = true;
 356                                 }
 357                         }
 358
 359                         // Set of original pages to pass on to further manipulation...
 360                         $inputPages = array_keys( $pageSet );
 361
 362                         // Look up any linked pages if asked...
 363                         if ( $this->templates ) {
 364                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 365                         }
 366                         $linkDepth = $this->pageLinkDepth;
 367                         if ( $linkDepth ) {
 368                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 369                         }
 370
 371                         $pages = array_keys( $pageSet );
 372
 373                         // Normalize titles to the same format and remove dupes, see T19374
 374                         foreach ( $pages as $k => $v ) {
 375                                 $pages[$k] = str_replace( " ", "_", $v );
 376                         }
 377
 378                         $pages = array_unique( $pages );
 379                 }
 380
 381                 /* Ok, let's get to it... */
 382                 if ( $history == WikiExporter::CURRENT ) {
 383                         $lb = false;
 384                         $db = wfGetDB( DB_REPLICA );
 385                         $buffer = WikiExporter::BUFFER;
 386                 } else {
 387                         // Use an unbuffered query; histories may be very long!
 388                         $lb = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->newMainLB();
 389                         $db = $lb->getConnection( DB_REPLICA );
 390                         $buffer = WikiExporter::STREAM;
 391
 392                         // This might take a while... :D
 393                         Wikimedia\suppressWarnings();
 394                         set_time_limit( 0 );
 395                         Wikimedia\restoreWarnings();
 396                 }
 397
 398                 $exporter = new WikiExporter( $db, $history, $buffer );
 399                 $exporter->list_authors = $list_authors;
 400                 $exporter->openStream();
 401
 402                 if ( $exportall ) {
 403                         $exporter->allPages();
 404                 } else {
 405                         foreach ( $pages as $page ) {
 406                                 # T10824: Only export pages the user can read
 407                                 $title = Title::newFromText( $page );
 408                                 if ( is_null( $title ) ) {
 409                                         // @todo Perhaps output an <error> tag or something.
 410                                         continue;
 411                                 }
 412
 413                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 414                                         // @todo Perhaps output an <error> tag or something.
 415                                         continue;
 416                                 }
 417
 418                                 $exporter->pageByTitle( $title );
 419                         }
 420                 }
 421
 422                 $exporter->closeStream();
 423
 424                 if ( $lb ) {
 425                         $lb->closeAll();
 426                 }
 427         }
 428
 429         /**
 430          * @param Title $title
 431          * @return string[]
 432          */
 433         private function getPagesFromCategory( $title ) {
 434                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 435
 436                 $name = $title->getDBkey();
 437
 438                 $dbr = wfGetDB( DB_REPLICA );
 439                 $res = $dbr->select(
 440                         [ 'page', 'categorylinks' ],
 441                         [ 'page_namespace', 'page_title' ],
 442                         [ 'cl_from=page_id', 'cl_to' => $name ],
 443                         __METHOD__,
 444                         [ 'LIMIT' => $maxPages ]
 445                 );
 446
 447                 $pages = [];
 448
 449                 foreach ( $res as $row ) {
 450                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 451                 }
 452
 453                 return $pages;
 454         }
 455
 456         /**
 457          * @param int $nsindex
 458          * @return string[]
 459          */
 460         private function getPagesFromNamespace( $nsindex ) {
 461                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 462
 463                 $dbr = wfGetDB( DB_REPLICA );
 464                 $res = $dbr->select(
 465                         'page',
 466                         [ 'page_namespace', 'page_title' ],
 467                         [ 'page_namespace' => $nsindex ],
 468                         __METHOD__,
 469                         [ 'LIMIT' => $maxPages ]
 470                 );
 471
 472                 $pages = [];
 473
 474                 foreach ( $res as $row ) {
 475                         $pages[] = Title::makeName( $row->page_namespace, $row->page_title );
 476                 }
 477
 478                 return $pages;
 479         }
 480
 481         /**
 482          * Expand a list of pages to include templates used in those pages.
 483          * @param array $inputPages List of titles to look up
 484          * @param array $pageSet Associative array indexed by titles for output
 485          * @return array Associative array index by titles
 486          */
 487         private function getTemplates( $inputPages, $pageSet ) {
 488                 return $this->getLinks( $inputPages, $pageSet,
 489                         'templatelinks',
 490                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 491                         [ 'page_id=tl_from' ]
 492                 );
 493         }
 494
 495         /**
 496          * Validate link depth setting, if available.
 497          * @param int $depth
 498          * @return int
 499          */
 500         private function validateLinkDepth( $depth ) {
 501                 if ( $depth < 0 ) {
 502                         return 0;
 503                 }
 504
 505                 if ( !$this->userCanOverrideExportDepth() ) {
 506                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 507                         if ( $depth > $maxLinkDepth ) {
 508                                 return $maxLinkDepth;
 509                         }
 510                 }
 511
 512                 /*
 513                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 514                  * crazy-big export from being done by someone setting the depth
 515                  * number too high. In other words, last resort safety net.
 516                  */
 517
 518                 return intval( min( $depth, 5 ) );
 519         }
 520
 521         /**
 522          * Expand a list of pages to include pages linked to from that page.
 523          * @param array $inputPages
 524          * @param array $pageSet
 525          * @param int $depth
 526          * @return array
 527          */
 528         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 529                 for ( ; $depth > 0; --$depth ) {
 530                         $pageSet = $this->getLinks(
 531                                 $inputPages, $pageSet, 'pagelinks',
 532                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 533                                 [ 'page_id=pl_from' ]
 534                         );
 535                         $inputPages = array_keys( $pageSet );
 536                 }
 537
 538                 return $pageSet;
 539         }
 540
 541         /**
 542          * Expand a list of pages to include items used in those pages.
 543          * @param array $inputPages Array of page titles
 544          * @param array $pageSet
 545          * @param string $table
 546          * @param array $fields Array of field names
 547          * @param array $join
 548          * @return array
 549          */
 550         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 551                 $dbr = wfGetDB( DB_REPLICA );
 552
 553                 foreach ( $inputPages as $page ) {
 554                         $title = Title::newFromText( $page );
 555
 556                         if ( $title ) {
 557                                 $pageSet[$title->getPrefixedText()] = true;
 558                                 /// @todo FIXME: May or may not be more efficient to batch these
 559                                 ///        by namespace when given multiple input pages.
 560                                 $result = $dbr->select(
 561                                         [ 'page', $table ],
 562                                         $fields,
 563                                         array_merge(
 564                                                 $join,
 565                                                 [
 566                                                         'page_namespace' => $title->getNamespace(),
 567                                                         'page_title' => $title->getDBkey()
 568                                                 ]
 569                                         ),
 570                                         __METHOD__
 571                                 );
 572
 573                                 foreach ( $result as $row ) {
 574                                         $template = Title::makeTitle( $row->namespace, $row->title );
 575                                         $pageSet[$template->getPrefixedText()] = true;
 576                                 }
 577                         }
 578                 }
 579
 580                 return $pageSet;
 581         }
 582
 583         protected function getGroupName() {
 584                 return 'pagetools';
 585         }
 586 }