includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 /**
  27  * A special page that allows users to export pages in a XML file
  28  *
  29  * @ingroup SpecialPage
  30  */
  31 class SpecialExport extends SpecialPage {
  32         private $curonly, $doExport, $pageLinkDepth, $templates;
  33
  34         public function __construct() {
  35                 parent::__construct( 'Export' );
  36         }
  37
  38         public function execute( $par ) {
  39                 $this->setHeaders();
  40                 $this->outputHeader();
  41                 $config = $this->getConfig();
  42
  43                 // Set some variables
  44                 $this->curonly = true;
  45                 $this->doExport = false;
  46                 $request = $this->getRequest();
  47                 $this->templates = $request->getCheck( 'templates' );
  48                 $this->pageLinkDepth = $this->validateLinkDepth(
  49                         $request->getIntOrNull( 'pagelink-depth' )
  50                 );
  51                 $nsindex = '';
  52                 $exportall = false;
  53
  54                 if ( $request->getCheck( 'addcat' ) ) {
  55                         $page = $request->getText( 'pages' );
  56                         $catname = $request->getText( 'catname' );
  57
  58                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  59                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  60                                 if ( $t ) {
  61                                         /**
  62                                          * @todo FIXME: This can lead to hitting memory limit for very large
  63                                          * categories. Ideally we would do the lookup synchronously
  64                                          * during the export in a single query.
  65                                          */
  66                                         $catpages = $this->getPagesFromCategory( $t );
  67                                         if ( $catpages ) {
  68                                                 if ( $page !== '' ) {
  69                                                         $page .= "\n";
  70                                                 }
  71                                                 $page .= implode( "\n", $catpages );
  72                                         }
  73                                 }
  74                         }
  75                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  76                         $page = $request->getText( 'pages' );
  77                         $nsindex = $request->getText( 'nsindex', '' );
  78
  79                         if ( strval( $nsindex ) !== '' ) {
  80                                 /**
  81                                  * Same implementation as above, so same @todo
  82                                  */
  83                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  84                                 if ( $nspages ) {
  85                                         $page .= "\n" . implode( "\n", $nspages );
  86                                 }
  87                         }
  88                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  89                         $this->doExport = true;
  90                         $exportall = true;
  91
  92                         /* Although $page and $history are not used later on, we
  93                         nevertheless set them to avoid that PHP notices about using
  94                         undefined variables foul up our XML output (see call to
  95                         doExport(...) further down) */
  96                         $page = '';
  97                         $history = '';
  98                 } elseif ( $request->wasPosted() && $par == '' ) {
  99                         $page = $request->getText( 'pages' );
 100                         $this->curonly = $request->getCheck( 'curonly' );
 101                         $rawOffset = $request->getVal( 'offset' );
 102
 103                         if ( $rawOffset ) {
 104                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 105                         } else {
 106                                 $offset = null;
 107                         }
 108
 109                         $maxHistory = $config->get( 'ExportMaxHistory' );
 110                         $limit = $request->getInt( 'limit' );
 111                         $dir = $request->getVal( 'dir' );
 112                         $history = [
 113                                 'dir' => 'asc',
 114                                 'offset' => false,
 115                                 'limit' => $maxHistory,
 116                         ];
 117                         $historyCheck = $request->getCheck( 'history' );
 118
 119                         if ( $this->curonly ) {
 120                                 $history = WikiExporter::CURRENT;
 121                         } elseif ( !$historyCheck ) {
 122                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 123                                         $history['limit'] = $limit;
 124                                 }
 125
 126                                 if ( !is_null( $offset ) ) {
 127                                         $history['offset'] = $offset;
 128                                 }
 129
 130                                 if ( strtolower( $dir ) == 'desc' ) {
 131                                         $history['dir'] = 'desc';
 132                                 }
 133                         }
 134
 135                         if ( $page != '' ) {
 136                                 $this->doExport = true;
 137                         }
 138                 } else {
 139                         // Default to current-only for GET requests.
 140                         $page = $request->getText( 'pages', $par );
 141                         $historyCheck = $request->getCheck( 'history' );
 142
 143                         if ( $historyCheck ) {
 144                                 $history = WikiExporter::FULL;
 145                         } else {
 146                                 $history = WikiExporter::CURRENT;
 147                         }
 148
 149                         if ( $page != '' ) {
 150                                 $this->doExport = true;
 151                         }
 152                 }
 153
 154                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 155                         // Override
 156                         $history = WikiExporter::CURRENT;
 157                 }
 158
 159                 $list_authors = $request->getCheck( 'listauthors' );
 160                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 161                         $list_authors = false;
 162                 }
 163
 164                 if ( $this->doExport ) {
 165                         $this->getOutput()->disable();
 166
 167                         // Cancel output buffering and gzipping if set
 168                         // This should provide safer streaming for pages with history
 169                         wfResetOutputBuffers();
 170                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 171
 172                         if ( $request->getCheck( 'wpDownload' ) ) {
 173                                 // Provide a sane filename suggestion
 174                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 175                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 176                         }
 177
 178                         $this->doExport( $page, $history, $list_authors, $exportall );
 179
 180                         return;
 181                 }
 182
 183                 $out = $this->getOutput();
 184                 $out->addWikiMsg( 'exporttext' );
 185
 186                 if ( $page == '' ) {
 187                         $categoryName = $request->getText( 'catname' );
 188                 } else {
 189                         $categoryName = '';
 190                 }
 191
 192                 $formDescriptor = [
 193                         'catname' => [
 194                                 'type' => 'textwithbutton',
 195                                 'name' => 'catname',
 196                                 'horizontal-label' => true,
 197                                 'label-message' => 'export-addcattext',
 198                                 'default' => $categoryName,
 199                                 'size' => 40,
 200                                 'buttontype' => 'submit',
 201                                 'buttonname' => 'addcat',
 202                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 203                         ],
 204                 ];
 205                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 206                         $formDescriptor += [
 207                                 'nsindex' => [
 208                                         'type' => 'namespaceselectwithbutton',
 209                                         'default' => $nsindex,
 210                                         'label-message' => 'export-addnstext',
 211                                         'horizontal-label' => true,
 212                                         'name' => 'nsindex',
 213                                         'id' => 'namespace',
 214                                         'cssclass' => 'namespaceselector',
 215                                         'buttontype' => 'submit',
 216                                         'buttonname' => 'addns',
 217                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 218                                 ],
 219                         ];
 220                 }
 221
 222                 if ( $config->get( 'ExportAllowAll' ) ) {
 223                         $formDescriptor += [
 224                                 'exportall' => [
 225                                         'type' => 'check',
 226                                         'label-message' => 'exportall',
 227                                         'name' => 'exportall',
 228                                         'id' => 'exportall',
 229                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 230                                 ],
 231                         ];
 232                 }
 233
 234                 $formDescriptor += [
 235                         'textarea' => [
 236                                 'class' => 'HTMLTextAreaField',
 237                                 'name' => 'pages',
 238                                 'label-message' => 'export-manual',
 239                                 'nodata' => true,
 240                                 'rows' => 10,
 241                                 'default' => $page,
 242                         ],
 243                 ];
 244
 245                 if ( $config->get( 'ExportAllowHistory' ) ) {
 246                         $formDescriptor += [
 247                                 'curonly' => [
 248                                         'type' => 'check',
 249                                         'label-message' => 'exportcuronly',
 250                                         'name' => 'curonly',
 251                                         'id' => 'curonly',
 252                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 253                                 ],
 254                         ];
 255                 } else {
 256                         $out->addWikiMsg( 'exportnohistory' );
 257                 }
 258
 259                 $formDescriptor += [
 260                         'templates' => [
 261                                 'type' => 'check',
 262                                 'label-message' => 'export-templates',
 263                                 'name' => 'templates',
 264                                 'id' => 'wpExportTemplates',
 265                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 266                         ],
 267                 ];
 268
 269                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 270                         $formDescriptor += [
 271                                 'pagelink-depth' => [
 272                                         'type' => 'text',
 273                                         'name' => 'pagelink-depth',
 274                                         'id' => 'pagelink-depth',
 275                                         'label-message' => 'export-pagelinks',
 276                                         'default' => '0',
 277                                         'size' => 20,
 278                                 ],
 279                         ];
 280                 }
 281
 282                 $formDescriptor += [
 283                         'wpDownload' => [
 284                                 'type' => 'check',
 285                                 'name' =>'wpDownload',
 286                                 'id' => 'wpDownload',
 287                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 288                                 'label-message' => 'export-download',
 289                         ],
 290                 ];
 291
 292                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 293                         $formDescriptor += [
 294                                 'listauthors' => [
 295                                         'type' => 'check',
 296                                         'label-message' => 'exportlistauthors',
 297                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 298                                         'name' => 'listauthors',
 299                                         'id' => 'listauthors',
 300                                 ],
 301                         ];
 302                 }
 303
 304                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 305                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 306                 $htmlForm->prepareForm()->displayForm( false );
 307                 $this->addHelpLink( 'Help:Export' );
 308         }
 309
 310         /**
 311          * @return bool
 312          */
 313         private function userCanOverrideExportDepth() {
 314                 return $this->getUser()->isAllowed( 'override-export-depth' );
 315         }
 316
 317         /**
 318          * Do the actual page exporting
 319          *
 320          * @param string $page User input on what page(s) to export
 321          * @param int $history One of the WikiExporter history export constants
 322          * @param bool $list_authors Whether to add distinct author list (when
 323          *   not returning full history)
 324          * @param bool $exportall Whether to export everything
 325          */
 326         private function doExport( $page, $history, $list_authors, $exportall ) {
 327
 328                 // If we are grabbing everything, enable full history and ignore the rest
 329                 if ( $exportall ) {
 330                         $history = WikiExporter::FULL;
 331                 } else {
 332                         $pageSet = []; // Inverted index of all pages to look up
 333
 334                         // Split up and normalize input
 335                         foreach ( explode( "\n", $page ) as $pageName ) {
 336                                 $pageName = trim( $pageName );
 337                                 $title = Title::newFromText( $pageName );
 338                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 339                                         // Only record each page once!
 340                                         $pageSet[$title->getPrefixedText()] = true;
 341                                 }
 342                         }
 343
 344                         // Set of original pages to pass on to further manipulation...
 345                         $inputPages = array_keys( $pageSet );
 346
 347                         // Look up any linked pages if asked...
 348                         if ( $this->templates ) {
 349                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 350                         }
 351                         $linkDepth = $this->pageLinkDepth;
 352                         if ( $linkDepth ) {
 353                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 354                         }
 355
 356                         $pages = array_keys( $pageSet );
 357
 358                         // Normalize titles to the same format and remove dupes, see bug 17374
 359                         foreach ( $pages as $k => $v ) {
 360                                 $pages[$k] = str_replace( " ", "_", $v );
 361                         }
 362
 363                         $pages = array_unique( $pages );
 364                 }
 365
 366                 /* Ok, let's get to it... */
 367                 if ( $history == WikiExporter::CURRENT ) {
 368                         $lb = false;
 369                         $db = wfGetDB( DB_SLAVE );
 370                         $buffer = WikiExporter::BUFFER;
 371                 } else {
 372                         // Use an unbuffered query; histories may be very long!
 373                         $lb = wfGetLBFactory()->newMainLB();
 374                         $db = $lb->getConnection( DB_SLAVE );
 375                         $buffer = WikiExporter::STREAM;
 376
 377                         // This might take a while... :D
 378                         MediaWiki\suppressWarnings();
 379                         set_time_limit( 0 );
 380                         MediaWiki\restoreWarnings();
 381                 }
 382
 383                 $exporter = new WikiExporter( $db, $history, $buffer );
 384                 $exporter->list_authors = $list_authors;
 385                 $exporter->openStream();
 386
 387                 if ( $exportall ) {
 388                         $exporter->allPages();
 389                 } else {
 390                         foreach ( $pages as $page ) {
 391                                 # Bug 8824: Only export pages the user can read
 392                                 $title = Title::newFromText( $page );
 393                                 if ( is_null( $title ) ) {
 394                                         // @todo Perhaps output an <error> tag or something.
 395                                         continue;
 396                                 }
 397
 398                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 399                                         // @todo Perhaps output an <error> tag or something.
 400                                         continue;
 401                                 }
 402
 403                                 $exporter->pageByTitle( $title );
 404                         }
 405                 }
 406
 407                 $exporter->closeStream();
 408
 409                 if ( $lb ) {
 410                         $lb->closeAll();
 411                 }
 412         }
 413
 414         /**
 415          * @param Title $title
 416          * @return array
 417          */
 418         private function getPagesFromCategory( $title ) {
 419                 global $wgContLang;
 420
 421                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 422
 423                 $name = $title->getDBkey();
 424
 425                 $dbr = wfGetDB( DB_SLAVE );
 426                 $res = $dbr->select(
 427                         [ 'page', 'categorylinks' ],
 428                         [ 'page_namespace', 'page_title' ],
 429                         [ 'cl_from=page_id', 'cl_to' => $name ],
 430                         __METHOD__,
 431                         [ 'LIMIT' => $maxPages ]
 432                 );
 433
 434                 $pages = [];
 435
 436                 foreach ( $res as $row ) {
 437                         $n = $row->page_title;
 438                         if ( $row->page_namespace ) {
 439                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 440                                 $n = $ns . ':' . $n;
 441                         }
 442
 443                         $pages[] = $n;
 444                 }
 445
 446                 return $pages;
 447         }
 448
 449         /**
 450          * @param int $nsindex
 451          * @return array
 452          */
 453         private function getPagesFromNamespace( $nsindex ) {
 454                 global $wgContLang;
 455
 456                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 457
 458                 $dbr = wfGetDB( DB_SLAVE );
 459                 $res = $dbr->select(
 460                         'page',
 461                         [ 'page_namespace', 'page_title' ],
 462                         [ 'page_namespace' => $nsindex ],
 463                         __METHOD__,
 464                         [ 'LIMIT' => $maxPages ]
 465                 );
 466
 467                 $pages = [];
 468
 469                 foreach ( $res as $row ) {
 470                         $n = $row->page_title;
 471
 472                         if ( $row->page_namespace ) {
 473                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 474                                 $n = $ns . ':' . $n;
 475                         }
 476
 477                         $pages[] = $n;
 478                 }
 479
 480                 return $pages;
 481         }
 482
 483         /**
 484          * Expand a list of pages to include templates used in those pages.
 485          * @param array $inputPages List of titles to look up
 486          * @param array $pageSet Associative array indexed by titles for output
 487          * @return array Associative array index by titles
 488          */
 489         private function getTemplates( $inputPages, $pageSet ) {
 490                 return $this->getLinks( $inputPages, $pageSet,
 491                         'templatelinks',
 492                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 493                         [ 'page_id=tl_from' ]
 494                 );
 495         }
 496
 497         /**
 498          * Validate link depth setting, if available.
 499          * @param int $depth
 500          * @return int
 501          */
 502         private function validateLinkDepth( $depth ) {
 503                 if ( $depth < 0 ) {
 504                         return 0;
 505                 }
 506
 507                 if ( !$this->userCanOverrideExportDepth() ) {
 508                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 509                         if ( $depth > $maxLinkDepth ) {
 510                                 return $maxLinkDepth;
 511                         }
 512                 }
 513
 514                 /*
 515                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 516                  * crazy-big export from being done by someone setting the depth
 517                  * number too high. In other words, last resort safety net.
 518                  */
 519
 520                 return intval( min( $depth, 5 ) );
 521         }
 522
 523         /**
 524          * Expand a list of pages to include pages linked to from that page.
 525          * @param array $inputPages
 526          * @param array $pageSet
 527          * @param int $depth
 528          * @return array
 529          */
 530         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 531                 // @codingStandardsIgnoreStart Squiz.WhiteSpace.SemicolonSpacing.Incorrect
 532                 for ( ; $depth > 0; --$depth ) {
 533                         // @codingStandardsIgnoreEnd
 534                         $pageSet = $this->getLinks(
 535                                 $inputPages, $pageSet, 'pagelinks',
 536                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 537                                 [ 'page_id=pl_from' ]
 538                         );
 539                         $inputPages = array_keys( $pageSet );
 540                 }
 541
 542                 return $pageSet;
 543         }
 544
 545         /**
 546          * Expand a list of pages to include items used in those pages.
 547          * @param array $inputPages Array of page titles
 548          * @param array $pageSet
 549          * @param string $table
 550          * @param array $fields Array of field names
 551          * @param array $join
 552          * @return array
 553          */
 554         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 555                 $dbr = wfGetDB( DB_SLAVE );
 556
 557                 foreach ( $inputPages as $page ) {
 558                         $title = Title::newFromText( $page );
 559
 560                         if ( $title ) {
 561                                 $pageSet[$title->getPrefixedText()] = true;
 562                                 /// @todo FIXME: May or may not be more efficient to batch these
 563                                 ///        by namespace when given multiple input pages.
 564                                 $result = $dbr->select(
 565                                         [ 'page', $table ],
 566                                         $fields,
 567                                         array_merge(
 568                                                 $join,
 569                                                 [
 570                                                         'page_namespace' => $title->getNamespace(),
 571                                                         'page_title' => $title->getDBkey()
 572                                                 ]
 573                                         ),
 574                                         __METHOD__
 575                                 );
 576
 577                                 foreach ( $result as $row ) {
 578                                         $template = Title::makeTitle( $row->namespace, $row->title );
 579                                         $pageSet[$template->getPrefixedText()] = true;
 580                                 }
 581                         }
 582                 }
 583
 584                 return $pageSet;
 585         }
 586
 587         protected function getGroupName() {
 588                 return 'pagetools';
 589         }
 590 }