maintenance/importImages.php

   1 <?php
   2 /**
   3  * Import one or more images from the local file system into the wiki without
   4  * using the web-based interface.
   5  *
   6  * "Smart import" additions:
   7  * - aim: preserve the essential metadata (user, description) when importing media
   8  *   files from an existing wiki.
   9  * - process:
  10  *      - interface with the source wiki, don't use bare files only (see --source-wiki-url).
  11  *      - fetch metadata from source wiki for each file to import.
  12  *      - commit the fetched metadata to the destination wiki while submitting.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License along
  25  * with this program; if not, write to the Free Software Foundation, Inc.,
  26  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  27  * http://www.gnu.org/copyleft/gpl.html
  28  *
  29  * @file
  30  * @ingroup Maintenance
  31  * @author Rob Church <robchur@gmail.com>
  32  * @author Mij <mij@bitchx.it>
  33  */
  34
  35 require_once __DIR__ . '/Maintenance.php';
  36
  37 class ImportImages extends Maintenance {
  38
  39         public function __construct() {
  40                 parent::__construct();
  41
  42                 $this->addDescription( 'Imports images and other media files into the wiki' );
  43                 $this->addArg( 'dir', 'Path to the directory containing images to be imported' );
  44
  45                 $this->addOption( 'extensions',
  46                         'Comma-separated list of allowable extensions, defaults to $wgFileExtensions',
  47                         false,
  48                         true
  49                 );
  50                 $this->addOption( 'overwrite',
  51                         'Overwrite existing images with the same name (default is to skip them)' );
  52                 $this->addOption( 'limit',
  53                         'Limit the number of images to process. Ignored or skipped images are not counted',
  54                         false,
  55                         true
  56                 );
  57                 $this->addOption( 'from',
  58                         "Ignore all files until the one with the given name. Useful for resuming aborted "
  59                                 . "imports. The name should be the file's canonical database form.",
  60                         false,
  61                         true
  62                 );
  63                 $this->addOption( 'skip-dupes',
  64                         'Skip images that were already uploaded under a different name (check SHA1)' );
  65                 $this->addOption( 'search-recursively', 'Search recursively for files in subdirectories' );
  66                 $this->addOption( 'sleep',
  67                         'Sleep between files. Useful mostly for debugging',
  68                         false,
  69                         true
  70                 );
  71                 $this->addOption( 'user',
  72                         "Set username of uploader, default 'Maintenance script'",
  73                         false,
  74                         true
  75                 );
  76                 // This parameter can optionally have an argument. If none specified, getOption()
  77                 // returns 1 which is precisely what we need.
  78                 $this->addOption( 'check-userblock', 'Check if the user got blocked during import' );
  79                 $this->addOption( 'comment',
  80                         "Set file description, default 'Importing file'",
  81                         false,
  82                         true
  83                 );
  84                 $this->addOption( 'comment-file',
  85                         'Set description to the content of this file',
  86                         false,
  87                         true
  88                 );
  89                 $this->addOption( 'comment-ext',
  90                         'Causes the description for each file to be loaded from a file with the same name, but '
  91                                 . 'the extension provided. If a global description is also given, it is appended.',
  92                         false,
  93                         true
  94                 );
  95                 $this->addOption( 'summary',
  96                         'Upload summary, description will be used if not provided',
  97                         false,
  98                         true
  99                 );
 100                 $this->addOption( 'license',
 101                         'Use an optional license template',
 102                         false,
 103                         true
 104                 );
 105                 $this->addOption( 'timestamp',
 106                         'Override upload time/date, all MediaWiki timestamp formats are accepted',
 107                         false,
 108                         true
 109                 );
 110                 $this->addOption( 'protect',
 111                         'Specify the protect value (autoconfirmed,sysop)',
 112                         false,
 113                         true
 114                 );
 115                 $this->addOption( 'unprotect', 'Unprotects all uploaded images' );
 116                 $this->addOption( 'source-wiki-url',
 117                         'If specified, take User and Comment data for each imported file from this URL. '
 118                                 . 'For example, --source-wiki-url="http://en.wikipedia.org/',
 119                         false,
 120                         true
 121                 );
 122                 $this->addOption( 'dry', "Dry run, don't import anything" );
 123         }
 124
 125         public function execute() {
 126                 global $wgFileExtensions, $wgUser, $wgRestrictionLevels;
 127
 128                 $processed = $added = $ignored = $skipped = $overwritten = $failed = 0;
 129
 130                 $this->output( "Import Images\n\n" );
 131
 132                 $dir = $this->getArg( 0 );
 133
 134                 # Check Protection
 135                 if ( $this->hasOption( 'protect' ) && $this->hasOption( 'unprotect' ) ) {
 136                         $this->error( "Cannot specify both protect and unprotect.  Only 1 is allowed.\n", 1 );
 137                 }
 138
 139                 if ( $this->hasOption( 'protect' ) && trim( $this->getOption( 'protect' ) ) ) {
 140                         $this->error( "You must specify a protection option.\n", 1 );
 141                 }
 142
 143                 # Prepare the list of allowed extensions
 144                 $extensions = $this->hasOption( 'extensions' )
 145                         ? explode( ',', strtolower( $this->getOption( 'extensions' ) ) )
 146                         : $wgFileExtensions;
 147
 148                 # Search the path provided for candidates for import
 149                 $files = $this->findFiles( $dir, $extensions, $this->hasOption( 'search-recursively' ) );
 150
 151                 # Initialise the user for this operation
 152                 $user = $this->hasOption( 'user' )
 153                         ? User::newFromName( $this->getOption( 'user' ) )
 154                         : User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
 155                 if ( !$user instanceof User ) {
 156                         $user = User::newSystemUser( 'Maintenance script', [ 'steal' => true ] );
 157                 }
 158                 $wgUser = $user;
 159
 160                 # Get block check. If a value is given, this specified how often the check is performed
 161                 $checkUserBlock = (int)$this->getOption( 'check-userblock' );
 162
 163                 $from = $this->getOption( 'from' );
 164                 $sleep = (int)$this->getOption( 'sleep' );
 165                 $limit = (int)$this->getOption( 'limit' );
 166                 $timestamp = $this->getOption( 'timestamp', false );
 167
 168                 # Get the upload comment. Provide a default one in case there's no comment given.
 169                 $commentFile = $this->getOption( 'comment-file' );
 170                 if ( $commentFile !== null ) {
 171                         $comment = file_get_contents( $commentFile );
 172                         if ( $comment === false || $comment === null ) {
 173                                 $this->error( "failed to read comment file: {$commentFile}\n", 1 );
 174                         }
 175                 } else {
 176                         $comment = $this->getOption( 'comment', 'Importing file' );
 177                 }
 178                 $commentExt = $this->getOption( 'comment-ext' );
 179                 $summary = $this->getOption( 'summary', '' );
 180
 181                 $license = $this->getOption( 'license', '' );
 182
 183                 $sourceWikiUrl = $this->getOption( 'source-wiki-url' );
 184
 185                 # Batch "upload" operation
 186                 $count = count( $files );
 187                 if ( $count > 0 ) {
 188
 189                         foreach ( $files as $file ) {
 190
 191                                 if ( $sleep && ( $processed > 0 ) ) {
 192                                         sleep( $sleep );
 193                                 }
 194
 195                                 $base = UtfNormal\Validator::cleanUp( wfBaseName( $file ) );
 196
 197                                 # Validate a title
 198                                 $title = Title::makeTitleSafe( NS_FILE, $base );
 199                                 if ( !is_object( $title ) ) {
 200                                         $this->output(
 201                                                 "{$base} could not be imported; a valid title cannot be produced\n" );
 202                                         continue;
 203                                 }
 204
 205                                 if ( $from ) {
 206                                         if ( $from == $title->getDBkey() ) {
 207                                                 $from = null;
 208                                         } else {
 209                                                 $ignored++;
 210                                                 continue;
 211                                         }
 212                                 }
 213
 214                                 if ( $checkUserBlock && ( ( $processed % $checkUserBlock ) == 0 ) ) {
 215                                         $user->clearInstanceCache( 'name' ); // reload from DB!
 216                                         if ( $user->isBlocked() ) {
 217                                                 $this->output( $user->getName() . " was blocked! Aborting.\n" );
 218                                                 break;
 219                                         }
 220                                 }
 221
 222                                 # Check existence
 223                                 $image = wfLocalFile( $title );
 224                                 if ( $image->exists() ) {
 225                                         if ( $this->hasOption( 'overwrite' ) ) {
 226                                                 $this->output( "{$base} exists, overwriting..." );
 227                                                 $svar = 'overwritten';
 228                                         } else {
 229                                                 $this->output( "{$base} exists, skipping\n" );
 230                                                 $skipped++;
 231                                                 continue;
 232                                         }
 233                                 } else {
 234                                         if ( $this->hasOption( 'skip-dupes' ) ) {
 235                                                 $repo = $image->getRepo();
 236                                                 # XXX: we end up calculating this again when actually uploading. that sucks.
 237                                                 $sha1 = FSFile::getSha1Base36FromPath( $file );
 238
 239                                                 $dupes = $repo->findBySha1( $sha1 );
 240
 241                                                 if ( $dupes ) {
 242                                                         $this->output(
 243                                                                 "{$base} already exists as {$dupes[0]->getName()}, skipping\n" );
 244                                                         $skipped++;
 245                                                         continue;
 246                                                 }
 247                                         }
 248
 249                                         $this->output( "Importing {$base}..." );
 250                                         $svar = 'added';
 251                                 }
 252
 253                                 if ( $sourceWikiUrl ) {
 254                                         /* find comment text directly from source wiki, through MW's API */
 255                                         $real_comment = $this->getFileCommentFromSourceWiki( $sourceWikiUrl, $base );
 256                                         if ( $real_comment === false ) {
 257                                                 $commentText = $comment;
 258                                         } else {
 259                                                 $commentText = $real_comment;
 260                                         }
 261
 262                                         /* find user directly from source wiki, through MW's API */
 263                                         $real_user = $this->getFileUserFromSourceWiki( $sourceWikiUrl, $base );
 264                                         if ( $real_user === false ) {
 265                                                 $wgUser = $user;
 266                                         } else {
 267                                                 $wgUser = User::newFromName( $real_user );
 268                                                 if ( $wgUser === false ) {
 269                                                         # user does not exist in target wiki
 270                                                         $this->output(
 271                                                                 "failed: user '$real_user' does not exist in target wiki." );
 272                                                         continue;
 273                                                 }
 274                                         }
 275                                 } else {
 276                                         # Find comment text
 277                                         $commentText = false;
 278
 279                                         if ( $commentExt ) {
 280                                                 $f = $this->findAuxFile( $file, $commentExt );
 281                                                 if ( !$f ) {
 282                                                         $this->output( " No comment file with extension {$commentExt} found "
 283                                                                  . "for {$file}, using default comment. " );
 284                                                 } else {
 285                                                         $commentText = file_get_contents( $f );
 286                                                         if ( !$commentText ) {
 287                                                                 $this->output(
 288                                                                         " Failed to load comment file {$f}, using default comment. " );
 289                                                         }
 290                                                 }
 291                                         }
 292
 293                                         if ( !$commentText ) {
 294                                                 $commentText = $comment;
 295                                         }
 296                                 }
 297
 298                                 # Import the file
 299                                 if ( $this->hasOption( 'dry' ) ) {
 300                                         $this->output(
 301                                                 " publishing {$file} by '{$wgUser->getName()}', comment '$commentText'... "
 302                                         );
 303                                 } else {
 304                                         $mwProps = new MWFileProps( MimeMagic::singleton() );
 305                                         $props = $mwProps->getPropsFromPath( $file, true );
 306                                         $flags = 0;
 307                                         $publishOptions = [];
 308                                         $handler = MediaHandler::getHandler( $props['mime'] );
 309                                         if ( $handler ) {
 310                                                 $metadata = MediaWiki\quietCall( 'unserialize', $props['metadata'] );
 311
 312                                                 $publishOptions['headers'] = $handler->getContentHeaders( $metadata );
 313                                         } else {
 314                                                 $publishOptions['headers'] = [];
 315                                         }
 316                                         $archive = $image->publish( $file, $flags, $publishOptions );
 317                                         if ( !$archive->isGood() ) {
 318                                                 $this->output( "failed. (" .
 319                                                          $archive->getWikiText( false, false, 'en' ) .
 320                                                          ")\n" );
 321                                                 $failed++;
 322                                                 continue;
 323                                         }
 324                                 }
 325
 326                                 $commentText = SpecialUpload::getInitialPageText( $commentText, $license );
 327                                 if ( !$this->hasOption( 'summary' ) ) {
 328                                         $summary = $commentText;
 329                                 }
 330
 331                                 if ( $this->hasOption( 'dry' ) ) {
 332                                         $this->output( "done.\n" );
 333                                 } elseif ( $image->recordUpload2(
 334                                         $archive->value,
 335                                         $summary,
 336                                         $commentText,
 337                                         $props,
 338                                         $timestamp
 339                                 ) ) {
 340                                         # We're done!
 341                                         $this->output( "done.\n" );
 342
 343                                         $doProtect = false;
 344
 345                                         $protectLevel = $this->getOption( 'protect' );
 346
 347                                         if ( $protectLevel && in_array( $protectLevel, $wgRestrictionLevels ) ) {
 348                                                 $doProtect = true;
 349                                         }
 350                                         if ( $this->hasOption( 'unprotect' ) ) {
 351                                                 $protectLevel = '';
 352                                                 $doProtect = true;
 353                                         }
 354
 355                                         if ( $doProtect ) {
 356                                                 # Protect the file
 357                                                 $this->output( "\nWaiting for replica DBs...\n" );
 358                                                 // Wait for replica DBs.
 359                                                 sleep( 2.0 ); # Why this sleep?
 360                                                 wfWaitForSlaves();
 361
 362                                                 $this->output( "\nSetting image restrictions ... " );
 363
 364                                                 $cascade = false;
 365                                                 $restrictions = [];
 366                                                 foreach ( $title->getRestrictionTypes() as $type ) {
 367                                                         $restrictions[$type] = $protectLevel;
 368                                                 }
 369
 370                                                 $page = WikiPage::factory( $title );
 371                                                 $status = $page->doUpdateRestrictions( $restrictions, [], $cascade, '', $user );
 372                                                 $this->output( ( $status->isOK() ? 'done' : 'failed' ) . "\n" );
 373                                         }
 374                                 } else {
 375                                         $this->output( "failed. (at recordUpload stage)\n" );
 376                                         $svar = 'failed';
 377                                 }
 378
 379                                 $$svar++;
 380                                 $processed++;
 381
 382                                 if ( $limit && $processed >= $limit ) {
 383                                         break;
 384                                 }
 385                         }
 386
 387                         # Print out some statistics
 388                         $this->output( "\n" );
 389                         foreach (
 390                                 [
 391                                         'count' => 'Found',
 392                                         'limit' => 'Limit',
 393                                         'ignored' => 'Ignored',
 394                                         'added' => 'Added',
 395                                         'skipped' => 'Skipped',
 396                                         'overwritten' => 'Overwritten',
 397                                         'failed' => 'Failed'
 398                                 ] as $var => $desc
 399                         ) {
 400                                 if ( $$var > 0 ) {
 401                                         $this->output( "{$desc}: {$$var}\n" );
 402                                 }
 403                         }
 404                 } else {
 405                         $this->output( "No suitable files could be found for import.\n" );
 406                 }
 407         }
 408
 409         /**
 410          * Search a directory for files with one of a set of extensions
 411          *
 412          * @param string $dir Path to directory to search
 413          * @param array $exts Array of extensions to search for
 414          * @param bool $recurse Search subdirectories recursively
 415          * @return array|bool Array of filenames on success, or false on failure
 416          */
 417         private function findFiles( $dir, $exts, $recurse = false ) {
 418                 if ( is_dir( $dir ) ) {
 419                         $dhl = opendir( $dir );
 420                         if ( $dhl ) {
 421                                 $files = [];
 422                                 while ( ( $file = readdir( $dhl ) ) !== false ) {
 423                                         if ( is_file( $dir . '/' . $file ) ) {
 424                                                 list( /* $name */, $ext ) = $this->splitFilename( $dir . '/' . $file );
 425                                                 if ( array_search( strtolower( $ext ), $exts ) !== false ) {
 426                                                         $files[] = $dir . '/' . $file;
 427                                                 }
 428                                         } elseif ( $recurse && is_dir( $dir . '/' . $file ) && $file !== '..' && $file !== '.' ) {
 429                                                 $files = array_merge( $files, $this->findFiles( $dir . '/' . $file, $exts, true ) );
 430                                         }
 431                                 }
 432
 433                                 return $files;
 434                         } else {
 435                                 return [];
 436                         }
 437                 } else {
 438                         return [];
 439                 }
 440         }
 441
 442         /**
 443          * Split a filename into filename and extension
 444          *
 445          * @param string $filename Filename
 446          * @return array
 447          */
 448         private function splitFilename( $filename ) {
 449                 $parts = explode( '.', $filename );
 450                 $ext = $parts[count( $parts ) - 1];
 451                 unset( $parts[count( $parts ) - 1] );
 452                 $fname = implode( '.', $parts );
 453
 454                 return [ $fname, $ext ];
 455         }
 456
 457         /**
 458          * Find an auxilliary file with the given extension, matching
 459          * the give base file path. $maxStrip determines how many extensions
 460          * may be stripped from the original file name before appending the
 461          * new extension. For example, with $maxStrip = 1 (the default),
 462          * file files acme.foo.bar.txt and acme.foo.txt would be auxilliary
 463          * files for acme.foo.bar and the extension ".txt". With $maxStrip = 2,
 464          * acme.txt would also be acceptable.
 465          *
 466          * @param string $file Base path
 467          * @param string $auxExtension The extension to be appended to the base path
 468          * @param int $maxStrip The maximum number of extensions to strip from the base path (default: 1)
 469          * @return string|bool
 470          */
 471         private function findAuxFile( $file, $auxExtension, $maxStrip = 1 ) {
 472                 if ( strpos( $auxExtension, '.' ) !== 0 ) {
 473                         $auxExtension = '.' . $auxExtension;
 474                 }
 475
 476                 $d = dirname( $file );
 477                 $n = basename( $file );
 478
 479                 while ( $maxStrip >= 0 ) {
 480                         $f = $d . '/' . $n . $auxExtension;
 481
 482                         if ( file_exists( $f ) ) {
 483                                 return $f;
 484                         }
 485
 486                         $idx = strrpos( $n, '.' );
 487                         if ( !$idx ) {
 488                                 break;
 489                         }
 490
 491                         $n = substr( $n, 0, $idx );
 492                         $maxStrip -= 1;
 493                 }
 494
 495                 return false;
 496         }
 497
 498         # @todo FIXME: Access the api in a saner way and performing just one query
 499         # (preferably batching files too).
 500         private function getFileCommentFromSourceWiki( $wiki_host, $file ) {
 501                 $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:'
 502                         . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=comment';
 503                 $body = Http::get( $url, [], __METHOD__ );
 504                 if ( preg_match( '#<ii comment="([^"]*)" />#', $body, $matches ) == 0 ) {
 505                         return false;
 506                 }
 507
 508                 return html_entity_decode( $matches[1] );
 509         }
 510
 511         private function getFileUserFromSourceWiki( $wiki_host, $file ) {
 512                 $url = $wiki_host . '/api.php?action=query&format=xml&titles=File:'
 513                         . rawurlencode( $file ) . '&prop=imageinfo&&iiprop=user';
 514                 $body = Http::get( $url, [], __METHOD__ );
 515                 if ( preg_match( '#<ii user="([^"]*)" />#', $body, $matches ) == 0 ) {
 516                         return false;
 517                 }
 518
 519                 return html_entity_decode( $matches[1] );
 520         }
 521
 522 }
 523
 524 $maintClass = 'ImportImages';
 525 require_once RUN_MAINTENANCE_IF_MAIN;