maintenance/refreshImageMetadata.php

   1 <?php
   2 /**
   3  * Refresh image metadata fields. See also rebuildImages.php
   4  *
   5  * Usage: php refreshImageMetadata.php
   6  *
   7  * Copyright © 2011 Brian Wolff
   8  * https://www.mediawiki.org/
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License along
  21  * with this program; if not, write to the Free Software Foundation, Inc.,
  22  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  23  * http://www.gnu.org/copyleft/gpl.html
  24  *
  25  * @file
  26  * @author Brian Wolff
  27  * @ingroup Maintenance
  28  */
  29
  30 require_once __DIR__ . '/Maintenance.php';
  31
  32 use Wikimedia\Rdbms\IDatabase;
  33 use Wikimedia\Rdbms\IMaintainableDatabase;
  34
  35 /**
  36  * Maintenance script to refresh image metadata fields.
  37  *
  38  * @ingroup Maintenance
  39  */
  40 class RefreshImageMetadata extends Maintenance {
  41
  42         /**
  43          * @var IMaintainableDatabase
  44          */
  45         protected $dbw;
  46
  47         function __construct() {
  48                 parent::__construct();
  49
  50                 $this->addDescription( 'Script to update image metadata records' );
  51                 $this->setBatchSize( 200 );
  52
  53                 $this->addOption(
  54                         'force',
  55                         'Reload metadata from file even if the metadata looks ok',
  56                         false,
  57                         false,
  58                         'f'
  59                 );
  60                 $this->addOption(
  61                         'broken-only',
  62                         'Only fix really broken records, leave old but still compatible records alone.'
  63                 );
  64                 $this->addOption(
  65                         'verbose',
  66                         'Output extra information about each upgraded/non-upgraded file.',
  67                         false,
  68                         false,
  69                         'v'
  70                 );
  71                 $this->addOption( 'start', 'Name of file to start with', false, true );
  72                 $this->addOption( 'end', 'Name of file to end with', false, true );
  73
  74                 $this->addOption(
  75                         'mediatype',
  76                         'Only refresh files with this media type, e.g. BITMAP, UNKNOWN etc.',
  77                         false,
  78                         true
  79                 );
  80                 $this->addOption(
  81                         'mime',
  82                         "Only refresh files with this MIME type. Can accept wild-card 'image/*'. "
  83                                 . "Potentially inefficient unless 'mediatype' is also specified",
  84                         false,
  85                         true
  86                 );
  87                 $this->addOption(
  88                         'metadata-contains',
  89                         '(Inefficient!) Only refresh files where the img_metadata field '
  90                                 . 'contains this string. Can be used if its known a specific '
  91                                 . 'property was being extracted incorrectly.',
  92                         false,
  93                         true
  94                 );
  95         }
  96
  97         public function execute() {
  98                 $force = $this->hasOption( 'force' );
  99                 $brokenOnly = $this->hasOption( 'broken-only' );
 100                 $verbose = $this->hasOption( 'verbose' );
 101                 $start = $this->getOption( 'start', false );
 102                 $this->setupParameters( $force, $brokenOnly );
 103
 104                 $upgraded = 0;
 105                 $leftAlone = 0;
 106                 $error = 0;
 107
 108                 $dbw = $this->getDB( DB_MASTER );
 109                 $batchSize = $this->getBatchSize();
 110                 if ( $batchSize <= 0 ) {
 111                         $this->fatalError( "Batch size is too low...", 12 );
 112                 }
 113
 114                 $repo = RepoGroup::singleton()->getLocalRepo();
 115                 $conds = $this->getConditions( $dbw );
 116
 117                 // For the WHERE img_name > 'foo' condition that comes after doing a batch
 118                 $conds2 = [];
 119                 if ( $start !== false ) {
 120                         $conds2[] = 'img_name >= ' . $dbw->addQuotes( $start );
 121                 }
 122
 123                 $options = [
 124                         'LIMIT' => $batchSize,
 125                         'ORDER BY' => 'img_name ASC',
 126                 ];
 127
 128                 $fileQuery = LocalFile::getQueryInfo();
 129
 130                 do {
 131                         $res = $dbw->select(
 132                                 $fileQuery['tables'],
 133                                 $fileQuery['fields'],
 134                                 array_merge( $conds, $conds2 ),
 135                                 __METHOD__,
 136                                 $options,
 137                                 $fileQuery['joins']
 138                         );
 139
 140                         if ( $res->numRows() > 0 ) {
 141                                 $row1 = $res->current();
 142                                 $this->output( "Processing next {$res->numRows()} row(s) starting with {$row1->img_name}.\n" );
 143                                 $res->rewind();
 144                         }
 145
 146                         foreach ( $res as $row ) {
 147                                 try {
 148                                         // LocalFile will upgrade immediately here if obsolete
 149                                         $file = $repo->newFileFromRow( $row );
 150                                         if ( $file->getUpgraded() ) {
 151                                                 // File was upgraded.
 152                                                 $upgraded++;
 153                                                 $newLength = strlen( $file->getMetadata() );
 154                                                 $oldLength = strlen( $row->img_metadata );
 155                                                 if ( $newLength < $oldLength - 5 ) {
 156                                                         // If after updating, the metadata is smaller then
 157                                                         // what it was before, that's probably not a good thing
 158                                                         // because we extract more data with time, not less.
 159                                                         // Thus this probably indicates an error of some sort,
 160                                                         // or at the very least is suspicious. Have the - 5 just
 161                                                         // to weed out any inconsequential changes.
 162                                                         $error++;
 163                                                         $this->output(
 164                                                                 "Warning: File:{$row->img_name} used to have " .
 165                                                                 "$oldLength bytes of metadata but now has $newLength bytes.\n"
 166                                                         );
 167                                                 } elseif ( $verbose ) {
 168                                                         $this->output( "Refreshed File:{$row->img_name}.\n" );
 169                                                 }
 170                                         } else {
 171                                                 $leftAlone++;
 172                                                 if ( $force ) {
 173                                                         $file->upgradeRow();
 174                                                         $newLength = strlen( $file->getMetadata() );
 175                                                         $oldLength = strlen( $row->img_metadata );
 176                                                         if ( $newLength < $oldLength - 5 ) {
 177                                                                 $error++;
 178                                                                 $this->output(
 179                                                                         "Warning: File:{$row->img_name} used to have " .
 180                                                                         "$oldLength bytes of metadata but now has $newLength bytes. (forced)\n"
 181                                                                 );
 182                                                         }
 183                                                         if ( $verbose ) {
 184                                                                 $this->output( "Forcibly refreshed File:{$row->img_name}.\n" );
 185                                                         }
 186                                                 } else {
 187                                                         if ( $verbose ) {
 188                                                                 $this->output( "Skipping File:{$row->img_name}.\n" );
 189                                                         }
 190                                                 }
 191                                         }
 192                                 } catch ( Exception $e ) {
 193                                         $this->output( "{$row->img_name} failed. {$e->getMessage()}\n" );
 194                                 }
 195                         }
 196                         $conds2 = [ 'img_name > ' . $dbw->addQuotes( $row->img_name ) ];
 197                         wfWaitForSlaves();
 198                 } while ( $res->numRows() === $batchSize );
 199
 200                 $total = $upgraded + $leftAlone;
 201                 if ( $force ) {
 202                         $this->output( "\nFinished refreshing file metadata for $total files. "
 203                                 . "$upgraded needed to be refreshed, $leftAlone did not need to "
 204                                 . "be but were refreshed anyways, and $error refreshes were suspicious.\n" );
 205                 } else {
 206                         $this->output( "\nFinished refreshing file metadata for $total files. "
 207                                 . "$upgraded were refreshed, $leftAlone were already up to date, "
 208                                 . "and $error refreshes were suspicious.\n" );
 209                 }
 210         }
 211
 212         /**
 213          * @param IDatabase $dbw
 214          * @return array
 215          */
 216         function getConditions( $dbw ) {
 217                 $conds = [];
 218
 219                 $end = $this->getOption( 'end', false );
 220                 $mime = $this->getOption( 'mime', false );
 221                 $mediatype = $this->getOption( 'mediatype', false );
 222                 $like = $this->getOption( 'metadata-contains', false );
 223
 224                 if ( $end !== false ) {
 225                         $conds[] = 'img_name <= ' . $dbw->addQuotes( $end );
 226                 }
 227                 if ( $mime !== false ) {
 228                         list( $major, $minor ) = File::splitMime( $mime );
 229                         $conds['img_major_mime'] = $major;
 230                         if ( $minor !== '*' ) {
 231                                 $conds['img_minor_mime'] = $minor;
 232                         }
 233                 }
 234                 if ( $mediatype !== false ) {
 235                         $conds['img_media_type'] = $mediatype;
 236                 }
 237                 if ( $like ) {
 238                         $conds[] = 'img_metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() );
 239                 }
 240
 241                 return $conds;
 242         }
 243
 244         /**
 245          * @param bool $force
 246          * @param bool $brokenOnly
 247          */
 248         function setupParameters( $force, $brokenOnly ) {
 249                 global $wgUpdateCompatibleMetadata;
 250
 251                 if ( $brokenOnly ) {
 252                         $wgUpdateCompatibleMetadata = false;
 253                 } else {
 254                         $wgUpdateCompatibleMetadata = true;
 255                 }
 256
 257                 if ( $brokenOnly && $force ) {
 258                         $this->fatalError( 'Cannot use --broken-only and --force together. ', 2 );
 259                 }
 260         }
 261 }
 262
 263 $maintClass = RefreshImageMetadata::class;
 264 require_once RUN_MAINTENANCE_IF_MAIN;