maintenance/rebuildImages.php

   1 <?php
   2 /*
   3  * Script to update image metadata records
   4  *
   5  * Usage: php rebuildImages.php [--missing] [--dry-run]
   6  * Options:
   7  *   --missing  Crawl the uploads dir for images without records, and
   8  *              add them only.
   9  *
  10  * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
  11  * http://www.mediawiki.org/
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License along
  24  * with this program; if not, write to the Free Software Foundation, Inc.,
  25  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  26  * http://www.gnu.org/copyleft/gpl.html
  27  *
  28  * @author Brion Vibber <brion at pobox.com>
  29  * @addtogroup maintenance
  30  */
  31
  32 $options = array( 'missing', 'dry-run' );
  33
  34 require_once( 'commandLine.inc' );
  35 require_once( 'FiveUpgrade.inc' );
  36
  37 class ImageBuilder extends FiveUpgrade {
  38         function ImageBuilder( $dryrun = false ) {
  39                 parent::FiveUpgrade();
  40
  41                 $this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
  42                 $this->dryrun = $dryrun;
  43         }
  44
  45         function build() {
  46                 $this->buildImage();
  47                 $this->buildOldImage();
  48         }
  49
  50         function init( $count, $table ) {
  51                 $this->processed = 0;
  52                 $this->updated = 0;
  53                 $this->count = $count;
  54                 $this->startTime = wfTime();
  55                 $this->table = $table;
  56         }
  57
  58         function progress( $updated ) {
  59                 $this->updated += $updated;
  60                 $this->processed++;
  61                 if( $this->processed % 100 != 0 ) {
  62                         return;
  63                 }
  64                 $portion = $this->processed / $this->count;
  65                 $updateRate = $this->updated / $this->processed;
  66
  67                 $now = wfTime();
  68                 $delta = $now - $this->startTime;
  69                 $estimatedTotalTime = $delta / $portion;
  70                 $eta = $this->startTime + $estimatedTotalTime;
  71
  72                 printf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  73                         wfTimestamp( TS_DB, intval( $now ) ),
  74                         $portion * 100.0,
  75                         $this->table,
  76                         wfTimestamp( TS_DB, intval( $eta ) ),
  77                         $completed,   // $completed does not appear to be defined.
  78                         $this->count,
  79                         $rate,        // $rate does not appear to be defined.
  80                         $updateRate * 100.0 );
  81                 flush();
  82         }
  83
  84         function buildTable( $table, $key, $callback ) {
  85                 $fname = 'ImageBuilder::buildTable';
  86
  87                 $count = $this->dbw->selectField( $table, 'count(*)', '', $fname );
  88                 $this->init( $count, $table );
  89                 $this->log( "Processing $table..." );
  90
  91                 $tableName = $this->dbr->tableName( $table );
  92                 $sql = "SELECT * FROM $tableName";
  93                 $result = $this->dbr->query( $sql, $fname );
  94
  95                 while( $row = $this->dbr->fetchObject( $result ) ) {
  96                         $update = call_user_func( $callback, $row );
  97                         if( is_array( $update ) ) {
  98                                 if( !$this->dryrun ) {
  99                                         $this->dbw->update( $table,
 100                                                 $update,
 101                                                 array( $key => $row->$key ),
 102                                                 $fname );
 103                                 }
 104                                 $this->progress( 1 );
 105                         } else {
 106                                 $this->progress( 0 );
 107                         }
 108                 }
 109                 $this->log( "Finished $table... $this->updated of $this->processed rows updated" );
 110                 $this->dbr->freeResult( $result );
 111         }
 112
 113         function buildImage() {
 114                 $callback = array( &$this, 'imageCallback' );
 115                 $this->buildTable( 'image', 'img_name', $callback );
 116         }
 117
 118         function imageCallback( $row ) {
 119                 if( $row->img_width ) {
 120                         // Already processed
 121                         return null;
 122                 }
 123
 124                 // Fill in the new image info fields
 125                 $info = $this->imageInfo( $row->img_name );
 126
 127                 global $wgMemc;
 128                 $key = wfMemcKey( "Image", md5( $row->img_name ) );
 129                 $wgMemc->delete( $key );
 130
 131                 return array(
 132                         'img_width'      => $info['width'],
 133                         'img_height'     => $info['height'],
 134                         'img_bits'       => $info['bits'],
 135                         'img_media_type' => $info['media'],
 136                         'img_major_mime' => $info['major'],
 137                         'img_minor_mime' => $info['minor'] );
 138         }
 139
 140
 141         function buildOldImage() {
 142                 $this->buildTable( 'oldimage', 'oi_archive_name',
 143                         array( &$this, 'oldimageCallback' ) );
 144         }
 145
 146         function oldimageCallback( $row ) {
 147                 if( $row->oi_width ) {
 148                         return null;
 149                 }
 150
 151                 // Fill in the new image info fields
 152                 $info = $this->imageInfo( $row->oi_archive_name, 'wfImageArchiveDir', $row->oi_name );
 153                 return array(
 154                         'oi_width'  => $info['width' ],
 155                         'oi_height' => $info['height'],
 156                         'oi_bits'   => $info['bits'  ] );
 157         }
 158
 159         function crawlMissing() {
 160                 global $wgUploadDirectory, $wgHashedUploadDirectory;
 161                 if( $wgHashedUploadDirectory ) {
 162                         for( $i = 0; $i < 16; $i++ ) {
 163                                 for( $j = 0; $j < 16; $j++ ) {
 164                                         $dir = sprintf( '%s%s%01x%s%02x',
 165                                                 $wgUploadDirectory,
 166                                                 DIRECTORY_SEPARATOR,
 167                                                 $i,
 168                                                 DIRECTORY_SEPARATOR,
 169                                                 $i * 16 + $j );
 170                                         $this->crawlDirectory( $dir );
 171                                 }
 172                         }
 173                 } else {
 174                         $this->crawlDirectory( $wgUploadDirectory );
 175                 }
 176         }
 177
 178         function crawlDirectory( $dir ) {
 179                 if( !file_exists( $dir ) ) {
 180                         return $this->log( "no directory, skipping $dir" );
 181                 }
 182                 if( !is_dir( $dir ) ) {
 183                         return $this->log( "not a directory?! skipping $dir" );
 184                 }
 185                 if( !is_readable( $dir ) ) {
 186                         return $this->log( "dir not readable, skipping $dir" );
 187                 }
 188                 $source = opendir( $dir );
 189                 if( $source === false ) {
 190                         return $this->log( "couldn't open dir, skipping $dir" );
 191                 }
 192
 193                 $this->log( "crawling $dir" );
 194                 while( false !== ( $filename = readdir( $source ) ) ) {
 195                         $fullpath = $dir . DIRECTORY_SEPARATOR . $filename;
 196                         if( is_dir( $fullpath ) ) {
 197                                 continue;
 198                         }
 199                         if( is_link( $fullpath ) ) {
 200                                 $this->log( "skipping symlink at $fullpath" );
 201                                 continue;
 202                         }
 203                         $this->checkMissingImage( $filename, $fullpath );
 204                 }
 205                 closedir( $source );
 206         }
 207
 208         function checkMissingImage( $filename, $fullpath ) {
 209                 $fname = 'ImageBuilder::checkMissingImage';
 210                 $row = $this->dbw->selectRow( 'image',
 211                         array( 'img_name' ),
 212                         array( 'img_name' => $filename ),
 213                         $fname );
 214
 215                 if( $row ) {
 216                         // already known, move on
 217                         return;
 218                 } else {
 219                         $this->addMissingImage( $filename, $fullpath );
 220                 }
 221         }
 222
 223         function addMissingImage( $filename, $fullpath ) {
 224                 $fname = 'ImageBuilder::addMissingImage';
 225
 226                 $size = filesize( $fullpath );
 227                 $info = $this->imageInfo( $filename );
 228                 $timestamp = $this->dbw->timestamp( filemtime( $fullpath ) );
 229
 230                 global $wgContLang;
 231                 $altname = $wgContLang->checkTitleEncoding( $filename );
 232                 if( $altname != $filename ) {
 233                         if( $this->dryrun ) {
 234                                 $filename = $altname;
 235                                 $this->log( "Estimating transcoding... $altname" );
 236                         } else {
 237                                 $filename = $this->renameFile( $filename );
 238                         }
 239                 }
 240
 241                 if( $filename == '' ) {
 242                         $this->log( "Empty filename for $fullpath" );
 243                         return;
 244                 }
 245
 246                 $fields = array(
 247                         'img_name'       => $filename,
 248                         'img_size'       => $size,
 249                         'img_width'      => $info['width'],
 250                         'img_height'     => $info['height'],
 251                         'img_metadata'   => '', // filled in on-demand
 252                         'img_bits'       => $info['bits'],
 253                         'img_media_type' => $info['media'],
 254                         'img_major_mime' => $info['major'],
 255                         'img_minor_mime' => $info['minor'],
 256                         'img_description' => '(recovered file, missing upload log entry)',
 257                         'img_user'        => 0,
 258                         'img_user_text'   => 'Conversion script',
 259                         'img_timestamp'   => $timestamp );
 260                 if( !$this->dryrun ) {
 261                         $this->dbw->insert( 'image', $fields, $fname );
 262                 }
 263                 $this->log( $fullpath );
 264         }
 265 }
 266
 267 $builder = new ImageBuilder( isset( $options['dry-run'] ) );
 268 if( isset( $options['missing'] ) ) {
 269         $builder->crawlMissing();
 270 } else {
 271         $builder->build();
 272 }
 273
 274 ?>