maintenance/purgeChangedFiles.php

   1 <?php
   2 /**
   3  * Scan the logging table and purge affected files within a timeframe.
   4  *
   5  * @section LICENSE
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along
  17  * with this program; if not, write to the Free Software Foundation, Inc.,
  18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19  * http://www.gnu.org/copyleft/gpl.html
  20  *
  21  * @file
  22  * @ingroup Maintenance
  23  */
  24
  25 require_once __DIR__ . '/Maintenance.php';
  26
  27 /**
  28  * Maintenance script that scans the deletion log and purges affected files
  29  * within a timeframe.
  30  *
  31  * @ingroup Maintenance
  32  */
  33 class PurgeChangedFiles extends Maintenance {
  34         /**
  35          * Mapping from type option to log type and actions.
  36          * @var array
  37          */
  38         private static $typeMappings = array(
  39                 'created' => array(
  40                         'upload' => array( 'upload' ),
  41                         'import' => array( 'upload', 'interwiki' ),
  42                 ),
  43                 'deleted' => array(
  44                         'delete' => array( 'delete', 'revision' ),
  45                         'suppress' => array( 'delete', 'revision' ),
  46                 ),
  47                 'modified' => array(
  48                         'upload' => array( 'overwrite', 'revert' ),
  49                         'move' => array( 'move', 'move_redir' ),
  50                 ),
  51         );
  52
  53         /**
  54          * @var string
  55          */
  56         private $startTimestamp;
  57
  58         /**
  59          * @var string
  60          */
  61         private $endTimestamp;
  62
  63         public function __construct() {
  64                 parent::__construct();
  65                 $this->mDescription = "Scan the logging table and purge files and thumbnails.";
  66                 $this->addOption( 'starttime', 'Starting timestamp', true, true );
  67                 $this->addOption( 'endtime', 'Ending timestamp', true, true );
  68                 $this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
  69                         implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
  70                 $this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
  71                 $this->addOption( 'dry-run', 'Do not send purge requests' );
  72                 $this->addOption( 'verbose', 'Show more output', false, false, 'v' );
  73         }
  74
  75         public function execute() {
  76                 global $wgHTCPRouting;
  77
  78                 if ( $this->hasOption( 'htcp-dest' ) ) {
  79                         $parts = explode( ':', $this->getOption( 'htcp-dest' ) );
  80                         if ( count( $parts ) < 2 ) {
  81                                 // Add default htcp port
  82                                 $parts[] = '4827';
  83                         }
  84
  85                         // Route all HTCP messages to provided host:port
  86                         $wgHTCPRouting = array(
  87                                 '' => array( 'host' => $parts[0], 'port' => $parts[1] ),
  88                         );
  89                         $this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
  90                 }
  91
  92                 // Find out which actions we should be concerned with
  93                 $typeOpt = $this->getOption( 'type', 'all' );
  94                 $validTypes = array_keys( self::$typeMappings );
  95                 if ( $typeOpt === 'all' ) {
  96                         // Convert 'all' to all registered types
  97                         $typeOpt = implode( ',', $validTypes );
  98                 }
  99                 $typeList = explode( ',', $typeOpt );
 100                 foreach ( $typeList as $type ) {
 101                         if ( !in_array( $type, $validTypes ) ) {
 102                                 $this->error( "\nERROR: Unknown type: {$type}\n" );
 103                                 $this->maybeHelp( true );
 104                         }
 105                 }
 106
 107                 // Validate the timestamps
 108                 $dbr = $this->getDB( DB_SLAVE );
 109                 $this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
 110                 $this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
 111
 112                 if ( $this->startTimestamp > $this->endTimestamp ) {
 113                         $this->error( "\nERROR: starttime after endtime\n" );
 114                         $this->maybeHelp( true );
 115                 }
 116
 117                 // Turn on verbose when dry-run is enabled
 118                 if ( $this->hasOption( 'dry-run' ) ) {
 119                         $this->mOptions['verbose'] = 1;
 120                 }
 121
 122                 $this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n");
 123                 foreach ( $typeList as $type ) {
 124                         $this->verbose( "Checking for {$type} files...\n" );
 125                         $this->purgeFromLogType( $type );
 126                         if ( !$this->hasOption( 'dry-run' ) ) {
 127                                 $this->verbose( "...{$type} files purged.\n\n" );
 128                         }
 129                 }
 130         }
 131
 132         /**
 133          * Purge cache and thumbnails for changes of the given type.
 134          *
 135          * @param string $type Type of change to find
 136          */
 137         protected function purgeFromLogType( $type ) {
 138                 $repo = RepoGroup::singleton()->getLocalRepo();
 139                 $dbr = $this->getDB( DB_SLAVE );
 140
 141                 foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
 142                         $this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
 143
 144                         $res = $dbr->select(
 145                                 'logging',
 146                                 array( 'log_title', 'log_timestamp', 'log_params' ),
 147                                 array(
 148                                         'log_namespace' => NS_FILE,
 149                                         'log_type' => $logType,
 150                                         'log_action' => $logActions,
 151                                         'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
 152                                         'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
 153                                 ),
 154                                 __METHOD__
 155                         );
 156
 157                         foreach ( $res as $row ) {
 158                                 $file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
 159
 160                                 if ( $this->hasOption( 'dry-run' ) ) {
 161                                         $this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
 162                                         continue;
 163                                 }
 164
 165                                 // Purge current version and any versions in oldimage table
 166                                 $file->purgeCache();
 167                                 $file->purgeHistory();
 168
 169                                 if ( $logType === 'delete' ) {
 170                                         // If there is an orphaned storage file... delete it
 171                                         if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
 172                                                 $dpath = $this->getDeletedPath( $repo, $file );
 173                                                 if ( $repo->fileExists( $dpath ) ) {
 174                                                         // Sanity check to avoid data loss
 175                                                         $repo->getBackend()->delete( array( 'src' => $file->getPath() ) );
 176                                                         $this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
 177
 178                                                 } else {
 179                                                         $this->error( "File was not deleted: {$file->getPath()}.\n" );
 180                                                 }
 181                                         }
 182
 183                                         // Purge items from fileachive table (rows are likely here)
 184                                         $this->purgeFromArchiveTable( $repo, $file );
 185
 186                                 } else if ( $logType === 'move' ) {
 187                                         // Purge the target file as well
 188
 189                                         $params = unserialize( $row->log_params );
 190                                         if ( isset( $params['4::target'] ) ) {
 191                                                 $target = $params['4::target'];
 192                                                 $targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
 193                                                 $targetFile->purgeCache();
 194                                                 $targetFile->purgeHistory();
 195                                                 $this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
 196                                         }
 197                                 }
 198
 199                                 $this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
 200                         }
 201                 }
 202         }
 203
 204         protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
 205                 $dbr = $repo->getSlaveDB();
 206                 $res = $dbr->select(
 207                         'filearchive',
 208                         array( 'fa_archive_name' ),
 209                         array( 'fa_name' => $file->getName() ),
 210                         __METHOD__
 211                 );
 212
 213                 foreach ( $res as $row ) {
 214                         if ( $row->fa_archive_name === null ) {
 215                                 // Was not an old version (current version names checked already)
 216                                 continue;
 217                         }
 218                         $ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
 219                         // If there is an orphaned storage file still there...delete it
 220                         if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
 221                                 $dpath = $this->getDeletedPath( $repo, $ofile );
 222                                 if ( $repo->fileExists( $dpath ) ) {
 223                                         // Sanity check to avoid data loss
 224                                         $repo->getBackend()->delete( array( 'src' => $ofile->getPath() ) );
 225                                         $this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
 226
 227                                 } else {
 228                                         $this->error( "File was not deleted: {$ofile->getPath()}.\n" );
 229                                 }
 230                         }
 231                         $file->purgeOldThumbnails( $row->fa_archive_name );
 232                 }
 233         }
 234
 235         protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
 236                 $hash = $repo->getFileSha1( $file->getPath() );
 237                 $key = "{$hash}.{$file->getExtension()}";
 238                 return $repo->getDeletedHashPath( $key ) . $key;
 239         }
 240
 241         /**
 242          * Send an output message iff the 'verbose' option has been provided.
 243          *
 244          * @param string $msg Message to output
 245          */
 246         protected function verbose( $msg ) {
 247                 if ( $this->hasOption( 'verbose' ) ) {
 248                         $this->output( $msg );
 249                 }
 250         }
 251
 252 }
 253
 254 $maintClass = "PurgeChangedFiles";
 255 require_once RUN_MAINTENANCE_IF_MAIN;