includes/ForeignResourceManager.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Maintenance
  20  */
  21
  22 use Wikimedia\AtEase\AtEase;
  23
  24 /**
  25  * Manage foreign resources registered with ResourceLoader.
  26  *
  27  * @since 1.32
  28  */
  29 class ForeignResourceManager {
  30         private $defaultAlgo = 'sha384';
  31         private $hasErrors = false;
  32         private $registryFile;
  33         private $libDir;
  34         private $tmpParentDir;
  35         private $cacheDir;
  36         private $infoPrinter;
  37         private $errorPrinter;
  38         private $verbosePrinter;
  39         private $action;
  40         private $registry;
  41
  42         /**
  43          * @param string $registryFile Path to YAML file
  44          * @param string $libDir Path to a modules directory
  45          * @param callable|null $infoPrinter Callback for printing info about the run.
  46          * @param callable|null $errorPrinter Callback for printing errors from the run.
  47          * @param callable|null $verbosePrinter Callback for printing extra verbose
  48          *  progress information from the run.
  49          */
  50         public function __construct(
  51                 $registryFile,
  52                 $libDir,
  53                 callable $infoPrinter = null,
  54                 callable $errorPrinter = null,
  55                 callable $verbosePrinter = null
  56         ) {
  57                 $this->registryFile = $registryFile;
  58                 $this->libDir = $libDir;
  59                 $this->infoPrinter = $infoPrinter ?? function () {
  60                 };
  61                 $this->errorPrinter = $errorPrinter ?? $this->infoPrinter;
  62                 $this->verbosePrinter = $verbosePrinter ?? function () {
  63                 };
  64
  65                 // Use a temporary directory under the destination directory instead
  66                 // of wfTempDir() because PHP's rename() does not work across file
  67                 // systems, and the user's /tmp and $IP may be on different filesystems.
  68                 $this->tmpParentDir = "{$this->libDir}/.foreign/tmp";
  69
  70                 $cacheHome = getenv( 'XDG_CACHE_HOME' ) ? realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
  71                 $this->cacheDir = $cacheHome ? "$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
  72         }
  73
  74         /**
  75          * @return bool
  76          * @throws Exception
  77          */
  78         public function run( $action, $module ) {
  79                 $actions = [ 'update', 'verify', 'make-sri' ];
  80                 if ( !in_array( $action, $actions ) ) {
  81                         $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
  82                         return false;
  83                 }
  84                 $this->action = $action;
  85
  86                 $this->registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) );
  87                 if ( $module === 'all' ) {
  88                         $modules = $this->registry;
  89                 } elseif ( isset( $this->registry[ $module ] ) ) {
  90                         $modules = [ $module => $this->registry[ $module ] ];
  91                 } else {
  92                         $this->error( "Unknown module name.\n\nMust be one of:\n" .
  93                                 wordwrap( implode( ', ', array_keys( $this->registry ) ), 80 ) .
  94                                 '.'
  95                         );
  96                         return false;
  97                 }
  98
  99                 foreach ( $modules as $moduleName => $info ) {
 100                         $this->verbose( "\n### {$moduleName}\n\n" );
 101                         $destDir = "{$this->libDir}/$moduleName";
 102
 103                         if ( $this->action === 'update' ) {
 104                                 $this->output( "... updating '{$moduleName}'\n" );
 105                                 $this->verbose( "... emptying directory for $moduleName\n" );
 106                                 wfRecursiveRemoveDir( $destDir );
 107                         } elseif ( $this->action === 'verify' ) {
 108                                 $this->output( "... verifying '{$moduleName}'\n" );
 109                         } else {
 110                                 $this->output( "... checking '{$moduleName}'\n" );
 111                         }
 112
 113                         $this->verbose( "... preparing {$this->tmpParentDir}\n" );
 114                         wfRecursiveRemoveDir( $this->tmpParentDir );
 115                         if ( !wfMkdirParents( $this->tmpParentDir ) ) {
 116                                 throw new Exception( "Unable to create {$this->tmpParentDir}" );
 117                         }
 118
 119                         if ( !isset( $info['type'] ) ) {
 120                                 throw new Exception( "Module '$moduleName' must have a 'type' key." );
 121                         }
 122                         switch ( $info['type'] ) {
 123                                 case 'tar':
 124                                         $this->handleTypeTar( $moduleName, $destDir, $info );
 125                                         break;
 126                                 case 'file':
 127                                         $this->handleTypeFile( $moduleName, $destDir, $info );
 128                                         break;
 129                                 case 'multi-file':
 130                                         $this->handleTypeMultiFile( $moduleName, $destDir, $info );
 131                                         break;
 132                                 default:
 133                                         throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
 134                         }
 135                 }
 136
 137                 $this->output( "\nDone!\n" );
 138                 $this->cleanUp();
 139                 if ( $this->hasErrors ) {
 140                         // The verify mode should check all modules/files and fail after, not during.
 141                         return false;
 142                 }
 143
 144                 return true;
 145         }
 146
 147         private function cacheKey( $src, $integrity ) {
 148                 $key = basename( $src ) . '_' . substr( $integrity, -12 );
 149                 $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
 150                 return rtrim( $key, '_' );
 151         }
 152
 153         /** @return string|false */
 154         private function cacheGet( $key ) {
 155                 return AtEase::quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
 156         }
 157
 158         private function cacheSet( $key, $data ) {
 159                 wfMkdirParents( $this->cacheDir );
 160                 file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX );
 161         }
 162
 163         private function fetch( $src, $integrity ) {
 164                 $key = $this->cacheKey( $src, $integrity );
 165                 $data = $this->cacheGet( $key );
 166                 if ( $data ) {
 167                         return $data;
 168                 }
 169
 170                 $req = MWHttpRequest::factory( $src, [ 'method' => 'GET', 'followRedirects' => false ] );
 171                 if ( !$req->execute()->isOK() ) {
 172                         throw new Exception( "Failed to download resource at {$src}" );
 173                 }
 174                 if ( $req->getStatus() !== 200 ) {
 175                         throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
 176                 }
 177                 $data = $req->getContent();
 178                 $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
 179                 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
 180                 if ( $integrity === $actualIntegrity ) {
 181                         $this->verbose( "... passed integrity check for {$src}\n" );
 182                         $this->cacheSet( $key, $data );
 183                 } elseif ( $this->action === 'make-sri' ) {
 184                         $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
 185                 } else {
 186                         throw new Exception( "Integrity check failed for {$src}\n" .
 187                                 "\tExpected: {$integrity}\n" .
 188                                 "\tActual: {$actualIntegrity}"
 189                         );
 190                 }
 191                 return $data;
 192         }
 193
 194         private function handleTypeFile( $moduleName, $destDir, array $info ) {
 195                 if ( !isset( $info['src'] ) ) {
 196                         throw new Exception( "Module '$moduleName' must have a 'src' key." );
 197                 }
 198                 $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
 199                 $dest = $info['dest'] ?? basename( $info['src'] );
 200                 $path = "$destDir/$dest";
 201                 if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 202                         throw new Exception( "File for '$moduleName' is different." );
 203                 }
 204                 if ( $this->action === 'update' ) {
 205                         wfMkdirParents( $destDir );
 206                         file_put_contents( "$destDir/$dest", $data );
 207                 }
 208         }
 209
 210         private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
 211                 if ( !isset( $info['files'] ) ) {
 212                         throw new Exception( "Module '$moduleName' must have a 'files' key." );
 213                 }
 214                 foreach ( $info['files'] as $dest => $file ) {
 215                         if ( !isset( $file['src'] ) ) {
 216                                 throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
 217                         }
 218                         $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
 219                         $path = "$destDir/$dest";
 220                         if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 221                                 throw new Exception( "File '$dest' for '$moduleName' is different." );
 222                         } elseif ( $this->action === 'update' ) {
 223                                 wfMkdirParents( $destDir );
 224                                 file_put_contents( "$destDir/$dest", $data );
 225                         }
 226                 }
 227         }
 228
 229         private function handleTypeTar( $moduleName, $destDir, array $info ) {
 230                 $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
 231                 if ( $info['src'] === null ) {
 232                         throw new Exception( "Module '$moduleName' must have a 'src' key." );
 233                 }
 234                 // Download the resource to a temporary file and open it
 235                 $data = $this->fetch( $info['src'], $info['integrity' ] );
 236                 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
 237                 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
 238                 file_put_contents( $tmpFile, $data );
 239                 $p = new PharData( $tmpFile );
 240                 $tmpDir = "{$this->tmpParentDir}/$moduleName";
 241                 $p->extractTo( $tmpDir );
 242                 unset( $data, $p );
 243
 244                 if ( $info['dest'] === null ) {
 245                         // Default: Replace the entire directory
 246                         $toCopy = [ $tmpDir => $destDir ];
 247                 } else {
 248                         // Expand and normalise the 'dest' entries
 249                         $toCopy = [];
 250                         foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
 251                                 // Use glob() to expand wildcards and check existence
 252                                 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
 253                                 if ( !$fromPaths ) {
 254                                         throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
 255                                 }
 256                                 foreach ( $fromPaths as $fromPath ) {
 257                                         $toCopy[$fromPath] = $toSubPath === null
 258                                                 ? "$destDir/" . basename( $fromPath )
 259                                                 : "$destDir/$toSubPath/" . basename( $fromPath );
 260                                 }
 261                         }
 262                 }
 263                 foreach ( $toCopy as $from => $to ) {
 264                         if ( $this->action === 'verify' ) {
 265                                 $this->verbose( "... verifying $to\n" );
 266                                 if ( is_dir( $from ) ) {
 267                                         $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
 268                                                 $from,
 269                                                 RecursiveDirectoryIterator::SKIP_DOTS
 270                                         ) );
 271                                         /** @var SplFileInfo $file */
 272                                         foreach ( $rii as $file ) {
 273                                                 $remote = $file->getPathname();
 274                                                 $local = strtr( $remote, [ $from => $to ] );
 275                                                 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
 276                                                         $this->error( "File '$local' is different." );
 277                                                         $this->hasErrors = true;
 278                                                 }
 279                                         }
 280                                 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
 281                                         $this->error( "File '$to' is different." );
 282                                         $this->hasErrors = true;
 283                                 }
 284                         } elseif ( $this->action === 'update' ) {
 285                                 $this->verbose( "... moving $from to $to\n" );
 286                                 wfMkdirParents( dirname( $to ) );
 287                                 if ( !rename( $from, $to ) ) {
 288                                         throw new Exception( "Could not move $from to $to." );
 289                                 }
 290                         }
 291                 }
 292         }
 293
 294         private function verbose( $text ) {
 295                 ( $this->verbosePrinter )( $text );
 296         }
 297
 298         private function output( $text ) {
 299                 ( $this->infoPrinter )( $text );
 300         }
 301
 302         private function error( $text ) {
 303                 ( $this->errorPrinter )( $text );
 304         }
 305
 306         private function cleanUp() {
 307                 wfRecursiveRemoveDir( $this->tmpParentDir );
 308
 309                 // Prune the cache of files we don't recognise.
 310                 $knownKeys = [];
 311                 foreach ( $this->registry as $info ) {
 312                         if ( $info['type'] === 'file' || $info['type'] === 'tar' ) {
 313                                 $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
 314                         } elseif ( $info['type'] === 'multi-file' ) {
 315                                 foreach ( $info['files'] as $file ) {
 316                                         $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
 317                                 }
 318                         }
 319                 }
 320                 foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
 321                         if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
 322                                 unlink( $cacheFile );
 323                         }
 324                 }
 325         }
 326
 327         /**
 328          * Basic YAML parser.
 329          *
 330          * Supports only string or object values, and 2 spaces indentation.
 331          *
 332          * @todo Just ship symfony/yaml.
 333          * @param string $input
 334          * @return array
 335          */
 336         private function parseBasicYaml( $input ) {
 337                 $lines = explode( "\n", $input );
 338                 $root = [];
 339                 $stack = [ &$root ];
 340                 $prev = 0;
 341                 foreach ( $lines as $i => $text ) {
 342                         $line = $i + 1;
 343                         $trimmed = ltrim( $text, ' ' );
 344                         if ( $trimmed === '' || $trimmed[0] === '#' ) {
 345                                 continue;
 346                         }
 347                         $indent = strlen( $text ) - strlen( $trimmed );
 348                         if ( $indent % 2 !== 0 ) {
 349                                 throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
 350                         }
 351                         $depth = $indent === 0 ? 0 : ( $indent / 2 );
 352                         if ( $depth < $prev ) {
 353                                 // Close previous branches we can't re-enter
 354                                 array_splice( $stack, $depth + 1 );
 355                         }
 356                         if ( !array_key_exists( $depth, $stack ) ) {
 357                                 throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
 358                         }
 359                         if ( strpos( $trimmed, ':' ) === false ) {
 360                                 throw new Exception( __METHOD__ . ": Missing colon on line $line." );
 361                         }
 362                         $dest =& $stack[ $depth ];
 363                         if ( $dest === null ) {
 364                                 // Promote from null to object
 365                                 $dest = [];
 366                         }
 367                         list( $key, $val ) = explode( ':', $trimmed, 2 );
 368                         $val = ltrim( $val, ' ' );
 369                         if ( $val !== '' ) {
 370                                 // Add string
 371                                 $dest[ $key ] = $val;
 372                         } else {
 373                                 // Add null (may become an object later)
 374                                 $val = null;
 375                                 $stack[] = &$val;
 376                                 $dest[ $key ] = &$val;
 377                         }
 378                         $prev = $depth;
 379                         unset( $dest, $val );
 380                 }
 381                 return $root;
 382         }
 383 }