maintenance/resources/manageForeignResources.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Maintenance
  20  */
  21
  22 require_once __DIR__ . '/../Maintenance.php';
  23
  24 /**
  25  * Manage foreign resources registered with ResourceLoader.
  26  *
  27  * @ingroup Maintenance
  28  * @since 1.32
  29  */
  30 class ManageForeignResources extends Maintenance {
  31         private $defaultAlgo = 'sha384';
  32         private $tmpParentDir;
  33         private $action;
  34         private $failAfterOutput = false;
  35
  36         public function __construct() {
  37                 global $IP;
  38                 parent::__construct();
  39                 $this->addDescription( <<<TEXT
  40 Manage foreign resources registered with ResourceLoader.
  41
  42 This helps developers to download, verify and update local copies of upstream
  43 libraries registered as ResourceLoader modules. See also foreign-resources.yaml.
  44
  45 For sources that don't publish an integrity hash, omit "integrity" (or leave empty)
  46 and run the "make-sri" action to compute the missing hashes.
  47
  48 This script runs in dry-run mode by default. Use --update to actually change,
  49 remove, or add files to resources/lib/.
  50 TEXT
  51                 );
  52                 $this->addArg( 'action', 'One of "update", "verify" or "make-sri"', true );
  53                 $this->addArg( 'module', 'Name of a single module (Default: all)', false );
  54                 $this->addOption( 'verbose', 'Be verbose', false, false, 'v' );
  55
  56                 // Use a directory in $IP instead of wfTempDir() because
  57                 // PHP's rename() does not work across file systems.
  58                 $this->tmpParentDir = "{$IP}/resources/tmp";
  59         }
  60
  61         public function execute() {
  62                 global $IP;
  63                 $this->action = $this->getArg( 0 );
  64                 if ( !in_array( $this->action, [ 'update', 'verify', 'make-sri' ] ) ) {
  65                         $this->fatalError( "Invalid action argument." );
  66                 }
  67
  68                 $registry = $this->parseBasicYaml(
  69                         file_get_contents( __DIR__ . '/foreign-resources.yaml' )
  70                 );
  71                 $module = $this->getArg( 1, 'all' );
  72                 if ( $module === 'all' ) {
  73                         $modules = $registry;
  74                 } elseif ( isset( $registry[ $module ] ) ) {
  75                         $modules = [ $module => $registry[ $module ] ];
  76                 } else {
  77                         $this->fatalError( 'Unknown module name.' );
  78                 }
  79
  80                 foreach ( $modules as $moduleName => $info ) {
  81                         $this->verbose( "\n### {$moduleName}\n\n" );
  82                         $destDir = "{$IP}/resources/lib/$moduleName";
  83
  84                         if ( $this->action === 'update' ) {
  85                                 $this->output( "... updating '{$moduleName}'\n" );
  86                                 $this->verbose( "... emptying /resources/lib/$moduleName\n" );
  87                                 wfRecursiveRemoveDir( $destDir );
  88                         } elseif ( $this->action === 'verify' ) {
  89                                 $this->output( "... verifying '{$moduleName}'\n" );
  90                         } else {
  91                                 $this->output( "... checking '{$moduleName}'\n" );
  92                         }
  93
  94                         $this->verbose( "... preparing {$this->tmpParentDir}\n" );
  95                         wfRecursiveRemoveDir( $this->tmpParentDir );
  96                         if ( !wfMkdirParents( $this->tmpParentDir ) ) {
  97                                 $this->fatalError( "Unable to create {$this->tmpParentDir}" );
  98                         }
  99
 100                         if ( !isset( $info['type'] ) ) {
 101                                 $this->fatalError( "Module '$moduleName' must have a 'type' key." );
 102                         }
 103                         switch ( $info['type'] ) {
 104                                 case 'tar':
 105                                         $this->handleTypeTar( $moduleName, $destDir, $info );
 106                                         break;
 107                                 case 'file':
 108                                         $this->handleTypeFile( $moduleName, $destDir, $info );
 109                                         break;
 110                                 case 'multi-file':
 111                                         $this->handleTypeMultiFile( $moduleName, $destDir, $info );
 112                                         break;
 113                                 default:
 114                                         $this->fatalError( "Unknown type '{$info['type']}' for '$moduleName'" );
 115                         }
 116                 }
 117
 118                 $this->cleanUp();
 119                 $this->output( "\nDone!\n" );
 120                 if ( $this->failAfterOutput ) {
 121                         // The verify mode should check all modules/files and fail after, not during.
 122                         return false;
 123                 }
 124         }
 125
 126         private function fetch( $src, $integrity ) {
 127                 $data = Http::get( $src, [ 'followRedirects' => false ] );
 128                 if ( $data === false ) {
 129                         $this->fatalError( "Failed to download resource at {$src}" );
 130                 }
 131                 $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
 132                 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
 133                 if ( $integrity === $actualIntegrity ) {
 134                         $this->verbose( "... passed integrity check for {$src}\n" );
 135                 } else {
 136                         if ( $this->action === 'make-sri' ) {
 137                                 $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
 138                         } else {
 139                                 $this->fatalError( "Integrity check failed for {$src}\n" .
 140                                         "\tExpected: {$integrity}\n" .
 141                                         "\tActual: {$actualIntegrity}"
 142                                 );
 143                         }
 144                 }
 145                 return $data;
 146         }
 147
 148         private function handleTypeFile( $moduleName, $destDir, array $info ) {
 149                 if ( !isset( $info['src'] ) ) {
 150                         $this->fatalError( "Module '$moduleName' must have a 'src' key." );
 151                 }
 152                 $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
 153                 $dest = $info['dest'] ?? basename( $info['src'] );
 154                 $path = "$destDir/$dest";
 155                 if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 156                         $this->fatalError( "File for '$moduleName' is different." );
 157                 } elseif ( $this->action === 'update' ) {
 158                         wfMkdirParents( $destDir );
 159                         file_put_contents( "$destDir/$dest", $data );
 160                 }
 161         }
 162
 163         private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
 164                 if ( !isset( $info['files'] ) ) {
 165                         $this->fatalError( "Module '$moduleName' must have a 'files' key." );
 166                 }
 167                 foreach ( $info['files'] as $dest => $file ) {
 168                         if ( !isset( $file['src'] ) ) {
 169                                 $this->fatalError( "Module '$moduleName' file '$dest' must have a 'src' key." );
 170                         }
 171                         $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
 172                         $path = "$destDir/$dest";
 173                         if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 174                                 $this->fatalError( "File '$dest' for '$moduleName' is different." );
 175                         } elseif ( $this->action === 'update' ) {
 176                                 wfMkdirParents( $destDir );
 177                                 file_put_contents( "$destDir/$dest", $data );
 178                         }
 179                 }
 180         }
 181
 182         private function handleTypeTar( $moduleName, $destDir, array $info ) {
 183                 $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
 184                 if ( $info['src'] === null ) {
 185                         $this->fatalError( "Module '$moduleName' must have a 'src' key." );
 186                 }
 187                 // Download the resource to a temporary file and open it
 188                 $data = $this->fetch( $info['src'], $info['integrity' ] );
 189                 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
 190                 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
 191                 file_put_contents( $tmpFile, $data );
 192                 $p = new PharData( $tmpFile );
 193                 $tmpDir = "{$this->tmpParentDir}/$moduleName";
 194                 $p->extractTo( $tmpDir );
 195                 unset( $data, $p );
 196
 197                 if ( $info['dest'] === null ) {
 198                         // Default: Replace the entire directory
 199                         $toCopy = [ $tmpDir => $destDir ];
 200                 } else {
 201                         // Expand and normalise the 'dest' entries
 202                         $toCopy = [];
 203                         foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
 204                                 // Use glob() to expand wildcards and check existence
 205                                 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
 206                                 if ( !$fromPaths ) {
 207                                         $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." );
 208                                 }
 209                                 foreach ( $fromPaths as $fromPath ) {
 210                                         $toCopy[$fromPath] = $toSubPath === null
 211                                                 ? "$destDir/" . basename( $fromPath )
 212                                                 : "$destDir/$toSubPath/" . basename( $fromPath );
 213                                 }
 214                         }
 215                 }
 216                 foreach ( $toCopy as $from => $to ) {
 217                         if ( $this->action === 'verify' ) {
 218                                 $this->verbose( "... verifying $to\n" );
 219                                 if ( is_dir( $from ) ) {
 220                                         $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
 221                                                 $from,
 222                                                 RecursiveDirectoryIterator::SKIP_DOTS
 223                                         ) );
 224                                         foreach ( $rii as $file ) {
 225                                                 $remote = $file->getPathname();
 226                                                 $local = strtr( $remote, [ $from => $to ] );
 227                                                 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
 228                                                         $this->error( "File '$local' is different." );
 229                                                         $this->failAfterOutput = true;
 230                                                 }
 231                                         }
 232                                 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
 233                                         $this->error( "File '$to' is different." );
 234                                         $this->failAfterOutput = true;
 235                                 }
 236                         } elseif ( $this->action === 'update' ) {
 237                                 $this->verbose( "... moving $from to $to\n" );
 238                                 wfMkdirParents( dirname( $to ) );
 239                                 if ( !rename( $from, $to ) ) {
 240                                         $this->fatalError( "Could not move $from to $to." );
 241                                 }
 242                         }
 243                 }
 244         }
 245
 246         private function verbose( $text ) {
 247                 if ( $this->hasOption( 'verbose' ) ) {
 248                         $this->output( $text );
 249                 }
 250         }
 251
 252         private function cleanUp() {
 253                 wfRecursiveRemoveDir( $this->tmpParentDir );
 254         }
 255
 256         protected function fatalError( $msg, $exitCode = 1 ) {
 257                 $this->cleanUp();
 258                 parent::fatalError( $msg, $exitCode );
 259         }
 260
 261         /**
 262          * Basic YAML parser.
 263          *
 264          * Supports only string or object values, and 2 spaces indentation.
 265          *
 266          * @todo Just ship symfony/yaml.
 267          * @param string $input
 268          * @return array
 269          */
 270         private function parseBasicYaml( $input ) {
 271                 $lines = explode( "\n", $input );
 272                 $root = [];
 273                 $stack = [ &$root ];
 274                 $prev = 0;
 275                 foreach ( $lines as $i => $text ) {
 276                         $line = $i + 1;
 277                         $trimmed = ltrim( $text, ' ' );
 278                         if ( $trimmed === '' || $trimmed[0] === '#' ) {
 279                                 continue;
 280                         }
 281                         $indent = strlen( $text ) - strlen( $trimmed );
 282                         if ( $indent % 2 !== 0 ) {
 283                                 throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
 284                         }
 285                         $depth = $indent === 0 ? 0 : ( $indent / 2 );
 286                         if ( $depth < $prev ) {
 287                                 // Close previous branches we can't re-enter
 288                                 array_splice( $stack, $depth + 1 );
 289                         }
 290                         if ( !array_key_exists( $depth, $stack ) ) {
 291                                 throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
 292                         }
 293                         if ( strpos( $trimmed, ':' ) === false ) {
 294                                 throw new Exception( __METHOD__ . ": Missing colon on line $line." );
 295                         }
 296                         $dest =& $stack[ $depth ];
 297                         if ( $dest === null ) {
 298                                 // Promote from null to object
 299                                 $dest = [];
 300                         }
 301                         list( $key, $val ) = explode( ':', $trimmed, 2 );
 302                         $val = ltrim( $val, ' ' );
 303                         if ( $val !== '' ) {
 304                                 // Add string
 305                                 $dest[ $key ] = $val;
 306                         } else {
 307                                 // Add null (may become an object later)
 308                                 $val = null;
 309                                 $stack[] = &$val;
 310                                 $dest[ $key ] = &$val;
 311                         }
 312                         $prev = $depth;
 313                         unset( $dest, $val );
 314                 }
 315                 return $root;
 316         }
 317 }
 318
 319 $maintClass = ManageForeignResources::class;
 320 require_once RUN_MAINTENANCE_IF_MAIN;