maintenance/resources/manageForeignResources.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Maintenance
  20  */
  21
  22 require_once __DIR__ . '/../Maintenance.php';
  23
  24 /**
  25  * Manage foreign resources registered with ResourceLoader.
  26  *
  27  * @ingroup Maintenance
  28  * @since 1.32
  29  */
  30 class ManageForeignResources extends Maintenance {
  31         private $defaultAlgo = 'sha384';
  32         private $tmpParentDir;
  33         private $action;
  34         private $failAfterOutput = false;
  35
  36         public function __construct() {
  37                 global $IP;
  38                 parent::__construct();
  39                 $this->addDescription( <<<TEXT
  40 Manage foreign resources registered with ResourceLoader.
  41
  42 This helps developers to download, verify and update local copies of upstream
  43 libraries registered as ResourceLoader modules. See also foreign-resources.yaml.
  44
  45 For sources that don't publish an integrity hash, omit "integrity" (or leave empty)
  46 and run the "make-sri" action to compute the missing hashes.
  47
  48 This script runs in dry-run mode by default. Use --update to actually change,
  49 remove, or add files to resources/lib/.
  50 TEXT
  51                 );
  52                 $this->addArg( 'action', 'One of "update", "verify" or "make-sri"', true );
  53                 $this->addArg( 'module', 'Name of a single module (Default: all)', false );
  54                 $this->addOption( 'verbose', 'Be verbose', false, false, 'v' );
  55
  56                 // Use a directory in $IP instead of wfTempDir() because
  57                 // PHP's rename() does not work across file systems.
  58                 $this->tmpParentDir = "{$IP}/resources/tmp";
  59         }
  60
  61         public function execute() {
  62                 global $IP;
  63                 $this->action = $this->getArg( 0 );
  64                 if ( !in_array( $this->action, [ 'update', 'verify', 'make-sri' ] ) ) {
  65                         $this->fatalError( "Invalid action argument." );
  66                 }
  67
  68                 $registry = $this->parseBasicYaml(
  69                         file_get_contents( __DIR__ . '/foreign-resources.yaml' )
  70                 );
  71                 $module = $this->getArg( 1, 'all' );
  72                 foreach ( $registry as $moduleName => $info ) {
  73                         if ( $module !== 'all' && $moduleName !== $module ) {
  74                                 continue;
  75                         }
  76                         $this->verbose( "\n### {$moduleName}\n\n" );
  77                         $destDir = "{$IP}/resources/lib/$moduleName";
  78
  79                         if ( $this->action === 'update' ) {
  80                                 $this->output( "... updating '{$moduleName}'\n" );
  81                                 $this->verbose( "... emptying /resources/lib/$moduleName\n" );
  82                                 wfRecursiveRemoveDir( $destDir );
  83                         } elseif ( $this->action === 'verify' ) {
  84                                 $this->output( "... verifying '{$moduleName}'\n" );
  85                         } else {
  86                                 $this->output( "... checking '{$moduleName}'\n" );
  87                         }
  88
  89                         $this->verbose( "... preparing {$this->tmpParentDir}\n" );
  90                         wfRecursiveRemoveDir( $this->tmpParentDir );
  91                         if ( !wfMkdirParents( $this->tmpParentDir ) ) {
  92                                 $this->fatalError( "Unable to create {$this->tmpParentDir}" );
  93                         }
  94
  95                         if ( !isset( $info['type'] ) ) {
  96                                 $this->fatalError( "Module '$moduleName' must have a 'type' key." );
  97                         }
  98                         switch ( $info['type'] ) {
  99                                 case 'tar':
 100                                         $this->handleTypeTar( $moduleName, $destDir, $info );
 101                                         break;
 102                                 case 'file':
 103                                         $this->handleTypeFile( $moduleName, $destDir, $info );
 104                                         break;
 105                                 case 'multi-file':
 106                                         $this->handleTypeMultiFile( $moduleName, $destDir, $info );
 107                                         break;
 108                                 default:
 109                                         $this->fatalError( "Unknown type '{$info['type']}' for '$moduleName'" );
 110                         }
 111                 }
 112
 113                 $this->cleanUp();
 114                 $this->output( "\nDone!\n" );
 115                 if ( $this->failAfterOutput ) {
 116                         // The verify mode should check all modules/files and fail after, not during.
 117                         return false;
 118                 }
 119         }
 120
 121         private function fetch( $src, $integrity ) {
 122                 $data = Http::get( $src, [ 'followRedirects' => false ] );
 123                 if ( $data === false ) {
 124                         $this->fatalError( "Failed to download resource at {$src}" );
 125                 }
 126                 $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
 127                 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
 128                 if ( $integrity === $actualIntegrity ) {
 129                         $this->verbose( "... passed integrity check for {$src}\n" );
 130                 } else {
 131                         if ( $this->action === 'make-sri' ) {
 132                                 $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
 133                         } else {
 134                                 $this->fatalError( "Integrity check failed for {$src}\n" .
 135                                         "\tExpected: {$integrity}\n" .
 136                                         "\tActual: {$actualIntegrity}"
 137                                 );
 138                         }
 139                 }
 140                 return $data;
 141         }
 142
 143         private function handleTypeFile( $moduleName, $destDir, array $info ) {
 144                 if ( !isset( $info['src'] ) ) {
 145                         $this->fatalError( "Module '$moduleName' must have a 'src' key." );
 146                 }
 147                 $data = $this->fetch( $info['src'], $info['integrity'] ?? null );
 148                 $dest = $info['dest'] ?? basename( $info['src'] );
 149                 $path = "$destDir/$dest";
 150                 if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 151                         $this->fatalError( "File for '$moduleName' is different." );
 152                 } elseif ( $this->action === 'update' ) {
 153                         wfMkdirParents( $destDir );
 154                         file_put_contents( "$destDir/$dest", $data );
 155                 }
 156         }
 157
 158         private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
 159                 if ( !isset( $info['files'] ) ) {
 160                         $this->fatalError( "Module '$moduleName' must have a 'files' key." );
 161                 }
 162                 foreach ( $info['files'] as $dest => $file ) {
 163                         if ( !isset( $file['src'] ) ) {
 164                                 $this->fatalError( "Module '$moduleName' file '$dest' must have a 'src' key." );
 165                         }
 166                         $data = $this->fetch( $file['src'], $file['integrity'] ?? null );
 167                         $path = "$destDir/$dest";
 168                         if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
 169                                 $this->fatalError( "File '$dest' for '$moduleName' is different." );
 170                         } elseif ( $this->action === 'update' ) {
 171                                 wfMkdirParents( $destDir );
 172                                 file_put_contents( "$destDir/$dest", $data );
 173                         }
 174                 }
 175         }
 176
 177         private function handleTypeTar( $moduleName, $destDir, array $info ) {
 178                 $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
 179                 if ( $info['src'] === null ) {
 180                         $this->fatalError( "Module '$moduleName' must have a 'src' key." );
 181                 }
 182                 // Download the resource to a temporary file and open it
 183                 $data = $this->fetch( $info['src'], $info['integrity' ] );
 184                 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
 185                 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
 186                 file_put_contents( $tmpFile, $data );
 187                 $p = new PharData( $tmpFile );
 188                 $tmpDir = "{$this->tmpParentDir}/$moduleName";
 189                 $p->extractTo( $tmpDir );
 190                 unset( $data, $p );
 191
 192                 if ( $info['dest'] === null ) {
 193                         // Default: Replace the entire directory
 194                         $toCopy = [ $tmpDir => $destDir ];
 195                 } else {
 196                         // Expand and normalise the 'dest' entries
 197                         $toCopy = [];
 198                         foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
 199                                 // Use glob() to expand wildcards and check existence
 200                                 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
 201                                 if ( !$fromPaths ) {
 202                                         $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." );
 203                                 }
 204                                 foreach ( $fromPaths as $fromPath ) {
 205                                         $toCopy[$fromPath] = $toSubPath === null
 206                                                 ? "$destDir/" . basename( $fromPath )
 207                                                 : "$destDir/$toSubPath/" . basename( $fromPath );
 208                                 }
 209                         }
 210                 }
 211                 foreach ( $toCopy as $from => $to ) {
 212                         if ( $this->action === 'verify' ) {
 213                                 $this->verbose( "... verifying $to\n" );
 214                                 if ( is_dir( $from ) ) {
 215                                         $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
 216                                                 $from,
 217                                                 RecursiveDirectoryIterator::SKIP_DOTS
 218                                         ) );
 219                                         foreach ( $rii as $file ) {
 220                                                 $remote = $file->getPathname();
 221                                                 $local = strtr( $remote, [ $from => $to ] );
 222                                                 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
 223                                                         $this->error( "File '$local' is different." );
 224                                                         $this->failAfterOutput = true;
 225                                                 }
 226                                         }
 227                                 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
 228                                         $this->error( "File '$to' is different." );
 229                                         $this->failAfterOutput = true;
 230                                 }
 231                         } elseif ( $this->action === 'update' ) {
 232                                 $this->verbose( "... moving $from to $to\n" );
 233                                 wfMkdirParents( dirname( $to ) );
 234                                 if ( !rename( $from, $to ) ) {
 235                                         $this->fatalError( "Could not move $from to $to." );
 236                                 }
 237                         }
 238                 }
 239         }
 240
 241         private function verbose( $text ) {
 242                 if ( $this->hasOption( 'verbose' ) ) {
 243                         $this->output( $text );
 244                 }
 245         }
 246
 247         private function cleanUp() {
 248                 wfRecursiveRemoveDir( $this->tmpParentDir );
 249         }
 250
 251         protected function fatalError( $msg, $exitCode = 1 ) {
 252                 $this->cleanUp();
 253                 parent::fatalError( $msg, $exitCode );
 254         }
 255
 256         /**
 257          * Basic YAML parser.
 258          *
 259          * Supports only string or object values, and 2 spaces indentation.
 260          *
 261          * @todo Just ship symfony/yaml.
 262          * @param string $input
 263          * @return array
 264          */
 265         private function parseBasicYaml( $input ) {
 266                 $lines = explode( "\n", $input );
 267                 $root = [];
 268                 $stack = [ &$root ];
 269                 $prev = 0;
 270                 foreach ( $lines as $i => $text ) {
 271                         $line = $i + 1;
 272                         $trimmed = ltrim( $text, ' ' );
 273                         if ( $trimmed === '' || $trimmed[0] === '#' ) {
 274                                 continue;
 275                         }
 276                         $indent = strlen( $text ) - strlen( $trimmed );
 277                         if ( $indent % 2 !== 0 ) {
 278                                 throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
 279                         }
 280                         $depth = $indent === 0 ? 0 : ( $indent / 2 );
 281                         if ( $depth < $prev ) {
 282                                 // Close previous branches we can't re-enter
 283                                 array_splice( $stack, $depth + 1 );
 284                         }
 285                         if ( !array_key_exists( $depth, $stack ) ) {
 286                                 throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
 287                         }
 288                         if ( strpos( $trimmed, ':' ) === false ) {
 289                                 throw new Exception( __METHOD__ . ": Missing colon on line $line." );
 290                         }
 291                         $dest =& $stack[ $depth ];
 292                         if ( $dest === null ) {
 293                                 // Promote from null to object
 294                                 $dest = [];
 295                         }
 296                         list( $key, $val ) = explode( ':', $trimmed, 2 );
 297                         $val = ltrim( $val, ' ' );
 298                         if ( $val !== '' ) {
 299                                 // Add string
 300                                 $dest[ $key ] = $val;
 301                         } else {
 302                                 // Add null (may become an object later)
 303                                 $val = null;
 304                                 $stack[] = &$val;
 305                                 $dest[ $key ] = &$val;
 306                         }
 307                         $prev = $depth;
 308                         unset( $dest, $val );
 309                 }
 310                 return $root;
 311         }
 312 }
 313
 314 $maintClass = ManageForeignResources::class;
 315 require_once RUN_MAINTENANCE_IF_MAIN;