From dd8cced9ffacaecb6c623a1ec2b4e4890dea2042 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Sun, 19 Aug 2018 06:14:05 +0100 Subject: [PATCH] maintenance: Add maintenance script for managing foreign resources Something for the short-term, perhaps. But at least an improvement over 'update-oojs.sh' and 'update-ooui.sh'. * Does not require any dependencies (no 'node' or 'npm'). * Performs integrity validation. Change-Id: I0f79b84ef3903756353c66d3c3ee7e492c60e648 --- autoload.php | 1 + maintenance/resources/foreign-resources.yaml | 50 ++++ .../resources/manageForeignResources.php | 246 ++++++++++++++++++ maintenance/resources/update-oojs.sh | 62 ----- maintenance/resources/update-ooui.sh | 108 -------- 5 files changed, 297 insertions(+), 170 deletions(-) create mode 100644 maintenance/resources/foreign-resources.yaml create mode 100644 maintenance/resources/manageForeignResources.php delete mode 100755 maintenance/resources/update-oojs.sh delete mode 100755 maintenance/resources/update-ooui.sh diff --git a/autoload.php b/autoload.php index 999d82dba9..b8b3a80e61 100644 --- a/autoload.php +++ b/autoload.php @@ -842,6 +842,7 @@ $wgAutoloadLocalClasses = [ 'Maintenance' => __DIR__ . '/maintenance/Maintenance.php', 'MakeTestEdits' => __DIR__ . '/maintenance/makeTestEdits.php', 'MalformedTitleException' => __DIR__ . '/includes/title/MalformedTitleException.php', + 'ManageForeignResources' => __DIR__ . '/maintenance/resources/manageForeignResources.php', 'ManageJobs' => __DIR__ . '/maintenance/manageJobs.php', 'ManualLogEntry' => __DIR__ . '/includes/logging/LogEntry.php', 'MapCacheLRU' => __DIR__ . '/includes/libs/MapCacheLRU.php', diff --git a/maintenance/resources/foreign-resources.yaml b/maintenance/resources/foreign-resources.yaml new file mode 100644 index 0000000000..b8d9848358 --- /dev/null +++ b/maintenance/resources/foreign-resources.yaml @@ -0,0 +1,50 @@ +### Format of this file +# +# The top-level keys are module names (as registered in Resources.php). +# The values of these keys are resource descriptors. +# +# In each resource descriptor object, the `src` and `integrity` keys are required. +# +# * `src`: Full URL to a remote resource. +# * `integrity`: Cryptographic hash used to verify the remote content. +# Uses the "integrity metadata" format defined at . +# * `dest`: An object mapping paths from the remote resource to a destination in +# `/resources/lib/$module/`. The value may be omitted to indicate that +# paths should be extracted to the destination directory itself. +oojs: + src: https://registry.npmjs.org/oojs/-/oojs-2.2.2.tgz + integrity: sha256-ebgQW2EGrSkBCnDJBGqDpsBDjA3PMN/M8U5DyLHt9mw= + dest: + package/dist/oojs.jquery.js: + package/AUTHORS.txt: + package/LICENSE-MIT: + package/README.md: +oojs-ui: + src: https://registry.npmjs.org/oojs-ui/-/oojs-ui-0.28.0.tgz + integrity: sha384-j8bzlCPrfS4sca+U9JO9tdcewDlLlDlOVOsLn+Vqlcg5GU59vLSd7TVm4FiuTowy + dest: + # Main stuff + package/dist/oojs-ui-core.js{,.map.json}: + package/dist/oojs-ui-core-{wikimediaui,apex}.css: + package/dist/oojs-ui-widgets.js{,.map.json}: + package/dist/oojs-ui-widgets-{wikimediaui,apex}.css: + package/dist/oojs-ui-toolbars.js{,.map.json}: + package/dist/oojs-ui-toolbars-{wikimediaui,apex}.css: + package/dist/oojs-ui-windows.js{,.map.json}: + package/dist/oojs-ui-windows-{wikimediaui,apex}.css: + package/dist/oojs-ui-{wikimediaui,apex}.js{,.map.json}: + package/dist/i18n: + package/dist/images: + # WikimediaUI theme + package/dist/themes/wikimediaui/images/icons/*.{svg,png}: themes/wikimediaui/images/icons + package/dist/themes/wikimediaui/images/indicators/*.{svg,png}: themes/wikimediaui/images/indicators + package/dist/themes/wikimediaui/images/textures/*.{gif,svg}: themes/wikimediaui/images/textures + package/src/themes/wikimediaui/*.json: themes/wikimediaui + package/dist/wikimedia-ui-base.less: + # Apex theme (icons, indicators, and textures) + package/src/themes/apex/*.json: themes/apex + # Misc stuff + package/dist/AUTHORS.txt: + package/dist/History.md: + package/dist/LICENSE-MIT: + package/dist/README.md: diff --git a/maintenance/resources/manageForeignResources.php b/maintenance/resources/manageForeignResources.php new file mode 100644 index 0000000000..528d6e7c4c --- /dev/null +++ b/maintenance/resources/manageForeignResources.php @@ -0,0 +1,246 @@ +addDescription( <<addArg( 'module', 'Name of a single module (Default: all)', false ); + $this->addOption( 'update', ' resources/lib/ missing integrity metadata' ); + $this->addOption( 'make-sri', 'Compute missing integrity metadata' ); + $this->addOption( 'verbose', 'Be verbose' ); + + // Use a directory in $IP instead of wfTempDir() because + // PHP's rename() does not work across file systems. + $this->tmpParentDir = "{$IP}/resources/tmp"; + } + + public function execute() { + global $IP; + $module = $this->getArg(); + $makeSRI = $this->hasOption( 'make-sri' ); + + $registry = $this->parseBasicYaml( + file_get_contents( __DIR__ . '/foreign-resources.yaml' ) + ); + foreach ( $registry as $moduleName => $info ) { + if ( $module !== null && $moduleName !== $module ) { + continue; + } + $this->verbose( "\n### {$moduleName}\n\n" ); + + // Validate required keys + $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; + if ( $info['src'] === null ) { + $this->fatalError( "Module '$moduleName' must have a 'src' key." ); + } + $integrity = is_string( $info['integrity'] ) ? $info['integrity'] : $makeSRI; + if ( $integrity === false ) { + $this->fatalError( "Module '$moduleName' must have an 'integrity' key." ); + } + + // Download the resource + $data = Http::get( $info['src'], [ 'followRedirects' => false ] ); + if ( $data === false ) { + $this->fatalError( "Failed to download resource for '$moduleName'." ); + } + + // Validate integrity metadata + $this->output( "... checking integrity of '{$moduleName}'\n" ); + $algo = $integrity === true ? $this->defaultAlgo : explode( '-', $integrity )[0]; + $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); + if ( $integrity === true ) { + $this->output( "Integrity for '{$moduleName}':\n\t${actualIntegrity}\n" ); + continue; + } elseif ( $integrity !== $actualIntegrity ) { + $this->fatalError( "Integrity check failed for '{$moduleName}:\n" . + "Expected: {$integrity}\n" . + "Actual: {$actualIntegrity}" + ); + } + + // Determine destination + $destDir = "{$IP}/resources/lib/$moduleName"; + $this->output( "... extracting files for '{$moduleName}'\n" ); + $this->handleTypeTar( $moduleName, $data, $destDir, $info ); + } + + // Clean up + wfRecursiveRemoveDir( $this->tmpParentDir ); + $this->output( "\nDone!\n" ); + } + + private function handleTypeTar( $moduleName, $data, $destDir, array $info ) { + global $IP; + wfRecursiveRemoveDir( $this->tmpParentDir ); + if ( !wfMkdirParents( $this->tmpParentDir ) ) { + $this->fatalError( "Unable to create {$this->tmpParentDir}" ); + } + + // Write resource to temporary file and open it + $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; + $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); + file_put_contents( $tmpFile, $data ); + $p = new PharData( $tmpFile ); + $tmpDir = "{$this->tmpParentDir}/$moduleName"; + $p->extractTo( $tmpDir ); + unset( $data, $p ); + + if ( $info['dest'] === null ) { + // Replace the entire directory as-is + if ( !$this->hasOption( 'update' ) ) { + $this->output( "[dry run] Would replace /resources/lib/$moduleName\n" ); + } else { + wfRecursiveRemoveDir( $destDir ); + if ( !rename( $tmpDir, $destDir ) ) { + $this->fatalError( "Could not move $destDir to $tmpDir." ); + } + } + return; + } + + // Create and/or empty the destination + if ( !$this->hasOption( 'update' ) ) { + $this->output( "... [dry run] would empty /resources/lib/$moduleName\n" ); + } else { + wfRecursiveRemoveDir( $destDir ); + wfMkdirParents( $destDir ); + } + + // Expand and normalise the 'dest' entries + $toCopy = []; + foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { + // Use glob() to expand wildcards and check existence + $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); + if ( !$fromPaths ) { + $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." ); + } + foreach ( $fromPaths as $fromPath ) { + $toCopy[$fromPath] = $toSubPath === null + ? "$destDir/" . basename( $fromPath ) + : "$destDir/$toSubPath/" . basename( $fromPath ); + } + } + foreach ( $toCopy as $from => $to ) { + if ( !$this->hasOption( 'update' ) ) { + $shortFrom = strtr( $from, [ "$tmpDir/" => '' ] ); + $shortTo = strtr( $to, [ "$IP/" => '' ] ); + $this->output( "... [dry run] would move $shortFrom to $shortTo\n" ); + } else { + $this->verbose( "... moving $from to $to\n" ); + wfMkdirParents( dirname( $to ) ); + if ( !rename( $from, $to ) ) { + $this->fatalError( "Could not move $from to $to." ); + } + } + } + } + + private function verbose( $text ) { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( $text ); + } + } + + /** + * Basic YAML parser. + * + * Supports only string or object values, and 2 spaces indentation. + * + * @todo Just ship symfony/yaml. + * @param string $input + * @return array + */ + private function parseBasicYaml( $input ) { + $lines = explode( "\n", $input ); + $root = []; + $stack = [ &$root ]; + $prev = 0; + foreach ( $lines as $i => $text ) { + $line = $i + 1; + $trimmed = ltrim( $text, ' ' ); + if ( $trimmed === '' || $trimmed[0] === '#' ) { + continue; + } + $indent = strlen( $text ) - strlen( $trimmed ); + if ( $indent % 2 !== 0 ) { + throw new Exception( __METHOD__ . ": Odd indentation on line $line." ); + } + $depth = $indent === 0 ? 0 : ( $indent / 2 ); + if ( $depth < $prev ) { + // Close previous branches we can't re-enter + array_splice( $stack, $depth + 1 ); + } + if ( !array_key_exists( $depth, $stack ) ) { + throw new Exception( __METHOD__ . ": Too much indentation on line $line." ); + } + if ( strpos( $trimmed, ':' ) === false ) { + throw new Exception( __METHOD__ . ": Missing colon on line $line." ); + } + $dest =& $stack[ $depth ]; + if ( $dest === null ) { + // Promote from null to object + $dest = []; + } + list( $key, $val ) = explode( ':', $trimmed, 2 ); + $val = ltrim( $val, ' ' ); + if ( $val !== '' ) { + // Add string + $dest[ $key ] = $val; + } else { + // Add null (may become an object later) + $val = null; + $stack[] = &$val; + $dest[ $key ] = &$val; + } + $prev = $depth; + unset( $dest, $val ); + } + return $root; + } +} + +$maintClass = ManageForeignResources::class; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/resources/update-oojs.sh b/maintenance/resources/update-oojs.sh deleted file mode 100755 index fd7b860d1d..0000000000 --- a/maintenance/resources/update-oojs.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -eu - -# This script generates a commit that updates our copy of OOjs - -if [ -n "${2:-}" ] -then - # Too many parameters - echo >&2 "Usage: $0 []" - exit 1 -fi - -REPO_DIR=$(cd "$(dirname $0)/../.."; pwd) # Root dir of the git repo working tree -TARGET_DIR="resources/lib/oojs" # Destination relative to the root of the repo -NPM_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-oojs') # e.g. /tmp/update-oojs.rI0I5Vir - -# Prepare working tree -cd "$REPO_DIR" -git reset -- $TARGET_DIR -git checkout -- $TARGET_DIR -git fetch origin -git checkout -B upstream-oojs origin/master - -# Fetch upstream version -cd $NPM_DIR -if [ -n "${1:-}" ] -then - npm install "oojs@$1" -else - npm install oojs -fi - -OOJS_VERSION=$(node -e 'console.log(require("./node_modules/oojs/package.json").version);') -if [ "$OOJS_VERSION" == "" ] -then - echo 'Could not find OOjs version' - exit 1 -fi - -# Copy file(s) -rsync --force ./node_modules/oojs/dist/oojs.jquery.js "$REPO_DIR/$TARGET_DIR" -rsync --force ./node_modules/oojs/AUTHORS.txt "$REPO_DIR/$TARGET_DIR" -rsync --force ./node_modules/oojs/LICENSE-MIT "$REPO_DIR/$TARGET_DIR" -rsync --force ./node_modules/oojs/README.md "$REPO_DIR/$TARGET_DIR" - -# Clean up temporary area -rm -rf "$NPM_DIR" - -# Generate commit -cd $REPO_DIR - -COMMITMSG=$(cat <&2 "Usage: $0 []" - exit 1 -fi - -REPO_DIR=$(cd "$(dirname $0)/../.."; pwd) # Root dir of the git repo working tree -TARGET_DIR="resources/lib/oojs-ui" # Destination relative to the root of the repo -NPM_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'update-ooui') # e.g. /tmp/update-ooui.rI0I5Vir - -# Prepare working tree -cd "$REPO_DIR" -git reset composer.json -git checkout composer.json -git reset -- $TARGET_DIR -git checkout -- $TARGET_DIR -git fetch origin -git checkout -B upstream-ooui origin/master - -# Fetch upstream version -cd $NPM_DIR -if [ -n "${1:-}" ] -then - npm install "oojs-ui@$1" -else - npm install oojs-ui -fi - -OOUI_VERSION=$(node -e 'console.log(require("./node_modules/oojs-ui/package.json").version);') -if [ "$OOUI_VERSION" == "" ] -then - echo 'Could not find OOUI version' - exit 1 -fi - -# Copy files, picking the necessary ones from source and distribution -rm -r "$REPO_DIR/$TARGET_DIR" - -# Core and thematic code and styling -mkdir -p "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-core.js{,.map.json} "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-core-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-widgets.js{,.map.json} "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-widgets-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-toolbars.js{,.map.json} "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-toolbars-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-windows.js{,.map.json} "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-windows-{wikimediaui,apex}.css "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/oojs-ui-{wikimediaui,apex}.js{,.map.json} "$REPO_DIR/$TARGET_DIR" - -# i18n -mkdir -p "$REPO_DIR/$TARGET_DIR/i18n" -cp -R ./node_modules/oojs-ui/dist/i18n "$REPO_DIR/$TARGET_DIR" - -# Core images (currently two .cur files) -mkdir -p "$REPO_DIR/$TARGET_DIR/images" -cp -R ./node_modules/oojs-ui/dist/images "$REPO_DIR/$TARGET_DIR" - -# WikimediaUI theme icons, indicators, and textures -mkdir -p "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/icons" -cp ./node_modules/oojs-ui/dist/themes/wikimediaui/images/icons/*.{svg,png} "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/icons" -mkdir -p "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/indicators" -cp ./node_modules/oojs-ui/dist/themes/wikimediaui/images/indicators/*.{svg,png} "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/indicators" -mkdir -p "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/textures" -cp ./node_modules/oojs-ui/dist/themes/wikimediaui/images/textures/*.{gif,svg} "$REPO_DIR/$TARGET_DIR/themes/wikimediaui/images/textures" - -cp ./node_modules/oojs-ui/src/themes/wikimediaui/*.json "$REPO_DIR/$TARGET_DIR/themes/wikimediaui" - -# Apex theme icons, indicators, and textures -mkdir -p "$REPO_DIR/$TARGET_DIR/themes/apex" -cp ./node_modules/oojs-ui/src/themes/apex/*.json "$REPO_DIR/$TARGET_DIR/themes/apex" - -# WikimediaUI LESS variables for sharing -cp ./node_modules/oojs-ui/dist/wikimedia-ui-base.less "$REPO_DIR/$TARGET_DIR" - -# Misc stuff -cp ./node_modules/oojs-ui/dist/AUTHORS.txt "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/History.md "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/LICENSE-MIT "$REPO_DIR/$TARGET_DIR" -cp ./node_modules/oojs-ui/dist/README.md "$REPO_DIR/$TARGET_DIR" - -# Clean up temporary area -rm -rf "$NPM_DIR" - -# Generate commit -cd $REPO_DIR - -COMMITMSG=$(cat <