Add a script to build an HHVM RepoAuthoritative bytecode file

This script compiles a list of known MW sources, including core and
extensions, and generates a RepoAuthoritative bytecode file. Some ideas
were taken from Bryan Davis's scap-hhvm-compile shell script.

Also, renamed the old maintenance/hiphop directory to maintenance/hhvm,
respecting the rename of the upstream project.

Change-Id: I55798729d0553d2840e8099e81d604a814e8664c

diff --git a/autoload.php b/autoload.php
index f6edd94..fc6577e 100644 (file)
--- a/autoload.php
+++ b/autoload.php
@@ -513,6 +513,7 @@ $wgAutoloadLocalClasses = [
        'GitInfo' => __DIR__ . '/includes/GitInfo.php',
        'GlobalDependency' => __DIR__ . '/includes/cache/CacheDependency.php',
        'GlobalVarConfig' => __DIR__ . '/includes/config/GlobalVarConfig.php',
+       'HHVMMakeRepo' => __DIR__ . '/maintenance/hhvm/makeRepo.php',
        'HTMLApiField' => __DIR__ . '/includes/htmlform/fields/HTMLApiField.php',
        'HTMLAutoCompleteSelectField' => __DIR__ . '/includes/htmlform/fields/HTMLAutoCompleteSelectField.php',
        'HTMLButtonField' => __DIR__ . '/includes/htmlform/fields/HTMLButtonField.php',

diff --git a/maintenance/Maintenance.php b/maintenance/Maintenance.php
index 8368aab..ab316c0 100644 (file)
--- a/maintenance/Maintenance.php
+++ b/maintenance/Maintenance.php
@@ -907,7 +907,7 @@ abstract class Maintenance {
 
                // Description ...
                if ( $this->mDescription ) {
-                       $this->output( "\n" . $this->mDescription . "\n" );
+                       $this->output( "\n" . wordwrap( $this->mDescription, $screenWidth ) . "\n" );
                }
                $output = "\nUsage: php " . basename( $this->mSelf );
 

diff --git a/maintenance/hhvm/makeRepo.php b/maintenance/hhvm/makeRepo.php
new file mode 100644 (file)
index 0000000..a0fe381
--- /dev/null
+++ b/maintenance/hhvm/makeRepo.php
@@ -0,0 +1,161 @@
+<?php
+
+require __DIR__ . '/../Maintenance.php';
+
+class HHVMMakeRepo extends Maintenance {
+       function __construct() {
+               parent::__construct();
+               $this->addDescription( 'Compile PHP sources for this MediaWiki instance, ' .
+                       'and generate an HHVM bytecode file to be used with HHVM\'s ' .
+                       'RepoAuthoritative mode. The MediaWiki core installation path and ' .
+                       'all registered extensions are automatically searched for the file ' .
+                       'extensions *.php, *.inc, *.php5 and *.phtml.' );
+               $this->addOption( 'output', 'Output filename', true, true, 'o' );
+               $this->addOption( 'input-dir', 'Add an input directory. ' .
+                       'This can be specified multiple times.', false, true, 'd', true );
+               $this->addOption( 'exclude-dir', 'Directory to exclude. ' .
+                       'This can be specified multiple times.', false, true, false, true );
+               $this->addOption( 'extension', 'Extra file extension', false, true, false, true );
+               $this->addOption( 'hhvm', 'Location of HHVM binary', false, true );
+               $this->addOption( 'base-dir', 'The root of all source files. ' .
+                       'This must match hhvm.server.source_root in the server\'s configuration file. ' .
+                       'By default, the MW core install path will be used.',
+                       false, true );
+               $this->addOption( 'verbose', 'Log level 0-3', false, true, 'v' );
+       }
+
+       private static function startsWith( $subject, $search ) {
+               return substr( $subject, 0, strlen( $search ) === $search );
+       }
+
+       function execute() {
+               global $wgExtensionCredits, $IP;
+
+               $dirs = [ $IP ];
+
+               foreach ( $wgExtensionCredits as $type => $extensions ) {
+                       foreach ( $extensions as $extension ) {
+                               if ( isset( $extension['path'] )
+                                       && !self::startsWith( $extension['path'], $IP )
+                               ) {
+                                       $dirs[] = dirname( $extension['path'] );
+                               }
+                       }
+               }
+
+               $dirs = array_merge( $dirs, $this->getOption( 'input-dir', [] ) );
+               $fileExts =
+                       [
+                               'php' => true,
+                               'inc' => true,
+                               'php5' => true,
+                               'phtml' => true
+                       ] +
+                       array_flip( $this->getOption( 'extension', [] ) );
+
+               $dirs = array_unique( $dirs );
+
+               $baseDir = $this->getOption( 'base-dir', $IP );
+               $excludeDirs = array_map( 'realpath', $this->getOption( 'exclude-dir', [] ) );
+
+               if ( $baseDir !== '' && substr( $baseDir, -1 ) !== '/' ) {
+                       $baseDir .= '/';
+               }
+
+               $unfilteredFiles = [ "$IP/LocalSettings.php" ];
+               foreach ( $dirs as $dir ) {
+                       $this->appendDir( $unfilteredFiles, $dir );
+               }
+
+               $files = [];
+               foreach ( $unfilteredFiles as $file ) {
+                       $dotPos = strrpos( $file, '.' );
+                       $slashPos = strrpos( $file, '/' );
+                       if ( $dotPos === false || $slashPos === false || $dotPos < $slashPos ) {
+                               continue;
+                       }
+                       $extension = substr( $file, $dotPos + 1 );
+                       if ( !isset( $fileExts[$extension] ) ) {
+                               continue;
+                       }
+                       $canonical = realpath( $file );
+                       foreach ( $excludeDirs as $excluded ) {
+                               if ( self::startsWith( $canonical, $excluded ) ) {
+                                       continue 2;
+                               }
+                       }
+                       if ( self::startsWith( $file, $baseDir ) ) {
+                               $file = substr( $file, strlen( $baseDir ) );
+                       }
+                       $files[] = $file;
+               }
+
+               $files = array_unique( $files );
+
+               print "Found " . count( $files ) . " files in " .
+                       count( $dirs ) . " directories\n";
+
+               $tmpDir = wfTempDir() . '/mw-make-repo' . mt_rand( 0, 1<<31 );
+               if ( !mkdir( $tmpDir ) ) {
+                       $this->error( 'Unable to create temporary directory', 1 );
+               }
+               file_put_contents( "$tmpDir/file-list", implode( "\n", $files ) );
+
+               $hhvm = $this->getOption( 'hhvm', 'hhvm' );
+               $verbose = $this->getOption( 'verbose', 3 );
+               $cmd = wfEscapeShellArg(
+                       $hhvm,
+                       '--hphp',
+                   '--target', 'hhbc',
+                       '--format', 'binary',
+                       '--force', '1',
+                       '--keep-tempdir', '1',
+                       '--log', $verbose,
+                       '-v', 'AllVolatile=true',
+                       '--input-dir', $baseDir,
+                       '--input-list', "$tmpDir/file-list",
+                       '--output-dir', $tmpDir );
+               print "$cmd\n";
+               passthru( $cmd, $ret );
+               if ( $ret ) {
+                       $this->cleanupTemp( $tmpDir );
+                       $this->error( "Error: HHVM returned error code $ret", 1 );
+               }
+               if ( !rename( "$tmpDir/hhvm.hhbc", $this->getOption( 'output' ) ) ) {
+                       $this->cleanupTemp( $tmpDir );
+                       $this->error( "Error: unable to rename output file", 1 );
+               }
+               $this->cleanupTemp( $tmpDir );
+               return 0;
+       }
+
+       private function cleanupTemp( $tmpDir ) {
+               if ( file_exists( "$tmpDir/hhvm.hhbc" ) ) {
+                       unlink( "$tmpDir/hhvm.hhbc" );
+               }
+               if ( file_exists( "$tmpDir/Stats.js" ) ) {
+                       unlink( "$tmpDir/Stats.js" );
+               }
+
+               unlink( "$tmpDir/file-list" );
+               rmdir( $tmpDir );
+       }
+
+       private function appendDir( &$files, $dir ) {
+               $iter = new RecursiveIteratorIterator(
+                       new RecursiveDirectoryIterator(
+                               $dir,
+                               FilesystemIterator::UNIX_PATHS
+                       ),
+                       RecursiveIteratorIterator::LEAVES_ONLY
+               );
+               foreach ( $iter as $file => $fileInfo ) {
+                       if ( $fileInfo->isFile() ) {
+                               $files[] = $file;
+                       }
+               }
+       }
+}
+
+$maintClass = 'HHVMMakeRepo';
+require RUN_MAINTENANCE_IF_MAIN;

diff --git a/maintenance/hhvm/run-server b/maintenance/hhvm/run-server
new file mode 100755 (executable)
index 0000000..2d71b87
--- /dev/null
+++ b/maintenance/hhvm/run-server
@@ -0,0 +1,28 @@
+#!/usr/bin/hhvm -f
+<?php
+
+require __DIR__ . '/../Maintenance.php';
+
+class RunHipHopServer extends Maintenance {
+       function __construct() {
+               parent::__construct();
+       }
+
+       function execute() {
+               global $IP;
+
+               passthru(
+                       'cd ' . wfEscapeShellArg( $IP ) . " && " .
+                       wfEscapeShellArg(
+                               'hhvm',
+                               '-c', __DIR__."/server.conf",
+                               '--mode=server',
+                               '--port=8080'
+                       ),
+                       $ret
+               );
+               exit( $ret );
+       }
+}
+$maintClass = 'RunHipHopServer';
+require_once RUN_MAINTENANCE_IF_MAIN;

diff --git a/maintenance/hhvm/server.conf b/maintenance/hhvm/server.conf
new file mode 100644 (file)
index 0000000..558bdad
--- /dev/null
+++ b/maintenance/hhvm/server.conf
@@ -0,0 +1,30 @@
+Log {
+       Level = Warning
+       UseLogFile = true
+       NativeStackTrace = true
+       InjectedStackTrace = true
+}
+Debug {
+       FullBacktrace = true
+       ServerStackTrace = true
+       ServerErrorMessage = true
+       TranslateSource = true
+}
+Server {
+       EnableStaticContentCache = false
+       EnableStaticContentFromDisk = true
+       AlwaysUseRelativePath = true
+}
+VirtualHost {
+       * {
+               ServerName = localhost
+               Pattern = .
+               RewriteRules {
+                       * {
+                               pattern = ^/wiki/(.*)$
+                               to = /index.php?title=$1
+                               qsa = true
+                       }
+               }
+       }
+}

diff --git a/maintenance/hiphop/run-server b/maintenance/hiphop/run-server
deleted file mode 100755 (executable)
index 2d71b87..0000000
--- a/maintenance/hiphop/run-server
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/hhvm -f
-<?php
-
-require __DIR__ . '/../Maintenance.php';
-
-class RunHipHopServer extends Maintenance {
-       function __construct() {
-               parent::__construct();
-       }
-
-       function execute() {
-               global $IP;
-
-               passthru(
-                       'cd ' . wfEscapeShellArg( $IP ) . " && " .
-                       wfEscapeShellArg(
-                               'hhvm',
-                               '-c', __DIR__."/server.conf",
-                               '--mode=server',
-                               '--port=8080'
-                       ),
-                       $ret
-               );
-               exit( $ret );
-       }
-}
-$maintClass = 'RunHipHopServer';
-require_once RUN_MAINTENANCE_IF_MAIN;

diff --git a/maintenance/hiphop/server.conf b/maintenance/hiphop/server.conf
deleted file mode 100644 (file)
index 558bdad..0000000
--- a/maintenance/hiphop/server.conf
+++ /dev/null
@@ -1,30 +0,0 @@
-Log {
-       Level = Warning
-       UseLogFile = true
-       NativeStackTrace = true
-       InjectedStackTrace = true
-}
-Debug {
-       FullBacktrace = true
-       ServerStackTrace = true
-       ServerErrorMessage = true
-       TranslateSource = true
-}
-Server {
-       EnableStaticContentCache = false
-       EnableStaticContentFromDisk = true
-       AlwaysUseRelativePath = true
-}
-VirtualHost {
-       * {
-               ServerName = localhost
-               Pattern = .
-               RewriteRules {
-                       * {
-                               pattern = ^/wiki/(.*)$
-                               to = /index.php?title=$1
-                               qsa = true
-                       }
-               }
-       }
-}