maintenance/hiphop/make

   1 #!/usr/bin/hphpi -f
   2 <?php
   3
   4 define( 'MW_CONFIG_CALLBACK', 'MakeHipHop::noConfigNeeded' );
   5 require( dirname( __FILE__ ) . '/../Maintenance.php' );
   6
   7 class MakeHipHop extends Maintenance {
   8         function noConfigNeeded() {}
   9
  10         function execute() {
  11                 global $wgHipHopBuildDirectory;
  12
  13                 $startTime = time();
  14
  15                 $thisDir = realpath( dirname( __FILE__ ) );
  16                 $IP = realpath( "$thisDir/../.." );
  17                 if ( strval( $wgHipHopBuildDirectory ) !== '' ) {
  18                         $buildDir = $wgHipHopBuildDirectory;
  19                 } else {
  20                         $buildDir = "$thisDir/build";
  21                 }
  22                 $extensionsDir = realpath( MWInit::getExtensionsDirectory() );
  23                 $outDir = "$buildDir/hiphop-output";
  24                 $persistentDir = "$buildDir/persistent";
  25
  26                 if ( !is_dir( $buildDir ) ) {
  27                         mkdir( $buildDir, 0777, true );
  28                 }
  29                 if ( !is_dir( $persistentDir ) ) {
  30                         mkdir( $persistentDir, 0777, true );
  31                 }
  32
  33                 if ( realpath( "$IP/../phase3" ) !== $IP
  34                         || realpath( "$IP/../extensions" ) !== $extensionsDir )
  35                 {
  36                         # Set up a fake source directory with the correct layout
  37                         $sourceBase = "$buildDir/source";
  38                         $this->setupFakeSourceBase( $IP, $extensionsDir, $sourceBase );
  39                 } else {
  40                         $sourceBase = realpath( "$IP/.." );
  41                         unlink( "$buildDir/source" );
  42                 }
  43
  44                 # With the CentOS RPMs, you just get g++44, no g++, so we have to
  45                 # use the environment
  46                 if ( isset( $_ENV['CXX'] ) ) {
  47                         $cxx = $_ENV['CXX'];
  48                 } else {
  49                         $cxx = 'g++';
  50                 }
  51
  52                 # Create a function that provides the HipHop compiler version, and
  53                 # doesn't exist when MediaWiki is invoked in interpreter mode.
  54                 $version = str_replace( PHP_EOL, ' ', trim( `hphp --version` ) );
  55                 file_put_contents(
  56                         "$buildDir/HipHopCompilerVersion.php",
  57                         "<" . "?php\n" .
  58                         "function wfHipHopCompilerVersion() {\n" .
  59                         "return " . var_export( $version, true ) . ";\n" .
  60                         "}\n"
  61                 );
  62
  63                 # Generate the file list
  64                 $files = $this->getFileList();
  65                 file_put_contents(
  66                         "$buildDir/file-list",
  67                         implode( "\n", $files ) . "\n" );
  68
  69                 # Generate the C++
  70                 passthru(
  71                         'hphp' .
  72                         ' --target=cpp' .
  73                         ' --format=file' .
  74                         ' --input-dir=' . wfEscapeShellArg( $sourceBase ) .
  75                         ' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) .
  76                         ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
  77                         ' -c ' . wfEscapeShellArg( "$thisDir/compiler.conf" ) .
  78                         ' --parse-on-demand=false' .
  79                         ' --program=mediawiki-hphp' .
  80                         ' --output-dir=' . wfEscapeShellArg( $outDir ) .
  81                         ' --log=3', $ret );
  82
  83                 if ( $ret ) {
  84                         $this->error( "hphp hit an error. Stopping build.\n" );
  85                         exit( 1 );
  86                 }
  87
  88                 # Sanity check, quickly make sure we've got an output directory
  89                 if( !is_dir( $outDir ) ) {
  90                         $this->error( "No output directory", true );
  91                 }
  92
  93                 # Warn about volatile classes
  94                 $this->checkVolatileClasses( $outDir );
  95
  96                 # Copy the generated C++ files into the source directory for cmake
  97                 $iter = new RecursiveIteratorIterator(
  98                         new RecursiveDirectoryIterator( $outDir ),
  99                         RecursiveIteratorIterator::SELF_FIRST );
 100                 $sourceFiles = array();
 101                 $regenerateMakefile = false;
 102                 $numFiles = 0;
 103                 $numFilesChanged = 0;
 104                 foreach ( $iter as $sourcePath => $file ) {
 105                         $name = substr( $sourcePath, strlen( $outDir ) + 1 );
 106                         $sourceFiles[$name] = true;
 107                         $destPath = "$persistentDir/$name";
 108                         if ( $file->isDir() ) {
 109                                 if ( !is_dir( $destPath ) ) {
 110                                         mkdir( $destPath );
 111                                 }
 112                                 continue;
 113                         }
 114
 115                         $numFiles++;
 116                         # Remove any files that weren't touched, these may have been removed
 117                         # from file-list, we should not compile them
 118                         if ( $file->getMTime() < $startTime ) {
 119                                 if ( file_exists( $destPath ) ) {
 120                                         unlink( $destPath );
 121                                         # Files removed, regenerate the makefile
 122                                         $regenerateMakefile = true;
 123                                 }
 124                                 unlink( $sourcePath );
 125                                 $numFilesChanged++;
 126                                 continue;
 127                         }
 128
 129                         if ( file_exists( $destPath ) ) {
 130                                 $sourceHash = md5( file_get_contents( $sourcePath ) );
 131                                 $destHash = md5( file_get_contents( $destPath ) );
 132                                 if ( $sourceHash == $destHash ) {
 133                                         continue;
 134                                 }
 135                         } else {
 136                                 # New files added, regenerate the makefile
 137                                 $regenerateMakefile = true;
 138                         }
 139                         $numFilesChanged++;
 140                         copy( $sourcePath, $destPath );
 141                 }
 142
 143                 echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n";
 144
 145                 if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) {
 146                         # Run cmake for the first time
 147                         $regenerateMakefile = true;
 148                 }
 149
 150                 # Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken.
 151                 # HipHop's RELEASE mode seems to be stuck always on, so symbols get
 152                 # stripped. Also we will try keeping the generated .o files instead of
 153                 # throwing away hours of CPU time every time you make a typo.
 154
 155                 chdir( $persistentDir );
 156
 157                 if ( $regenerateMakefile ) {
 158                         copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt',
 159                                 "$persistentDir/CMakeLists.txt" );
 160
 161                         if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) {
 162                                 unlink( "$persistentDir/CMakeCache.txt" );
 163                         }
 164
 165                         $cmd = 'cmake' .
 166                                 " -D CMAKE_BUILD_TYPE:string=" . wfEscapeShellArg( $GLOBALS['wgHipHopBuildType'] ) .
 167                                 ' -D PROGRAM_NAME:string=mediawiki-hphp';
 168
 169                         if ( file_exists( '/usr/bin/ccache' ) ) {
 170                                 $cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' .
 171                                         ' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx );
 172                         }
 173
 174                         $cmd .= ' .';
 175                         echo "$cmd\n";
 176                         passthru( $cmd );
 177                 }
 178
 179                 # Determine appropriate make concurrency
 180                 # Compilation can take a lot of memory, let's assume that that is limiting.
 181                 $procs = $this->getNumProcs();
 182
 183                 # Run make. This is the slow step.
 184                 passthru( 'make -j' . wfEscapeShellArg( $procs ) );
 185
 186                 $elapsed = time() - $startTime;
 187
 188                 echo "Completed in ";
 189                 if ( $elapsed >= 3600 ) {
 190                         $hours = floor( $elapsed / 3600 );
 191                         echo $hours . 'h ';
 192                         $elapsed -= $hours * 3600;
 193                 }
 194                 if ( $elapsed >= 60 ) {
 195                         $minutes = floor( $elapsed / 60 );
 196                         echo $minutes . 'm ';
 197                         $elapsed -= $minutes * 60;
 198                 }
 199                 echo $elapsed . "s\n";
 200                 echo "The MediaWiki executable is at $buildDir/persistent/mediawiki-hphp\n";
 201         }
 202
 203         function checkVolatileClasses( $dir ) {
 204                 $lines = file( "$dir/sys/dynamic_table_class.cpp" );
 205                 $classes = array();
 206                 foreach ( $lines as $line ) {
 207                         if ( preg_match( '/^\s+\(const char \*\)"([^"]*)", \(const char \*\)-1/', $line, $m ) ) {
 208                                 $classes[] = $m[1];
 209                         }
 210                 }
 211                 if ( !count( $classes ) ) {
 212                         print "No volatile classes found\n";
 213                         return;
 214                 }
 215                 sort( $classes );
 216                 $classes = array_unique( $classes );
 217                 print "WARNING: The following classes are volatile: " . implode( ', ', $classes ) . "\n";
 218         }
 219
 220         function getNumProcs() {
 221                 global $wgHipHopCompilerProcs;
 222                 if ( $wgHipHopCompilerProcs !== 'detect' ) {
 223                         return intval( $wgHipHopCompilerProcs );
 224                 }
 225
 226                 if ( !file_exists( '/proc/meminfo' ) ) {
 227                         return 1;
 228                 }
 229                 $mem = false;
 230                 foreach ( file( '/proc/meminfo' ) as $line ) {
 231                         if ( preg_match( '/^MemTotal:\s+(\d+)\s+kB/', $line, $m ) ) {
 232                                 $mem = intval( $m[1] );
 233                                 break;
 234                         }
 235                 }
 236                 if ( $mem ) {
 237                         // At least one process
 238                         return max( 1, floor( $mem / 1000000 ) );
 239                 } else {
 240                         return 1;
 241                 }
 242         }
 243
 244         function setupFakeSourceBase( $phase3, $extensions, $dest ) {
 245                 if ( !file_exists( $dest ) ) {
 246                         mkdir( $dest, 0777, true );
 247                 }
 248
 249                 $this->forceCreateLink( "$dest/phase3", $phase3 );
 250                 $this->forceCreateLink( "$dest/extensions", $extensions );
 251         }
 252
 253         function forceCreateLink( $target, $link ) {
 254                 if ( file_exists( $target ) ) {
 255                         if ( readlink( $target ) === $link ) {
 256                                 return;
 257                         }
 258                         unlink( $target );
 259                 }
 260                 symlink( $target, $link );
 261         }
 262
 263         function getFileList() {
 264                 global $wgAutoloadClasses, $wgAutoloadLocalClasses, $wgCompiledFiles;
 265                 $inputFiles = array_merge(
 266                         array_values( $wgAutoloadClasses ),
 267                         array_values( $wgAutoloadLocalClasses ),
 268                         $wgCompiledFiles
 269                 );
 270                 $processedFiles = array();
 271                 foreach ( $inputFiles as $file ) {
 272                         if ( substr( $file, 0, 1 ) === '/' ) {
 273                                 $processedFiles[] = $this->absoluteToRelative( $file );
 274                         } elseif ( preg_match( '/^extensions/', $file ) ) {
 275                                 $processedFiles[] = $file;
 276                         } else {
 277                                 $processedFiles[] = "phase3/$file";
 278                         }
 279                 }
 280
 281                 $extraCoreFiles = array_map( 'trim', file( dirname( __FILE__ ) . '/extra-files' ) );
 282                 foreach ( $extraCoreFiles as $file ) {
 283                         if ( $file === '' ) {
 284                                 continue;
 285                         }
 286                         $processedFiles[] = "phase3/$file";
 287                 }
 288                 return array_unique( $processedFiles );
 289         }
 290
 291         function absoluteToRelative( $file ) {
 292                 global $IP;
 293
 294                 $coreBase = realpath( $IP ) . '/';
 295                 $extBase = realpath( MWInit::getExtensionsDirectory() ) . '/';
 296                 $file = realpath( $file );
 297
 298                 if ( substr( $file, 0, strlen( $extBase ) ) === $extBase ) {
 299                         return 'extensions/' . substr( $file, strlen( $extBase ) );
 300                 } elseif ( substr( $file, 0, strlen( $coreBase ) ) === $coreBase ) {
 301                         return 'phase3/' . substr( $file, strlen( $coreBase ) );
 302                 } else {
 303                         $this->error( "The following file is registered for compilation but is not in \$IP or " .
 304                                 "\$wgExtensionsDirectory: $file \n" );
 305                         exit( 1 );
 306                 }
 307         }
 308 }
 309
 310 $maintClass = 'MakeHipHop';
 311 require_once( RUN_MAINTENANCE_IF_MAIN );