resourceloader: Improve caching for LESS file compilation

Caching the output of a LESS compiler is tricky, because a LESS file may
include additional LESS files via @imports, in which case the cache needs
to vary as the contents of those files vary (and not just the contents of
the primary LESS file).

To solve this, we first introduce a utility class, FileContentsHasher. This
class is essentially a smart version of md5_file() -- given one or more file
names, it computes a hash digest of their contents. It tries to avoid
re-reading files by caching the hash digest in APC and re-using it as long as
the files' mtimes have not changed. This is the same approach I used in
I5ceb8537c.

Next, we use this class in ResourceLoaderFileModule in the following way:
whenever we compile a LESS file, we cache the result as an associative array
with the following keys:

* `files` : the list of files whose contents influenced the compiled CSS.
* `hash`  : a hash digest of the combined contents of those files.
* `css`   : the CSS output of the compiler itself.

Before using a cached value, we verify that it is still current by asking
FileContentHasher for a hash of the combined contents of all referenced files,
and we compare that against the value of the `hash` key of the cached entry.

Bug: T112035
Change-Id: I1ff61153ddb95ed17e543bd4af7dd13fa3352861

diff --git a/autoload.php b/autoload.php
index 4bed014..4ad921a 100644 (file)
--- a/autoload.php
+++ b/autoload.php
@@ -436,6 +436,7 @@ $wgAutoloadLocalClasses = array(
        'FileBackendStoreShardListIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php',
        'FileBasedSiteLookup' => __DIR__ . '/includes/site/FileBasedSiteLookup.php',
        'FileCacheBase' => __DIR__ . '/includes/cache/FileCacheBase.php',
+       'FileContentsHasher' => __DIR__ . '/includes/FileContentsHasher.php',
        'FileDeleteForm' => __DIR__ . '/includes/FileDeleteForm.php',
        'FileDependency' => __DIR__ . '/includes/cache/CacheDependency.php',
        'FileDuplicateSearchPage' => __DIR__ . '/includes/specials/SpecialFileDuplicateSearch.php',

diff --git a/includes/FileContentsHasher.php b/includes/FileContentsHasher.php
new file mode 100644 (file)
index 0000000..67eb9d2
--- /dev/null
+++ b/includes/FileContentsHasher.php
@@ -0,0 +1,111 @@
+<?php
+/**
+ * Generate hash digests of file contents to help with cache invalidation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+class FileContentsHasher {
+
+       /** @var BagOStuff */
+       protected $cache;
+
+       /** @var FileContentsHasher */
+       private static $instance;
+
+       /**
+        * Constructor.
+        */
+       public function __construct() {
+               $this->cache = ObjectCache::newAccelerator( 'hash' );
+       }
+
+       /**
+        * Get the singleton instance of this class.
+        *
+        * @return FileContentsHasher
+        */
+       public static function singleton() {
+               if ( !self::$instance ) {
+                       self::$instance = new self;
+               }
+
+               return self::$instance;
+       }
+
+       /**
+        * Get a hash of a file's contents, either by retrieving a previously-
+        * computed hash from the cache, or by computing a hash from the file.
+        *
+        * @private
+        * @param string $filePath Full path to the file.
+        * @param string $algo Name of selected hashing algorithm.
+        * @return string|bool Hash of file contents, or false if the file could not be read.
+        */
+       public function getFileContentsHashInternal( $filePath, $algo = 'md4' ) {
+               $mtime = MediaWiki\quietCall( 'filemtime', $filePath );
+               if ( $mtime === false ) {
+                       return false;
+               }
+
+               $cacheKey = wfGlobalCacheKey( __CLASS__, $filePath, $mtime, $algo );
+               $hash = $this->cache->get( $cacheKey );
+
+               if ( $hash ) {
+                       return $hash;
+               }
+
+               $contents = MediaWiki\quietCall( 'file_get_contents', $filePath );
+               if ( $contents === false ) {
+                       return false;
+               }
+
+               $hash = hash( $algo, $contents );
+               $this->cache->set( $cacheKey, $hash, 60 * 60 * 24 );  // 24h
+
+               return $hash;
+       }
+
+       /**
+        * Get a hash of the combined contents of one or more files, either by
+        * retrieving a previously-computed hash from the cache, or by computing
+        * a hash from the files.
+        *
+        * @param string|string[] $filePaths One or more file paths.
+        * @param string $algo Name of selected hashing algorithm.
+        * @return string|bool Hash of files' contents, or false if no file could not be read.
+        */
+       public static function getFileContentsHash( $filePaths, $algo = 'md4' ) {
+               $instance = self::singleton();
+
+               if ( !is_array( $filePaths ) ) {
+                       $filePaths = (array) $filePaths;
+               }
+
+               if ( count( $filePaths ) === 1 ) {
+                       return $instance->getFileContentsHashInternal( $filePaths[0], $algo );
+               }
+
+               sort( $filePaths );
+               $hashes = array_map( function ( $filePath ) use ( $instance, $algo ) {
+                       return $instance->getFileContentsHashInternal( $filePath, $algo ) ?: '';
+               }, $filePaths );
+
+               $hashes = implode( '', $hashes );
+               return $hashes ? hash( $algo, $hashes ) : false;
+       }
+}

diff --git a/includes/resourceloader/ResourceLoaderFileModule.php b/includes/resourceloader/ResourceLoaderFileModule.php
index 7fbc1cb..51118b1 100644 (file)
--- a/includes/resourceloader/ResourceLoaderFileModule.php
+++ b/includes/resourceloader/ResourceLoaderFileModule.php
@@ -966,12 +966,44 @@ class ResourceLoaderFileModule extends ResourceLoaderModule {
         * @return string CSS source
         */
        protected function compileLessFile( $fileName, $compiler = null ) {
+               static $cache;
+
+               if ( !$cache ) {
+                       $cache = ObjectCache::newAccelerator( CACHE_ANYTHING );
+               }
+
+               // Construct a cache key from the LESS file name and a hash digest
+               // of the LESS variables used for compilation.
+               $varsHash = hash( 'md4', serialize( ResourceLoader::getLessVars( $this->getConfig() ) ) );
+               $cacheKey = wfGlobalCacheKey( 'LESS', $fileName, $varsHash );
+               $cachedCompile = $cache->get( $cacheKey );
+
+               // If we got a cached value, we have to validate it by getting a
+               // checksum of all the files that were loaded by the parser and
+               // ensuring it matches the cached entry's.
+               if ( isset( $cachedCompile['hash'] ) ) {
+                       $contentHash = FileContentsHasher::getFileContentsHash( $cachedCompile['files'] );
+                       if ( $contentHash === $cachedCompile['hash'] ) {
+                               $this->localFileRefs += $cachedCompile['files'];
+                               return $cachedCompile['css'];
+                       }
+               }
+
                if ( !$compiler ) {
                        $compiler = $this->getLessCompiler();
                }
-               $result = $compiler->parseFile( $fileName )->getCss();
-               $this->localFileRefs += array_keys( $compiler->AllParsedFiles() );
-               return $result;
+
+               $css = $compiler->parseFile( $fileName )->getCss();
+               $files = $compiler->AllParsedFiles();
+               $this->localFileRefs = array_merge( $this->localFileRefs, $files );
+
+               $cache->set( $cacheKey, array(
+                       'css'   => $css,
+                       'files' => $files,
+                       'hash'  => FileContentsHasher::getFileContentsHash( $files ),
+               ), 60 * 60 * 24 );  // 86400 seconds, or 24 hours.
+
+               return $css;
        }
 
        /**