[FileRepo] Lazy load large metadata from the DB.

* Previously, large values that could not fit into memcached would not be cached
  and the master DB would be hit for any file loading (djvu files hit this often).

Change-Id: I3ed8ad4a85d6e3084330b56c3b48ee76103bd2b8

diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php
index 6c20eb1..c08d192 100644 (file)
--- a/includes/filerepo/file/LocalFile.php
+++ b/includes/filerepo/file/LocalFile.php
@@ -24,7 +24,7 @@
 /**
  * Bump this number when serialized cache records may be incompatible.
  */
-define( 'MW_FILE_VERSION', 8 );
+define( 'MW_FILE_VERSION', 9 );
 
 /**
  * Class to represent a local file in the wiki's own database
@@ -67,7 +67,8 @@ class LocalFile extends File {
                $sha1,             # SHA-1 base 36 content hash
                $user, $user_text, # User, who uploaded the file
                $description,      # Description of current revision of the file
-               $dataLoaded,       # Whether or not all this has been loaded from the database (loadFromXxx)
+               $dataLoaded,       # Whether or not core data has been loaded from the database (loadFromXxx)
+               $extraDataLoaded,  # Whether or not lazy-loaded data has been loaded from the database
                $upgraded,         # Whether the row was upgraded on load
                $locked,           # True if the image row is locked
                $missing,          # True if file is not present in file system. Not to be cached in memcached
@@ -82,6 +83,8 @@ class LocalFile extends File {
 
        protected $repoClass = 'LocalRepo';
 
+       const LOAD_ALL = 1; // integer; load all the lazy fields too (like metadata)
+
        /**
         * Create a LocalFile from a title
         * Do not call this except from inside a repo class.
@@ -175,6 +178,7 @@ class LocalFile extends File {
                $this->historyLine = 0;
                $this->historyRes = null;
                $this->dataLoaded = false;
+               $this->extraDataLoaded = false;
 
                $this->assertRepoDefined();
                $this->assertTitleDefined();
@@ -200,6 +204,7 @@ class LocalFile extends File {
 
                wfProfileIn( __METHOD__ );
                $this->dataLoaded = false;
+               $this->extraDataLoaded = false;
                $key = $this->getCacheKey();
 
                if ( !$key ) {
@@ -210,13 +215,17 @@ class LocalFile extends File {
                $cachedValues = $wgMemc->get( $key );
 
                // Check if the key existed and belongs to this version of MediaWiki
-               if ( isset( $cachedValues['version'] ) && ( $cachedValues['version'] == MW_FILE_VERSION ) ) {
+               if ( isset( $cachedValues['version'] ) && $cachedValues['version'] == MW_FILE_VERSION ) {
                        wfDebug( "Pulling file metadata from cache key $key\n" );
                        $this->fileExists = $cachedValues['fileExists'];
                        if ( $this->fileExists ) {
                                $this->setProps( $cachedValues );
                        }
                        $this->dataLoaded = true;
+                       $this->extraDataLoaded = true;
+                       foreach ( $this->getLazyCacheFields( '' ) as $field ) {
+                               $this->extraDataLoaded = $this->extraDataLoaded && isset( $cachedValues[$field] );
+                       }
                }
 
                if ( $this->dataLoaded ) {
@@ -252,6 +261,15 @@ class LocalFile extends File {
                        }
                }
 
+               // Strip off excessive entries from the subset of fields that can become large.
+               // If the cache value gets to large it will not fit in memcached and nothing will
+               // get cached at all, causing master queries for any file access.
+               foreach ( $this->getLazyCacheFields( '' ) as $field ) {
+                       if ( isset( $cache[$field] ) && strlen( $cache[$field] ) > 100*1024 ) {
+                               unset( $cache[$field] ); // don't let the value get too big
+                       }
+               }
+
                // Cache presence for 1 week and negatives for 1 day
                $wgMemc->set( $key, $cache, $this->fileExists ? 86400 * 7 : 86400 );
        }
@@ -288,6 +306,28 @@ class LocalFile extends File {
                return $results[$prefix];
        }
 
+       /**
+        * @return array
+        */
+       function getLazyCacheFields( $prefix = 'img_' ) {
+               static $fields = array( 'metadata' );
+               static $results = array();
+
+               if ( $prefix == '' ) {
+                       return $fields;
+               }
+
+               if ( !isset( $results[$prefix] ) ) {
+                       $prefixedFields = array();
+                       foreach ( $fields as $field ) {
+                               $prefixedFields[] = $prefix . $field;
+                       }
+                       $results[$prefix] = $prefixedFields;
+               }
+
+               return $results[$prefix];
+       }
+
        /**
         * Load file metadata from the DB
         */
@@ -298,9 +338,9 @@ class LocalFile extends File {
 
                # Unconditionally set loaded=true, we don't want the accessors constantly rechecking
                $this->dataLoaded = true;
+               $this->extraDataLoaded = true;
 
                $dbr = $this->repo->getMasterDB();
-
                $row = $dbr->selectRow( 'image', $this->getCacheFields( 'img_' ),
                        array( 'img_name' => $this->getName() ), $fname );
 
@@ -314,14 +354,45 @@ class LocalFile extends File {
        }
 
        /**
-        * Decode a row from the database (either object or array) to an array
-        * with timestamps and MIME types decoded, and the field prefix removed.
-        * @param $row
+        * Load lazy file metadata from the DB.
+        * This covers fields that are sometimes not cached.
+        */
+       protected function loadExtraFromDB() {
+               # Polymorphic function name to distinguish foreign and local fetches
+               $fname = get_class( $this ) . '::' . __FUNCTION__;
+               wfProfileIn( $fname );
+
+               # Unconditionally set loaded=true, we don't want the accessors constantly rechecking
+               $this->extraDataLoaded = true;
+
+               $dbr = $this->repo->getSlaveDB();
+               // In theory the file could have just been renamed/deleted...oh well
+               $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ),
+                       array( 'img_name' => $this->getName() ), $fname );
+
+               if ( !$row ) { // fallback to master
+                       $dbr = $this->repo->getMasterDB();
+                       $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ),
+                               array( 'img_name' => $this->getName() ), $fname );
+               }
+
+               if ( $row ) {
+                       foreach ( $this->unprefixRow( $row, 'img_' ) as $name => $value ) {
+                               $this->$name = $value;
+                       }
+               } else {
+                       throw new MWException( "Could not find data for image '{$this->getName()}'." );
+               }
+
+               wfProfileOut( $fname );
+       }
+
+       /**
+        * @param Row $row
         * @param $prefix string
-        * @throws MWException
-        * @return array
+        * @return Array
         */
-       function decodeRow( $row, $prefix = 'img_' ) {
+       protected function unprefixRow( $row, $prefix = 'img_' ) {
                $array = (array)$row;
                $prefixLength = strlen( $prefix );
 
@@ -331,10 +402,22 @@ class LocalFile extends File {
                }
 
                $decoded = array();
-
                foreach ( $array as $name => $value ) {
                        $decoded[substr( $name, $prefixLength )] = $value;
                }
+               return $decoded;
+       }
+
+       /**
+        * Decode a row from the database (either object or array) to an array
+        * with timestamps and MIME types decoded, and the field prefix removed.
+        * @param $row
+        * @param $prefix string
+        * @throws MWException
+        * @return array
+        */
+       function decodeRow( $row, $prefix = 'img_' ) {
+               $decoded = $this->unprefixRow( $row, $prefix );
 
                $decoded['timestamp'] = wfTimestamp( TS_MW, $decoded['timestamp'] );
 
@@ -358,6 +441,8 @@ class LocalFile extends File {
         */
        function loadFromRow( $row, $prefix = 'img_' ) {
                $this->dataLoaded = true;
+               $this->extraDataLoaded = true;
+
                $array = $this->decodeRow( $row, $prefix );
 
                foreach ( $array as $name => $value ) {
@@ -370,8 +455,9 @@ class LocalFile extends File {
 
        /**
         * Load file metadata from cache or DB, unless already loaded
+        * @param integer $flags
         */
-       function load() {
+       function load( $flags = 0 ) {
                if ( !$this->dataLoaded ) {
                        if ( !$this->loadFromCache() ) {
                                $this->loadFromDB();
@@ -379,6 +465,9 @@ class LocalFile extends File {
                        }
                        $this->dataLoaded = true;
                }
+               if ( ( $flags & self::LOAD_ALL ) && !$this->extraDataLoaded ) {
+                       $this->loadExtraFromDB();
+               }
        }
 
        /**
@@ -398,7 +487,7 @@ class LocalFile extends File {
                } else {
                        $handler = $this->getHandler();
                        if ( $handler ) {
-                               $validity = $handler->isMetadataValid( $this, $this->metadata );
+                               $validity = $handler->isMetadataValid( $this, $this->getMetadata() );
                                if ( $validity === MediaHandler::METADATA_BAD
                                        || ( $validity === MediaHandler::METADATA_COMPATIBLE && $wgUpdateCompatibleMetadata )
                                ) {
@@ -465,6 +554,7 @@ class LocalFile extends File {
        /**
         * Set properties in this object to be equal to those given in the
         * associative array $info. Only cacheable fields can be set.
+        * All fields *must* be set in $info except for getLazyCacheFields().
         *
         * If 'mime' is given, it will be split into major_mime/minor_mime.
         * If major_mime/minor_mime are given, $this->mime will also be set.
@@ -571,7 +661,7 @@ class LocalFile extends File {
         * @return string
         */
        function getMetadata() {
-               $this->load();
+               $this->load( self::LOAD_ALL ); // large metadata is loaded in another step
                return $this->metadata;
        }
 

diff --git a/includes/filerepo/file/OldLocalFile.php b/includes/filerepo/file/OldLocalFile.php
index 40d7dca..3180835 100644 (file)
--- a/includes/filerepo/file/OldLocalFile.php
+++ b/includes/filerepo/file/OldLocalFile.php
@@ -169,13 +169,14 @@ class OldLocalFile extends LocalFile {
 
        function loadFromDB() {
                wfProfileIn( __METHOD__ );
+
                $this->dataLoaded = true;
                $dbr = $this->repo->getSlaveDB();
                $conds = array( 'oi_name' => $this->getName() );
                if ( is_null( $this->requestedTime ) ) {
                        $conds['oi_archive_name'] = $this->archive_name;
                } else {
-                       $conds[] = 'oi_timestamp = ' . $dbr->addQuotes( $dbr->timestamp( $this->requestedTime ) );
+                       $conds['oi_timestamp'] = $dbr->timestamp( $this->requestedTime );
                }
                $row = $dbr->selectRow( 'oldimage', $this->getCacheFields( 'oi_' ),
                        $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) );
@@ -184,6 +185,42 @@ class OldLocalFile extends LocalFile {
                } else {
                        $this->fileExists = false;
                }
+
+               wfProfileOut( __METHOD__ );
+       }
+
+       /**
+        * Load lazy file metadata from the DB
+        */
+       protected function loadExtraFromDB() {
+               wfProfileIn( __METHOD__ );
+
+               $this->extraDataLoaded = true;
+               $dbr = $this->repo->getSlaveDB();
+               $conds = array( 'oi_name' => $this->getName() );
+               if ( is_null( $this->requestedTime ) ) {
+                       $conds['oi_archive_name'] = $this->archive_name;
+               } else {
+                       $conds['oi_timestamp'] = $dbr->timestamp( $this->requestedTime );
+               }
+               // In theory the file could have just been renamed/deleted...oh well
+               $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ),
+                       $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) );
+
+               if ( !$row ) { // fallback to master
+                       $dbr = $this->repo->getMasterDB();
+                       $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ),
+                               $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) );
+               }
+
+               if ( $row ) {
+                       foreach ( $this->unprefixRow( $row, 'oi_' ) as $name => $value ) {
+                               $this->$name = $value;
+                       }
+               } else {
+                       throw new MWException( "Could not find data for image '{$this->archive_name}'." );
+               }
+
                wfProfileOut( __METHOD__ );
        }