X-Git-Url: https://git.heureux-cyclage.org/?a=blobdiff_plain;f=includes%2Ffilerepo%2Fbackend%2FSwiftFileBackend.php;h=41d30dd6974cd3d2227daeed3bb55c35a6583456;hb=a8076245aab9ed81a67d73fcd8f2504f5b514e90;hp=ca5e1298715abc8025609276e731eac7f139d860;hpb=b3a3f86a5aa43705f90afe1bb80b4bcb50115e3a;p=lhc%2Fweb%2Fwiklou.git

diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php
index ca5e129871..41d30dd697 100644
--- a/includes/filerepo/backend/SwiftFileBackend.php
+++ b/includes/filerepo/backend/SwiftFileBackend.php
@@ -7,24 +7,24 @@
  */
 
 /**
- * Class for an OpenStack Swift based file backend.
+ * @brief Class for an OpenStack Swift based file backend.
  *
  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  *
- * Status messages should avoid mentioning the Swift account name
+ * Status messages should avoid mentioning the Swift account name.
  * Likewise, error suppression should be used to avoid path disclosure.
  *
  * @ingroup FileBackend
  * @since 1.19
  */
-class SwiftFileBackend extends FileBackend {
+class SwiftFileBackend extends FileBackendStore {
 	/** @var CF_Authentication */
 	protected $auth; // Swift authentication handler
 	protected $authTTL; // integer seconds
 	protected $swiftAnonUser; // string; username to handle unauthenticated requests
-	protected $maxContCacheSize = 20; // integer; max containers with entries
+	protected $maxContCacheSize = 300; // integer; max containers with entries
 
 	/** @var CF_Connection */
 	protected $conn; // Swift connection handle
@@ -32,38 +32,45 @@ class SwiftFileBackend extends FileBackend {
 	protected $connContainers = array(); // container object cache
 
 	/**
-	 * @see FileBackend::__construct()
+	 * @see FileBackendStore::__construct()
 	 * Additional $config params include:
 	 *    swiftAuthUrl       : Swift authentication server URL
 	 *    swiftUser          : Swift user used by MediaWiki (account:username)
 	 *    swiftKey           : Swift authentication key for the above user
 	 *    swiftAuthTTL       : Swift authentication TTL (seconds)
 	 *    swiftAnonUser      : Swift user used for end-user requests (account:username)
-	 *    shardViaHashLevels : Map of container names to the number of hash levels
+	 *    shardViaHashLevels : Map of container names to sharding config with:
+	 *                         'base'   : base of hash characters, 16 or 36
+	 *                         'levels' : the number of hash levels (and digits)
+	 *                         'repeat' : hash subdirectories are prefixed with all the
+	 *                                    parent hash directory names (e.g. "a/ab/abc")
 	 */
 	public function __construct( array $config ) {
 		parent::__construct( $config );
 		// Required settings
 		$this->auth = new CF_Authentication(
-			$config['swiftUser'], 
-			$config['swiftKey'], 
+			$config['swiftUser'],
+			$config['swiftKey'],
 			null, // account; unused
-			$config['swiftAuthUrl'] 
+			$config['swiftAuthUrl']
 		);
 		// Optional settings
 		$this->authTTL = isset( $config['swiftAuthTTL'] )
 			? $config['swiftAuthTTL']
-			: 120; // some sane number
+			: 5 * 60; // some sane number
 		$this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 			? $config['swiftAnonUser']
 			: '';
 		$this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 			? $config['shardViaHashLevels']
 			: '';
+		// Cache container info to mask latency
+		$this->memCache = wfGetMainCache();
 	}
 
 	/**
-	 * @see FileBackend::resolveContainerPath()
+	 * @see FileBackendStore::resolveContainerPath()
+	 * @return null
 	 */
 	protected function resolveContainerPath( $container, $relStoragePath ) {
 		if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
@@ -73,7 +80,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::isPathUsableInternal()
+	 * @see FileBackendStore::isPathUsableInternal()
+	 * @return bool
 	 */
 	public function isPathUsableInternal( $storagePath ) {
 		list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
@@ -94,7 +102,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doCopyInternal()
+	 * @see FileBackendStore::doCreateInternal()
+	 * @return Status
 	 */
 	protected function doCreateInternal( array $params ) {
 		$status = Status::newGood();
@@ -109,7 +118,7 @@ class SwiftFileBackend extends FileBackend {
 		try {
 			$dContObj = $this->getContainer( $dstCont );
 			if ( empty( $params['overwrite'] ) &&
-				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) 
+				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 			{
 				$status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 				return $status;
@@ -156,7 +165,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doStoreInternal()
+	 * @see FileBackendStore::doStoreInternal()
+	 * @return Status
 	 */
 	protected function doStoreInternal( array $params ) {
 		$status = Status::newGood();
@@ -171,7 +181,7 @@ class SwiftFileBackend extends FileBackend {
 		try {
 			$dContObj = $this->getContainer( $dstCont );
 			if ( empty( $params['overwrite'] ) &&
-				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) 
+				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 			{
 				$status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 				return $status;
@@ -224,7 +234,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doCopyInternal()
+	 * @see FileBackendStore::doCopyInternal()
+	 * @return Status
 	 */
 	protected function doCopyInternal( array $params ) {
 		$status = Status::newGood();
@@ -246,7 +257,7 @@ class SwiftFileBackend extends FileBackend {
 			$sContObj = $this->getContainer( $srcCont );
 			$dContObj = $this->getContainer( $dstCont );
 			if ( empty( $params['overwrite'] ) &&
-				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) 
+				$this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 			{
 				$status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 				return $status;
@@ -279,7 +290,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doDeleteInternal()
+	 * @see FileBackendStore::doDeleteInternal()
+	 * @return Status
 	 */
 	protected function doDeleteInternal( array $params ) {
 		$status = Status::newGood();
@@ -310,7 +322,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doPrepareInternal()
+	 * @see FileBackendStore::doPrepareInternal()
+	 * @return Status
 	 */
 	protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 		$status = Status::newGood();
@@ -355,7 +368,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doSecureInternal()
+	 * @see FileBackendStore::doSecureInternal()
+	 * @return Status
 	 */
 	protected function doSecureInternal( $fullCont, $dir, array $params ) {
 		$status = Status::newGood();
@@ -389,7 +403,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doCleanInternal()
+	 * @see FileBackendStore::doCleanInternal()
+	 * @return Status
 	 */
 	protected function doCleanInternal( $fullCont, $dir, array $params ) {
 		$status = Status::newGood();
@@ -433,7 +448,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doFileExists()
+	 * @see FileBackendStore::doFileExists()
+	 * @return array|bool|null
 	 */
 	protected function doGetFileStat( array $params ) {
 		list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
@@ -445,6 +461,7 @@ class SwiftFileBackend extends FileBackend {
 		try {
 			$contObj = $this->getContainer( $srcCont );
 			$srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
+			$this->addMissingMetadata( $srcObj, $params['src'] );
 			$stat = array(
 				// Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 				'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
@@ -464,7 +481,37 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackendBase::getFileContents()
+	 * Fill in any missing object metadata and save it to Swift
+	 *
+	 * @param $obj CF_Object
+	 * @param $path string Storage path to object
+	 * @return bool Success
+	 * @throws Exception cloudfiles exceptions
+	 */
+	protected function addMissingMetadata( CF_Object $obj, $path ) {
+		if ( isset( $obj->metadata['Sha1base36'] ) ) {
+			return true; // nothing to do
+		}
+		$status = Status::newGood();
+		$scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
+		if ( $status->isOK() ) {
+			$tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
+			if ( $tmpFile ) {
+				$hash = $tmpFile->getSha1Base36();
+				if ( $hash !== false ) {
+					$obj->metadata['Sha1base36'] = $hash;
+					$obj->sync_metadata(); // save to Swift
+					return true; // success
+				}
+			}
+		}
+		$obj->metadata['Sha1base36'] = false;
+		return false; // failed
+	}
+
+	/**
+	 * @see FileBackend::getFileContents()
+	 * @return bool|null|string
 	 */
 	public function getFileContents( array $params ) {
 		list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
@@ -491,30 +538,136 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::getFileListInternal()
+	 * @see FileBackendStore::doDirectoryExists()
+	 * @return bool|null
+	 */
+	protected function doDirectoryExists( $fullCont, $dir, array $params ) {
+		try {
+			$container = $this->getContainer( $fullCont );
+			$prefix = ( $dir == '' ) ? null : "{$dir}/";
+			return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
+		} catch ( NoSuchContainerException $e ) {
+			return false;
+		} catch ( InvalidResponseException $e ) {
+		} catch ( Exception $e ) { // some other exception?
+			$this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
+		}
+
+		return null; // error
+	}
+
+	/**
+	 * @see FileBackendStore::getDirectoryListInternal()
+	 * @return SwiftFileBackendDirList
+	 */
+	public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
+		return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
+	}
+
+	/**
+	 * @see FileBackendStore::getFileListInternal()
+	 * @return SwiftFileBackendFileList
 	 */
 	public function getFileListInternal( $fullCont, $dir, array $params ) {
-		return new SwiftFileBackendFileList( $this, $fullCont, $dir );
+		return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 	}
 
 	/**
 	 * Do not call this function outside of SwiftFileBackendFileList
-	 * 
+	 *
 	 * @param $fullCont string Resolved container name
 	 * @param $dir string Resolved storage directory with no trailing slash
-	 * @param $after string Storage path of file to list items after
+	 * @param $after string|null Storage path of file to list items after
 	 * @param $limit integer Max number of items to list
-	 * @return Array
+	 * @param $params Array Includes flag for 'topOnly'
+	 * @return Array List of relative paths of dirs directly under $dir
+	 */
+	public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
+		$dirs = array();
+
+		try {
+			$container = $this->getContainer( $fullCont );
+			$prefix = ( $dir == '' ) ? null : "{$dir}/";
+			// Non-recursive: only list dirs right under $dir
+			if ( !empty( $params['topOnly'] ) ) {
+				$objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
+				foreach ( $objects as $object ) { // files and dirs
+					if ( substr( $object, -1 ) === '/' ) {
+						$dirs[] = $object; // directories end in '/'
+					}
+					$after = $object; // update last item
+				}
+			// Recursive: list all dirs under $dir and its subdirs
+			} else {
+				// Get directory from last item of prior page
+				$lastDir = $this->getParentDir( $after ); // must be first page
+				$objects = $container->list_objects( $limit, $after, $prefix );
+				foreach ( $objects as $object ) { // files
+					$objectDir = $this->getParentDir( $object ); // directory of object
+					if ( $objectDir !== false ) { // file has a parent dir
+						// Swift stores paths in UTF-8, using binary sorting.
+						// See function "create_container_table" in common/db.py.
+						// If a directory is not "greater" than the last one,
+						// then it was already listed by the calling iterator.
+						if ( $objectDir > $lastDir ) {
+							$pDir = $objectDir;
+							do { // add dir and all its parent dirs
+								$dirs[] = "{$pDir}/";
+								$pDir = $this->getParentDir( $pDir );
+							} while ( $pDir !== false // sanity
+								&& $pDir > $lastDir // not done already
+								&& strlen( $pDir ) > strlen( $dir ) // within $dir
+							);
+						}
+						$lastDir = $objectDir;
+					}
+					$after = $object; // update last item
+				}
+			}
+		} catch ( NoSuchContainerException $e ) {
+		} catch ( InvalidResponseException $e ) {
+		} catch ( Exception $e ) { // some other exception?
+			$this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
+		}
+
+		return $dirs;
+	}
+
+	protected function getParentDir( $path ) {
+		return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
+	}
+
+	/**
+	 * Do not call this function outside of SwiftFileBackendFileList
+	 *
+	 * @param $fullCont string Resolved container name
+	 * @param $dir string Resolved storage directory with no trailing slash
+	 * @param $after string|null Storage path of file to list items after
+	 * @param $limit integer Max number of items to list
+	 * @param $params Array Includes flag for 'topOnly'
+	 * @return Array List of relative paths of files under $dir
 	 */
-	public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) {
+	public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 		$files = array();
 
 		try {
 			$container = $this->getContainer( $fullCont );
 			$prefix = ( $dir == '' ) ? null : "{$dir}/";
-			$files = $container->list_objects( $limit, $after, $prefix );
+			// Non-recursive: only list files right under $dir
+			if ( !empty( $params['topOnly'] ) ) { // files and dirs
+				$objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
+				foreach ( $objects as $object ) {
+					if ( substr( $object, -1 ) !== '/' ) {
+						$files[] = $object; // directories end in '/'
+					}
+				}
+			// Recursive: list all files under $dir and its subdirs
+			} else { // files
+				$files = $container->list_objects( $limit, $after, $prefix );
+			}
+			$after = end( $files ); // update last item
+			reset( $files ); // reset pointer
 		} catch ( NoSuchContainerException $e ) {
-		} catch ( NoSuchObjectException $e ) {
 		} catch ( InvalidResponseException $e ) {
 		} catch ( Exception $e ) { // some other exception?
 			$this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
@@ -524,9 +677,10 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doGetFileSha1base36()
+	 * @see FileBackendStore::doGetFileSha1base36()
+	 * @return bool
 	 */
-	public function doGetFileSha1base36( array $params ) {
+	protected function doGetFileSha1base36( array $params ) {
 		$stat = $this->getFileStat( $params );
 		if ( $stat ) {
 			return $stat['sha1'];
@@ -536,7 +690,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doStreamFile()
+	 * @see FileBackendStore::doStreamFile()
+	 * @return Status
 	 */
 	protected function doStreamFile( array $params ) {
 		$status = Status::newGood();
@@ -561,8 +716,7 @@ class SwiftFileBackend extends FileBackend {
 		}
 
 		try {
-			$output = fopen( 'php://output', 'w' );
-			// FileBackend::streamFile() already checks existence
+			$output = fopen( 'php://output', 'wb' );
 			$obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 			$obj->stream( $output, $this->headersFromParams( $params ) );
 		} catch ( InvalidResponseException $e ) { // 404? connection problem?
@@ -576,7 +730,8 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::getLocalCopy()
+	 * @see FileBackendStore::getLocalCopy()
+	 * @return null|TempFSFile
 	 */
 	public function getLocalCopy( array $params ) {
 		list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
@@ -617,13 +772,21 @@ class SwiftFileBackend extends FileBackend {
 		return $tmpFile;
 	}
 
+	/**
+	 * @see FileBackendStore::directoriesAreVirtual()
+	 * @return bool
+	 */
+	protected function directoriesAreVirtual() {
+		return true;
+	}
+
 	/**
 	 * Get headers to send to Swift when reading a file based
-	 * on a FileBackend params array, e.g. that of getLocalCopy(). 
+	 * on a FileBackend params array, e.g. that of getLocalCopy().
 	 * $params is currently only checked for a 'latest' flag.
-	 * 
+	 *
 	 * @param $params Array
-	 * @return Array 
+	 * @return Array
 	 */
 	protected function headersFromParams( array $params ) {
 		$hdrs = array();
@@ -649,7 +812,7 @@ class SwiftFileBackend extends FileBackend {
 		$url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 
 		// Note: 10 second timeout consistent with php-cloudfiles
-		$req = new CurlHttpRequest( $url, array( 'method'  => 'POST', 'timeout' => 10 ) );
+		$req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 		$req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 		$req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 		$req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
@@ -660,7 +823,7 @@ class SwiftFileBackend extends FileBackend {
 	/**
 	 * Get a connection to the Swift proxy
 	 *
-	 * @return CF_Connection|false
+	 * @return CF_Connection|bool False on failure
 	 * @throws InvalidResponseException
 	 */
 	protected function getConnection() {
@@ -692,7 +855,7 @@ class SwiftFileBackend extends FileBackend {
 	}
 
 	/**
-	 * @see FileBackend::doClearCache()
+	 * @see FileBackendStore::doClearCache()
 	 */
 	protected function doClearCache( array $paths = null ) {
 		$this->connContainers = array(); // clear container object cache
@@ -703,23 +866,30 @@ class SwiftFileBackend extends FileBackend {
 	 * Use $reCache if the file count or byte count is needed.
 	 *
 	 * @param $container string Container name
-	 * @param $reCache bool Refresh the process cache
+	 * @param $bypassCache bool Bypass all caches and load from Swift
 	 * @return CF_Container
+	 * @throws InvalidResponseException
 	 */
-	protected function getContainer( $container, $reCache = false ) {
+	protected function getContainer( $container, $bypassCache = false ) {
 		$conn = $this->getConnection(); // Swift proxy connection
-		if ( $reCache ) {
-			unset( $this->connContainers[$container] ); // purge cache
+		if ( $bypassCache ) { // purge cache
+			unset( $this->connContainers[$container] );
+		} elseif ( !isset( $this->connContainers[$container] ) ) {
+			$this->primeContainerCache( array( $container ) ); // check persistent cache
 		}
 		if ( !isset( $this->connContainers[$container] ) ) {
 			$contObj = $conn->get_container( $container );
 			// NoSuchContainerException not thrown: container must exist
 			if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 				reset( $this->connContainers );
-				$key = key( $this->connContainers );
-				unset( $this->connContainers[$key] );
+				unset( $this->connContainers[key( $this->connContainers )] );
 			}
 			$this->connContainers[$container] = $contObj; // cache it
+			if ( !$bypassCache ) {
+				$this->setContainerCache( $container, // update persistent cache
+					array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
+				);
+			}
 		}
 		return $this->connContainers[$container];
 	}
@@ -729,6 +899,7 @@ class SwiftFileBackend extends FileBackend {
 	 *
 	 * @param $container string Container name
 	 * @return CF_Container
+	 * @throws InvalidResponseException
 	 */
 	protected function createContainer( $container ) {
 		$conn = $this->getConnection(); // Swift proxy connection
@@ -742,6 +913,7 @@ class SwiftFileBackend extends FileBackend {
 	 *
 	 * @param $container string Container name
 	 * @return void
+	 * @throws InvalidResponseException
 	 */
 	protected function deleteContainer( $container ) {
 		$conn = $this->getConnection(); // Swift proxy connection
@@ -749,9 +921,31 @@ class SwiftFileBackend extends FileBackend {
 		unset( $this->connContainers[$container] ); // purge cache
 	}
 
+	/**
+	 * @see FileBackendStore::doPrimeContainerCache()
+	 * @return void
+	 */
+	protected function doPrimeContainerCache( array $containerInfo ) {
+		try {
+			$conn = $this->getConnection(); // Swift proxy connection
+			foreach ( $containerInfo as $container => $info ) {
+				$this->connContainers[$container] = new CF_Container(
+					$conn->cfs_auth,
+					$conn->cfs_http,
+					$container,
+					$info['count'],
+					$info['bytes']
+				);
+			}
+		} catch ( InvalidResponseException $e ) {
+		} catch ( Exception $e ) { // some other exception?
+			$this->logException( $e, __METHOD__, array() );
+		}
+	}
+
 	/**
 	 * Log an unexpected exception for this backend
-	 * 
+	 *
 	 * @param $e Exception
 	 * @param $func string
 	 * @param $params Array
@@ -759,28 +953,34 @@ class SwiftFileBackend extends FileBackend {
 	 */
 	protected function logException( Exception $e, $func, array $params ) {
 		wfDebugLog( 'SwiftBackend',
-			get_class( $e ) . " in '{$this->name}': '{$func}' with " . serialize( $params ) 
+			get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
+			( $e instanceof InvalidResponseException
+				? ": {$e->getMessage()}"
+				: ""
+			)
 		);
 	}
 }
 
 /**
- * SwiftFileBackend helper class to page through object listings.
+ * SwiftFileBackend helper class to page through listings.
  * Swift also has a listing limit of 10,000 objects for sanity.
  * Do not use this class from places outside SwiftFileBackend.
  *
  * @ingroup FileBackend
  */
-class SwiftFileBackendFileList implements Iterator {
+abstract class SwiftFileBackendList implements Iterator {
 	/** @var Array */
 	protected $bufferIter = array();
 	protected $bufferAfter = null; // string; list items *after* this path
 	protected $pos = 0; // integer
+	/** @var Array */
+	protected $params = array();
 
 	/** @var SwiftFileBackend */
-	protected $backend; 
-	protected $container; //
-	protected $dir; // string storage directory
+	protected $backend;
+	protected $container; // string; container name
+	protected $dir; // string; storage directory
 	protected $suffixStart; // integer
 
 	const PAGE_SIZE = 5000; // file listing buffer size
@@ -789,8 +989,9 @@ class SwiftFileBackendFileList implements Iterator {
 	 * @param $backend SwiftFileBackend
 	 * @param $fullCont string Resolved container name
 	 * @param $dir string Resolved directory relative to container
+	 * @param $params Array
 	 */
-	public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) {
+	public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
 		$this->backend = $backend;
 		$this->container = $fullCont;
 		$this->dir = $dir;
@@ -802,16 +1003,21 @@ class SwiftFileBackendFileList implements Iterator {
 		} else { // dir within container
 			$this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
 		}
+		$this->params = $params;
 	}
 
-	public function current() {
-		return substr( current( $this->bufferIter ), $this->suffixStart );
-	}
-
+	/**
+	 * @see Iterator::key()
+	 * @return integer
+	 */
 	public function key() {
 		return $this->pos;
 	}
 
+	/**
+	 * @see Iterator::next()
+	 * @return void
+	 */
 	public function next() {
 		// Advance to the next file in the page
 		next( $this->bufferIter );
@@ -819,22 +1025,87 @@ class SwiftFileBackendFileList implements Iterator {
 		// Check if there are no files left in this page and
 		// advance to the next page if this page was not empty.
 		if ( !$this->valid() && count( $this->bufferIter ) ) {
-			$this->bufferAfter = end( $this->bufferIter );
-			$this->bufferIter = $this->backend->getFileListPageInternal(
-				$this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
-			);
+			$this->bufferIter = $this->pageFromList(
+				$this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
+			); // updates $this->bufferAfter
 		}
 	}
 
+	/**
+	 * @see Iterator::rewind()
+	 * @return void
+	 */
 	public function rewind() {
 		$this->pos = 0;
 		$this->bufferAfter = null;
-		$this->bufferIter = $this->backend->getFileListPageInternal(
-			$this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
-		);
+		$this->bufferIter = $this->pageFromList(
+			$this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
+		); // updates $this->bufferAfter
 	}
 
+	/**
+	 * @see Iterator::valid()
+	 * @return bool
+	 */
 	public function valid() {
-		return ( current( $this->bufferIter ) !== false ); // no paths can have this value
+		if ( $this->bufferIter === null ) {
+			return false; // some failure?
+		} else {
+			return ( current( $this->bufferIter ) !== false ); // no paths can have this value
+		}
+	}
+
+	/**
+	 * Get the given list portion (page)
+	 *
+	 * @param $container string Resolved container name
+	 * @param $dir string Resolved path relative to container
+	 * @param $after string|null
+	 * @param $limit integer
+	 * @param $params Array
+	 * @return Traversable|Array|null Returns null on failure
+	 */
+	abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
+}
+
+/**
+ * Iterator for listing directories
+ */
+class SwiftFileBackendDirList extends SwiftFileBackendList {
+	/**
+	 * @see Iterator::current()
+	 * @return string|bool String (relative path) or false
+	 */
+	public function current() {
+		return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
+	}
+
+	/**
+	 * @see SwiftFileBackendList::pageFromList()
+	 * @return Array|null
+	 */
+	protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
+		return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
+	}
+}
+
+/**
+ * Iterator for listing regular files
+ */
+class SwiftFileBackendFileList extends SwiftFileBackendList {
+	/**
+	 * @see Iterator::current()
+	 * @return string|bool String (relative path) or false
+	 */
+	public function current() {
+		return substr( current( $this->bufferIter ), $this->suffixStart );
+	}
+
+	/**
+	 * @see SwiftFileBackendList::pageFromList()
+	 * @return Array|null
+	 */
+	protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
+		return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
 	}
 }