includes/externalstore/ExternalStore.php

   1 <?php
   2 /**
   3  * @defgroup ExternalStorage ExternalStorage
   4  */
   5
   6 use MediaWiki\MediaWikiServices;
   7
   8 /**
   9  * Interface for data storage in external repositories.
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License along
  22  * with this program; if not, write to the Free Software Foundation, Inc.,
  23  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  24  * http://www.gnu.org/copyleft/gpl.html
  25  *
  26  * @file
  27  */
  28
  29 /**
  30  * Constructor class for key/value blob data kept in external repositories.
  31  *
  32  * Objects in external stores are defined by a special URL. The URL is of
  33  * the form "<store protocol>://<location>/<object name>". The protocol is used
  34  * to determine what ExternalStoreMedium class is used. The location identifies
  35  * particular storage instances or database clusters for store class to use.
  36  *
  37  * When an object is inserted into a store, the calling code uses a partial URL of
  38  * the form "<store protocol>://<location>" and receives the full object URL on success.
  39  * This is useful since object names can be sequential IDs, UUIDs, or hashes.
  40  * Callers are not responsible for unique name generation.
  41  *
  42  * External repositories might be populated by maintenance/async
  43  * scripts, thus partial moving of data may be possible, as well
  44  * as the possibility to have any storage format (i.e. for archives).
  45  *
  46  * @ingroup ExternalStorage
  47  */
  48 class ExternalStore {
  49         /**
  50          * Get an external store object of the given type, with the given parameters
  51          *
  52          * @param string $proto Type of external storage, should be a value in $wgExternalStores
  53          * @param array $params Associative array of ExternalStoreMedium parameters
  54          * @return ExternalStoreMedium|bool The store class or false on error
  55          */
  56         public static function getStoreObject( $proto, array $params = [] ) {
  57                 return MediaWikiServices::getInstance()
  58                         ->getExternalStoreFactory()
  59                         ->getStoreObject( $proto, $params );
  60         }
  61
  62         /**
  63          * Fetch data from given URL
  64          *
  65          * @param string $url The URL of the text to get
  66          * @param array $params Associative array of ExternalStoreMedium parameters
  67          * @return string|bool The text stored or false on error
  68          * @throws MWException
  69          */
  70         public static function fetchFromURL( $url, array $params = [] ) {
  71                 $parts = explode( '://', $url, 2 );
  72                 if ( count( $parts ) != 2 ) {
  73                         return false; // invalid URL
  74                 }
  75
  76                 list( $proto, $path ) = $parts;
  77                 if ( $path == '' ) { // bad URL
  78                         return false;
  79                 }
  80
  81                 $store = self::getStoreObject( $proto, $params );
  82                 if ( $store === false ) {
  83                         return false;
  84                 }
  85
  86                 return $store->fetchFromURL( $url );
  87         }
  88
  89         /**
  90          * Fetch data from multiple URLs with a minimum of round trips
  91          *
  92          * @param array $urls The URLs of the text to get
  93          * @return array Map from url to its data.  Data is either string when found
  94          *     or false on failure.
  95          * @throws MWException
  96          */
  97         public static function batchFetchFromURLs( array $urls ) {
  98                 $batches = [];
  99                 foreach ( $urls as $url ) {
 100                         $scheme = parse_url( $url, PHP_URL_SCHEME );
 101                         if ( $scheme ) {
 102                                 $batches[$scheme][] = $url;
 103                         }
 104                 }
 105                 $retval = [];
 106                 foreach ( $batches as $proto => $batchedUrls ) {
 107                         $store = self::getStoreObject( $proto );
 108                         if ( $store === false ) {
 109                                 continue;
 110                         }
 111                         $retval += $store->batchFetchFromURLs( $batchedUrls );
 112                 }
 113                 // invalid, not found, db dead, etc.
 114                 $missing = array_diff( $urls, array_keys( $retval ) );
 115                 if ( $missing ) {
 116                         foreach ( $missing as $url ) {
 117                                 $retval[$url] = false;
 118                         }
 119                 }
 120
 121                 return $retval;
 122         }
 123
 124         /**
 125          * Store a data item to an external store, identified by a partial URL
 126          * The protocol part is used to identify the class, the rest is passed to the
 127          * class itself as a parameter.
 128          *
 129          * @param string $url A partial external store URL ("<store type>://<location>")
 130          * @param string $data
 131          * @param array $params Associative array of ExternalStoreMedium parameters
 132          * @return string|bool The URL of the stored data item, or false on error
 133          * @throws MWException
 134          */
 135         public static function insert( $url, $data, array $params = [] ) {
 136                 $parts = explode( '://', $url, 2 );
 137                 if ( count( $parts ) != 2 ) {
 138                         return false; // invalid URL
 139                 }
 140
 141                 list( $proto, $path ) = $parts;
 142                 if ( $path == '' ) { // bad URL
 143                         return false;
 144                 }
 145
 146                 $store = self::getStoreObject( $proto, $params );
 147                 if ( $store === false ) {
 148                         return false;
 149                 } else {
 150                         return $store->store( $path, $data );
 151                 }
 152         }
 153
 154         /**
 155          * Like insert() above, but does more of the work for us.
 156          * This function does not need a url param, it builds it by
 157          * itself. It also fails-over to the next possible clusters
 158          * provided by $wgDefaultExternalStore.
 159          *
 160          * @param string $data
 161          * @param array $params Map of ExternalStoreMedium::__construct context parameters
 162          * @return string|bool The URL of the stored data item, or false on error
 163          * @throws MWException
 164          */
 165         public static function insertToDefault( $data, array $params = [] ) {
 166                 global $wgDefaultExternalStore;
 167
 168                 return self::insertWithFallback( (array)$wgDefaultExternalStore, $data, $params );
 169         }
 170
 171         /**
 172          * Like insert() above, but does more of the work for us.
 173          * This function does not need a url param, it builds it by
 174          * itself. It also fails-over to the next possible clusters
 175          * as provided in the first parameter.
 176          *
 177          * @param array $tryStores Refer to $wgDefaultExternalStore
 178          * @param string $data
 179          * @param array $params Map of ExternalStoreMedium::__construct context parameters
 180          * @return string|bool The URL of the stored data item, or false on error
 181          * @throws MWException
 182          */
 183         public static function insertWithFallback( array $tryStores, $data, array $params = [] ) {
 184                 $error = false;
 185                 while ( count( $tryStores ) > 0 ) {
 186                         $index = mt_rand( 0, count( $tryStores ) - 1 );
 187                         $storeUrl = $tryStores[$index];
 188                         wfDebug( __METHOD__ . ": trying $storeUrl\n" );
 189                         list( $proto, $path ) = explode( '://', $storeUrl, 2 );
 190                         $store = self::getStoreObject( $proto, $params );
 191                         if ( $store === false ) {
 192                                 throw new MWException( "Invalid external storage protocol - $storeUrl" );
 193                         }
 194
 195                         try {
 196                                 if ( $store->isReadOnly( $path ) ) {
 197                                         $msg = 'read only';
 198                                 } else {
 199                                         $url = $store->store( $path, $data );
 200                                         if ( strlen( $url ) ) {
 201                                                 return $url; // a store accepted the write; done!
 202                                         }
 203                                         $msg = 'operation failed';
 204                                 }
 205                         } catch ( Exception $error ) {
 206                                 $msg = 'caught exception';
 207                         }
 208
 209                         unset( $tryStores[$index] ); // Don't try this one again!
 210                         $tryStores = array_values( $tryStores ); // Must have consecutive keys
 211                         wfDebugLog( 'ExternalStorage',
 212                                 "Unable to store text to external storage $storeUrl ($msg)" );
 213                 }
 214                 // All stores failed
 215                 if ( $error ) {
 216                         throw $error; // rethrow the last error
 217                 } else {
 218                         throw new MWException( "Unable to store text to external storage" );
 219                 }
 220         }
 221
 222         /**
 223          * @return bool Whether all the default insertion stores are marked as read-only
 224          * @since 1.31
 225          */
 226         public static function defaultStoresAreReadOnly() {
 227                 global $wgDefaultExternalStore;
 228
 229                 $tryStores = (array)$wgDefaultExternalStore;
 230                 if ( !$tryStores ) {
 231                         return false; // no stores exists which can be "read only"
 232                 }
 233
 234                 foreach ( $tryStores as $storeUrl ) {
 235                         list( $proto, $path ) = explode( '://', $storeUrl, 2 );
 236                         $store = self::getStoreObject( $proto, [] );
 237                         if ( !$store->isReadOnly( $path ) ) {
 238                                 return false; // at least one store is not read-only
 239                         }
 240                 }
 241
 242                 return true; // all stores are read-only
 243         }
 244
 245         /**
 246          * @param string $data
 247          * @param string $wiki
 248          * @return string|bool The URL of the stored data item, or false on error
 249          * @throws MWException
 250          */
 251         public static function insertToForeignDefault( $data, $wiki ) {
 252                 return self::insertToDefault( $data, [ 'wiki' => $wiki ] );
 253         }
 254 }