includes/filerepo/ForeignAPIRepo.php

   1 <?php
   2 /**
   3  * Foreign repository accessible through api.php requests.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileRepo
  22  */
  23
  24 /**
  25  * A foreign repository with a remote MediaWiki with an API thingy
  26  *
  27  * Example config:
  28  *
  29  * $wgForeignFileRepos[] = array(
  30  *   'class'                  => 'ForeignAPIRepo',
  31  *   'name'                   => 'shared',
  32  *   'apibase'                => 'http://en.wikipedia.org/w/api.php',
  33  *   'fetchDescription'       => true, // Optional
  34  *   'descriptionCacheExpiry' => 3600,
  35  * );
  36  *
  37  * @ingroup FileRepo
  38  */
  39 class ForeignAPIRepo extends FileRepo {
  40         /* This version string is used in the user agent for requests and will help
  41          * server maintainers in identify ForeignAPI usage.
  42          * Update the version every time you make breaking or significant changes. */
  43         const VERSION = "2.1";
  44
  45         /**
  46          * List of iiprop values for the thumbnail fetch queries.
  47          * @since 1.23
  48          */
  49         protected static $imageInfoProps = array(
  50                 'url',
  51                 'thumbnail',
  52                 'timestamp',
  53         );
  54
  55         var $fileFactory = array( 'ForeignAPIFile', 'newFromTitle' );
  56         /* Check back with Commons after a day */
  57         var $apiThumbCacheExpiry = 86400; /* 24*60*60 */
  58         /* Redownload thumbnail files after a month */
  59         var $fileCacheExpiry = 2592000; /* 86400*30 */
  60
  61         protected $mQueryCache = array();
  62         protected $mFileExists = array();
  63
  64         /**
  65          * @param $info array|null
  66          */
  67         function __construct( $info ) {
  68                 global $wgLocalFileRepo;
  69                 parent::__construct( $info );
  70
  71                 // http://commons.wikimedia.org/w/api.php
  72                 $this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null;
  73
  74                 if ( isset( $info['apiThumbCacheExpiry'] ) ) {
  75                         $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry'];
  76                 }
  77                 if ( isset( $info['fileCacheExpiry'] ) ) {
  78                         $this->fileCacheExpiry = $info['fileCacheExpiry'];
  79                 }
  80                 if ( !$this->scriptDirUrl ) {
  81                         // hack for description fetches
  82                         $this->scriptDirUrl = dirname( $this->mApiBase );
  83                 }
  84                 // If we can cache thumbs we can guess sane defaults for these
  85                 if ( $this->canCacheThumbs() && !$this->url ) {
  86                         $this->url = $wgLocalFileRepo['url'];
  87                 }
  88                 if ( $this->canCacheThumbs() && !$this->thumbUrl ) {
  89                         $this->thumbUrl = $this->url . '/thumb';
  90                 }
  91         }
  92
  93         /**
  94          * @return string
  95          * @since 1.22
  96          */
  97         function getApiUrl() {
  98                 return $this->mApiBase;
  99         }
 100
 101         /**
 102          * Per docs in FileRepo, this needs to return false if we don't support versioned
 103          * files. Well, we don't.
 104          *
 105          * @param $title Title
 106          * @param $time string|bool
 107          * @return File
 108          */
 109         function newFile( $title, $time = false ) {
 110                 if ( $time ) {
 111                         return false;
 112                 }
 113                 return parent::newFile( $title, $time );
 114         }
 115
 116         /**
 117          * @param $files array
 118          * @return array
 119          */
 120         function fileExistsBatch( array $files ) {
 121                 $results = array();
 122                 foreach ( $files as $k => $f ) {
 123                         if ( isset( $this->mFileExists[$f] ) ) {
 124                                 $results[$k] = $this->mFileExists[$f];
 125                                 unset( $files[$k] );
 126                         } elseif ( self::isVirtualUrl( $f ) ) {
 127                                 # @todo FIXME: We need to be able to handle virtual
 128                                 # URLs better, at least when we know they refer to the
 129                                 # same repo.
 130                                 $results[$k] = false;
 131                                 unset( $files[$k] );
 132                         } elseif ( FileBackend::isStoragePath( $f ) ) {
 133                                 $results[$k] = false;
 134                                 unset( $files[$k] );
 135                                 wfWarn( "Got mwstore:// path '$f'." );
 136                         }
 137                 }
 138
 139                 $data = $this->fetchImageQuery( array( 'titles' => implode( $files, '|' ),
 140                                                                                         'prop' => 'imageinfo' ) );
 141                 if ( isset( $data['query']['pages'] ) ) {
 142                         # First, get results from the query. Note we only care whether the image exists,
 143                         # not whether it has a description page.
 144                         foreach ( $data['query']['pages'] as $p ) {
 145                                 $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' );
 146                         }
 147                         # Second, copy the results to any redirects that were queried
 148                         if ( isset( $data['query']['redirects'] ) ) {
 149                                 foreach ( $data['query']['redirects'] as $r ) {
 150                                         $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']];
 151                                 }
 152                         }
 153                         # Third, copy the results to any non-normalized titles that were queried
 154                         if ( isset( $data['query']['normalized'] ) ) {
 155                                 foreach ( $data['query']['normalized'] as $n ) {
 156                                         $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']];
 157                                 }
 158                         }
 159                         # Finally, copy the results to the output
 160                         foreach ( $files as $key => $file ) {
 161                                 $results[$key] = $this->mFileExists[$file];
 162                         }
 163                 }
 164                 return $results;
 165         }
 166
 167         /**
 168          * @param $virtualUrl string
 169          * @return bool
 170          */
 171         function getFileProps( $virtualUrl ) {
 172                 return false;
 173         }
 174
 175         /**
 176          * @param $query array
 177          * @return string
 178          */
 179         function fetchImageQuery( $query ) {
 180                 global $wgMemc, $wgLanguageCode;
 181
 182                 $query = array_merge( $query,
 183                         array(
 184                                 'format' => 'json',
 185                                 'action' => 'query',
 186                                 'redirects' => 'true'
 187                         ) );
 188
 189                 if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
 190                         $query['uselang'] = $wgLanguageCode;
 191                 }
 192
 193                 $data = $this->httpGetCached( 'Metadata', $query );
 194
 195                 if ( $data ) {
 196                         return FormatJson::decode( $data, true );
 197                 } else {
 198                         return null;
 199                 }
 200         }
 201
 202         /**
 203          * @param $data array
 204          * @return bool|array
 205          */
 206         function getImageInfo( $data ) {
 207                 if ( $data && isset( $data['query']['pages'] ) ) {
 208                         foreach ( $data['query']['pages'] as $info ) {
 209                                 if ( isset( $info['imageinfo'][0] ) ) {
 210                                         return $info['imageinfo'][0];
 211                                 }
 212                         }
 213                 }
 214                 return false;
 215         }
 216
 217         /**
 218          * @param $hash string
 219          * @return array
 220          */
 221         function findBySha1( $hash ) {
 222                 $results = $this->fetchImageQuery( array(
 223                         'aisha1base36' => $hash,
 224                         'aiprop' => ForeignAPIFile::getProps(),
 225                         'list' => 'allimages',
 226                 ) );
 227                 $ret = array();
 228                 if ( isset( $results['query']['allimages'] ) ) {
 229                         foreach ( $results['query']['allimages'] as $img ) {
 230                                 // 1.14 was broken, doesn't return name attribute
 231                                 if ( !isset( $img['name'] ) ) {
 232                                         continue;
 233                                 }
 234                                 $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img );
 235                         }
 236                 }
 237                 return $ret;
 238         }
 239
 240         /**
 241          * @param $name string
 242          * @param $width int
 243          * @param $height int
 244          * @param $result null
 245          * @param $otherParams string
 246          * @return bool
 247          */
 248         function getThumbUrl( $name, $width = -1, $height = -1, &$result = null, $otherParams = '' ) {
 249                 $data = $this->fetchImageQuery( array(
 250                         'titles' => 'File:' . $name,
 251                         'iiprop' => self::getIIProps(),
 252                         'iiurlwidth' => $width,
 253                         'iiurlheight' => $height,
 254                         'iiurlparam' => $otherParams,
 255                         'prop' => 'imageinfo' ) );
 256                 $info = $this->getImageInfo( $data );
 257
 258                 if ( $data && $info && isset( $info['thumburl'] ) ) {
 259                         wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] . "\n" );
 260                         $result = $info;
 261                         return $info['thumburl'];
 262                 } else {
 263                         return false;
 264                 }
 265         }
 266
 267         /**
 268          * @param $name string
 269          * @param $width int
 270          * @param $height int
 271          * @param $otherParams string
 272          * @return bool|MediaTransformError
 273          * @since 1.22
 274          */
 275         function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) {
 276                 $data = $this->fetchImageQuery( array(
 277                         'titles' => 'File:' . $name,
 278                         'iiprop' => self::getIIProps(),
 279                         'iiurlwidth' => $width,
 280                         'iiurlheight' => $height,
 281                         'iiurlparam' => $otherParams,
 282                         'prop' => 'imageinfo',
 283                         'uselang' => $lang,
 284                 ) );
 285                 $info = $this->getImageInfo( $data );
 286
 287                 if ( $data && $info && isset( $info['thumberror'] ) ) {
 288                         wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] . "\n" );
 289                         return new MediaTransformError(
 290                                 'thumbnail_error_remote',
 291                                 $width,
 292                                 $height,
 293                                 $this->getDisplayName(),
 294                                 $info['thumberror'] // already parsed message from foreign repo
 295                         );
 296                 } else {
 297                         return false;
 298                 }
 299         }
 300
 301         /**
 302          * Return the imageurl from cache if possible
 303          *
 304          * If the url has been requested today, get it from cache
 305          * Otherwise retrieve remote thumb url, check for local file.
 306          *
 307          * @param string $name is a dbkey form of a title
 308          * @param $width
 309          * @param $height
 310          * @param string $params Other rendering parameters (page number, etc) from handler's makeParamString.
 311          * @return bool|string
 312          */
 313         function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
 314                 global $wgMemc;
 315                 // We can't check the local cache using FileRepo functions because
 316                 // we override fileExistsBatch(). We have to use the FileBackend directly.
 317                 $backend = $this->getBackend(); // convenience
 318
 319                 if ( !$this->canCacheThumbs() ) {
 320                         $result = null; // can't pass "null" by reference, but it's ok as default value
 321                         return $this->getThumbUrl( $name, $width, $height, $result, $params );
 322                 }
 323                 $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
 324                 $sizekey = "$width:$height:$params";
 325
 326                 /* Get the array of urls that we already know */
 327                 $knownThumbUrls = $wgMemc->get( $key );
 328                 if ( !$knownThumbUrls ) {
 329                         /* No knownThumbUrls for this file */
 330                         $knownThumbUrls = array();
 331                 } else {
 332                         if ( isset( $knownThumbUrls[$sizekey] ) ) {
 333                                 wfDebug( __METHOD__ . ': Got thumburl from local cache: ' .
 334                                         "{$knownThumbUrls[$sizekey]} \n" );
 335                                 return $knownThumbUrls[$sizekey];
 336                         }
 337                         /* This size is not yet known */
 338                 }
 339
 340                 $metadata = null;
 341                 $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
 342
 343                 if ( !$foreignUrl ) {
 344                         wfDebug( __METHOD__ . " Could not find thumburl\n" );
 345                         return false;
 346                 }
 347
 348                 // We need the same filename as the remote one :)
 349                 $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) );
 350                 if ( !$this->validateFilename( $fileName ) ) {
 351                         wfDebug( __METHOD__ . " The deduced filename $fileName is not safe\n" );
 352                         return false;
 353                 }
 354                 $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
 355                 $localFilename = $localPath . "/" . $fileName;
 356                 $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) . rawurlencode( $name ) . "/" . rawurlencode( $fileName );
 357
 358                 if ( $backend->fileExists( array( 'src' => $localFilename ) )
 359                         && isset( $metadata['timestamp'] ) ) {
 360                         wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" );
 361                         $modified = $backend->getFileTimestamp( array( 'src' => $localFilename ) );
 362                         $remoteModified = strtotime( $metadata['timestamp'] );
 363                         $current = time();
 364                         $diff = abs( $modified - $current );
 365                         if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) {
 366                                 /* Use our current and already downloaded thumbnail */
 367                                 $knownThumbUrls[$sizekey] = $localUrl;
 368                                 $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
 369                                 return $localUrl;
 370                         }
 371                         /* There is a new Commons file, or existing thumbnail older than a month */
 372                 }
 373                 $thumb = self::httpGet( $foreignUrl );
 374                 if ( !$thumb ) {
 375                         wfDebug( __METHOD__ . " Could not download thumb\n" );
 376                         return false;
 377                 }
 378
 379                 # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
 380                 $backend->prepare( array( 'dir' => dirname( $localFilename ) ) );
 381                 $params = array( 'dst' => $localFilename, 'content' => $thumb );
 382                 if ( !$backend->quickCreate( $params )->isOK() ) {
 383                         wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'\n" );
 384                         return $foreignUrl;
 385                 }
 386                 $knownThumbUrls[$sizekey] = $localUrl;
 387                 $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry );
 388                 wfDebug( __METHOD__ . " got local thumb $localUrl, saving to cache \n" );
 389                 return $localUrl;
 390         }
 391
 392         /**
 393          * @see FileRepo::getZoneUrl()
 394          * @param $zone String
 395          * @param string|null $ext Optional file extension
 396          * @return String
 397          */
 398         function getZoneUrl( $zone, $ext = null ) {
 399                 switch ( $zone ) {
 400                         case 'public':
 401                                 return $this->url;
 402                         case 'thumb':
 403                                 return $this->thumbUrl;
 404                         default:
 405                                 return parent::getZoneUrl( $zone, $ext );
 406                 }
 407         }
 408
 409         /**
 410          * Get the local directory corresponding to one of the basic zones
 411          * @param $zone string
 412          * @return bool|null|string
 413          */
 414         function getZonePath( $zone ) {
 415                 $supported = array( 'public', 'thumb' );
 416                 if ( in_array( $zone, $supported ) ) {
 417                         return parent::getZonePath( $zone );
 418                 }
 419                 return false;
 420         }
 421
 422         /**
 423          * Are we locally caching the thumbnails?
 424          * @return bool
 425          */
 426         public function canCacheThumbs() {
 427                 return ( $this->apiThumbCacheExpiry > 0 );
 428         }
 429
 430         /**
 431          * The user agent the ForeignAPIRepo will use.
 432          * @return string
 433          */
 434         public static function getUserAgent() {
 435                 return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION;
 436         }
 437
 438         /**
 439          * Get information about the repo - overrides/extends the parent
 440          * class's information.
 441          * @return array
 442          * @since 1.22
 443          */
 444         function getInfo() {
 445                 $info = parent::getInfo();
 446                 $info['apiurl'] = $this->getApiUrl();
 447
 448                 $query = array(
 449                         'format' => 'json',
 450                         'action' => 'query',
 451                         'meta' => 'siteinfo',
 452                         'siprop' => 'general',
 453                 );
 454
 455                 $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
 456
 457                 if ( $data ) {
 458                         $siteInfo = FormatJson::decode( $data, true );
 459                         $general = $siteInfo['query']['general'];
 460
 461                         $info['articlepath'] = $general['articlepath'];
 462                         $info['server'] = $general['server'];
 463                 }
 464
 465                 return $info;
 466         }
 467
 468         /**
 469          * Like a Http:get request, but with custom User-Agent.
 470          * @see Http:get
 471          * @param $url string
 472          * @param $timeout string
 473          * @param $options array
 474          * @return bool|String
 475          */
 476         public static function httpGet( $url, $timeout = 'default', $options = array() ) {
 477                 $options['timeout'] = $timeout;
 478                 /* Http::get */
 479                 $url = wfExpandUrl( $url, PROTO_HTTP );
 480                 wfDebug( "ForeignAPIRepo: HTTP GET: $url\n" );
 481                 $options['method'] = "GET";
 482
 483                 if ( !isset( $options['timeout'] ) ) {
 484                         $options['timeout'] = 'default';
 485                 }
 486
 487                 $req = MWHttpRequest::factory( $url, $options );
 488                 $req->setUserAgent( ForeignAPIRepo::getUserAgent() );
 489                 $status = $req->execute();
 490
 491                 if ( $status->isOK() ) {
 492                         return $req->getContent();
 493                 } else {
 494                         return false;
 495                 }
 496         }
 497
 498         /**
 499          * @return string
 500          * @since 1.23
 501          */
 502         protected static function getIIProps() {
 503                 return join( '|', self::$imageInfoProps );
 504         }
 505
 506         /**
 507          * HTTP GET request to a mediawiki API (with caching)
 508          * @param $target string Used in cache key creation, mostly
 509          * @param $query array The query parameters for the API request
 510          * @param $cacheTTL int Time to live for the memcached caching
 511          */
 512         public function httpGetCached( $target, $query, $cacheTTL = 3600 ) {
 513                 if ( $this->mApiBase ) {
 514                         $url = wfAppendQuery( $this->mApiBase, $query );
 515                 } else {
 516                         $url = $this->makeUrl( $query, 'api' );
 517                 }
 518
 519                 if ( !isset( $this->mQueryCache[$url] ) ) {
 520                         global $wgMemc;
 521
 522                         $key = $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) );
 523                         $data = $wgMemc->get( $key );
 524
 525                         if ( !$data ) {
 526                                 $data = self::httpGet( $url );
 527
 528                                 if ( !$data ) {
 529                                         return null;
 530                                 }
 531
 532                                 $wgMemc->set( $key, $data, $cacheTTL );
 533                         }
 534
 535                         if ( count( $this->mQueryCache ) > 100 ) {
 536                                 // Keep the cache from growing infinitely
 537                                 $this->mQueryCache = array();
 538                         }
 539
 540                         $this->mQueryCache[$url] = $data;
 541                 }
 542
 543                 return $this->mQueryCache[$url];
 544         }
 545
 546         /**
 547          * @param $callback Array|string
 548          * @throws MWException
 549          */
 550         function enumFiles( $callback ) {
 551                 throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) );
 552         }
 553
 554         /**
 555          * @throws MWException
 556          */
 557         protected function assertWritableRepo() {
 558                 throw new MWException( get_class( $this ) . ': write operations are not supported.' );
 559         }
 560 }