includes/HttpFunctions.php

   1 <?php
   2 /**
   3  * HTTP handling class
   4  * @defgroup HTTP HTTP
   5  * @file
   6  * @ingroup HTTP
   7  */
   8
   9 class Http {
  10         const SYNC_DOWNLOAD = 1;  // syncronys upload (in a single request)
  11         const ASYNC_DOWNLOAD = 2; // asynchronous upload we should spawn out another process and monitor progress if possible)
  12
  13         var $body = '';
  14         public static function request( $url, $opts = array() ) {
  15                 $req = new HttpRequest( $url, $opts );
  16                 $status = $req->doRequest();
  17                 if( $status->isOK() ){
  18                         return $status->value;
  19                 } else {
  20                         return false;
  21                 }
  22         }
  23
  24         /**
  25          * Simple wrapper for Http::request( 'GET' )
  26          */
  27         public static function get( $url, $opts = array() ) {
  28                 $opt['method'] = 'GET';
  29                 return Http::request( $url, $opts );
  30         }
  31
  32         /**
  33          * Simple wrapper for Http::request( 'POST' )
  34          */
  35         public static function post( $url, $opts = array() ) {
  36                 $opts['method'] = 'POST';
  37                 return Http::request( $url, $opts );
  38         }
  39
  40         public static function doDownload( $url, $target_file_path , $dl_mode = self::SYNC_DOWNLOAD , $redirectCount = 0 ){
  41                 global $wgPhpCliPath, $wgMaxUploadSize, $wgMaxRedirects;
  42                 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
  43                 $head = get_headers( $url, 1 );
  44
  45                 // check for redirects:
  46                 if( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ){
  47                         if( $redirectCount < $wgMaxRedirects ){
  48                                 if( UploadFromUrl::isValidURI( $head['Location'] ) ){
  49                                         return self::doDownload( $head['Location'], $target_file_path , $dl_mode, $redirectCount++ );
  50                                 } else {
  51                                         return Status::newFatal( 'upload-proto-error' );
  52                                 }
  53                         } else {
  54                                 return Status::newFatal( 'upload-too-many-redirects' );
  55                         }
  56                 }
  57                 // we did not get a 200 ok response:
  58                 if( strrpos( $head[0], '200 OK' ) === false ){
  59                         return Status::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
  60                 }
  61
  62                 $content_length = ( isset( $head['Content-Length'] ) ) ? $head['Content-Length'] : null;
  63                 if( $content_length ){
  64                         if( $content_length > $wgMaxUploadSize ){
  65                                 return Status::newFatal( 'requested file length ' . $content_length . ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
  66                         }
  67                 }
  68
  69                 // check if we can find phpCliPath (for doing a background shell request to php to do the download:
  70                 if( $wgPhpCliPath && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD ){
  71                         wfDebug( __METHOD__ . "\ASYNC_DOWNLOAD\n" );
  72                         // setup session and shell call:
  73                         return self::initBackgroundDownload( $url, $target_file_path, $content_length );
  74                 } else if( $dl_mode == self::SYNC_DOWNLOAD ){
  75                         wfDebug( __METHOD__ . "\nSYNC_DOWNLOAD\n" );
  76                         // SYNC_DOWNLOAD download as much as we can in the time we have to execute
  77                         $opts['method'] = 'GET';
  78                         $opts['target_file_path'] = $target_file_path;
  79                         $req = new HttpRequest( $url, $opts );
  80                         return $req->doRequest();
  81                 }
  82         }
  83
  84         /**
  85          * a non blocking request (generally an exit point in the application)
  86          * should write to a file location and give updates
  87          *
  88          */
  89         private function initBackgroundDownload( $url, $target_file_path, $content_length = null ){
  90                 global $wgMaxUploadSize, $IP, $wgPhpCliPath;
  91                 $status = Status::newGood();
  92
  93                 // generate a session id with all the details for the download (pid, target_file_path )
  94                 $upload_session_key = self::getUploadSessionKey();
  95                 $session_id = session_id();
  96
  97                 // store the url and target path:
  98                 $_SESSION['wsDownload'][$upload_session_key]['url'] = $url;
  99                 $_SESSION['wsDownload'][$upload_session_key]['target_file_path'] = $target_file_path;
 100
 101                 if( $content_length )
 102                         $_SESSION['wsDownload'][$upload_session_key]['content_length'] = $content_length;
 103
 104                 // set initial loaded bytes:
 105                 $_SESSION['wsDownload'][$upload_session_key]['loaded'] = 0;
 106
 107                 // run the background download request:
 108                 $cmd = $wgPhpCliPath . ' ' . $IP . "/maintenance/http_session_download.php --sid {$session_id} --usk {$upload_session_key}";
 109                 $pid = wfShellBackgroundExec( $cmd, $retval );
 110                 // the pid is not of much use since we won't be visiting this same apache any-time soon.
 111                 if( !$pid )
 112                         return Status::newFatal( 'could not run background shell exec' );
 113
 114                 // update the status value with the $upload_session_key (for the user to check on the status of the upload)
 115                 $status->value = $upload_session_key;
 116
 117                 // return good status
 118                 return $status;
 119         }
 120
 121         function getUploadSessionKey(){
 122                 $key = mt_rand( 0, 0x7fffffff );
 123                 $_SESSION['wsUploadData'][$key] = array();
 124                 return $key;
 125         }
 126
 127         /**
 128          * used to run a session based download. Is initiated via the shell.
 129          *
 130          * @param $session_id  String: the session id to grab download details from
 131          * @param $upload_session_key String: the key of the given upload session
 132          *                      (a given client could have started a few http uploads at once)
 133          */
 134         public static function doSessionIdDownload( $session_id, $upload_session_key ){
 135                 global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout;
 136                 wfDebug( __METHOD__ . "\n\ndoSessionIdDownload\n\n" );
 137                 // set session to the provided key:
 138                 session_id( $session_id );
 139                 // start the session
 140                 if( session_start() === false ){
 141                         wfDebug( __METHOD__ . ' could not start session' );
 142                 }
 143                 //get all the vars we need from session_id
 144                 if(!isset($_SESSION[ 'wsDownload' ][$upload_session_key])){
 145                         wfDebug(  __METHOD__ .' Error:could not find upload session');
 146                         exit();
 147                 }
 148                 // setup the global user from the session key we just inherited
 149                 $wgUser = User::newFromSession();
 150
 151                 // grab the session data to setup the request:
 152                 $sd =& $_SESSION['wsDownload'][$upload_session_key];
 153                 // close down the session so we can other http queries can get session updates:
 154                 session_write_close();
 155
 156                 $req = new HttpRequest( $sd['url'], array(
 157                         'target_file_path'      => $sd['target_file_path'],
 158                         'upload_session_key'=> $upload_session_key,
 159                         'timeout'                       => $wgAsyncHTTPTimeout
 160                 ) );
 161                 // run the actual request .. (this can take some time)
 162                 wfDebug( __METHOD__ . "do Request: " . $sd['url'] . ' tf: ' . $sd['target_file_path'] );
 163                 $status = $req->doRequest();
 164                 //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
 165
 166                 // start up the session again:
 167                 if( session_start() === false ){
 168                         wfDebug( __METHOD__ . ' ERROR:: Could not start session');
 169                 }
 170                 // grab the updated session data pointer
 171                 $sd =& $_SESSION['wsDownload'][$upload_session_key];
 172                 // if error update status:
 173                 if( !$status->isOK() ){
 174                         $sd['apiUploadResult'] = ApiFormatJson::getJsonEncode(
 175                                 array( 'error' => $status->getWikiText() )
 176                         );
 177                 }
 178                 // if status okay process upload using fauxReq to api:
 179                 if( $status->isOK() ){
 180                         // setup the FauxRequest
 181                         $fauxReqData = $sd['mParams'];
 182                         $fauxReqData['action'] = 'upload';
 183                         $fauxReqData['format'] = 'json';
 184                         $fauxReqData['internalhttpsession'] = $upload_session_key;
 185
 186                         // evil but no other clean way about it:
 187                         $faxReq = new FauxRequest( $fauxReqData, true );
 188                         $processor = new ApiMain( $faxReq, $wgEnableWriteAPI );
 189
 190                         //init the mUpload var for the $processor
 191                         $processor->execute();
 192                         $processor->getResult()->cleanUpUTF8();
 193                         $printer = $processor->createPrinterByName( 'json' );
 194                         $printer->initPrinter( false );
 195                         ob_start();
 196                         $printer->execute();
 197                         $apiUploadResult = ob_get_clean();
 198
 199                         wfDebug( __METHOD__ . "\n\n got api result:: $apiUploadResult \n" );
 200                         // the status updates runner will grab the result form the session:
 201                         $sd['apiUploadResult'] = $apiUploadResult;
 202                 }
 203                 // close the session:
 204                 session_write_close();
 205         }
 206
 207         /**
 208          * Check if the URL can be served by localhost
 209          * @param $url string Full url to check
 210          * @return bool
 211          */
 212         public static function isLocalURL( $url ) {
 213                 global $wgCommandLineMode, $wgConf;
 214                 if ( $wgCommandLineMode ) {
 215                         return false;
 216                 }
 217
 218                 // Extract host part
 219                 $matches = array();
 220                 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
 221                         $host = $matches[1];
 222                         // Split up dotwise
 223                         $domainParts = explode( '.', $host );
 224                         // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
 225                         $domainParts = array_reverse( $domainParts );
 226                         for ( $i = 0; $i < count( $domainParts ); $i++ ) {
 227                                 $domainPart = $domainParts[$i];
 228                                 if ( $i == 0 ) {
 229                                         $domain = $domainPart;
 230                                 } else {
 231                                         $domain = $domainPart . '.' . $domain;
 232                                 }
 233                                 if ( $wgConf->isLocalVHost( $domain ) ) {
 234                                         return true;
 235                                 }
 236                         }
 237                 }
 238                 return false;
 239         }
 240
 241         /**
 242          * Return a standard user-agent we can use for external requests.
 243          */
 244         public static function userAgent() {
 245                 global $wgVersion;
 246                 return "MediaWiki/$wgVersion";
 247         }
 248 }
 249 class HttpRequest{
 250         var $target_file_path;
 251         var $upload_session_key;
 252
 253         function __construct( $url, $opt ){
 254                 global $wgSyncHTTPTimeout;
 255                 $this->url = $url;
 256                 // set the timeout to default sync timeout (unless the timeout option is provided)
 257                 $this->timeout = ( isset( $opt['timeout'] ) ) ? $opt['timeout'] : $wgSyncHTTPTimeout;
 258                 $this->method = ( isset( $opt['method'] ) ) ? $opt['method'] : 'GET';
 259                 $this->target_file_path = ( isset( $opt['target_file_path'] ) ) ? $opt['target_file_path'] : false;
 260                 $this->upload_session_key = ( isset( $opt['upload_session_key'] ) ) ? $opt['upload_session_key'] : false;
 261         }
 262
 263         /**
 264          * Get the contents of a file by HTTP
 265          * @param $url string Full URL to act on
 266          * @param $Opt associative array Optional array of options:
 267          *              'method'          => 'GET', 'POST' etc.
 268          *              'target_file_path' => if curl should output to a target file
 269          *              'adapter'         => 'curl', 'soket'
 270          */
 271         public function doRequest() {
 272                 # Use curl if available
 273                 if ( function_exists( 'curl_init' ) ) {
 274                         return $this->doCurlReq();
 275                 } else {
 276                         return $this->doPhpReq();
 277                 }
 278         }
 279
 280         private function doCurlReq(){
 281                 global $wgHTTPProxy, $wgTitle;
 282
 283                 $status = Status::newGood();
 284                 $c = curl_init( $this->url );
 285
 286                 // proxy setup:
 287                 if ( Http::isLocalURL( $this->url ) ) {
 288                         curl_setopt( $c, CURLOPT_PROXY, 'localhost:80' );
 289                 } else if ( $wgHTTPProxy ) {
 290                         curl_setopt( $c, CURLOPT_PROXY, $wgHTTPProxy );
 291                 }
 292
 293                 curl_setopt( $c, CURLOPT_TIMEOUT, $this->timeout );
 294                 curl_setopt( $c, CURLOPT_USERAGENT, Http::userAgent() );
 295
 296                 if ( $this->method == 'POST' ) {
 297                         curl_setopt( $c, CURLOPT_POST, true );
 298                         curl_setopt( $c, CURLOPT_POSTFIELDS, '' );
 299                 } else {
 300                         curl_setopt( $c, CURLOPT_CUSTOMREQUEST, $this->method );
 301                 }
 302
 303                 # Set the referer to $wgTitle, even in command-line mode
 304                 # This is useful for interwiki transclusion, where the foreign
 305                 # server wants to know what the referring page is.
 306                 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
 307                 # referring page.
 308                 if ( is_object( $wgTitle ) ) {
 309                         curl_setopt( $c, CURLOPT_REFERER, $wgTitle->getFullURL() );
 310                 }
 311
 312                 // set the write back function (if we are writing to a file)
 313                 if( $this->target_file_path ){
 314                         $cwrite = new simpleFileWriter( $this->target_file_path, $this->upload_session_key );
 315                         if( !$cwrite->status->isOK() ){
 316                                 wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
 317                                 $status = $cwrite->status;
 318                         }
 319                         curl_setopt( $c, CURLOPT_WRITEFUNCTION, array( $cwrite, 'callbackWriteBody' ) );
 320                 }
 321
 322                 // start output grabber:
 323                 if( !$this->target_file_path )
 324                         ob_start();
 325
 326                 //run the actual curl_exec:
 327                 try {
 328                         if ( false === curl_exec( $c ) ) {
 329                                 $error_txt ='Error sending request: #' . curl_errno( $c ) .' '. curl_error( $c );
 330                                 wfDebug( __METHOD__ . $error_txt . "\n" );
 331                                 $status = Status::newFatal( $error_txt );
 332                         }
 333                 } catch ( Exception $e ) {
 334                         // do something with curl exec error?
 335                 }
 336                 // if direct request output the results to the stats value:
 337                 if( !$this->target_file_path && $status->isOK() ){
 338                         $status->value = ob_get_contents();
 339                         ob_end_clean();
 340                 }
 341                 // if we wrote to a target file close up or return error
 342                 if( $this->target_file_path ){
 343                         $cwrite->close();
 344                         if( !$cwrite->status->isOK() ){
 345                                 return $cwrite->status;
 346                         }
 347                 }
 348
 349                 # Don't return the text of error messages, return false on error
 350                 $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE );
 351                 if ( $retcode != 200 ) {
 352                         wfDebug( __METHOD__ . ": HTTP return code $retcode\n" );
 353                         $status = Status::newFatal( "HTTP return code $retcode\n" );
 354                 }
 355                 # Don't return truncated output
 356                 $errno = curl_errno( $c );
 357                 if ( $errno != CURLE_OK ) {
 358                         $errstr = curl_error( $c );
 359                         wfDebug( __METHOD__ . ": CURL error code $errno: $errstr\n" );
 360                                 $status = Status::newFatal( " CURL error code $errno: $errstr\n" );
 361                 }
 362                 curl_close( $c );
 363
 364                 // return the result obj
 365                 return $status;
 366         }
 367
 368         public function doPhpReq(){
 369                 #$use file_get_contents...
 370                 # This doesn't have local fetch capabilities...
 371
 372                 $headers = array( "User-Agent: " . self :: userAgent() );
 373                 if( strcasecmp( $method, 'post' ) == 0 ) {
 374                         // Required for HTTP 1.0 POSTs
 375                         $headers[] = "Content-Length: 0";
 376                 }
 377                 $opts = array(
 378                         'http' => array(
 379                                 'method' => $method,
 380                                 'header' => implode( "\r\n", $headers ),
 381                                 'timeout' => $timeout ) );
 382                 $ctx = stream_context_create( $opts );
 383
 384                 $status->value = file_get_contents( $url, false, $ctx );
 385                 if( !$status->value ){
 386                         $status->error( 'file_get_contents-failed' );
 387                 }
 388                 return $status;
 389         }
 390
 391 }
 392
 393 /**
 394  * a simpleFileWriter with session id updates
 395  */
 396 class simpleFileWriter {
 397         var $target_file_path;
 398         var $status = null;
 399         var $session_id = null;
 400         var $session_update_interval = 0; // how often to update the session while downloading
 401
 402         function simpleFileWriter( $target_file_path, $upload_session_key ){
 403                 $this->target_file_path = $target_file_path;
 404                 $this->upload_session_key = $upload_session_key;
 405                 $this->status = Status::newGood();
 406                 // open the file:
 407                 $this->fp = fopen( $this->target_file_path, 'w' );
 408                 if( $this->fp === false ){
 409                         $this->status = Status::newFatal( 'HTTP::could-not-open-file-for-writing' );
 410                 }
 411                 // true start time
 412                 $this->prevTime = time();
 413         }
 414
 415         public function callbackWriteBody($ch, $data_packet){
 416                 global $wgMaxUploadSize;
 417
 418                 // write out the content
 419                 if( fwrite( $this->fp, $data_packet ) === false ){
 420                         wfDebug( __METHOD__ ." ::could-not-write-to-file\n" );
 421                         $this->status = Status::newFatal( 'HTTP::could-not-write-to-file' );
 422                         return 0;
 423                 }
 424
 425                 // check file size:
 426                 clearstatcache();
 427                 $this->current_fsize = filesize( $this->target_file_path );
 428
 429                 if( $this->current_fsize > $wgMaxUploadSize ){
 430                         wfDebug( __METHOD__ . " ::http download too large\n" );
 431                         $this->status = Status::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: downloaded more than ' .
 432                                 Language::formatSize( $wgMaxUploadSize ) . ' ' );
 433                         return 0;
 434                 }
 435
 436                 // if more than session_update_interval second have passed update_session_progress
 437                 if( $this->upload_session_key && ( ( time() - $this->prevTime ) > $this->session_update_interval ) ) {
 438                         $this->prevTime = time();
 439                         $session_status = $this->update_session_progress();
 440                         if( !$session_status->isOK() ){
 441                                 $this->status = $session_status;
 442                                 wfDebug( __METHOD__ . ' update session failed or was canceled');
 443                                 return 0;
 444                         }
 445                 }
 446                 return strlen( $data_packet );
 447         }
 448
 449         public function update_session_progress(){
 450                 $status = Status::newGood();
 451                 // start the session
 452                 if( session_start() === false){
 453                         wfDebug( __METHOD__ . ' could not start session' );
 454                         exit( 0 );
 455                 }
 456                 $sd =& $_SESSION['wsDownload'][$this->upload_session_key];
 457                 // check if the user canceled the request:
 458                 if( $sd['user_cancel'] == true ){
 459                         // kill the download
 460                         return Status::newFatal( 'user-canceled-request' );
 461                 }
 462                 // update the progress bytes download so far:
 463                 $sd['loaded'] = $this->current_fsize;
 464                 wfDebug( __METHOD__ . ': set session loaded amount to: ' . $sd['loaded'] . "\n");
 465                 // close down the session so we can other http queries can get session updates:
 466                 session_write_close();
 467                 return $status;
 468         }
 469
 470         public function close(){
 471                 // do a final session update:
 472                 $this->update_session_progress();
 473                 // close up the file handle:
 474                 if( false === fclose( $this->fp ) ){
 475                         $this->status = Status::newFatal( 'HTTP::could-not-close-file' );
 476                 }
 477         }
 478
 479 }