includes/shell/Command.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 namespace MediaWiki\Shell;
  22
  23 use Exception;
  24 use MediaWiki\ProcOpenError;
  25 use MediaWiki\ShellDisabledError;
  26 use Profiler;
  27 use Psr\Log\LoggerAwareTrait;
  28 use Psr\Log\NullLogger;
  29 use Wikimedia\AtEase\AtEase;
  30
  31 /**
  32  * Class used for executing shell commands
  33  *
  34  * @since 1.30
  35  */
  36 class Command {
  37         use LoggerAwareTrait;
  38
  39         /** @var string */
  40         protected $command = '';
  41
  42         /** @var array */
  43         private $limits = [
  44                 // seconds
  45                 'time' => 180,
  46                 // seconds
  47                 'walltime' => 180,
  48                 // KB
  49                 'memory' => 307200,
  50                 // KB
  51                 'filesize' => 102400,
  52         ];
  53
  54         /** @var string[] */
  55         private $env = [];
  56
  57         /** @var string */
  58         private $method;
  59
  60         /** @var string|null */
  61         private $inputString;
  62
  63         /** @var bool */
  64         private $doIncludeStderr = false;
  65
  66         /** @var bool */
  67         private $doLogStderr = false;
  68
  69         /** @var bool */
  70         private $everExecuted = false;
  71
  72         /** @var string|false */
  73         private $cgroup = false;
  74
  75         /**
  76          * Bitfield with restrictions
  77          *
  78          * @var int
  79          */
  80         protected $restrictions = 0;
  81
  82         /**
  83          * Don't call directly, instead use Shell::command()
  84          *
  85          * @throws ShellDisabledError
  86          */
  87         public function __construct() {
  88                 if ( Shell::isDisabled() ) {
  89                         throw new ShellDisabledError();
  90                 }
  91
  92                 $this->setLogger( new NullLogger() );
  93         }
  94
  95         /**
  96          * Makes sure the programmer didn't forget to execute the command after all
  97          */
  98         public function __destruct() {
  99                 if ( !$this->everExecuted ) {
 100                         $context = [ 'command' => $this->command ];
 101                         $message = __CLASS__ . " was instantiated, but execute() was never called.";
 102                         if ( $this->method ) {
 103                                 $message .= ' Calling method: {method}.';
 104                                 $context['method'] = $this->method;
 105                         }
 106                         $message .= ' Command: {command}';
 107                         $this->logger->warning( $message, $context );
 108                 }
 109         }
 110
 111         /**
 112          * Adds parameters to the command. All parameters are sanitized via Shell::escape().
 113          * Null values are ignored.
 114          *
 115          * @param string|string[] ...$args
 116          * @return $this
 117          */
 118         public function params( ...$args ): Command {
 119                 if ( count( $args ) === 1 && is_array( reset( $args ) ) ) {
 120                         // If only one argument has been passed, and that argument is an array,
 121                         // treat it as a list of arguments
 122                         $args = reset( $args );
 123                 }
 124                 $this->command = trim( $this->command . ' ' . Shell::escape( $args ) );
 125
 126                 return $this;
 127         }
 128
 129         /**
 130          * Adds unsafe parameters to the command. These parameters are NOT sanitized in any way.
 131          * Null values are ignored.
 132          *
 133          * @param string|string[] ...$args
 134          * @return $this
 135          */
 136         public function unsafeParams( ...$args ): Command {
 137                 if ( count( $args ) === 1 && is_array( reset( $args ) ) ) {
 138                         // If only one argument has been passed, and that argument is an array,
 139                         // treat it as a list of arguments
 140                         $args = reset( $args );
 141                 }
 142                 $args = array_filter( $args,
 143                         function ( $value ) {
 144                                 return $value !== null;
 145                         }
 146                 );
 147                 $this->command = trim( $this->command . ' ' . implode( ' ', $args ) );
 148
 149                 return $this;
 150         }
 151
 152         /**
 153          * Sets execution limits
 154          *
 155          * @param array $limits Associative array of limits. Keys (all optional):
 156          *   filesize (for ulimit -f), memory, time, walltime.
 157          * @return $this
 158          */
 159         public function limits( array $limits ): Command {
 160                 if ( !isset( $limits['walltime'] ) && isset( $limits['time'] ) ) {
 161                         // Emulate the behavior of old wfShellExec() where walltime fell back on time
 162                         // if the latter was overridden and the former wasn't
 163                         $limits['walltime'] = $limits['time'];
 164                 }
 165                 $this->limits = $limits + $this->limits;
 166
 167                 return $this;
 168         }
 169
 170         /**
 171          * Sets environment variables which should be added to the executed command environment
 172          *
 173          * @param string[] $env array of variable name => value
 174          * @return $this
 175          */
 176         public function environment( array $env ): Command {
 177                 $this->env = $env;
 178
 179                 return $this;
 180         }
 181
 182         /**
 183          * Sets calling function for profiler. By default, the caller for execute() will be used.
 184          *
 185          * @param string $method
 186          * @return $this
 187          */
 188         public function profileMethod( $method ): Command {
 189                 $this->method = $method;
 190
 191                 return $this;
 192         }
 193
 194         /**
 195          * Sends the provided input to the command.
 196          * When set to null (default), the command will use the standard input.
 197          * @param string|null $inputString
 198          * @return $this
 199          */
 200         public function input( $inputString ): Command {
 201                 $this->inputString = is_null( $inputString ) ? null : (string)$inputString;
 202
 203                 return $this;
 204         }
 205
 206         /**
 207          * Controls whether stderr should be included in stdout, including errors from limit.sh.
 208          * Default: don't include.
 209          *
 210          * @param bool $yesno
 211          * @return $this
 212          */
 213         public function includeStderr( $yesno = true ): Command {
 214                 $this->doIncludeStderr = $yesno;
 215
 216                 return $this;
 217         }
 218
 219         /**
 220          * When enabled, text sent to stderr will be logged with a level of 'error'.
 221          *
 222          * @param bool $yesno
 223          * @return $this
 224          */
 225         public function logStderr( $yesno = true ): Command {
 226                 $this->doLogStderr = $yesno;
 227
 228                 return $this;
 229         }
 230
 231         /**
 232          * Sets cgroup for this command
 233          *
 234          * @param string|false $cgroup Absolute file path to the cgroup, or false to not use a cgroup
 235          * @return $this
 236          */
 237         public function cgroup( $cgroup ): Command {
 238                 $this->cgroup = $cgroup;
 239
 240                 return $this;
 241         }
 242
 243         /**
 244          * Set additional restrictions for this request
 245          *
 246          * @since 1.31
 247          * @param int $restrictions
 248          * @return $this
 249          */
 250         public function restrict( $restrictions ): Command {
 251                 $this->restrictions |= $restrictions;
 252
 253                 return $this;
 254         }
 255
 256         /**
 257          * Bitfield helper on whether a specific restriction is enabled
 258          *
 259          * @param int $restriction
 260          *
 261          * @return bool
 262          */
 263         protected function hasRestriction( $restriction ) {
 264                 return ( $this->restrictions & $restriction ) === $restriction;
 265         }
 266
 267         /**
 268          * If called, only the files/directories that are
 269          * whitelisted will be available to the shell command.
 270          *
 271          * limit.sh will always be whitelisted
 272          *
 273          * @param string[] $paths
 274          *
 275          * @return $this
 276          */
 277         public function whitelistPaths( array $paths ): Command {
 278                 // Default implementation is a no-op
 279                 return $this;
 280         }
 281
 282         /**
 283          * String together all the options and build the final command
 284          * to execute
 285          *
 286          * @param string $command Already-escaped command to run
 287          * @return array [ command, whether to use log pipe ]
 288          */
 289         protected function buildFinalCommand( $command ) {
 290                 $envcmd = '';
 291                 foreach ( $this->env as $k => $v ) {
 292                         if ( wfIsWindows() ) {
 293                                 /* Surrounding a set in quotes (method used by wfEscapeShellArg) makes the quotes themselves
 294                                  * appear in the environment variable, so we must use carat escaping as documented in
 295                                  * https://technet.microsoft.com/en-us/library/cc723564.aspx
 296                                  * Note however that the quote isn't listed there, but is needed, and the parentheses
 297                                  * are listed there but doesn't appear to need it.
 298                                  */
 299                                 $envcmd .= "set $k=" . preg_replace( '/([&|()<>^"])/', '^\\1', $v ) . '&& ';
 300                         } else {
 301                                 /* Assume this is a POSIX shell, thus required to accept variable assignments before the command
 302                                  * http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_09_01
 303                                  */
 304                                 $envcmd .= "$k=" . escapeshellarg( $v ) . ' ';
 305                         }
 306                 }
 307
 308                 $useLogPipe = false;
 309                 $cmd = $envcmd . trim( $command );
 310
 311                 if ( is_executable( '/bin/bash' ) ) {
 312                         $time = intval( $this->limits['time'] );
 313                         $wallTime = intval( $this->limits['walltime'] );
 314                         $mem = intval( $this->limits['memory'] );
 315                         $filesize = intval( $this->limits['filesize'] );
 316
 317                         if ( $time > 0 || $mem > 0 || $filesize > 0 || $wallTime > 0 ) {
 318                                 $cmd = '/bin/bash ' . escapeshellarg( __DIR__ . '/limit.sh' ) . ' ' .
 319                                         escapeshellarg( $cmd ) . ' ' .
 320                                         escapeshellarg(
 321                                                 "MW_INCLUDE_STDERR=" . ( $this->doIncludeStderr ? '1' : '' ) . ';' .
 322                                                 "MW_CPU_LIMIT=$time; " .
 323                                                 'MW_CGROUP=' . escapeshellarg( $this->cgroup ) . '; ' .
 324                                                 "MW_MEM_LIMIT=$mem; " .
 325                                                 "MW_FILE_SIZE_LIMIT=$filesize; " .
 326                                                 "MW_WALL_CLOCK_LIMIT=$wallTime; " .
 327                                                 "MW_USE_LOG_PIPE=yes"
 328                                         );
 329                                 $useLogPipe = true;
 330                         }
 331                 }
 332                 if ( !$useLogPipe && $this->doIncludeStderr ) {
 333                         $cmd .= ' 2>&1';
 334                 }
 335
 336                 return [ $cmd, $useLogPipe ];
 337         }
 338
 339         /**
 340          * Executes command. Afterwards, getExitCode() and getOutput() can be used to access execution
 341          * results.
 342          *
 343          * @return Result
 344          * @throws Exception
 345          * @throws ProcOpenError
 346          * @throws ShellDisabledError
 347          */
 348         public function execute() {
 349                 $this->everExecuted = true;
 350
 351                 $profileMethod = $this->method ?: wfGetCaller();
 352
 353                 list( $cmd, $useLogPipe ) = $this->buildFinalCommand( $this->command );
 354
 355                 $this->logger->debug( __METHOD__ . ": $cmd" );
 356
 357                 // Don't try to execute commands that exceed Linux's MAX_ARG_STRLEN.
 358                 // Other platforms may be more accomodating, but we don't want to be
 359                 // accomodating, because very long commands probably include user
 360                 // input. See T129506.
 361                 if ( strlen( $cmd ) > SHELL_MAX_ARG_STRLEN ) {
 362                         throw new Exception( __METHOD__ .
 363                                 '(): total length of $cmd must not exceed SHELL_MAX_ARG_STRLEN' );
 364                 }
 365
 366                 $desc = [
 367                         0 => $this->inputString === null ? [ 'file', 'php://stdin', 'r' ] : [ 'pipe', 'r' ],
 368                         1 => [ 'pipe', 'w' ],
 369                         2 => [ 'pipe', 'w' ],
 370                 ];
 371                 if ( $useLogPipe ) {
 372                         $desc[3] = [ 'pipe', 'w' ];
 373                 }
 374                 $pipes = null;
 375                 $scoped = Profiler::instance()->scopedProfileIn( __FUNCTION__ . '-' . $profileMethod );
 376                 $proc = proc_open( $cmd, $desc, $pipes );
 377                 if ( !$proc ) {
 378                         $this->logger->error( "proc_open() failed: {command}", [ 'command' => $cmd ] );
 379                         throw new ProcOpenError();
 380                 }
 381
 382                 $buffers = [
 383                         0 => $this->inputString, // input
 384                         1 => '', // stdout
 385                         2 => null, // stderr
 386                         3 => '', // log
 387                 ];
 388                 $emptyArray = [];
 389                 $status = false;
 390                 $logMsg = false;
 391
 392                 /* According to the documentation, it is possible for stream_select()
 393                  * to fail due to EINTR. I haven't managed to induce this in testing
 394                  * despite sending various signals. If it did happen, the error
 395                  * message would take the form:
 396                  *
 397                  * stream_select(): unable to select [4]: Interrupted system call (max_fd=5)
 398                  *
 399                  * where [4] is the value of the macro EINTR and "Interrupted system
 400                  * call" is string which according to the Linux manual is "possibly"
 401                  * localised according to LC_MESSAGES.
 402                  */
 403                 $eintr = defined( 'SOCKET_EINTR' ) ? SOCKET_EINTR : 4;
 404                 $eintrMessage = "stream_select(): unable to select [$eintr]";
 405
 406                 /* The select(2) system call only guarantees a "sufficiently small write"
 407                  * can be made without blocking. And on Linux the read might block too
 408                  * in certain cases, although I don't know if any of them can occur here.
 409                  * Regardless, set all the pipes to non-blocking to avoid T184171.
 410                  */
 411                 foreach ( $pipes as $pipe ) {
 412                         stream_set_blocking( $pipe, false );
 413                 }
 414
 415                 $running = true;
 416                 $timeout = null;
 417                 $numReadyPipes = 0;
 418
 419                 while ( $pipes && ( $running === true || $numReadyPipes !== 0 ) ) {
 420                         if ( $running ) {
 421                                 $status = proc_get_status( $proc );
 422                                 // If the process has terminated, switch to nonblocking selects
 423                                 // for getting any data still waiting to be read.
 424                                 if ( !$status['running'] ) {
 425                                         $running = false;
 426                                         $timeout = 0;
 427                                 }
 428                         }
 429
 430                         // clear get_last_error without actually raising an error
 431                         // from https://www.php.net/manual/en/function.error-get-last.php#113518
 432                         // TODO replace with error_clear_last after dropping HHVM
 433                         // @phan-suppress-next-line PhanTypeMismatchArgumentInternal
 434                         set_error_handler( function () {
 435                         }, 0 );
 436                         AtEase::suppressWarnings();
 437                         trigger_error( '' );
 438                         AtEase::restoreWarnings();
 439                         restore_error_handler();
 440
 441                         $readPipes = array_filter( $pipes, function ( $fd ) use ( $desc ) {
 442                                 return $desc[$fd][0] === 'pipe' && $desc[$fd][1] === 'r';
 443                         }, ARRAY_FILTER_USE_KEY );
 444                         $writePipes = array_filter( $pipes, function ( $fd ) use ( $desc ) {
 445                                 return $desc[$fd][0] === 'pipe' && $desc[$fd][1] === 'w';
 446                         }, ARRAY_FILTER_USE_KEY );
 447                         // stream_select parameter names are from the POV of us being able to do the operation;
 448                         // proc_open desriptor types are from the POV of the process doing it.
 449                         // So $writePipes is passed as the $read parameter and $readPipes as $write.
 450                         AtEase::suppressWarnings();
 451                         $numReadyPipes = stream_select( $writePipes, $readPipes, $emptyArray, $timeout );
 452                         AtEase::restoreWarnings();
 453                         if ( $numReadyPipes === false ) {
 454                                 $error = error_get_last();
 455                                 if ( strncmp( $error['message'], $eintrMessage, strlen( $eintrMessage ) ) == 0 ) {
 456                                         continue;
 457                                 } else {
 458                                         trigger_error( $error['message'], E_USER_WARNING );
 459                                         $logMsg = $error['message'];
 460                                         break;
 461                                 }
 462                         }
 463                         foreach ( $writePipes + $readPipes as $fd => $pipe ) {
 464                                 // True if a pipe is unblocked for us to write into, false if for reading from
 465                                 $isWrite = array_key_exists( $fd, $readPipes );
 466
 467                                 if ( $isWrite ) {
 468                                         // Don't bother writing if the buffer is empty
 469                                         if ( $buffers[$fd] === '' ) {
 470                                                 fclose( $pipes[$fd] );
 471                                                 unset( $pipes[$fd] );
 472                                                 continue;
 473                                         }
 474                                         $res = fwrite( $pipe, $buffers[$fd], 65536 );
 475                                 } else {
 476                                         $res = fread( $pipe, 65536 );
 477                                 }
 478
 479                                 if ( $res === false ) {
 480                                         $logMsg = 'Error ' . ( $isWrite ? 'writing to' : 'reading from' ) . ' pipe';
 481                                         break 2;
 482                                 }
 483
 484                                 if ( $res === '' || $res === 0 ) {
 485                                         // End of file?
 486                                         if ( feof( $pipe ) ) {
 487                                                 fclose( $pipes[$fd] );
 488                                                 unset( $pipes[$fd] );
 489                                         }
 490                                 } elseif ( $isWrite ) {
 491                                         $buffers[$fd] = (string)substr( $buffers[$fd], $res );
 492                                         if ( $buffers[$fd] === '' ) {
 493                                                 fclose( $pipes[$fd] );
 494                                                 unset( $pipes[$fd] );
 495                                         }
 496                                 } else {
 497                                         $buffers[$fd] .= $res;
 498                                         if ( $fd === 3 && strpos( $res, "\n" ) !== false ) {
 499                                                 // For the log FD, every line is a separate log entry.
 500                                                 $lines = explode( "\n", $buffers[3] );
 501                                                 $buffers[3] = array_pop( $lines );
 502                                                 foreach ( $lines as $line ) {
 503                                                         $this->logger->info( $line );
 504                                                 }
 505                                         }
 506                                 }
 507                         }
 508                 }
 509
 510                 foreach ( $pipes as $pipe ) {
 511                         fclose( $pipe );
 512                 }
 513
 514                 // Use the status previously collected if possible, since proc_get_status()
 515                 // just calls waitpid() which will not return anything useful the second time.
 516                 if ( $running ) {
 517                         $status = proc_get_status( $proc );
 518                 }
 519
 520                 if ( $logMsg !== false ) {
 521                         // Read/select error
 522                         $retval = -1;
 523                         proc_close( $proc );
 524                 } elseif ( $status['signaled'] ) {
 525                         $logMsg = "Exited with signal {$status['termsig']}";
 526                         $retval = 128 + $status['termsig'];
 527                         proc_close( $proc );
 528                 } else {
 529                         if ( $status['running'] ) {
 530                                 $retval = proc_close( $proc );
 531                         } else {
 532                                 $retval = $status['exitcode'];
 533                                 proc_close( $proc );
 534                         }
 535                         if ( $retval == 127 ) {
 536                                 $logMsg = "Possibly missing executable file";
 537                         } elseif ( $retval >= 129 && $retval <= 192 ) {
 538                                 $logMsg = "Probably exited with signal " . ( $retval - 128 );
 539                         }
 540                 }
 541
 542                 if ( $logMsg !== false ) {
 543                         $this->logger->warning( "$logMsg: {command}", [ 'command' => $cmd ] );
 544                 }
 545
 546                 if ( $buffers[2] && $this->doLogStderr ) {
 547                         $this->logger->error( "Error running {command}: {error}", [
 548                                 'command' => $cmd,
 549                                 'error' => $buffers[2],
 550                                 'exitcode' => $retval,
 551                                 'exception' => new Exception( 'Shell error' ),
 552                         ] );
 553                 }
 554
 555                 return new Result( $retval, $buffers[1], $buffers[2] );
 556         }
 557
 558         /**
 559          * Returns the final command line before environment/limiting, etc are applied.
 560          * Use string conversion only for debugging, don't try to pass this to
 561          * some other execution medium.
 562          *
 563          * @return string
 564          */
 565         public function __toString() {
 566                 return "#Command: {$this->command}";
 567         }
 568 }