includes/parser/MWTidy.php

   1 <?php
   2 /**
   3  * HTML validation and correction
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Parser
  22  */
  23
  24 /**
  25  * Class used to hide mw:editsection tokens from Tidy so that it doesn't break them
  26  * or break on them. This is a bit of a hack for now, but hopefully in the future
  27  * we may create a real postprocessor or something that will replace this.
  28  * It's called wrapper because for now it basically takes over MWTidy::tidy's task
  29  * of wrapping the text in a xhtml block
  30  *
  31  * This re-uses some of the parser's UNIQ tricks, though some of it is private so it's
  32  * duplicated. Perhaps we should create an abstract marker hiding class.
  33  *
  34  * @ingroup Parser
  35  */
  36 class MWTidyWrapper {
  37
  38         /**
  39          * @var ReplacementArray
  40          */
  41         protected $mTokens;
  42
  43         protected $mMarkerIndex;
  44
  45         public function __construct() {
  46                 $this->mTokens = null;
  47         }
  48
  49         /**
  50          * @param string $text
  51          * @return string
  52          */
  53         public function getWrapped( $text ) {
  54                 $this->mTokens = new ReplacementArray;
  55                 $this->mMarkerIndex = 0;
  56
  57                 // Replace <mw:editsection> elements with placeholders
  58                 $wrappedtext = preg_replace_callback( ParserOutput::EDITSECTION_REGEX,
  59                         array( &$this, 'replaceCallback' ), $text );
  60                 // ...and <mw:toc> markers
  61                 $wrappedtext = preg_replace_callback( '/\<\\/?mw:toc\>/',
  62                         array( &$this, 'replaceCallback' ), $wrappedtext );
  63                 // ... and <math> tags
  64                 $wrappedtext = preg_replace_callback( '/\<math(.*?)\<\\/math\>/s',
  65                         array( &$this, 'replaceCallback' ), $wrappedtext );
  66                 // Modify inline Microdata <link> and <meta> elements so they say <html-link> and <html-meta> so
  67                 // we can trick Tidy into not stripping them out by including them in tidy's new-empty-tags config
  68                 $wrappedtext = preg_replace( '!<(link|meta)([^>]*?)(/{0,1}>)!', '<html-$1$2$3', $wrappedtext );
  69
  70                 // Wrap the whole thing in a doctype and body for Tidy.
  71                 $wrappedtext = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"' .
  72                         ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html>' .
  73                         '<head><title>test</title></head><body>' . $wrappedtext . '</body></html>';
  74
  75                 return $wrappedtext;
  76         }
  77
  78         /**
  79          * @param array $m
  80          *
  81          * @return string
  82          */
  83         public function replaceCallback( $m ) {
  84                 $marker = Parser::MARKER_PREFIX . "-item-{$this->mMarkerIndex}" . Parser::MARKER_SUFFIX;
  85                 $this->mMarkerIndex++;
  86                 $this->mTokens->setPair( $marker, $m[0] );
  87                 return $marker;
  88         }
  89
  90         /**
  91          * @param string $text
  92          * @return string
  93          */
  94         public function postprocess( $text ) {
  95                 // Revert <html-{link,meta}> back to <{link,meta}>
  96                 $text = preg_replace( '!<html-(link|meta)([^>]*?)(/{0,1}>)!', '<$1$2$3', $text );
  97
  98                 // Restore the contents of placeholder tokens
  99                 $text = $this->mTokens->replace( $text );
 100
 101                 return $text;
 102         }
 103
 104 }
 105
 106 /**
 107  * Class to interact with HTML tidy
 108  *
 109  * Either the external tidy program or the in-process tidy extension
 110  * will be used depending on availability. Override the default
 111  * $wgTidyInternal setting to disable the internal if it's not working.
 112  *
 113  * @ingroup Parser
 114  */
 115 class MWTidy {
 116         /**
 117          * Interface with html tidy, used if $wgUseTidy = true.
 118          * If tidy isn't able to correct the markup, the original will be
 119          * returned in all its glory with a warning comment appended.
 120          *
 121          * @param string $text Hideous HTML input
 122          * @return string Corrected HTML output
 123          */
 124         public static function tidy( $text ) {
 125                 $wrapper = new MWTidyWrapper;
 126                 $wrappedtext = $wrapper->getWrapped( $text );
 127
 128                 $retVal = null;
 129                 $correctedtext = self::clean( $wrappedtext, false, $retVal );
 130
 131                 if ( $retVal < 0 ) {
 132                         wfDebug( "Possible tidy configuration error!\n" );
 133                         return $text . "\n<!-- Tidy was unable to run -->\n";
 134                 } elseif ( is_null( $correctedtext ) ) {
 135                         wfDebug( "Tidy error detected!\n" );
 136                         return $text . "\n<!-- Tidy found serious XHTML errors -->\n";
 137                 }
 138
 139                 $correctedtext = $wrapper->postprocess( $correctedtext ); // restore any hidden tokens
 140
 141                 return $correctedtext;
 142         }
 143
 144         /**
 145          * Check HTML for errors, used if $wgValidateAllHtml = true.
 146          *
 147          * @param string $text
 148          * @param string &$errorStr Return the error string
 149          * @return bool Whether the HTML is valid
 150          */
 151         public static function checkErrors( $text, &$errorStr = null ) {
 152                 $retval = 0;
 153                 $errorStr = self::clean( $text, true, $retval );
 154                 return ( $retval < 0 && $errorStr == '' ) || $retval == 0;
 155         }
 156
 157         /**
 158          * Perform a clean/repair operation
 159          * @param string $text HTML to check
 160          * @param bool $stderr Whether to read result from STDERR rather than STDOUT
 161          * @param int &$retval Exit code (-1 on internal error)
 162          * @return null|string
 163          * @throws MWException
 164          */
 165         private static function clean( $text, $stderr = false, &$retval = null ) {
 166                 global $wgTidyInternal;
 167
 168                 if ( $wgTidyInternal ) {
 169                         if ( wfIsHHVM() ) {
 170                                 if ( $stderr ) {
 171                                         throw new MWException( __METHOD__ . ": error text return from HHVM tidy is not supported" );
 172                                 }
 173                                 return self::hhvmClean( $text, $retval );
 174                         } else {
 175                                 return self::phpClean( $text, $stderr, $retval );
 176                         }
 177                 } else {
 178                         return self::externalClean( $text, $stderr, $retval );
 179                 }
 180         }
 181
 182         /**
 183          * Spawn an external HTML tidy process and get corrected markup back from it.
 184          * Also called in OutputHandler.php for full page validation
 185          *
 186          * @param string $text HTML to check
 187          * @param bool $stderr Whether to read result from STDERR rather than STDOUT
 188          * @param int &$retval Exit code (-1 on internal error)
 189          * @return string|null
 190          */
 191         private static function externalClean( $text, $stderr = false, &$retval = null ) {
 192                 global $wgTidyConf, $wgTidyBin, $wgTidyOpts;
 193
 194                 $cleansource = '';
 195                 $opts = ' -utf8';
 196
 197                 if ( $stderr ) {
 198                         $descriptorspec = array(
 199                                 0 => array( 'pipe', 'r' ),
 200                                 1 => array( 'file', wfGetNull(), 'a' ),
 201                                 2 => array( 'pipe', 'w' )
 202                         );
 203                 } else {
 204                         $descriptorspec = array(
 205                                 0 => array( 'pipe', 'r' ),
 206                                 1 => array( 'pipe', 'w' ),
 207                                 2 => array( 'file', wfGetNull(), 'a' )
 208                         );
 209                 }
 210
 211                 $readpipe = $stderr ? 2 : 1;
 212                 $pipes = array();
 213
 214                 $process = proc_open(
 215                         "$wgTidyBin -config $wgTidyConf $wgTidyOpts$opts", $descriptorspec, $pipes );
 216
 217                 //NOTE: At least on linux, the process will be created even if tidy is not installed.
 218                 //      This means that missing tidy will be treated as a validation failure.
 219
 220                 if ( is_resource( $process ) ) {
 221                         // Theoretically, this style of communication could cause a deadlock
 222                         // here. If the stdout buffer fills up, then writes to stdin could
 223                         // block. This doesn't appear to happen with tidy, because tidy only
 224                         // writes to stdout after it's finished reading from stdin. Search
 225                         // for tidyParseStdin and tidySaveStdout in console/tidy.c
 226                         fwrite( $pipes[0], $text );
 227                         fclose( $pipes[0] );
 228                         while ( !feof( $pipes[$readpipe] ) ) {
 229                                 $cleansource .= fgets( $pipes[$readpipe], 1024 );
 230                         }
 231                         fclose( $pipes[$readpipe] );
 232                         $retval = proc_close( $process );
 233                 } else {
 234                         wfWarn( "Unable to start external tidy process" );
 235                         $retval = -1;
 236                 }
 237
 238                 if ( !$stderr && $cleansource == '' && $text != '' ) {
 239                         // Some kind of error happened, so we couldn't get the corrected text.
 240                         // Just give up; we'll use the source text and append a warning.
 241                         $cleansource = null;
 242                 }
 243
 244                 return $cleansource;
 245         }
 246
 247         /**
 248          * Use the HTML tidy extension to use the tidy library in-process,
 249          * saving the overhead of spawning a new process.
 250          *
 251          * @param string $text HTML to check
 252          * @param bool $stderr Whether to read result from error status instead of output
 253          * @param int &$retval Exit code (-1 on internal error)
 254          * @return string|null
 255          */
 256         private static function phpClean( $text, $stderr = false, &$retval = null ) {
 257                 global $wgTidyConf, $wgDebugTidy;
 258
 259                 if ( ( !wfIsHHVM() && !class_exists( 'tidy' ) ) ||
 260                         ( wfIsHHVM() && !function_exists( 'tidy_repair_string' ) )
 261                 ) {
 262                         wfWarn( "Unable to load internal tidy class." );
 263                         $retval = -1;
 264
 265                         return null;
 266                 }
 267
 268                 $tidy = new tidy;
 269                 $tidy->parseString( $text, $wgTidyConf, 'utf8' );
 270
 271                 if ( $stderr ) {
 272                         $retval = $tidy->getStatus();
 273                         return $tidy->errorBuffer;
 274                 }
 275
 276                 $tidy->cleanRepair();
 277                 $retval = $tidy->getStatus();
 278                 if ( $retval == 2 ) {
 279                         // 2 is magic number for fatal error
 280                         // http://www.php.net/manual/en/function.tidy-get-status.php
 281                         $cleansource = null;
 282                 } else {
 283                         $cleansource = tidy_get_output( $tidy );
 284                         if ( $wgDebugTidy && $retval > 0 ) {
 285                                 $cleansource .= "<!--\nTidy reports:\n" .
 286                                         str_replace( '-->', '--&gt;', $tidy->errorBuffer ) .
 287                                         "\n-->";
 288                         }
 289                 }
 290
 291                 return $cleansource;
 292         }
 293
 294         /**
 295          * Use the tidy extension for HHVM from
 296          * https://github.com/wikimedia/mediawiki-php-tidy
 297          *
 298          * This currently does not support the object-oriented interface, but
 299          * tidy_repair_string() can be used for the most common tasks.
 300          *
 301          * @param string $text HTML to check
 302          * @param int &$retval Exit code (-1 on internal error)
 303          * @return string|null
 304          */
 305         private static function hhvmClean( $text, &$retval ) {
 306                 global $wgTidyConf;
 307
 308                 $cleansource = tidy_repair_string( $text, $wgTidyConf, 'utf8' );
 309                 if ( $cleansource === false ) {
 310                         $cleansource = null;
 311                         $retval = -1;
 312                 } else {
 313                         $retval = 0;
 314                 }
 315
 316                 return $cleansource;
 317         }
 318 }