includes/parser/MWTidy.php

   1 <?php
   2 /**
   3  * HTML validation and correction
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Parser
  22  */
  23
  24 /**
  25  * Class used to hide mw:editsection tokens from Tidy so that it doesn't break them
  26  * or break on them. This is a bit of a hack for now, but hopefully in the future
  27  * we may create a real postprocessor or something that will replace this.
  28  * It's called wrapper because for now it basically takes over MWTidy::tidy's task
  29  * of wrapping the text in a xhtml block
  30  *
  31  * This re-uses some of the parser's UNIQ tricks, though some of it is private so it's
  32  * duplicated. Perhaps we should create an abstract marker hiding class.
  33  *
  34  * @ingroup Parser
  35  */
  36 class MWTidyWrapper {
  37
  38         /**
  39          * @var ReplacementArray
  40          */
  41         protected $mTokens;
  42
  43         protected $mUniqPrefix;
  44
  45         protected $mMarkerIndex;
  46
  47         public function __construct() {
  48                 $this->mTokens = null;
  49                 $this->mUniqPrefix = null;
  50         }
  51
  52         /**
  53          * @param string $text
  54          * @return string
  55          */
  56         public function getWrapped( $text ) {
  57                 $this->mTokens = new ReplacementArray;
  58                 $this->mUniqPrefix = "\x7fUNIQ" .
  59                         dechex( mt_rand( 0, 0x7fffffff ) ) . dechex( mt_rand( 0, 0x7fffffff ) );
  60                 $this->mMarkerIndex = 0;
  61
  62                 // Replace <mw:editsection> elements with placeholders
  63                 $wrappedtext = preg_replace_callback( ParserOutput::EDITSECTION_REGEX,
  64                         array( &$this, 'replaceCallback' ), $text );
  65                 // ...and <mw:toc> markers
  66                 $wrappedtext = preg_replace_callback( '/\<\\/?mw:toc\>/',
  67                         array( &$this, 'replaceCallback' ), $wrappedtext );
  68                 // ... and <math> tags
  69                 $wrappedtext = preg_replace_callback( '/\<math(.*?)\<\\/math\>/s',
  70                         array( &$this, 'replaceCallback' ), $wrappedtext );
  71                 // Modify inline Microdata <link> and <meta> elements so they say <html-link> and <html-meta> so
  72                 // we can trick Tidy into not stripping them out by including them in tidy's new-empty-tags config
  73                 $wrappedtext = preg_replace( '!<(link|meta)([^>]*?)(/{0,1}>)!', '<html-$1$2$3', $wrappedtext );
  74
  75                 // Wrap the whole thing in a doctype and body for Tidy.
  76                 $wrappedtext = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"' .
  77                         ' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html>' .
  78                         '<head><title>test</title></head><body>' . $wrappedtext . '</body></html>';
  79
  80                 return $wrappedtext;
  81         }
  82
  83         /**
  84          * @param array $m
  85          *
  86          * @return string
  87          */
  88         public function replaceCallback( $m ) {
  89                 $marker = "{$this->mUniqPrefix}-item-{$this->mMarkerIndex}" . Parser::MARKER_SUFFIX;
  90                 $this->mMarkerIndex++;
  91                 $this->mTokens->setPair( $marker, $m[0] );
  92                 return $marker;
  93         }
  94
  95         /**
  96          * @param string $text
  97          * @return string
  98          */
  99         public function postprocess( $text ) {
 100                 // Revert <html-{link,meta}> back to <{link,meta}>
 101                 $text = preg_replace( '!<html-(link|meta)([^>]*?)(/{0,1}>)!', '<$1$2$3', $text );
 102
 103                 // Restore the contents of placeholder tokens
 104                 $text = $this->mTokens->replace( $text );
 105
 106                 return $text;
 107         }
 108
 109 }
 110
 111 /**
 112  * Class to interact with HTML tidy
 113  *
 114  * Either the external tidy program or the in-process tidy extension
 115  * will be used depending on availability. Override the default
 116  * $wgTidyInternal setting to disable the internal if it's not working.
 117  *
 118  * @ingroup Parser
 119  */
 120 class MWTidy {
 121         /**
 122          * Interface with html tidy, used if $wgUseTidy = true.
 123          * If tidy isn't able to correct the markup, the original will be
 124          * returned in all its glory with a warning comment appended.
 125          *
 126          * @param string $text Hideous HTML input
 127          * @return string Corrected HTML output
 128          */
 129         public static function tidy( $text ) {
 130                 $wrapper = new MWTidyWrapper;
 131                 $wrappedtext = $wrapper->getWrapped( $text );
 132
 133                 $retVal = null;
 134                 list( $correctedtext, $errors ) = self::clean( $wrappedtext, $retVal );
 135
 136                 if ( $retVal < 0 ) {
 137                         wfDebug( "Possible tidy configuration error!\n" );
 138                         return $text . "\n<!-- Tidy was unable to run -->\n";
 139                 } elseif ( $correctedtext === '' && $text !== '' ) {
 140                         wfDebug( "Tidy error detected!\n" );
 141                         return $text . "\n<!-- Tidy found serious XHTML errors -->\n";
 142                 }
 143
 144                 $correctedtext = $wrapper->postprocess( $correctedtext ); // restore any hidden tokens
 145
 146                 return $correctedtext;
 147         }
 148
 149         /**
 150          * Check HTML for errors, used if $wgValidateAllHtml = true.
 151          *
 152          * @param string $text
 153          * @param string &$errorStr Return the error string
 154          * @return bool Whether the HTML is valid
 155          */
 156         public static function checkErrors( $text, &$errorStr = null ) {
 157                 $retval = 0;
 158                 list( $outputStr, $errorStr ) = self::clean( $text, $retval );
 159                 return ( $retval < 0 && $errorStr == '' ) || $retval == 0;
 160         }
 161
 162         /**
 163          * Perform a clean/repair operation
 164          * @param string $text HTML to check
 165          * @param int &$retval Exit code (-1 on internal error)
 166          * @return string|null
 167          */
 168         private static function clean( $text, &$retval = null ) {
 169                 global $wgTidyInternal;
 170
 171                 if ( $wgTidyInternal ) {
 172                         return self::internalClean( $text, $retval );
 173                 } else {
 174                         return self::externalClean( $text, $retval );
 175                 }
 176         }
 177
 178         /**
 179          * Spawn an external HTML tidy process and get corrected markup back from it.
 180          * Also called in OutputHandler.php for full page validation
 181          *
 182          * @param string $text HTML to check
 183          * @param int &$retval Exit code (-1 on internal error)
 184          * @return string|null
 185          */
 186         private static function externalClean( $text, &$retval = null ) {
 187                 global $wgTidyConf, $wgTidyBin, $wgTidyOpts;
 188                 wfProfileIn( __METHOD__ );
 189
 190                 $opts = ' -utf8';
 191
 192                 $descriptorspec = array(
 193                         0 => array( 'pipe', 'r' ),
 194                         1 => array( 'pipe', 'w' ),
 195                         2 => array( 'pipe', 'w' ),
 196                 );
 197
 198                 $outputBuffer = '';
 199                 $errorBuffer = '';
 200                 $pipes = array();
 201
 202                 $process = proc_open(
 203                         "$wgTidyBin -config $wgTidyConf $wgTidyOpts$opts", $descriptorspec, $pipes );
 204
 205                 //NOTE: At least on linux, the process will be created even if tidy is not installed.
 206                 //      This means that missing tidy will be treated as a validation failure.
 207
 208                 if ( is_resource( $process ) ) {
 209                         // Theoretically, this style of communication could cause a deadlock
 210                         // here. If the stdout buffer fills up, then writes to stdin could
 211                         // block. This doesn't appear to happen with tidy, because tidy only
 212                         // writes to stdout after it's finished reading from stdin. Search
 213                         // for tidyParseStdin and tidySaveStdout in console/tidy.c
 214                         fwrite( $pipes[0], $text );
 215                         fclose( $pipes[0] );
 216
 217                         while ( !feof( $pipes[1] ) ) {
 218                                 $outputBuffer .= fgets( $pipes[1], 1024 );
 219                         }
 220                         fclose( $pipes[1] );
 221
 222                         while ( !feof( $pipes[2] ) ) {
 223                                 $errorBuffer .= fgets( $pipes[2], 1024 );
 224                         }
 225                         fclose( $pipes[2] );
 226
 227                         $retval = proc_close( $process );
 228                 } else {
 229                         wfWarn( "Unable to start external tidy process" );
 230                         $retval = -1;
 231                 }
 232
 233                 wfProfileOut( __METHOD__ );
 234                 return array( $outputBuffer, $errorBuffer );
 235         }
 236
 237         /**
 238          * Use the HTML tidy extension to use the tidy library in-process,
 239          * saving the overhead of spawning a new process.
 240          *
 241          * @param string $text HTML to check
 242          * @param int &$retval Exit code (-1 on internal error)
 243          * @return string|null
 244          */
 245         private static function internalClean( $text, &$retval = null ) {
 246                 global $wgTidyConf, $wgDebugTidy;
 247                 wfProfileIn( __METHOD__ );
 248
 249                 if ( ( !wfIsHHVM() && !class_exists( 'tidy' ) ) ||
 250                         ( wfIsHHVM() && !function_exists( 'tidy_repair_string' ) )
 251                 ) {
 252                         wfWarn( "Unable to load internal tidy class." );
 253                         $retval = -1;
 254
 255                         wfProfileOut( __METHOD__ );
 256                         return null;
 257                 }
 258
 259                 $outputBuffer = '';
 260                 $errorBuffer = '';
 261
 262                 if ( wfIsHHVM() ) {
 263                         // Use the tidy extension for HHVM from
 264                         // https://github.com/wikimedia/mediawiki-php-tidy
 265                         //
 266                         // This currently does not support the object-oriented interface, but
 267                         // tidy_repair_string() can be used for the most common tasks.
 268                         $result = tidy_repair_string( $text, $wgTidyConf, 'utf8' );
 269                         $outputBuffer .= $result;
 270                         $retval = $result === false ? -1 : 0;
 271                 } else {
 272                         $tidy = new tidy;
 273                         $tidy->parseString( $text, $wgTidyConf, 'utf8' );
 274                         $tidy->cleanRepair();
 275                         $retval = $tidy->getStatus();
 276                         $outputBuffer .= tidy_get_output( $tidy );
 277                         if ( $retval > 0 ) {
 278                                 $errorBuffer .= $tidy->errorBuffer;
 279                         }
 280                 }
 281
 282                 if ( $wgDebugTidy && $errorBuffer && $retval > 0 ) {
 283                         $outputBuffer .= "<!--\nTidy reports:\n" .
 284                                 str_replace( '-->', '--&gt;', $tidy->errorBuffer ) .
 285                                 "\n-->";
 286                 }
 287
 288                 wfProfileOut( __METHOD__ );
 289                 return array( $outputBuffer, $errorBuffer );
 290         }
 291 }