Don't armor french spaces before punctuation followed by word characters

This makes Sanitizer::armorFrenchSpaces() more selective about where it
inserts  , avoiding the need to protect common "not actually French"
cases like `color: red !important` and `foo :bar`.

We also added the single-guillemet to the rules, to accomodate Swiss French.

Bug: T197902
Change-Id: I42e747f17c17c1513fec96cdd2d3285da7da05a4

diff --git a/includes/parser/Sanitizer.php b/includes/parser/Sanitizer.php
index 21498f8..7a0d5f6 100644 (file)
--- a/includes/parser/Sanitizer.php
+++ b/includes/parser/Sanitizer.php
@@ -1155,9 +1155,10 @@ class Sanitizer {
                $fixtags = [
                        # French spaces, last one Guillemet-left
                        # only if there is something before the space
-                       '/(.) (?=[?:;!%»])/u' => "\\1$space",
+                       # and a non-word character after the punctuation.
+                       '/(\S) (?=[?:;!%»›](?!\w))/u' => "\\1$space",
                        # French spaces, Guillemet-right
-                       '/(«) /u' => "\\1$space",
+                       '/([«‹]) /u' => "\\1$space",
                ];
                return preg_replace( array_keys( $fixtags ), array_values( $fixtags ), $text );
        }

diff --git a/tests/parser/parserTests.txt b/tests/parser/parserTests.txt
index e06a732..d19a397 100644 (file)
--- a/tests/parser/parserTests.txt
+++ b/tests/parser/parserTests.txt
@@ -3880,7 +3880,7 @@ Simple definition
 !! wikitext
 ;name :Definition
 !! html
-<dl><dt>name&#160;</dt>
+<dl><dt>name</dt>
 <dd>Definition</dd></dl>
 
 !! end
@@ -3909,7 +3909,7 @@ Definition list with URL link
 !! wikitext
 ;http://example.com/ :definition
 !! html
-<dl><dt><a rel="nofollow" class="external free" href="http://example.com/">http://example.com/</a>&#160;</dt>
+<dl><dt><a rel="nofollow" class="external free" href="http://example.com/">http://example.com/</a></dt>
 <dd>definition</dd></dl>
 
 !! end
@@ -3940,7 +3940,7 @@ Definition list with news link containing colon
 !! wikitext
 ;news:alt.wikipedia.rox :This isn't even a real newsgroup!
 !! html/php
-<dl><dt><a rel="nofollow" class="external free" href="news:alt.wikipedia.rox">news:alt.wikipedia.rox</a>&#160;</dt>
+<dl><dt><a rel="nofollow" class="external free" href="news:alt.wikipedia.rox">news:alt.wikipedia.rox</a></dt>
 <dd>This isn't even a real newsgroup!</dd></dl>
 
 !! html/parsoid
@@ -3961,7 +3961,7 @@ Definition lists: colon in external link text
 !! wikitext
 ;[http://www.wikipedia2.org/ Wikipedia :The Next Generation] :OK, I made that up
 !! html
-<dl><dt><a rel="nofollow" class="external text" href="http://www.wikipedia2.org/">Wikipedia&#160;:The Next Generation</a>&#160;</dt>
+<dl><dt><a rel="nofollow" class="external text" href="http://www.wikipedia2.org/">Wikipedia :The Next Generation</a></dt>
 <dd>OK, I made that up</dd></dl>
 
 !! end
@@ -3980,7 +3980,7 @@ Definition lists: self-closed tag
 !! wikitext
 ;one<br/>two :two-line fun
 !! html
-<dl><dt>one<br />two&#160;</dt>
+<dl><dt>one<br />two</dt>
 <dd>two-line fun</dd></dl>
 
 !! end
@@ -4000,7 +4000,7 @@ Definition lists: excess closed tags
 !! wikitext
 ;one</b>two :bad tag fun
 !! html/php+tidy
-<dl><dt>onetwo&#160;</dt>
+<dl><dt>onetwo</dt>
 <dd>bad tag fun</dd></dl>
 !! html/parsoid
 <dl>
@@ -4037,7 +4037,7 @@ Definition and unordered list using wiki syntax nested in unordered list using h
 </li></ul>
 !! html
 <ul><li>
-<dl><dt>term&#160;</dt>
+<dl><dt>term</dt>
 <dd>description</dd></dl>
 <ul><li>unordered</li></ul>
 </li></ul>
@@ -4437,9 +4437,9 @@ Definition Lists: Mixed Lists: Test 4
 *;d1 :d2
 *;d3 :d4
 !! html
-<ul><li><dl><dt>d1&#160;</dt>
+<ul><li><dl><dt>d1</dt>
 <dd>d2</dd>
-<dt>d3&#160;</dt>
+<dt>d3</dt>
 <dd>d4</dd></dl></li></ul>
 
 !! end
@@ -4498,7 +4498,7 @@ Definition Lists: Mixed Lists: Test 9
 !! wikitext
 *;foo :bar
 !! html
-<ul><li><dl><dt>foo&#160;</dt>
+<ul><li><dl><dt>foo</dt>
 <dd>bar</dd></dl></li></ul>
 
 !! end
@@ -4509,7 +4509,7 @@ Definition Lists: Mixed Lists: Test 10
 !! wikitext
 *#;foo :bar
 !! html
-<ul><li><ol><li><dl><dt>foo&#160;</dt>
+<ul><li><ol><li><dl><dt>foo</dt>
 <dd>bar</dd></dl></li></ol></li></ul>
 
 !! end
@@ -4542,15 +4542,15 @@ Definition Lists: Mixed Lists: Test 12
 *#*#;*;;foo :bar
 *#*#;boo :baz
 !! html/php
-<ul><li><ol><li><ul><li><ol><li><dl><dt>foo&#160;</dt>
+<ul><li><ol><li><ul><li><ol><li><dl><dt>foo</dt>
 <dd><ul><li><dl><dt><dl><dt>bar</dt></dl></dd></dl></li></ul></dd></dl>
-<dl><dt>boo&#160;</dt>
+<dl><dt>boo</dt>
 <dd>baz</dd></dl></li></ol></li></ul></li></ol></li></ul>
 
 !! html/php+tidy
-<ul><li><ol><li><ul><li><ol><li><dl><dt>foo&#160;</dt>
+<ul><li><ol><li><ul><li><ol><li><dl><dt>foo</dt>
 <dd><ul><li><dl><dt><dl><dt>bar</dt></dl></dt></dl></li></ul></dd></dl></li></ol></li></ul>
-<dl><dt>boo&#160;</dt>
+<dl><dt>boo</dt>
 <dd>baz</dd></dl></li></ol></li></ul>
 !! html/parsoid
 <ul>
@@ -4568,12 +4568,12 @@ Definition Lists: Mixed Lists: Test 12
 <dl>
 <dt>
 <dl>
-<dt>foo<span typeof="mw:Placeholder" data-parsoid='{"src":" "}'>&nbsp;</span></dt>
+<dt>foo</dt>
 <dd data-parsoid='{"stx":"row"}'>bar</dd>
 </dl></dt>
 </dl></li>
 </ul></dt>
-<dt>boo<span typeof="mw:Placeholder" data-parsoid='{"src":" "}'>&nbsp;</span></dt>
+<dt>boo</dt>
 <dd data-parsoid='{"stx":"row"}'>baz</dd>
 </dl></li>
 </ol></li>
@@ -4582,6 +4582,18 @@ Definition Lists: Mixed Lists: Test 12
 </ul>
 !! end
 
+!! test
+Definition Lists: Mixed Lists: Test 13
+!! wikitext
+*#*#;*;;foo : bar
+*#*#;boo : baz
+!! html+tidy
+<ul><li><ol><li><ul><li><ol><li><dl><dt>foo&#160;</dt>
+<dd><ul><li><dl><dt><dl><dt>bar</dt></dl></dt></dl></li></ul></dd></dl></li></ol></li></ul>
+<dl><dt>boo&#160;</dt>
+<dd>baz</dd></dl></li></ol></li></ul>
+!! end
+
 # FIXME: Maybe get rid of this test?
 # From whitelist:
 # * The test is wrong, there are two colons where there should be :;
@@ -4591,7 +4603,7 @@ Definition Lists: Weird Ones: Test 1
 !! wikitext
 *#;*::;;foo :bar (who uses this?)
 !! html/php+tidy
-<ul><li><ol><li><dl><dt>foo&#160;</dt>
+<ul><li><ol><li><dl><dt>foo</dt>
 <dd><ul><li><dl><dd><dl><dd><dl><dt><dl><dt>bar (who uses this?)</dt></dl></dt></dl></dd></dl></dd></dl></li></ul></dd></dl></li></ol></li></ul>
 !! html/parsoid
 <ul>
@@ -4609,7 +4621,7 @@ Definition Lists: Weird Ones: Test 1
 <dl>
 <dt>
 <dl>
-<dt>foo<span typeof="mw:DisplaySpace mw:Placeholder" data-parsoid='{"src":" ","isDisplayHack":true}'> </span></dt>
+<dt>foo</dt>
 <dd data-parsoid='{"stx":"row"}'>bar (who uses this?)</dd>
 </dl></dt>
 </dl></dd>
@@ -6716,7 +6728,7 @@ Element attributes with double ! should not be broken up by <th>
 !! html/php
 <table>
 <tr>
-<td><div style="color: red&#32;!important;" data-contrived="put this here &#124;&#124;">hi</div>
+<td><div style="color: red !important;" data-contrived="put this here &#124;&#124;">hi</div>
 </td></tr></table>
 
 !! html/parsoid
@@ -6737,7 +6749,7 @@ parsoid=wt2html
 !! html/php
 <table>
 <tr>
-<td>style="color: red&#160;!important;" data-contrived="put this here</td>
+<td>style="color: red !important;" data-contrived="put this here</td>
 <td>foo
 </td></tr></table>
 
@@ -18758,7 +18770,7 @@ Punctuation: CSS !important (T13874)
 !! wikitext
 <div style="width:50% !important">important</div>
 !! html
-<div style="width:50%&#32;!important">important</div>
+<div style="width:50% !important">important</div>
 
 !!end
 
@@ -21536,16 +21548,16 @@ Definition list code coverage
 ;title :def
 ;title:def
 !! html/php
-<dl><dt>title  &#160;</dt>
+<dl><dt>title</dt>
 <dd>def</dd>
-<dt>title&#160;</dt>
+<dt>title</dt>
 <dd>def</dd>
 <dt>title</dt>
 <dd>def</dd></dl>
 
 !! html/parsoid
-<dl><dt>title  <span typeof="mw:Placeholder"> </span></dt><dd>def</dd>
-<dt>title<span typeof="mw:Placeholder"> </span></dt><dd>def</dd>
+<dl><dt>title   </dt><dd>def</dd>
+<dt>title </dt><dd>def</dd>
 <dt>title</dt><dd>def</dd></dl>
 !! end
 
@@ -24050,7 +24062,7 @@ Play a bit with r67090 and T5158
 <div style="width:50%&#160;!important">&nbsp;</div>
 <div style="border : solid;">&nbsp;</div>
 !! html/php
-<div style="width:50%&#32;!important">&#160;</div>
+<div style="width:50% !important">&#160;</div>
 <div style="width:50% !important">&#160;</div>
 <div style="width:50% !important">&#160;</div>
 <div style="border&#32;: solid;">&#160;</div>
@@ -24063,6 +24075,29 @@ Play a bit with r67090 and T5158
 
 !! end
 
+!! test
+French spaces in wikitext
+!! wikitext
+foo ! bar ? bat 50 % is less than 75 %.
+
+Hello : this ; is « something ‹ else › again »
+!! html
+<p>foo&#160;! bar&#160;? bat 50&#160;% is less than 75&#160;%.
+</p><p>Hello&#160;: this&#160;; is «&#160;something ‹&#160;else&#160;› again&#160;»
+</p>
+!! end
+
+# It would be reasonable for Parsoid and PHP to differ here.
+# The PHP behavior is arguably a bug.
+!! test
+Corner case: french spaces in definition list
+!! wikitext
+;foo : bar
+!! html+tidy
+<dl><dt>foo&#160;</dt>
+<dd>bar</dd></dl>
+!! end
+
 !! test
 T5158: Test for French spaces in attributes
 !! wikitext