[lazylex/html] Lex > separately, and convert it to > in XML.

Added tests.
oils-for-unix · Jan 13, 2025 · b450511 · b450511
1 parent 03e5cd0
commit b450511
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 6 deletions.
diff --git a/lazylex/html.py b/lazylex/html.py
@@ -113,7 +113,7 @@ def Print(self, s):
 
 # HTML Tokens
 # CommentBegin, ProcessingBegin, CDataBegin are "pseudo-tokens", not visible
-TOKENS = 'Decl Comment CommentBegin Processing ProcessingBegin CData CDataBegin StartTag StartEndTag EndTag DecChar HexChar CharEntity RawData HtmlCData BadAmpersand Invalid EndOfStream'.split(
+TOKENS = 'Decl Comment CommentBegin Processing ProcessingBegin CData CDataBegin StartTag StartEndTag EndTag DecChar HexChar CharEntity RawData HtmlCData BadAmpersand BadGreaterThan BadLessThan Invalid EndOfStream'.split(
 )
 
 
@@ -226,8 +226,10 @@ def MakeLexer(rules):
     #
     # - My early blog has THREE errors when disallowing >
     # - So do some .wwz files
-    (r'[^&<\x00]+', Tok.RawData),
-    (r'.', Tok.Invalid),  # error!
+    (r'[^&<>\x00]+', Tok.RawData),
+    (r'>', Tok.BadGreaterThan),
+    # < is an error
+    (r'.', Tok.Invalid),
 ]
 
 #  Old notes:
@@ -781,7 +783,8 @@ def ToText(s, left_pos=0, right_pos=-1):
 
     pos = left_pos
     for tok_id, end_pos in ValidTokens(s, left_pos, right_pos):
-        if tok_id in (Tok.RawData, Tok.BadAmpersand):
+        if tok_id in (Tok.RawData, Tok.BadAmpersand, Tok.BadGreaterThan,
+                      Tok.BadLessThan):
             out.SkipTo(pos)
             out.PrintUntil(end_pos)
 
@@ -974,6 +977,11 @@ def ToXml(htm8_str):
             #out.SkipTo(pos)
             out.Print('&amp;')
             out.SkipTo(end_pos)
+
+        elif tok_id == Tok.BadGreaterThan:
+            #out.SkipTo(pos)
+            out.Print('&gt;')
+            out.SkipTo(end_pos)
         else:
             out.PrintUntil(end_pos)
 

diff --git a/lazylex/html_test.py b/lazylex/html_test.py
@@ -336,6 +336,25 @@ def testStartTag(self):
             (Tok.EndOfStream, 9),
         ], tokens)
 
+    def testBad(self):
+        Tok = html.Tok
+
+        h = '&'
+        tokens = Lex(h)
+
+        self.assertEqual([
+            (Tok.BadAmpersand, 1),
+            (Tok.EndOfStream, 1),
+        ], tokens)
+
+        h = '>'
+        tokens = Lex(h)
+
+        self.assertEqual([
+            (Tok.BadGreaterThan, 1),
+            (Tok.EndOfStream, 1),
+        ], tokens)
+
     def testInvalid(self):
         Tok = html.Tok
 
@@ -406,10 +425,12 @@ def testValid(self):
     ('<script><![CDATA[ <wtf> >< ]]></script>', UNCHANGED),
 
     # allowed, but 3 < 4 is not allowed
-    ('<a> 3 > 4 </a>', ''),
+    ('<a> 3 > 4 </a>', '<a> 3 &gt; 4 </a>'),
     # allowed, but 3 > 4 is not allowed
     ('<p x="3 < 4"></p>', ''),
-    ('<b><a href="foo">link</a></b>', ''),
+    ('<b><a href="foo">link</a></b>', UNCHANGED),
+
+    # TODO: should be self-closing
     #('<meta><a></a>', '<meta/><a></a>'),
     ('<meta><a></a>', ''),