fix(tokenizer): Allow XML tags to start with any character (#778)

Fixes cheeriojs/cheerio#1804
fb55 · Apr 8, 2021 · 0b94ab5 · 0b94ab5
1 parent ee68790
commit 0b94ab5
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 2 deletions.
diff --git a/src/Tokenizer.ts b/src/Tokenizer.ts
@@ -324,6 +324,18 @@ export default class Tokenizer {
             this.sectionStart = this._index;
         }
     }
+    /**
+     * HTML only allows ASCII alpha characters (a-z and A-Z) at the beginning of a tag name.
+     *
+     * XML allows a lot more characters here (@see https://www.w3.org/TR/REC-xml/#NT-NameStartChar).
+     * We allow anything that wouldn't end the tag.
+     */
+    private isTagStartChar(c: string) {
+        return (
+            isASCIIAlpha(c) ||
+            (this.xmlMode && !whitespace(c) && c !== "/" && c !== ">")
+        );
+    }
     private stateBeforeTagName(c: string) {
         if (c === "/") {
             this._state = State.BeforeClosingTagName;
@@ -342,7 +354,7 @@ export default class Tokenizer {
         } else if (c === "?") {
             this._state = State.InProcessingInstruction;
             this.sectionStart = this._index + 1;
-        } else if (!isASCIIAlpha(c)) {
+        } else if (!this.isTagStartChar(c)) {
             this._state = State.Text;
         } else {
             this._state =
@@ -378,7 +390,7 @@ export default class Tokenizer {
                 this._state = State.Text;
                 this._index--;
             }
-        } else if (!isASCIIAlpha(c)) {
+        } else if (!this.isTagStartChar(c)) {
             this._state = State.InSpecialComment;
             this.sectionStart = this._index;
         } else {

diff --git a/src/__fixtures__/Events/20-xml_entities.json → ...xtures__/Events/20-xml_entities copy.json b/src/__fixtures__/Events/20-xml_entities.json → ...xtures__/Events/20-xml_entities copy.json
diff --git a/src/__fixtures__/Events/40-xml_tags.json b/src/__fixtures__/Events/40-xml_tags.json
@@ -0,0 +1,33 @@
+{
+    "name": "XML tags",
+    "options": {
+        "parser": { "xmlMode": true }
+    },
+    "html": "<:foo><_bar>",
+    "expected": [
+        {
+            "data": [":foo"],
+            "event": "opentagname"
+        },
+        {
+            "data": [":foo", {}],
+            "event": "opentag"
+        },
+        {
+            "data": ["_bar"],
+            "event": "opentagname"
+        },
+        {
+            "data": ["_bar", {}],
+            "event": "opentag"
+        },
+        {
+            "data": ["_bar"],
+            "event": "closetag"
+        },
+        {
+            "data": [":foo"],
+            "event": "closetag"
+        }
+    ]
+}