From 12ac8a87ef1b17a8acb6f0768daccf57b1be452c Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Fri, 3 Nov 2023 14:48:27 +0100
Subject: [PATCH 01/14] Refactor of the tree-construction test harness

---
 src/bin/html5-parser-test.rs               |  36 +-
 src/bin/parser-test.rs                     | 305 +++++-------
 src/html5/parser.rs                        | 256 +++++-----
 src/html5/parser/helper.rs                 |  29 +-
 src/testing.rs                             |   1 +
 src/testing/tree_construction.rs           | 551 +++++++++------------
 src/testing/tree_construction/fixture.rs   |  95 ++++
 src/testing/tree_construction/generator.rs | 110 ++++
 src/testing/tree_construction/parser.rs    |  18 +-
 src/testing/tree_construction/result.rs    |  69 +++
 tests/tree_construction.rs                 |  53 +-
 11 files changed, 841 insertions(+), 682 deletions(-)
 create mode 100644 src/testing/tree_construction/fixture.rs
 create mode 100644 src/testing/tree_construction/generator.rs
 create mode 100644 src/testing/tree_construction/result.rs

diff --git a/src/bin/html5-parser-test.rs b/src/bin/html5-parser-test.rs
index fe2118581..4d80f0788 100755
--- a/src/bin/html5-parser-test.rs
+++ b/src/bin/html5-parser-test.rs
@@ -1,28 +1,40 @@
-use gosub_engine::testing::tree_construction::fixture_from_filename;
-use gosub_engine::testing::FIXTURE_ROOT;
+use gosub_engine::testing::tree_construction::fixture::{
+    get_fixture_root_path, read_fixture_from_path,
+};
+use gosub_engine::testing::tree_construction::Harness;
 use gosub_engine::types::Result;
 use std::io::Write;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use walkdir::WalkDir;
 
 fn main() -> Result<()> {
-    let mut files = get_files_from_path(format!("{}/tree-construction", FIXTURE_ROOT).as_str());
+    let mut files = get_files_from_path(get_fixture_root_path());
     files.sort();
 
     let mut total = 0;
     let mut failed = 0;
 
     for file in files.iter() {
-        let fixture = fixture_from_filename(file)?;
+        // if file != "math.dat" {
+        //     continue;
+        // }
+        let fixture = read_fixture_from_path(&get_fixture_root_path().join(file))?;
+
         print!("Test: ({:3}) {} [", fixture.tests.len(), file);
         let _ = std::io::stdout().flush();
 
+        let mut harness = Harness::new();
+
         // Run tests
-        for test in fixture.tests {
-            let results = test.run().expect("problem running tree construction test");
-            for result in results {
+        for test in fixture.tests.iter() {
+            for &scripting_enabled in test.script_modes() {
+                let result = harness
+                    .run_test(test.clone(), scripting_enabled)
+                    .expect("problem parsing");
+
                 total += 1;
-                if result.success() {
+
+                if result.is_success() {
                     print!(".");
                 } else {
                     print!("X");
@@ -45,10 +57,10 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-fn get_files_from_path(root_dir: &str) -> Vec<String> {
+fn get_files_from_path(dir: PathBuf) -> Vec<String> {
     let mut files = Vec::new();
 
-    for entry in WalkDir::new(root_dir)
+    for entry in WalkDir::new(dir.clone())
         .follow_links(true)
         .into_iter()
         .flatten()
@@ -58,7 +70,7 @@ fn get_files_from_path(root_dir: &str) -> Vec<String> {
                 if extension == "dat" {
                     if let Ok(relative_path) = entry
                         .path()
-                        .strip_prefix(root_dir)
+                        .strip_prefix(dir.clone())
                         .map(Path::to_str)
                         .map(|s| s.unwrap().to_string())
                     {
diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 1ac1c7ec7..63776e3a1 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -1,12 +1,10 @@
-use gosub_engine::{
-    testing::{
-        self,
-        tree_construction::{ErrorResult, NodeResult, SubtreeResult, Test, TestResult},
-    },
-    types::Result,
-};
+use gosub_engine::testing::tree_construction::fixture::read_fixtures;
+use gosub_engine::testing::tree_construction::result::ResultStatus;
+use gosub_engine::testing::tree_construction::Harness;
+use gosub_engine::testing::tree_construction::Test;
 
-pub struct TestResults {
+/// Holds the results from all tests that are executed
+pub struct GlobalTestResults {
     /// Number of tests (as defined in the suite)
     tests: usize,
     /// Number of assertions (different combinations of input/output per test)
@@ -21,8 +19,8 @@ pub struct TestResults {
     tests_failed: Vec<(usize, usize, String)>,
 }
 
-fn main() -> Result<()> {
-    let mut results = TestResults {
+fn main() {
+    let mut results = GlobalTestResults {
         tests: 0,
         assertions: 0,
         succeeded: 0,
@@ -31,8 +29,8 @@ fn main() -> Result<()> {
         tests_failed: Vec::new(),
     };
 
-    let filenames = Some(&["doctype01.dat"][..]);
-    let fixtures = testing::tree_construction::fixtures(filenames).expect("fixtures");
+    let filenames = Some(&["template.dat"][..]);
+    let fixtures = read_fixtures(filenames).expect("fixtures");
 
     for fixture_file in fixtures {
         println!(
@@ -43,9 +41,9 @@ fn main() -> Result<()> {
 
         let mut test_idx = 1;
         for test in fixture_file.tests {
-            // if test_idx == 81 {
-            run_tree_test(test_idx, &test, &mut results);
-            // }
+            if test_idx == 1 {
+                run_test(test_idx, test, &mut results);
+            }
             test_idx += 1;
         }
 
@@ -67,10 +65,9 @@ fn main() -> Result<()> {
             println!("    {}", data);
         }
     }
-    Ok(())
 }
 
-fn run_tree_test(test_idx: usize, test: &Test, all_results: &mut TestResults) {
+fn run_test(_test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
     #[cfg(feature = "debug_parser_verbose")]
     println!(
         "🧪 Running test #{test_idx}: {}:{}",
@@ -79,189 +76,127 @@ fn run_tree_test(test_idx: usize, test: &Test, all_results: &mut TestResults) {
 
     all_results.tests += 1;
 
-    let results = test.run().expect("problem running tree construction test");
+    let mut harness = Harness::new();
+    let result = harness
+        .run_test(test.clone(), false)
+        .expect("problem parsing");
 
-    for result in results {
-        #[cfg(feature = "debug_parser")]
-        print_test_result(&result);
+    // #[cfg(feature = "debug_parser")]
+    // print_test_result(&result);
 
-        // Check the document tree, which counts as a single assertion
+    for entry in &result.tree_results {
         all_results.assertions += 1;
-        if result.success() {
-            all_results.succeeded += 1;
-        } else {
-            all_results.failed += 1;
-        }
-
-        let errors = test.errors();
 
-        if result.actual_errors.len() != errors.len() {
-            #[cfg(feature = "debug_parser")]
-            println!(
-                "⚠️ Unexpected errors found (wanted {}, got {}): ",
-                errors.len(),
-                result.actual_errors.len()
-            );
-
-            // for want_err in &test.errors {
-            //     println!(
-            //         "     * Want: '{}' at {}:{}",
-            //         want_err.code, want_err.line, want_err.col
-            //     );
-            // }
-            // for got_err in &parse_errors {
-            //     println!(
-            //         "     * Got: '{}' at {}:{}",
-            //         got_err.message, got_err.line, got_err.col
-            //     );
-            // }
-            // results.assertions += 1;
-            // results.failed += 1;
-        } else {
-            #[cfg(feature = "debug_parser")]
-            println!("✅  Found {} errors", result.actual_errors.len());
-        }
+        match entry.result {
+            ResultStatus::Success => {
+                all_results.succeeded += 1;
 
-        // For now, we skip the tests that checks for errors as most of the errors do not match
-        // with the actual tests, as these errors as specific from html5lib. Either we reuse them
-        // or have some kind of mapping to our own errors if we decide to use our custom errors.
+                #[cfg(feature = "debug_parser")]
+                println!("✅  {}", entry.actual);
+            }
+            ResultStatus::Missing => {
+                all_results.failed += 1;
 
-        // // Check each error messages
-        // let mut idx = 0;
-        // for error in &test.errors {
-        //     if parse_errors.get(idx).is_none() {
-        //         println!("❌ Expected error '{}' at {}:{}", error.code, error.line, error.col);
-        //         results.assertions += 1;
-        //         results.failed += 1;
-        //         continue;
-        //     }
-        //
-        //     let err = parse_errors.get(idx).unwrap();
-        //     let got_error = Error{
-        //         code: err.message.to_string(),
-        //         line: err.line as i64,
-        //         col: err.col as i64,
-        //     };
-        //
-        //     match match_error(&got_error, &error) {
-        //         ErrorResult::Failure => {
-        //             results.assertions += 1;
-        //             results.failed += 1;
-        //         },
-        //         ErrorResult::PositionFailure => {
-        //             results.assertions += 1;
-        //             results.failed += 1;
-        //             results.failed_position += 1;
-        //         },
-        //         ErrorResult::Success => {
-        //             results.assertions += 1;
-        //             results.succeeded += 1;
-        //         }
-        //     }
-        //
-        //     idx += 1;
-        // }
+                #[cfg(feature = "debug_parser")]
+                println!("❌ {} (missing)", entry.expected);
+            }
+            ResultStatus::Additional => {
+                all_results.failed += 1;
 
-        // Display additional data if there a failure is found
-        if !result.success() {
-            all_results
-                .tests_failed
-                .push((test_idx, test.line, test.data().to_string()));
+                #[cfg(feature = "debug_parser")]
+                println!("❌ {} (unexpected)", entry.actual);
+            }
+            ResultStatus::Mismatch => {
+                all_results.failed += 1;
 
-            if cfg!(feature = "debug_parser") {
-                println!("----------------------------------------");
-                println!("📄 Input stream: ");
-                println!("{}", test.data());
-                println!("----------------------------------------");
-                println!("🌳 Generated tree: ");
-                println!("{}", result.actual_document);
-                println!("----------------------------------------");
-                println!("🌳 Expected tree: ");
-                for line in &test.document {
-                    println!("{line}");
-                }
+                #[cfg(feature = "debug_parser")]
+                println!("❌ {} (wanted: {})", entry.actual, entry.expected);
             }
+            _ => {}
         }
-
-        #[cfg(feature = "debug_parser")]
-        println!("----------------------------------------");
     }
-}
 
-#[allow(dead_code)]
-fn print_test_result(result: &TestResult) {
-    // We need a better tree match system. Right now we match the tree based on the (debug) output
-    // of the tree. Instead, we should generate a document-tree from the expected output and compare
-    // it against the current generated tree.
-    print_node_result(&result.root)
-}
-
-#[allow(dead_code)]
-fn print_node_result(result: &SubtreeResult) {
-    match &result.node {
-        Some(NodeResult::ElementMatchSuccess { actual }) => {
-            println!("✅  {actual}");
-        }
-
-        Some(NodeResult::AttributeMatchFailure { name, expected, .. }) => {
-            println!("❌ {expected}, Found unexpected attribute: {name}");
-        }
-
-        Some(NodeResult::ElementMatchFailure {
-            actual, expected, ..
-        }) => {
-            println!("❌ {expected}, Found unexpected element node: {actual}");
-        }
-
-        Some(NodeResult::TextMatchSuccess { expected }) => {
-            println!("✅  {expected}");
-        }
+    for entry in &result.error_results {
+        all_results.assertions += 1;
 
-        Some(NodeResult::TextMatchFailure { expected, text, .. }) => {
-            println!("❌ {expected}, Found unexpected text node: {text}");
-        }
+        match entry.result {
+            ResultStatus::Success => {
+                all_results.succeeded += 1;
 
-        Some(NodeResult::DocTypeMatchFailure {
-            actual, expected, ..
-        }) => {
-            println!("❌ {actual}, Found unexpected doctype node: {expected}");
-        }
-
-        Some(NodeResult::CommentMatchFailure {
-            expected, comment, ..
-        }) => {
-            println!("❌ {expected}, Found unexpected comment node: {comment}");
+                #[cfg(feature = "debug_parser")]
+                println!(
+                    "✅  ({}:{}) {}",
+                    entry.actual.line, entry.actual.col, entry.actual.message
+                );
+            }
+            ResultStatus::Missing => {
+                all_results.failed += 1;
+
+                #[cfg(feature = "debug_parser")]
+                println!(
+                    "❌ ({}:{}) {} (missing)",
+                    entry.expected.line, entry.expected.col, entry.expected.message
+                );
+            }
+            ResultStatus::Additional => {
+                all_results.failed += 1;
+
+                #[cfg(feature = "debug_parser")]
+                println!(
+                    "❌ ({}:{}) {} (unexpected)",
+                    entry.actual.line, entry.actual.col, entry.actual.message
+                );
+            }
+            ResultStatus::Mismatch => {
+                all_results.failed += 1;
+
+                #[cfg(feature = "debug_parser")]
+                println!(
+                    "❌ ({}:{}) {} (wanted: {})",
+                    entry.actual.line,
+                    entry.actual.col,
+                    entry.actual.message,
+                    entry.expected.message
+                );
+            }
+            ResultStatus::IncorrectPosition => {
+                all_results.failed += 1;
+                all_results.failed_position += 1;
+
+                #[cfg(feature = "debug_parser")]
+                println!(
+                    "❌ ({}:{}) (wanted: ({}::{})) {}",
+                    entry.actual.line,
+                    entry.actual.col,
+                    entry.expected.line,
+                    entry.expected.col,
+                    entry.expected.message
+                );
+            }
         }
-
-        None => {}
     }
 
-    result.children.iter().for_each(print_node_result);
-}
-
-#[allow(dead_code)]
-fn match_error(result: ErrorResult) {
-    match result {
-        ErrorResult::Success { actual } => {
-            println!(
-                "✅  Found parse error '{}' at {}:{}",
-                actual.code, actual.line, actual.col
-            );
-        }
-
-        ErrorResult::Failure { expected, .. } => {
-            println!(
-                "❌ Expected error '{}' at {}:{}",
-                expected.code, expected.line, expected.col
-            );
-        }
-
-        ErrorResult::PositionFailure { actual, expected } => {
-            // Found an error with the same code, but different line/pos
-            println!(
-                "⚠️ Unexpected error position '{}' at {}:{} (got: {}:{})",
-                expected.code, expected.line, expected.col, actual.line, actual.col
-            );
-        }
-    }
+    // // Display additional data if there a failure is found
+    // if !result.success() {
+    //     all_results
+    //         .tests_failed
+    //         .push((test_idx, test.line, test.data().to_string()));
+    //
+    //     if cfg!(feature = "debug_parser") {
+    //         println!("----------------------------------------");
+    //         println!("📄 Input stream: ");
+    //         println!("{}", test.data());
+    //         println!("----------------------------------------");
+    //         println!("🌳 Generated tree: ");
+    //         println!("{}", result.actual_document);
+    //         println!("----------------------------------------");
+    //         println!("🌳 Expected tree: ");
+    //         for line in &test.document {
+    //             println!("{line}");
+    //         }
+    //     }
+    // }
+    //
+    // #[cfg(feature = "debug_parser")]
+    // println!("----------------------------------------");
 }
diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index 615df3087..3d07c1aab 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -401,7 +401,7 @@ impl<'chars> Html5Parser<'chars> {
     }
 
     /// Internal parser function that does the actual parsing
-    pub fn do_parse(&mut self) -> Result<Vec<ParseError>> {
+    fn do_parse(&mut self) -> Result<Vec<ParseError>> {
         let mut dispatcher_mode = DispatcherMode::Html;
 
         loop {
@@ -1203,7 +1203,7 @@ impl<'chars> Html5Parser<'chars> {
                     Token::EndTag { name, .. }
                         if name == "tbody" || name == "tfoot" || name == "thead" =>
                     {
-                        if !self.is_in_scope(name, Scope::Table) {
+                        if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("tbody, tfoot or thead tag not allowed in in table body insertion mode");
                             // ignore token
                             return;
@@ -1218,9 +1218,9 @@ impl<'chars> Html5Parser<'chars> {
                         if ["caption", "col", "colgroup", "tbody", "tfoot", "thead"]
                             .contains(&name.as_str()) =>
                     {
-                        if !self.is_in_scope("tbody", Scope::Table)
-                            && !self.is_in_scope("tfoot", Scope::Table)
-                            && !self.is_in_scope("thead", Scope::Table)
+                        if !self.is_in_scope("tbody", HTML_NAMESPACE, Scope::Table)
+                            && !self.is_in_scope("tfoot", HTML_NAMESPACE, Scope::Table)
+                            && !self.is_in_scope("thead", HTML_NAMESPACE, Scope::Table)
                         {
                             self.parse_error("caption, col, colgroup, tbody, tfoot or thead tag not allowed in in table body insertion mode");
                             // ignore token
@@ -1234,9 +1234,9 @@ impl<'chars> Html5Parser<'chars> {
                         self.reprocess_token = true;
                     }
                     Token::EndTag { name, .. } if name == "table" => {
-                        if !self.is_in_scope("tbody", Scope::Table)
-                            && !self.is_in_scope("tfoot", Scope::Table)
-                            && !self.is_in_scope("thead", Scope::Table)
+                        if !self.is_in_scope("tbody", HTML_NAMESPACE, Scope::Table)
+                            && !self.is_in_scope("tfoot", HTML_NAMESPACE, Scope::Table)
+                            && !self.is_in_scope("thead", HTML_NAMESPACE, Scope::Table)
                         {
                             self.parse_error(
                                 "table end tag not allowed in in table body insertion mode",
@@ -1275,7 +1275,7 @@ impl<'chars> Html5Parser<'chars> {
                         self.active_formatting_elements_push_marker();
                     }
                     Token::EndTag { name, .. } if name == "tr" => {
-                        if !self.is_in_scope("tr", Scope::Table) {
+                        if !self.is_in_scope("tr", HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("tr tag not allowed in in row insertion mode");
                             // ignore token
                             return;
@@ -1292,7 +1292,7 @@ impl<'chars> Html5Parser<'chars> {
                         ]
                         .contains(&name.as_str()) =>
                     {
-                        if !self.is_in_scope("tr", Scope::Table) {
+                        if !self.is_in_scope("tr", HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("caption, col, colgroup, tbody, tfoot or thead tag not allowed in in row insertion mode");
                             // ignore token
                             return;
@@ -1305,7 +1305,7 @@ impl<'chars> Html5Parser<'chars> {
                         self.reprocess_token = true;
                     }
                     Token::EndTag { name, .. } if name == "table" => {
-                        if !self.is_in_scope("tr", Scope::Table) {
+                        if !self.is_in_scope("tr", HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("table tag not allowed in in row insertion mode");
                             // ignore token
                             return;
@@ -1320,13 +1320,13 @@ impl<'chars> Html5Parser<'chars> {
                     Token::EndTag { name, .. }
                         if name == "tbody" || name == "tfoot" || name == "thead" =>
                     {
-                        if !self.is_in_scope(name, Scope::Table) {
+                        if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("tbody, tfoot or thead tag not allowed in in table body insertion mode");
                             // ignore token
                             return;
                         }
 
-                        if !self.is_in_scope("tr", Scope::Table) {
+                        if !self.is_in_scope("tr", HTML_NAMESPACE, Scope::Table) {
                             // ignore token
                             return;
                         }
@@ -1357,7 +1357,7 @@ impl<'chars> Html5Parser<'chars> {
                     Token::EndTag { name, .. } if name == "th" || name == "td" => {
                         let token_name = name.clone();
 
-                        if !self.is_in_scope(name.as_str(), Scope::Table) {
+                        if !self.is_in_scope(name.as_str(), HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("th or td tag not allowed in in cell insertion mode");
                             // ignore token
                             return;
@@ -1381,8 +1381,8 @@ impl<'chars> Html5Parser<'chars> {
                         ]
                         .contains(&name.as_str()) =>
                     {
-                        if !self.is_in_scope("td", Scope::Table)
-                            && !self.is_in_scope("th", Scope::Table)
+                        if !self.is_in_scope("td", HTML_NAMESPACE, Scope::Table)
+                            && !self.is_in_scope("th", HTML_NAMESPACE, Scope::Table)
                         {
                             // fragment case
                             self.parse_error("caption, col, colgroup, tbody, tfoot or thead tag not allowed in in cell insertion mode");
@@ -1410,7 +1410,7 @@ impl<'chars> Html5Parser<'chars> {
                             || name == "thead"
                             || name == "tr" =>
                     {
-                        if !self.is_in_scope(name.as_str(), Scope::Table) {
+                        if !self.is_in_scope(name.as_str(), HTML_NAMESPACE, Scope::Table) {
                             self.parse_error("tbody, tfoot or thead tag not allowed in in table body insertion mode");
                             // ignore token
                             return;
@@ -1453,7 +1453,7 @@ impl<'chars> Html5Parser<'chars> {
                     {
                         self.parse_error("caption, table, tbody, tfoot, thead, tr, td or th tag not allowed in in select in table insertion mode");
 
-                        if !self.is_in_scope(name, Scope::Table) {
+                        if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Table) {
                             // ignore token
                             return;
                         }
@@ -1982,10 +1982,10 @@ impl<'chars> Html5Parser<'chars> {
     }
 
     /// Checks if the given element is in given scope
-    fn is_in_scope(&self, tag: &str, scope: Scope) -> bool {
+    fn is_in_scope(&self, tag: &str, namespace: &str, scope: Scope) -> bool {
         for &node_id in self.open_elements.iter().rev() {
             let node = get_node_by_id!(self.document, node_id).clone();
-            if node.name == tag {
+            if node.name == tag && node.is_namespace(namespace) {
                 return true;
             }
             let default_html_scope = [
@@ -2188,7 +2188,7 @@ impl<'chars> Html5Parser<'chars> {
                 }
             }
             Token::EndTag { name, .. } if name == "body" => {
-                if !self.is_in_scope("body", Scope::Regular) {
+                if !self.is_in_scope("body", HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("body end tag not in scope");
                     // ignore token
                     return;
@@ -2199,7 +2199,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.insertion_mode = InsertionMode::AfterBody;
             }
             Token::EndTag { name, .. } if name == "html" => {
-                if !self.is_in_scope("body", Scope::Regular) {
+                if !self.is_in_scope("body", HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("body end tag not in scope");
                     // ignore token
                     return;
@@ -2237,7 +2237,7 @@ impl<'chars> Html5Parser<'chars> {
                     || name == "summary"
                     || name == "ul" =>
             {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2251,7 +2251,7 @@ impl<'chars> Html5Parser<'chars> {
                     || name == "h5"
                     || name == "h6" =>
             {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2264,7 +2264,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.insert_html_element(&self.current_token.clone());
             }
             Token::StartTag { name, .. } if name == "pre" || name == "listing" => {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2281,7 +2281,7 @@ impl<'chars> Html5Parser<'chars> {
                     return;
                 }
 
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2316,7 +2316,7 @@ impl<'chars> Html5Parser<'chars> {
                     idx -= 1;
                 }
 
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2348,14 +2348,14 @@ impl<'chars> Html5Parser<'chars> {
                     idx -= 1;
                 }
 
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
                 self.insert_html_element(&self.current_token.clone());
             }
             Token::StartTag { name, .. } if name == "plaintext" => {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2364,7 +2364,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.tokenizer.state = State::PLAINTEXT;
             }
             Token::StartTag { name, .. } if name == "button" => {
-                if self.is_in_scope("button", Scope::Regular) {
+                if self.is_in_scope("button", HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("button tag not allowed in in body insertion mode");
                     self.generate_implied_end_tags(None, false);
                     self.pop_until_named("button");
@@ -2403,7 +2403,7 @@ impl<'chars> Html5Parser<'chars> {
                     || name == "summary"
                     || name == "ul" =>
             {
-                if !self.is_in_scope(name, Scope::Regular) {
+                if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("end tag not in scope");
                     // ignore token
                     return;
@@ -2423,7 +2423,8 @@ impl<'chars> Html5Parser<'chars> {
                     let node_id = self.form_element;
                     self.form_element = None;
 
-                    if node_id.is_none() || !self.is_in_scope(name, Scope::Regular) {
+                    if node_id.is_none() || !self.is_in_scope(name, HTML_NAMESPACE, Scope::Regular)
+                    {
                         self.parse_error("end tag not in scope");
                         // ignore token
                         return;
@@ -2442,7 +2443,7 @@ impl<'chars> Html5Parser<'chars> {
                     }
                     self.open_elements_remove(node_id);
                 } else {
-                    if !self.is_in_scope(name, Scope::Regular) {
+                    if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Regular) {
                         self.parse_error("end tag not in scope");
                         // ignore token
                         return;
@@ -2459,7 +2460,7 @@ impl<'chars> Html5Parser<'chars> {
                 }
             }
             Token::EndTag { name, .. } if name == "p" => {
-                if !self.is_in_scope(name, Scope::Button) {
+                if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Button) {
                     self.parse_error("end tag not in scope");
 
                     let token = Token::StartTag {
@@ -2473,7 +2474,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.close_p_element();
             }
             Token::EndTag { name, .. } if name == "li" => {
-                if !self.is_in_scope(name, Scope::ListItem) {
+                if !self.is_in_scope(name, HTML_NAMESPACE, Scope::ListItem) {
                     self.parse_error("end tag not in scope");
                     // ignore token
                     return;
@@ -2488,7 +2489,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.pop_until_named(name);
             }
             Token::EndTag { name, .. } if name == "dd" || name == "dt" => {
-                if !self.is_in_scope(name, Scope::Regular) {
+                if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("end tag not in scope");
                     // ignore token
                     return;
@@ -2512,7 +2513,7 @@ impl<'chars> Html5Parser<'chars> {
             {
                 if ["h1", "h2", "h3", "h4", "h5", "h6"]
                     .iter()
-                    .map(|tag| self.is_in_scope(tag, Scope::Regular))
+                    .map(|tag| self.is_in_scope(tag, HTML_NAMESPACE, Scope::Regular))
                     .any(|res| res)
                 {
                     self.generate_implied_end_tags(Some(name), false);
@@ -2568,7 +2569,7 @@ impl<'chars> Html5Parser<'chars> {
             Token::StartTag { name, .. } if name == "nobr" => {
                 self.reconstruct_formatting();
 
-                if self.is_in_scope("nobr", Scope::Regular) {
+                if self.is_in_scope("nobr", HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("nobr tag in scope");
                     self.adoption_agency_algorithm(&self.current_token.clone());
                     self.reconstruct_formatting();
@@ -2611,7 +2612,7 @@ impl<'chars> Html5Parser<'chars> {
             Token::EndTag { name, .. }
                 if name == "applet" || name == "marquee" || name == "object" =>
             {
-                if !self.is_in_scope(name, Scope::Regular) {
+                if !self.is_in_scope(name, HTML_NAMESPACE, Scope::Regular) {
                     self.parse_error("end tag not in scope");
                     // ignore token
                     return;
@@ -2628,7 +2629,7 @@ impl<'chars> Html5Parser<'chars> {
             }
             Token::StartTag { name, .. } if name == "table" => {
                 if self.document.get_mut().quirks_mode != QuirksMode::Quirks
-                    && self.is_in_scope("p", Scope::Button)
+                    && self.is_in_scope("p", HTML_NAMESPACE, Scope::Button)
                 {
                     self.close_p_element();
                 }
@@ -2710,7 +2711,7 @@ impl<'chars> Html5Parser<'chars> {
                 is_self_closing,
                 ..
             } if name == "hr" => {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2744,7 +2745,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.insertion_mode = InsertionMode::Text;
             }
             Token::StartTag { name, .. } if name == "xmp" => {
-                if self.is_in_scope("p", Scope::Button) {
+                if self.is_in_scope("p", HTML_NAMESPACE, Scope::Button) {
                     self.close_p_element();
                 }
 
@@ -2790,7 +2791,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.insert_html_element(&self.current_token.clone());
             }
             Token::StartTag { name, .. } if name == "rb" || name == "rtc" => {
-                if self.is_in_scope("ruby", Scope::Regular) {
+                if self.is_in_scope("ruby", HTML_NAMESPACE, Scope::Regular) {
                     self.generate_implied_end_tags(None, false);
                 }
 
@@ -2801,7 +2802,7 @@ impl<'chars> Html5Parser<'chars> {
                 self.insert_html_element(&self.current_token.clone());
             }
             Token::StartTag { name, .. } if name == "rp" || name == "rt" => {
-                if self.is_in_scope("ruby", Scope::Regular) {
+                if self.is_in_scope("ruby", HTML_NAMESPACE, Scope::Regular) {
                     self.generate_implied_end_tags(Some("rtc"), false);
                 }
 
@@ -2966,6 +2967,22 @@ impl<'chars> Html5Parser<'chars> {
             Token::StartTag { name, .. } if name == "template" => {
                 let node_id = self.insert_html_element(&self.current_token.clone());
 
+                self.active_formatting_elements_push_marker();
+                self.frameset_ok = false;
+                self.insertion_mode = InsertionMode::InTemplate;
+                self.template_insertion_mode.push(InsertionMode::InTemplate);
+
+                // Let adjusted insert location
+                // intended parent
+                // document = indented parent's node document
+
+                // check shadow root != none
+                // or allow declarative shadow roots == true
+                // or adjusted current node is not topmost element in stack open elements
+                // then insert html element for token
+                // else:
+                //
+
                 {
                     let current_node_id = current_node!(self).id;
 
@@ -2976,11 +2993,6 @@ impl<'chars> Html5Parser<'chars> {
                         data.template_contents = Some(DocumentFragment::new(doc, current_node_id));
                     }
                 }
-
-                self.active_formatting_elements_push_marker();
-                self.frameset_ok = false;
-                self.insertion_mode = InsertionMode::InTemplate;
-                self.template_insertion_mode.push(InsertionMode::InTemplate);
             }
             Token::EndTag { name, .. } if name == "template" => {
                 if !self.open_elements_has("template") {
@@ -3352,7 +3364,7 @@ impl<'chars> Html5Parser<'chars> {
                 }
             }
             Token::EndTag { name, .. } if name == "select" => {
-                if !self.is_in_scope("select", Scope::Select) {
+                if !self.is_in_scope("select", HTML_NAMESPACE, Scope::Select) {
                     // fragment case
                     self.parse_error("select end tag not allowed in in select insertion mode");
                     // ignore token
@@ -3365,7 +3377,7 @@ impl<'chars> Html5Parser<'chars> {
             Token::StartTag { name, .. } if name == "select" => {
                 self.parse_error("select tag not allowed in in select insertion mode");
 
-                if !self.is_in_scope("select", Scope::Select) {
+                if !self.is_in_scope("select", HTML_NAMESPACE, Scope::Select) {
                     // fragment case
                     // ignore token
                     return;
@@ -3381,7 +3393,7 @@ impl<'chars> Html5Parser<'chars> {
                     "input, keygen or textarea tag not allowed in in select insertion mode",
                 );
 
-                if !self.is_in_scope("select", Scope::Select) {
+                if !self.is_in_scope("select", HTML_NAMESPACE, Scope::Select) {
                     // fragment case
                     // ignore token
                     return;
@@ -3884,10 +3896,10 @@ mod test {
         node_create!(parser, "div");
         node_create!(parser, "p");
         node_create!(parser, "button");
-        assert!(parser.is_in_scope("p", Scope::Regular));
-        assert!(!parser.is_in_scope("p", Scope::Button));
-        assert!(parser.is_in_scope("p", Scope::ListItem));
-        assert!(!parser.is_in_scope("p", Scope::Select));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -3896,10 +3908,10 @@ mod test {
         let mut parser = Html5Parser::new_parser(chars);
 
         parser.open_elements.clear();
-        assert!(!parser.is_in_scope("p", Scope::Regular));
-        assert!(!parser.is_in_scope("p", Scope::Button));
-        assert!(!parser.is_in_scope("p", Scope::ListItem));
-        assert!(!parser.is_in_scope("p", Scope::Select));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Button));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -3912,10 +3924,10 @@ mod test {
         node_create!(parser, "p");
         node_create!(parser, "button");
 
-        assert!(!parser.is_in_scope("foo", Scope::Regular));
-        assert!(!parser.is_in_scope("foo", Scope::Button));
-        assert!(!parser.is_in_scope("foo", Scope::ListItem));
-        assert!(!parser.is_in_scope("foo", Scope::Select));
+        assert!(!parser.is_in_scope("foo", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("foo", HTML_NAMESPACE, Scope::Button));
+        assert!(!parser.is_in_scope("foo", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("foo", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -3931,29 +3943,29 @@ mod test {
         node_create!(parser, "p");
         node_create!(parser, "span");
 
-        assert!(parser.is_in_scope("p", Scope::Regular));
-        assert!(parser.is_in_scope("p", Scope::ListItem));
-        assert!(parser.is_in_scope("p", Scope::Button));
-        assert!(parser.is_in_scope("p", Scope::Table));
-        assert!(!parser.is_in_scope("p", Scope::Select));
-
-        assert!(!parser.is_in_scope("div", Scope::Regular));
-        assert!(!parser.is_in_scope("div", Scope::ListItem));
-        assert!(!parser.is_in_scope("div", Scope::Button));
-        assert!(!parser.is_in_scope("div", Scope::Table));
-        assert!(!parser.is_in_scope("div", Scope::Select));
-
-        assert!(!parser.is_in_scope("tr", Scope::Regular));
-        assert!(!parser.is_in_scope("tr", Scope::ListItem));
-        assert!(!parser.is_in_scope("tr", Scope::Button));
-        assert!(parser.is_in_scope("tr", Scope::Table));
-        assert!(!parser.is_in_scope("tr", Scope::Select));
-
-        assert!(!parser.is_in_scope("xmp", Scope::Regular));
-        assert!(!parser.is_in_scope("xmp", Scope::ListItem));
-        assert!(!parser.is_in_scope("xmp", Scope::Button));
-        assert!(!parser.is_in_scope("xmp", Scope::Table));
-        assert!(!parser.is_in_scope("xmp", Scope::Select));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::Regular));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::ListItem));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("p", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("p", HTML_NAMESPACE, Scope::Select));
+
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Button));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Select));
+
+        assert!(!parser.is_in_scope("tr", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("tr", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("tr", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("tr", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("tr", HTML_NAMESPACE, Scope::Select));
+
+        assert!(!parser.is_in_scope("xmp", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("xmp", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("xmp", HTML_NAMESPACE, Scope::Button));
+        assert!(!parser.is_in_scope("xmp", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("xmp", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -3968,11 +3980,11 @@ mod test {
         node_create!(parser, "div");
         node_create!(parser, "button");
 
-        assert!(parser.is_in_scope("li", Scope::Regular));
-        assert!(parser.is_in_scope("li", Scope::ListItem));
-        assert!(!parser.is_in_scope("li", Scope::Button));
-        assert!(parser.is_in_scope("li", Scope::Table));
-        assert!(!parser.is_in_scope("li", Scope::Select));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::Regular));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("li", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("li", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -3987,11 +3999,11 @@ mod test {
         node_create!(parser, "li");
         node_create!(parser, "p");
 
-        assert!(parser.is_in_scope("li", Scope::Regular));
-        assert!(parser.is_in_scope("li", Scope::ListItem));
-        assert!(parser.is_in_scope("li", Scope::Button));
-        assert!(parser.is_in_scope("li", Scope::Table));
-        assert!(!parser.is_in_scope("li", Scope::Select));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::Regular));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::ListItem));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("li", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("li", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -4008,11 +4020,11 @@ mod test {
         node_create!(parser, "button");
         node_create!(parser, "span");
 
-        assert!(parser.is_in_scope("td", Scope::Regular));
-        assert!(parser.is_in_scope("td", Scope::ListItem));
-        assert!(!parser.is_in_scope("td", Scope::Button));
-        assert!(parser.is_in_scope("td", Scope::Table));
-        assert!(!parser.is_in_scope("td", Scope::Select));
+        assert!(parser.is_in_scope("td", HTML_NAMESPACE, Scope::Regular));
+        assert!(parser.is_in_scope("td", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("td", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("td", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("td", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -4028,11 +4040,11 @@ mod test {
         node_create!(parser, "a");
         node_create!(parser, "span");
 
-        assert!(!parser.is_in_scope("div", Scope::Regular));
-        assert!(!parser.is_in_scope("div", Scope::ListItem));
-        assert!(!parser.is_in_scope("div", Scope::Button));
-        assert!(parser.is_in_scope("div", Scope::Table));
-        assert!(!parser.is_in_scope("div", Scope::Select));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("div", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("div", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -4048,11 +4060,11 @@ mod test {
         node_create!(parser, "marquee");
         node_create!(parser, "p");
 
-        assert!(!parser.is_in_scope("ul", Scope::Regular));
-        assert!(!parser.is_in_scope("ul", Scope::ListItem));
-        assert!(!parser.is_in_scope("ul", Scope::Button));
-        assert!(parser.is_in_scope("ul", Scope::Table));
-        assert!(!parser.is_in_scope("ul", Scope::Select));
+        assert!(!parser.is_in_scope("ul", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("ul", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("ul", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("ul", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("ul", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -4067,11 +4079,11 @@ mod test {
         node_create!(parser, "caption");
         node_create!(parser, "p");
 
-        assert!(!parser.is_in_scope("table", Scope::Regular));
-        assert!(!parser.is_in_scope("table", Scope::ListItem));
-        assert!(!parser.is_in_scope("table", Scope::Button));
-        assert!(parser.is_in_scope("table", Scope::Table));
-        assert!(!parser.is_in_scope("table", Scope::Select));
+        assert!(!parser.is_in_scope("table", HTML_NAMESPACE, Scope::Regular));
+        assert!(!parser.is_in_scope("table", HTML_NAMESPACE, Scope::ListItem));
+        assert!(!parser.is_in_scope("table", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("table", HTML_NAMESPACE, Scope::Table));
+        assert!(!parser.is_in_scope("table", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
@@ -4085,11 +4097,11 @@ mod test {
         node_create!(parser, "optgroup");
         node_create!(parser, "option");
 
-        assert!(parser.is_in_scope("select", Scope::Regular));
-        assert!(parser.is_in_scope("select", Scope::ListItem));
-        assert!(parser.is_in_scope("select", Scope::Button));
-        assert!(parser.is_in_scope("select", Scope::Table));
-        assert!(parser.is_in_scope("select", Scope::Select));
+        assert!(parser.is_in_scope("select", HTML_NAMESPACE, Scope::Regular));
+        assert!(parser.is_in_scope("select", HTML_NAMESPACE, Scope::ListItem));
+        assert!(parser.is_in_scope("select", HTML_NAMESPACE, Scope::Button));
+        assert!(parser.is_in_scope("select", HTML_NAMESPACE, Scope::Table));
+        assert!(parser.is_in_scope("select", HTML_NAMESPACE, Scope::Select));
     }
 
     #[test]
diff --git a/src/html5/parser/helper.rs b/src/html5/parser/helper.rs
index 889f2f052..2582614c4 100644
--- a/src/html5/parser/helper.rs
+++ b/src/html5/parser/helper.rs
@@ -90,12 +90,7 @@ impl Html5Parser<'_> {
         }
     }
 
-    pub fn insert_text_helper(
-        &mut self,
-        node: NodeId,
-        position: InsertionPositionMode<NodeId>,
-        token: &Token,
-    ) {
+    pub fn insert_text_helper(&mut self, position: InsertionPositionMode<NodeId>, token: &Token) {
         match position {
             InsertionPositionMode::Sibling {
                 handle,
@@ -107,7 +102,8 @@ impl Html5Parser<'_> {
                 let position = parent_node.children.iter().position(|&x| x == before);
                 match position {
                     None | Some(0) => {
-                        doc.attach_node_to_parent(node, parent, position);
+                        let node = self.create_node(token, HTML_NAMESPACE);
+                        doc.add_node(node, parent, position);
                     }
                     Some(index) => {
                         let last_node_id = parent_node.children[index - 1];
@@ -120,7 +116,9 @@ impl Html5Parser<'_> {
                             value.push_str(&token.to_string());
                             return;
                         };
-                        doc.attach_node_to_parent(node, parent, Some(index));
+
+                        let node = self.create_node(token, HTML_NAMESPACE);
+                        doc.add_node(node, parent, Some(index));
                     }
                 }
             }
@@ -138,10 +136,13 @@ impl Html5Parser<'_> {
                         value.push_str(&token.to_string());
                         return;
                     };
-                    doc.attach_node_to_parent(node, parent, None);
+                    let node = self.create_node(token, HTML_NAMESPACE);
+                    doc.add_node(node, parent, None);
                     return;
                 }
-                doc.attach_node_to_parent(node, parent, None);
+
+                let node = self.create_node(token, HTML_NAMESPACE);
+                doc.add_node(node, parent, None);
             }
         }
     }
@@ -239,11 +240,9 @@ impl Html5Parser<'_> {
     }
 
     pub fn insert_text_element(&mut self, token: &Token) {
-        let node = self.create_node(token, HTML_NAMESPACE);
-        let node_id = self.document.get_mut().add_new_node(node);
         let insertion_position = self.appropriate_place_insert(None);
-        // TODO, for text element, if the insertion_position is Docuement, should not do next step.
-        self.insert_text_helper(node_id, insertion_position, token);
+        // TODO, for text element, if the insertion_position is Document, should not do next step.
+        self.insert_text_helper(insertion_position, token);
     }
 
     // @todo: where is the fragment case handled? (substep 4: https://html.spec.whatwg.org/multipage/parsing.html#appropriate-place-for-inserting-a-node)
@@ -363,7 +362,7 @@ impl Html5Parser<'_> {
             };
 
             // step 4.5
-            if !self.is_in_scope(&format_elem_node.name, Scope::Regular) {
+            if !self.is_in_scope(&format_elem_node.name, HTML_NAMESPACE, Scope::Regular) {
                 self.parse_error("format_element_node not in regular scope");
                 return;
             }
diff --git a/src/testing.rs b/src/testing.rs
index 3d66b2abf..e0deb4be0 100644
--- a/src/testing.rs
+++ b/src/testing.rs
@@ -3,3 +3,4 @@ pub mod tokenizer;
 pub mod tree_construction;
 
 pub const FIXTURE_ROOT: &str = "./tests/data/html5lib-tests";
+pub const TREE_CONSTRUCTION_PATH: &str = "tree-construction";
diff --git a/src/testing/tree_construction.rs b/src/testing/tree_construction.rs
index e953ec5bb..5030c4bbb 100644
--- a/src/testing/tree_construction.rs
+++ b/src/testing/tree_construction.rs
@@ -1,16 +1,19 @@
-mod parser;
+pub mod fixture;
+mod generator;
+pub(crate) mod parser;
+pub mod result;
 
-use self::parser::{ErrorSpec, ScriptMode, TestSpec, QUOTED_DOUBLE_NEWLINE};
-use super::FIXTURE_ROOT;
-use crate::html5::node::data::doctype::DocTypeData;
 use crate::html5::node::{HTML_NAMESPACE, MATHML_NAMESPACE, SVG_NAMESPACE};
 use crate::html5::parser::document::DocumentBuilder;
 use crate::html5::parser::tree_builder::TreeBuilder;
 use crate::html5::parser::Html5ParserOptions;
+use crate::testing::tree_construction::generator::TreeOutputGenerator;
+use crate::testing::tree_construction::parser::{ScriptMode, TestSpec};
+use crate::testing::tree_construction::result::{ResultStatus, TreeLineResult};
 use crate::{
     bytes::CharIterator,
     html5::{
-        node::{NodeData, NodeId},
+        node::NodeId,
         parser::{
             document::{Document, DocumentHandle},
             Html5Parser,
@@ -18,28 +21,7 @@ use crate::{
     },
     types::{ParseError, Result},
 };
-use std::{
-    fs,
-    path::{Path, PathBuf},
-};
-
-/// Holds all tests as found in the given fixture file
-#[derive(Debug, PartialEq)]
-pub struct FixtureFile {
-    pub tests: Vec<Test>,
-    pub path: String,
-}
-
-/// Holds information about an error
-#[derive(Clone, Debug, PartialEq)]
-pub struct TestError {
-    /// The code or message of the error
-    pub code: String,
-    /// The line number (1-based) where the error occurred
-    pub line: i64,
-    /// The column number (1-based) where the error occurred
-    pub col: i64,
-}
+use result::TestResult;
 
 /// Holds a single parser test
 #[derive(Debug, PartialEq)]
@@ -50,188 +32,80 @@ pub struct Test {
     pub line: usize,
     /// The specification of the test provided in the test file
     pub spec: TestSpec,
-    /// The document tree that is expected to be parsed
+    /// The document tree as found in the spec converted to an array
     pub document: Vec<String>,
 }
 
-/// Holds the result of a single "node" (which is either an element, text or comment)
-pub enum NodeResult {
-    /// An attribute of an element node did not match
-    AttributeMatchFailure {
-        name: String,
-        actual: String,
-        expected: String,
-    },
-
-    /// The actual element does not match the expected element
-    ElementMatchFailure {
-        name: String,
-        actual: String,
-        expected: String,
-    },
-
-    /// The element matches the expected element
-    ElementMatchSuccess {
-        actual: String,
-    },
-
-    /// A text node did not match
-    TextMatchFailure {
-        actual: String,
-        expected: String,
-        text: String,
-    },
-
-    // A doctype node did not match
-    DocTypeMatchFailure {
-        actual: String,
-        expected: String,
-    },
-
-    /// A comment node did not match
-    CommentMatchFailure {
-        actual: String,
-        expected: String,
-        comment: String,
-    },
-
-    /// A text node matches
-    TextMatchSuccess {
-        expected: String,
-    },
-}
-
-pub struct SubtreeResult {
-    pub node: Option<NodeResult>,
-    pub children: Vec<SubtreeResult>,
-    next_expected_idx: Option<usize>,
-}
-
-impl SubtreeResult {
-    pub fn valid(&self) -> bool {
-        self.next_expected_idx.is_some()
-    }
-}
-
-#[derive(PartialEq)]
-pub enum ErrorResult {
-    /// Found the correct error
-    Success { actual: TestError },
-    /// Didn't find the error (not even with incorrect position)
-    Failure {
-        actual: TestError,
-        expected: TestError,
-    },
-    /// Found the error, but on an incorrect position
-    PositionFailure {
-        actual: TestError,
-        expected: TestError,
-    },
-}
-
-pub struct TestResult {
-    pub root: SubtreeResult,
-    pub actual_document: DocumentHandle,
-    pub actual_errors: Vec<ParseError>,
-}
-
-impl TestResult {
-    pub fn success(&self) -> bool {
-        self.root.valid()
+impl Clone for Test {
+    fn clone(&self) -> Self {
+        Self {
+            file_path: self.file_path.clone(),
+            line: self.line,
+            spec: self.spec.clone(),
+            document: self.document.clone(),
+        }
     }
 }
 
 impl Test {
-    pub fn data(&self) -> &str {
-        self.spec.data.strip_suffix('\n').unwrap_or_default()
+    /// Returns the script modes that should be tested as an array
+    pub fn script_modes(&self) -> &[bool] {
+        match self.spec.script_mode {
+            ScriptMode::ScriptOff => &[false],
+            ScriptMode::ScriptOn => &[true],
+            ScriptMode::Both => &[false, true],
+        }
     }
 
-    pub fn errors(&self) -> &Vec<ErrorSpec> {
-        &self.spec.errors
+    pub fn get_document_as_str(&self) -> &str {
+        return self.spec.document.as_str();
     }
+}
 
-    /// Runs the test and returns the result
-    pub fn run(&self) -> Result<Vec<TestResult>> {
-        let mut results = vec![];
-
-        for &scripting_enabled in self.script_modes() {
-            let (actual_document, actual_errors) = self.parse(scripting_enabled)?;
-            let root = self.match_document_tree(&actual_document.get());
-            results.push(TestResult {
-                root,
-                actual_document,
-                actual_errors,
-            });
+impl Default for Test {
+    fn default() -> Self {
+        Self {
+            file_path: "".to_string(),
+            line: 0,
+            spec: TestSpec::default(),
+            document: vec![],
         }
-
-        Ok(results)
     }
+}
 
-    /// Verifies that the tree construction code obtains the right result
-    pub fn assert_valid(&self) {
-        let results = self.run().expect("failed to parse");
-
-        fn assert_tree(tree: &SubtreeResult) {
-            match &tree.node {
-                Some(NodeResult::ElementMatchSuccess { .. })
-                | Some(NodeResult::TextMatchSuccess { .. })
-                | None => {}
-
-                Some(NodeResult::TextMatchFailure {
-                    actual, expected, ..
-                }) => {
-                    panic!("text match failed, wanted: [{expected}], got: [{actual}]");
-                }
-
-                Some(NodeResult::DocTypeMatchFailure {
-                    actual, expected, ..
-                }) => {
-                    panic!("doctype match failed, wanted: [{expected}], got: [{actual}]");
-                }
-
-                Some(NodeResult::ElementMatchFailure {
-                    actual,
-                    expected,
-                    name,
-                }) => {
-                    panic!("element [{name}] match failed, wanted: [{expected}], got: [{actual}]");
-                }
+/// Harness is a wrapper to run tree-construction tests
+#[derive(Default)]
+pub struct Harness {
+    // Test that is currently being run
+    test: Test,
+    /// Next line in the document array
+    next_document_line: usize,
+}
 
-                Some(NodeResult::AttributeMatchFailure {
-                    name,
-                    actual,
-                    expected,
-                }) => {
-                    panic!(
-                        "attribute [{name}] match failed, wanted: [{expected}], got: [{actual}]"
-                    );
-                }
+impl Harness {
+    /// Generated a new harness instance. It uses a dummy test that is replaced when run_test is called
+    pub fn new() -> Self {
+        Self::default()
+    }
 
-                Some(NodeResult::CommentMatchFailure {
-                    actual, expected, ..
-                }) => {
-                    panic!("comment match failed, wanted: [{expected}], got: [{actual}]");
-                }
-            }
+    /// Runs a single test and returns the test result of that run
+    pub fn run_test(&mut self, test: Test, scripting_enabled: bool) -> Result<TestResult> {
+        self.test = test;
+        self.next_document_line = 0;
 
-            tree.children.iter().for_each(assert_tree);
-        }
+        let (actual_document, actual_errors) = self.do_parse(scripting_enabled)?;
+        let result = self.generate_test_result(Document::clone(&actual_document), &actual_errors);
 
-        for result in results {
-            assert_tree(&result.root);
-            assert!(result.success(), "invalid tree-construction result");
-        }
+        Ok(result)
     }
 
-    /// Run the parser and return the document and errors
-    pub fn parse(&self, scripting_enabled: bool) -> Result<(DocumentHandle, Vec<ParseError>)> {
-        // let mut is_fragment = false;
+    /// Run the html5 parser and return the document tree and errors
+    fn do_parse(&mut self, scripting_enabled: bool) -> Result<(DocumentHandle, Vec<ParseError>)> {
         let mut context_node = None;
         let document;
-
         let is_fragment;
 
-        if let Some(fragment) = self.spec.document_fragment.clone() {
+        if let Some(fragment) = self.test.spec.document_fragment.clone() {
             // First, create a (fake) main document that contains only the fragment as node
             let main_document = DocumentBuilder::new_document();
             let mut main_document = Document::clone(&main_document);
@@ -267,11 +141,10 @@ impl Test {
             document = DocumentBuilder::new_document();
         };
 
-        // Create a new parser
         let options = Html5ParserOptions { scripting_enabled };
 
         let mut chars = CharIterator::new();
-        chars.read_from_str(self.data(), None);
+        chars.read_from_str(self.test.spec.data.as_str(), None);
 
         let parse_errors = if is_fragment {
             Html5Parser::parse_fragment(
@@ -287,8 +160,171 @@ impl Test {
         Ok((document, parse_errors))
     }
 
+    /// Retrieves the next line from the spec document
+    fn get_next_line(&mut self) -> Option<String> {
+        let mut line = String::new();
+        let mut is_multi_line_text = false;
+
+        loop {
+            // If we are at the end of the document, we return None
+            if self.next_document_line >= self.test.document.len() {
+                return None;
+            }
+
+            // Get the next line
+            let tmp = self.test.document[self.next_document_line].to_owned();
+            self.next_document_line += 1;
+
+            // If we have a starting quote, but not an ending quote, we are a multi-line text
+            if tmp.starts_with('\"') && !tmp.ends_with('\"') {
+                is_multi_line_text = true;
+            }
+
+            // Add the line to the current line if we are a multiline
+            if is_multi_line_text {
+                line.push_str(&tmp);
+            } else {
+                line = tmp;
+            }
+
+            // Only break if we're in a multi-line text and we found the ending double-quote
+            if is_multi_line_text && line.ends_with('\"') {
+                break;
+            }
+
+            // if we are not a multi-line, we can just break
+            if !is_multi_line_text {
+                break;
+            }
+
+            // Otherwise we continue with the next line (multi-line text)
+        }
+
+        Some(line.to_string())
+    }
+
+    fn generate_test_result(
+        &mut self,
+        document: DocumentHandle,
+        _parse_errors: &[ParseError],
+    ) -> TestResult {
+        let mut result = TestResult::default();
+
+        let generator = TreeOutputGenerator::new(document);
+        let actual = generator.generate();
+
+        let mut line_idx = 1;
+        for actual_line in actual {
+            let mut status = ResultStatus::Success;
+
+            let expected_line = self.get_next_line();
+            match expected_line.clone() {
+                Some(expected_line) => {
+                    if actual_line != expected_line {
+                        status = ResultStatus::Mismatch;
+                    }
+                }
+                None => {
+                    status = ResultStatus::Missing;
+                }
+            }
+
+            result.tree_results.push(TreeLineResult {
+                index: line_idx,
+                result: status,
+                expected: expected_line.unwrap_or_default().to_string(),
+                actual: actual_line.to_string(),
+            });
+            line_idx += 1;
+        }
+
+        // Check if we have additional lines and if so, add as errors
+        loop {
+            let expected_line = self.get_next_line();
+            if expected_line.is_none() {
+                break;
+            }
+
+            result.tree_results.push(TreeLineResult {
+                index: line_idx,
+                result: ResultStatus::Additional,
+                expected: expected_line.expect("").to_string(),
+                actual: "".into(),
+            });
+            line_idx += 1;
+        }
+
+        // if ! result.is_success() {
+        //     let actual = generator.generate();
+        //     let expected = self.test.document.clone();
+        //     println!("\n\nactual   : {:?}", actual);
+        //     println!("expected : {:?}\n\n", expected);
+        // }
+
+        result
+    }
+}
+
+/*
+    /// Verifies that the tree construction code obtains the right result
+    pub fn assert_valid(&self) {
+        let results = self.run().expect("failed to parse");
+
+        fn assert_tree(tree: &SubtreeResult) {
+            match &tree.node {
+                Some(NodeResult::ElementMatchSuccess { .. })
+                | Some(NodeResult::TextMatchSuccess { .. })
+                | None => {}
+
+                Some(NodeResult::TextMatchFailure {
+                         actual, expected, ..
+                     }) => {
+                    panic!("text match failed, wanted: [{expected}], got: [{actual}]");
+                }
+
+                Some(NodeResult::DocTypeMatchFailure {
+                         actual, expected, ..
+                     }) => {
+                    panic!("doctype match failed, wanted: [{expected}], got: [{actual}]");
+                }
+
+                Some(NodeResult::ElementMatchFailure {
+                         actual,
+                         expected,
+                         name,
+                     }) => {
+                    panic!("element [{name}] match failed, wanted: [{expected}], got: [{actual}]");
+                }
+
+                Some(NodeResult::AttributeMatchFailure {
+                         name,
+                         actual,
+                         expected,
+                     }) => {
+                    panic!(
+                        "attribute [{name}] match failed, wanted: [{expected}], got: [{actual}]"
+                    );
+                }
+
+                Some(NodeResult::CommentMatchFailure {
+                         actual, expected, ..
+                     }) => {
+                    panic!("comment match failed, wanted: [{expected}], got: [{actual}]");
+                }
+            }
+
+            tree.children.iter().for_each(assert_tree);
+        }
+
+        for result in results {
+            assert_tree(&result.result);
+            assert!(result.success(), "invalid tree-construction result");
+        }
+    }
+
+
     /// Returns true if the whole document tree matches the expected result
-    pub fn match_document_tree(&self, document: &Document) -> SubtreeResult {
+    fn match_document_tree(&self, document: &Document) -> SubtreeResult {
         if self.spec.document_fragment.is_some() {
             self.match_node(NodeId::from(1), 0, 0, document)
         } else {
@@ -310,10 +346,10 @@ impl Test {
 
         let node_result = match &node.data {
             NodeData::DocType(DocTypeData {
-                name,
-                pub_identifier,
-                sys_identifier,
-            }) => {
+                                  name,
+                                  pub_identifier,
+                                  sys_identifier,
+                              }) => {
                 let doctype_text = if pub_identifier.is_empty() && sys_identifier.is_empty() {
                     // <!DOCTYPE html>
                     name.to_string()
@@ -328,9 +364,7 @@ impl Test {
                     doctype_text.trim(),
                 );
 
-                let expected = self.document[next_expected_idx as usize].to_owned();
-                next_expected_idx += 1;
-
+                let expected = self.get_next_line().expect("line");
                 if actual != expected {
                     let node = Some(NodeResult::DocTypeMatchFailure {
                         actual,
@@ -366,8 +400,7 @@ impl Test {
                     element.name()
                 );
 
-                let expected = self.document[next_expected_idx as usize].to_owned();
-                next_expected_idx += 1;
+                let expected = self.get_next_line().expect("line");
 
                 if actual != expected {
                     let node = Some(NodeResult::ElementMatchFailure {
@@ -393,8 +426,7 @@ impl Test {
                 sorted_attrs.sort_by(|a, b| a.0.cmp(b.0));
 
                 for attr in sorted_attrs {
-                    let expected = self.document[next_expected_idx as usize].to_owned();
-                    next_expected_idx += 1;
+                    let expected = self.get_next_line().expect("line");
 
                     let actual = format!(
                         "|{}{}=\"{}\"",
@@ -427,23 +459,7 @@ impl Test {
                     text.value()
                 );
 
-                // Text might be split over multiple lines, read all lines until we find the closing
-                // quote.
-                let mut expected = String::new();
-                loop {
-                    let tmp = self.document[next_expected_idx as usize].to_owned();
-                    next_expected_idx += 1;
-
-                    expected.push_str(&tmp);
-
-                    if tmp.ends_with('\"') {
-                        break;
-                    } else {
-                        // each line is terminated with a newline
-                        expected.push('\n');
-                    }
-                }
-
+                let expected = self.get_next_line().expect("line");
                 if actual != expected {
                     let node = Some(NodeResult::TextMatchFailure {
                         actual,
@@ -466,8 +482,7 @@ impl Test {
                     " ".repeat(indent as usize * 2 + 1),
                     comment.value()
                 );
-                let expected = self.document[next_expected_idx as usize].to_owned();
-                next_expected_idx += 1;
+                let expected = self.get_next_line().expect("line");
 
                 if actual != expected {
                     let node = Some(NodeResult::CommentMatchFailure {
@@ -535,103 +550,5 @@ impl Test {
             actual: actual.to_owned(),
         }
     }
-
-    pub fn script_modes(&self) -> &[bool] {
-        match self.spec.script_mode {
-            ScriptMode::ScriptOff => &[false],
-            ScriptMode::ScriptOn => &[true],
-            ScriptMode::Both => &[false, true],
-        }
-    }
-}
-
-pub fn fixture_from_filename(filename: &str) -> Result<FixtureFile> {
-    let path = PathBuf::from(FIXTURE_ROOT)
-        .join("tree-construction")
-        .join(filename);
-    fixture_from_path(&path)
-}
-
-// Split into an array of lines.  Combine lines in cases where a subsequent line does not
-// have a "|" prefix using an "\n" delimiter.  Otherwise strip "\n" from lines.
-fn document(s: &str) -> Vec<String> {
-    let mut document = s
-        .replace(QUOTED_DOUBLE_NEWLINE, "\"\n\n\"")
-        .split('|')
-        .skip(1)
-        .filter_map(|l| {
-            if l.is_empty() {
-                None
-            } else {
-                Some(format!("|{}", l.trim_end()))
-            }
-        })
-        .collect::<Vec<_>>();
-
-    // TODO: drop the following line
-    document.push("".into());
-    document
-}
-
-/// Read a given test file and extract all test data
-pub fn fixture_from_path(path: &PathBuf) -> Result<FixtureFile> {
-    let input = fs::read_to_string(path)?;
-    let path = path.to_string_lossy().into_owned();
-
-    let tests = parser::parse_str(&input)?
-        .into_iter()
-        .map(|spec| Test {
-            file_path: path.to_string(),
-            line: spec.position.line,
-            document: document(&spec.document),
-            spec,
-        })
-        .collect::<Vec<_>>();
-
-    Ok(FixtureFile {
-        tests,
-        path: path.to_string(),
-    })
-}
-
-fn use_fixture(filenames: &[&str], path: &Path) -> bool {
-    if !path.is_file() || path.extension().expect("file ending") != "dat" {
-        return false;
-    }
-
-    if filenames.is_empty() {
-        return true;
-    }
-
-    filenames.iter().any(|filename| path.ends_with(filename))
-}
-
-pub fn fixtures(filenames: Option<&[&str]>) -> Result<Vec<FixtureFile>> {
-    let root = PathBuf::from(FIXTURE_ROOT).join("tree-construction");
-    let filenames = filenames.unwrap_or_default();
-    let mut files = vec![];
-
-    for entry in fs::read_dir(root)? {
-        let path = entry?.path();
-
-        if path.is_file() {
-            if !use_fixture(filenames, &path) {
-                continue;
-            }
-
-            let file = fixture_from_path(&path)?;
-            files.push(file);
-        } else {
-            for subentry in fs::read_dir(path)? {
-                let path = subentry?.path();
-                if !use_fixture(filenames, &path) {
-                    continue;
-                }
-                let file = fixture_from_path(&path)?;
-                files.push(file);
-            }
-        }
-    }
-
-    Ok(files)
 }
+ */
diff --git a/src/testing/tree_construction/fixture.rs b/src/testing/tree_construction/fixture.rs
new file mode 100644
index 000000000..e8e6dc39c
--- /dev/null
+++ b/src/testing/tree_construction/fixture.rs
@@ -0,0 +1,95 @@
+use crate::testing::tree_construction::parser::{parse_fixture, QUOTED_DOUBLE_NEWLINE};
+use crate::testing::tree_construction::Test;
+use crate::testing::{FIXTURE_ROOT, TREE_CONSTRUCTION_PATH};
+use crate::types::Error;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+/// Holds all tests as found in the given fixture file
+#[derive(Debug, PartialEq)]
+pub struct FixtureFile {
+    /// All the tests extracted from this fixture file
+    pub tests: Vec<Test>,
+    /// Path to the fixture file
+    pub path: String,
+}
+
+/// Reads a given test file and extract all test data
+pub fn read_fixture_from_path(path: &PathBuf) -> Result<FixtureFile, Error> {
+    let input = fs::read_to_string(path)?;
+    let path = path.to_string_lossy().into_owned();
+
+    let tests = parse_fixture(&input)?
+        .into_iter()
+        .map(|spec| Test {
+            file_path: path.to_string(),
+            line: spec.position.line,
+            document: create_document_array(&spec.document),
+            spec,
+        })
+        .collect::<Vec<_>>();
+
+    Ok(FixtureFile {
+        tests,
+        path: path.to_string(),
+    })
+}
+
+/// Returns true when the fixture at 'path' is a correct fixture file and is allowed to be used
+/// according to the list of given filenames. If no filenames are given, all fixtures are used.
+fn use_fixture(filenames: &[&str], path: &Path) -> bool {
+    if !path.is_file() || path.extension().expect("file ending") != "dat" {
+        return false;
+    }
+
+    if filenames.is_empty() {
+        return true;
+    }
+
+    filenames.iter().any(|filename| path.ends_with(filename))
+}
+
+/// Returns the root path for the fixtures
+pub fn get_fixture_root_path() -> PathBuf {
+    PathBuf::from(FIXTURE_ROOT).join(TREE_CONSTRUCTION_PATH)
+}
+
+/// Read tree construction fixtures from the given path. If no filenames are given, all
+/// fixtures are read, otherwise only the fixes with the given filenames are read.
+pub fn read_fixtures(filenames: Option<&[&str]>) -> Result<Vec<FixtureFile>, Error> {
+    let filenames = filenames.unwrap_or_default();
+    let mut files = vec![];
+
+    for entry in fs::read_dir(get_fixture_root_path())? {
+        let path = entry?.path();
+
+        // Check if the fixture is a correct fixture file and if it's allowed to be used
+        if !use_fixture(filenames, &path) {
+            continue;
+        }
+
+        let file = read_fixture_from_path(&path)?;
+        files.push(file);
+    }
+
+    Ok(files)
+}
+
+// Split a string into an array of lines.  Combine lines in cases where a subsequent line does not
+// have a "|" prefix using an "\n" delimiter.  Otherwise strip "\n" from lines.
+fn create_document_array(s: &str) -> Vec<String> {
+    let document = s
+        .replace(QUOTED_DOUBLE_NEWLINE, "\"\n\n\"")
+        .split('|')
+        .skip(1)
+        .filter_map(|l| {
+            if l.is_empty() {
+                None
+            } else {
+                Some(format!("|{}", l.trim_end()))
+            }
+        })
+        .collect::<Vec<_>>();
+
+    document
+}
diff --git a/src/testing/tree_construction/generator.rs b/src/testing/tree_construction/generator.rs
new file mode 100644
index 000000000..d5a5db53f
--- /dev/null
+++ b/src/testing/tree_construction/generator.rs
@@ -0,0 +1,110 @@
+use crate::html5::node::{Node, NodeData, NodeTrait, NodeType};
+use crate::html5::node::{MATHML_NAMESPACE, SVG_NAMESPACE, XLINK_NAMESPACE, XMLNS_NAMESPACE};
+use crate::html5::parser::document::DocumentHandle;
+
+/// Generates a tree output that can be used for matching with the expected output
+pub struct TreeOutputGenerator {
+    document: DocumentHandle,
+}
+
+impl TreeOutputGenerator {
+    /// Initializes a new tree output generator
+    pub fn new(document: DocumentHandle) -> Self {
+        Self { document }
+    }
+
+    /// Generates a tree
+    pub fn generate(&self) -> Vec<String> {
+        self.output_treeline(self.document.get().get_root(), 0)
+    }
+
+    /// Generates an array of indented tree line and its children. Note that text lines can have newlines in them
+    fn output_treeline(&self, node: &Node, indent_level: usize) -> Vec<String> {
+        let mut indent_level = indent_level;
+        let mut output = Vec::new();
+
+        // We can skip the document node, as it is always the root node (either a document node, or
+        // a html node when it's a fragment)
+        if indent_level > 0 {
+            output.push(format!(
+                "| {}{}",
+                "  ".repeat(indent_level - 1),
+                self.output_node(node)
+            ));
+
+            if node.type_of() == NodeType::Element {
+                if let NodeData::Element(element) = &node.data {
+                    let mut sorted_attrs = vec![];
+                    for attr in element.attributes.iter() {
+                        sorted_attrs.push(attr);
+                    }
+                    sorted_attrs.sort_by(|a, b| a.0.cmp(b.0));
+
+                    for attr in sorted_attrs.iter() {
+                        output.push(format!(
+                            "| {}{}=\"{}\"",
+                            "  ".repeat(indent_level),
+                            attr.0,
+                            attr.1
+                        ));
+                    }
+                }
+            }
+
+            // Template tags have an extra "content" node in the test tree ouput
+            if node.name == "template" {
+                output.push(format!("| {}content", "  ".repeat(indent_level)));
+                indent_level += 1;
+            }
+        }
+
+        for child_id in node.children.iter() {
+            let doc = self.document.get();
+            let child_node = doc.get_node_by_id(*child_id).expect("node not found");
+
+            output.append(&mut self.output_treeline(child_node, indent_level + 1));
+        }
+
+        output
+    }
+
+    /// Generate the output for a single node
+    fn output_node(&self, node: &Node) -> String {
+        match node.data.clone() {
+            NodeData::Element(element) => {
+                match node.namespace.clone() {
+                    Some(ns) => {
+                        let ns_prefix = match ns.as_str() {
+                            MATHML_NAMESPACE => "math ",
+                            SVG_NAMESPACE => "svg ",
+                            XMLNS_NAMESPACE => "xml ",
+                            XLINK_NAMESPACE => "xlink ",
+                            _ => "",
+                        };
+                        format!("<{}{}>", ns_prefix, element.name())
+                    }
+                    None => format!("<{}>", element.name()),
+                }
+                // format!("<{}{}>", ns_prefix, element.name())
+            }
+            NodeData::Text(text) => format!("\"{}\"", text.value()),
+            NodeData::Comment(comment) => format!("<!-- {} -->", comment.value()),
+            NodeData::DocType(doctype) => {
+                let doctype_text =
+                    if doctype.pub_identifier.is_empty() && doctype.sys_identifier.is_empty() {
+                        // <!DOCTYPE html>
+                        doctype.name.to_string()
+                    } else {
+                        // <!DOCTYPE html "pubid" "sysid">
+                        format!(
+                            r#"{0} "{1}" "{2}""#,
+                            doctype.name, doctype.pub_identifier, doctype.sys_identifier
+                        )
+                    };
+
+                format!("<!DOCTYPE {}>", doctype_text.trim())
+            }
+            _ => "".to_string(),
+        }
+    }
+}
diff --git a/src/testing/tree_construction/parser.rs b/src/testing/tree_construction/parser.rs
index 3c7839c4b..6456ab540 100644
--- a/src/testing/tree_construction/parser.rs
+++ b/src/testing/tree_construction/parser.rs
@@ -15,13 +15,13 @@ pub const QUOTED_DOUBLE_NEWLINE: &str = ":quoted-double-newline:";
 
 type Span<'a> = LocatedSpan<&'a str>;
 
-#[derive(Copy, Clone, Debug, PartialEq)]
+#[derive(Copy, Clone, Debug, PartialEq, Default)]
 pub struct Position {
     pub line: usize,
     pub col: usize,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum ErrorSpec {
     Message(String),
 
@@ -42,14 +42,15 @@ pub enum ErrorSpec {
     },
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Default)]
 pub enum ScriptMode {
     ScriptOn,
     ScriptOff,
+    #[default]
     Both,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Default, Clone)]
 pub struct TestSpec {
     /// #data section
     pub data: String,
@@ -296,7 +297,7 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
 
             TestSpec {
                 position,
-                data: data.to_string(),
+                data: data.to_string().trim_end().to_string(),
                 errors,
                 new_errors: new_errors.unwrap_or_default(),
                 script_mode,
@@ -307,7 +308,7 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
     )(i)
 }
 
-pub fn parse_str(i: &str) -> Result<Vec<TestSpec>> {
+pub fn parse_fixture(i: &str) -> Result<Vec<TestSpec>> {
     // Deal with a corner case that makes it hard to parse tricky01.dat.
     let input = i.replace("\"\n\n\"", QUOTED_DOUBLE_NEWLINE).to_owned() + "\n";
 
@@ -333,6 +334,9 @@ mod tests {
 
     #[test]
     fn parse_data() {
+        let (_, s) = data("#data\n         Test \n#errors\n".into()).unwrap();
+        assert_eq!(*s.fragment(), "Test \n");
+
         let (_, s) = data("#data\n         Test \n#errors".into()).unwrap();
         assert_eq!(*s.fragment(), "Test \n");
 
@@ -387,7 +391,7 @@ Test
 "#,
         );
 
-        assert_eq!(test.data, "Test\n");
+        assert_eq!(test.data, "Test");
         assert_eq!(
             test.errors,
             &[ErrorSpec::Location {
diff --git a/src/testing/tree_construction/result.rs b/src/testing/tree_construction/result.rs
new file mode 100644
index 000000000..5fc983ca8
--- /dev/null
+++ b/src/testing/tree_construction/result.rs
@@ -0,0 +1,69 @@
+#[derive(PartialEq)]
+pub enum ResultStatus {
+    /// This is a correct match
+    Success,
+    /// This line is found in expected, but wasn't found in actual (too few lines)
+    Missing,
+    /// This line is found in actual, but wasn't found in expected (too many lines)
+    Additional,
+    /// The error is correct, but is throwed on an incorrect position
+    IncorrectPosition,
+    // The tree line or message mismatches between expected and actual
+    Mismatch,
+}
+
+/// The result for a single line in the tree
+pub struct TreeLineResult {
+    /// Index (line number) inside the generated tree
+    pub index: usize,
+    /// Actual found line
+    pub actual: String,
+    /// Expected line
+    pub expected: String,
+    /// Result
+    pub result: ResultStatus,
+}
+
+/// Defines a message that is generated on a certain line and column
+pub struct ErrorMessagePosition {
+    /// The error message
+    pub message: String,
+    /// The line on which the error was thrown by the parser
+    pub line: usize,
+    /// The column on which the error was thrown by the parser
+    pub col: usize,
+}
+
+/// The result for a single error
+pub struct ErrorResult {
+    /// The actual error found during testing
+    pub actual: ErrorMessagePosition,
+    /// The expected error that should have been found
+    pub expected: ErrorMessagePosition,
+    /// Index of the error (0 based)
+    pub index: usize,
+    /// The actual result
+    pub result: ResultStatus,
+}
+
+/// A combined result from a single test with all the result from the tree and parser errors
+#[derive(Default)]
+pub struct TestResult {
+    /// Results of each line in the node tree (as generated by the parser)
+    pub tree_results: Vec<TreeLineResult>,
+    /// Results of each error generated by the parser
+    pub error_results: Vec<ErrorResult>,
+}
+
+impl TestResult {
+    /// Returns true when both results and error_results are either empty or have a result that is not Success
+    pub fn is_success(&self) -> bool {
+        self.tree_results
+            .iter()
+            .all(|r| r.result == ResultStatus::Success)
+            && self
+                .error_results
+                .iter()
+                .all(|r| r.result == ResultStatus::Success)
+    }
+}
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 85005b784..45a1a3f99 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -1,6 +1,7 @@
-use gosub_engine::testing::tree_construction::fixture_from_filename;
-use lazy_static::lazy_static;
-use std::collections::HashSet;
+use gosub_engine::testing::tree_construction::fixture::{
+    get_fixture_root_path, read_fixture_from_path,
+};
+use gosub_engine::testing::tree_construction::Harness;
 use test_case::test_case;
 
 const DISABLED_CASES: &[&str] = &[
@@ -8,20 +9,13 @@ const DISABLED_CASES: &[&str] = &[
     "<!doctype html><template><plaintext>a</template>b",
 ];
 
-lazy_static! {
-    static ref DISABLED: HashSet<String> = DISABLED_CASES
-        .iter()
-        .map(|s| s.to_string())
-        .collect::<HashSet<_>>();
-}
-
 // See tests/data/html5lib-tests/tree-construction/ for other test files.
 #[test_case("tests1.dat")]
-#[test_case("tests2.dat")]
+// #[test_case("tests2.dat")]
 #[test_case("tests3.dat")]
 #[test_case("tests4.dat")]
 #[test_case("tests5.dat")]
-#[test_case("tests6.dat")]
+// #[test_case("tests6.dat")]
 #[test_case("tests7.dat")]
 #[test_case("tests8.dat")]
 #[test_case("tests9.dat")]
@@ -30,17 +24,17 @@ lazy_static! {
 #[test_case("tests12.dat")]
 #[test_case("tests14.dat")]
 #[test_case("tests15.dat")]
-#[test_case("tests16.dat")]
+// #[test_case("tests16.dat")]
 #[test_case("tests17.dat")]
 #[test_case("tests18.dat")]
-#[test_case("tests19.dat")]
+// #[test_case("tests19.dat")]
 #[test_case("tests20.dat")]
 // #[test_case("tests21.dat")]
 #[test_case("tests22.dat")]
 #[test_case("tests23.dat")]
 #[test_case("tests24.dat")]
 #[test_case("tests25.dat")]
-#[test_case("tests26.dat")]
+// #[test_case("tests26.dat")]
 #[test_case("adoption01.dat")]
 #[test_case("adoption02.dat")]
 #[test_case("blocks.dat")]
@@ -70,21 +64,32 @@ lazy_static! {
 // #[test_case("template.dat")]
 #[test_case("tests_innerHTML_1.dat")]
 #[test_case("tricky01.dat")]
-#[test_case("webkit01.dat")]
+// #[test_case("webkit01.dat")]
 #[test_case("webkit02.dat")]
 fn tree_construction(filename: &str) {
-    let fixture_file = fixture_from_filename(filename).expect("fixture");
+    let fixture_file =
+        read_fixture_from_path(&get_fixture_root_path().join(filename)).expect("fixture");
+    let mut harness = Harness::new();
 
     for test in fixture_file.tests {
-        if DISABLED.contains(test.data()) {
-            for &scripting_enabled in test.script_modes() {
-                // Check that we don't panic
-                let _ = test.parse(scripting_enabled).expect("problem parsing");
-            }
+        // skip disabled tests
+        if DISABLED_CASES.contains(&test.get_document_as_str()) {
             continue;
         }
 
-        println!("tree construction: {}", test.data());
-        test.assert_valid();
+        // for each test, run it with and without scripting enabled based on the test file
+        for &scripting_enabled in test.script_modes() {
+            let result = harness
+                .run_test(test.clone(), scripting_enabled)
+                .expect("problem parsing");
+
+            println!(
+                "tree construction: {}:{} {}",
+                test.file_path,
+                test.line,
+                test.get_document_as_str()
+            );
+            assert!(result.is_success());
+        }
     }
 }

From 68336351912a2bbc5657781a5564eb3b3aaa101f Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sat, 4 Nov 2023 14:55:42 +0100
Subject: [PATCH 02/14] Copying attributes from extra body tags

---
 src/bin/parser-test.rs                  | 39 ++++++++-----------------
 src/html5/parser.rs                     | 25 ++++++++++++++--
 src/testing/tree_construction/parser.rs |  2 +-
 tests/tree_construction.rs              |  2 +-
 4 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 63776e3a1..e3b4eea28 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -29,7 +29,7 @@ fn main() {
         tests_failed: Vec::new(),
     };
 
-    let filenames = Some(&["template.dat"][..]);
+    let filenames = Some(&["tests2.dat"][..]);
     let fixtures = read_fixtures(filenames).expect("fixtures");
 
     for fixture_file in fixtures {
@@ -41,9 +41,9 @@ fn main() {
 
         let mut test_idx = 1;
         for test in fixture_file.tests {
-            if test_idx == 1 {
-                run_test(test_idx, test, &mut results);
-            }
+            // if test_idx == 57 {
+            run_test(test_idx, test, &mut results);
+            // }
             test_idx += 1;
         }
 
@@ -67,7 +67,7 @@ fn main() {
     }
 }
 
-fn run_test(_test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
+fn run_test(test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
     #[cfg(feature = "debug_parser_verbose")]
     println!(
         "🧪 Running test #{test_idx}: {}:{}",
@@ -177,26 +177,11 @@ fn run_test(_test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
     }
 
     // // Display additional data if there a failure is found
-    // if !result.success() {
-    //     all_results
-    //         .tests_failed
-    //         .push((test_idx, test.line, test.data().to_string()));
-    //
-    //     if cfg!(feature = "debug_parser") {
-    //         println!("----------------------------------------");
-    //         println!("📄 Input stream: ");
-    //         println!("{}", test.data());
-    //         println!("----------------------------------------");
-    //         println!("🌳 Generated tree: ");
-    //         println!("{}", result.actual_document);
-    //         println!("----------------------------------------");
-    //         println!("🌳 Expected tree: ");
-    //         for line in &test.document {
-    //             println!("{line}");
-    //         }
-    //     }
-    // }
-    //
-    // #[cfg(feature = "debug_parser")]
-    // println!("----------------------------------------");
+    if !result.is_success() {
+        all_results.tests_failed.push((
+            test_idx,
+            test.line,
+            test.get_document_as_str().to_string(),
+        ));
+    }
 }
diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index 3d07c1aab..3dc516f1f 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -2131,7 +2131,9 @@ impl<'chars> Html5Parser<'chars> {
             Token::EndTag { name, .. } if name == "template" => {
                 self.handle_in_head();
             }
-            Token::StartTag { name, .. } if name == "body" => {
+            Token::StartTag {
+                name, attributes, ..
+            } if name == "body" => {
                 self.parse_error("body tag not allowed in in body insertion mode");
 
                 if self.open_elements.len() == 1
@@ -2145,8 +2147,25 @@ impl<'chars> Html5Parser<'chars> {
 
                 self.frameset_ok = false;
 
-                // Add attributes to body element
-                // @TODO add body attributes
+                let body_node_id = self.open_elements.iter().find(|node_id| {
+                    let node = get_node_by_id!(self.document, **node_id);
+                    node.name == "body" && node.is_namespace(HTML_NAMESPACE)
+                });
+
+                if let Some(body_node_id) = body_node_id {
+                    let mut doc = self.document.get_mut();
+                    let body_node = doc
+                        .get_node_by_id_mut(*body_node_id)
+                        .expect("node not found");
+
+                    if let NodeData::Element(element) = &mut body_node.data {
+                        for (key, value) in attributes {
+                            if !element.attributes.contains_key(key) {
+                                element.attributes.insert(key.to_owned(), value.to_owned());
+                            }
+                        }
+                    }
+                }
             }
             Token::StartTag { name, .. } if name == "frameset" => {
                 self.parse_error("frameset tag not allowed in in body insertion mode");
diff --git a/src/testing/tree_construction/parser.rs b/src/testing/tree_construction/parser.rs
index 6456ab540..4f50c20ab 100644
--- a/src/testing/tree_construction/parser.rs
+++ b/src/testing/tree_construction/parser.rs
@@ -297,7 +297,7 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
 
             TestSpec {
                 position,
-                data: data.to_string().trim_end().to_string(),
+                data: data.to_string().trim_matches(|c| c == '\n').to_string(),
                 errors,
                 new_errors: new_errors.unwrap_or_default(),
                 script_mode,
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 45a1a3f99..817658424 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -11,7 +11,7 @@ const DISABLED_CASES: &[&str] = &[
 
 // See tests/data/html5lib-tests/tree-construction/ for other test files.
 #[test_case("tests1.dat")]
-// #[test_case("tests2.dat")]
+#[test_case("tests2.dat")]
 #[test_case("tests3.dat")]
 #[test_case("tests4.dat")]
 #[test_case("tests5.dat")]

From 52f5d63bbd69fb835e3b124fc7ac6ee11fb1590b Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 10:14:52 +0100
Subject: [PATCH 03/14] removed some get_ prefixes from functions

---
 src/bin/html5-parser-test.rs             | 6 +++---
 src/bin/parser-test.rs                   | 2 +-
 src/html5/tokenizer.rs                   | 2 ++
 src/testing/tokenizer.rs                 | 9 +++++++--
 src/testing/tree_construction.rs         | 8 ++++----
 src/testing/tree_construction/fixture.rs | 4 ++--
 tests/tree_construction.rs               | 4 ++--
 7 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/bin/html5-parser-test.rs b/src/bin/html5-parser-test.rs
index 4d80f0788..29a4202f4 100755
--- a/src/bin/html5-parser-test.rs
+++ b/src/bin/html5-parser-test.rs
@@ -1,5 +1,5 @@
 use gosub_engine::testing::tree_construction::fixture::{
-    get_fixture_root_path, read_fixture_from_path,
+    fixture_root_path, read_fixture_from_path,
 };
 use gosub_engine::testing::tree_construction::Harness;
 use gosub_engine::types::Result;
@@ -8,7 +8,7 @@ use std::path::{Path, PathBuf};
 use walkdir::WalkDir;
 
 fn main() -> Result<()> {
-    let mut files = get_files_from_path(get_fixture_root_path());
+    let mut files = get_files_from_path(fixture_root_path());
     files.sort();
 
     let mut total = 0;
@@ -18,7 +18,7 @@ fn main() -> Result<()> {
         // if file != "math.dat" {
         //     continue;
         // }
-        let fixture = read_fixture_from_path(&get_fixture_root_path().join(file))?;
+        let fixture = read_fixture_from_path(&fixture_root_path().join(file))?;
 
         print!("Test: ({:3}) {} [", fixture.tests.len(), file);
         let _ = std::io::stdout().flush();
diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index e3b4eea28..1f7435493 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -181,7 +181,7 @@ fn run_test(test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
         all_results.tests_failed.push((
             test_idx,
             test.line,
-            test.get_document_as_str().to_string(),
+            test.document_as_str().to_string(),
         ));
     }
 }
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index cea1f5ee5..aa6d50718 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -14,6 +14,8 @@ use crate::types::{Error, Result};
 use std::cell::{Ref, RefCell};
 use std::collections::HashMap;
 use std::rc::Rc;
+use crate::html5::node::HTML_NAMESPACE;
+use crate::html5::parser::Html5Parser;
 
 /// Constants that are not directly captured as visible chars
 pub const CHAR_NUL: char = '\u{0000}';
diff --git a/src/testing/tokenizer.rs b/src/testing/tokenizer.rs
index 252930828..8caa589b8 100644
--- a/src/testing/tokenizer.rs
+++ b/src/testing/tokenizer.rs
@@ -21,6 +21,7 @@ use std::{
     fs,
     path::{Path, PathBuf},
 };
+use crate::html5::parser::Html5Parser;
 
 pub struct TokenizerBuilder {
     chars: CharIterator,
@@ -205,18 +206,22 @@ impl TestSpec {
     }
 
     pub fn assert_valid(&self) {
+        // Dummy parser, as a tokenizer needs a parser to function correctly (for callbacks)
+        let mut ci = CharIterator::new();
+        let dummy_parser = Html5Parser::new_parser(&mut ci);
+
         for mut builder in self.builders() {
             let mut tokenizer = builder.build();
 
             // If there is no output, still do an (initial) next token so the parser can generate
             // errors.
             if self.output.is_empty() {
-                tokenizer.next_token().unwrap();
+                tokenizer.next_token(&dummy_parser).unwrap();
             }
 
             // There can be multiple tokens to match. Make sure we match all of them
             for expected in self.output.iter() {
-                let actual = tokenizer.next_token().unwrap();
+                let actual = tokenizer.next_token(&dummy_parser).unwrap();
                 assert_eq!(self.escape(&actual), self.escape(expected));
             }
 
diff --git a/src/testing/tree_construction.rs b/src/testing/tree_construction.rs
index 5030c4bbb..136e30843 100644
--- a/src/testing/tree_construction.rs
+++ b/src/testing/tree_construction.rs
@@ -57,7 +57,7 @@ impl Test {
         }
     }
 
-    pub fn get_document_as_str(&self) -> &str {
+    pub fn document_as_str(&self) -> &str {
         return self.spec.document.as_str();
     }
 }
@@ -161,7 +161,7 @@ impl Harness {
     }
 
     /// Retrieves the next line from the spec document
-    fn get_next_line(&mut self) -> Option<String> {
+    fn next_line(&mut self) -> Option<String> {
         let mut line = String::new();
         let mut is_multi_line_text = false;
 
@@ -217,7 +217,7 @@ impl Harness {
         for actual_line in actual {
             let mut status = ResultStatus::Success;
 
-            let expected_line = self.get_next_line();
+            let expected_line = self.next_line();
             match expected_line.clone() {
                 Some(expected_line) => {
                     if actual_line != expected_line {
@@ -240,7 +240,7 @@ impl Harness {
 
         // Check if we have additional lines and if so, add as errors
         loop {
-            let expected_line = self.get_next_line();
+            let expected_line = self.next_line();
             if expected_line.is_none() {
                 break;
             }
diff --git a/src/testing/tree_construction/fixture.rs b/src/testing/tree_construction/fixture.rs
index e8e6dc39c..93da63643 100644
--- a/src/testing/tree_construction/fixture.rs
+++ b/src/testing/tree_construction/fixture.rs
@@ -50,7 +50,7 @@ fn use_fixture(filenames: &[&str], path: &Path) -> bool {
 }
 
 /// Returns the root path for the fixtures
-pub fn get_fixture_root_path() -> PathBuf {
+pub fn fixture_root_path() -> PathBuf {
     PathBuf::from(FIXTURE_ROOT).join(TREE_CONSTRUCTION_PATH)
 }
 
@@ -60,7 +60,7 @@ pub fn read_fixtures(filenames: Option<&[&str]>) -> Result<Vec<FixtureFile>, Err
     let filenames = filenames.unwrap_or_default();
     let mut files = vec![];
 
-    for entry in fs::read_dir(get_fixture_root_path())? {
+    for entry in fs::read_dir(fixture_root_path())? {
         let path = entry?.path();
 
         // Check if the fixture is a correct fixture file and if it's allowed to be used
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 817658424..760f15ad6 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -73,7 +73,7 @@ fn tree_construction(filename: &str) {
 
     for test in fixture_file.tests {
         // skip disabled tests
-        if DISABLED_CASES.contains(&test.get_document_as_str()) {
+        if DISABLED_CASES.contains(&test.document_as_str()) {
             continue;
         }
 
@@ -87,7 +87,7 @@ fn tree_construction(filename: &str) {
                 "tree construction: {}:{} {}",
                 test.file_path,
                 test.line,
-                test.get_document_as_str()
+                test.document_as_str()
             );
             assert!(result.is_success());
         }

From fa84cf85847a4fbd692870876c8ac6833b0c45ac Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 10:30:14 +0100
Subject: [PATCH 04/14] renamed globaltests to totaltests

---
 src/bin/parser-test.rs | 6 +++---
 src/html5/tokenizer.rs | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 1f7435493..2ea7b9b4e 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -4,7 +4,7 @@ use gosub_engine::testing::tree_construction::Harness;
 use gosub_engine::testing::tree_construction::Test;
 
 /// Holds the results from all tests that are executed
-pub struct GlobalTestResults {
+pub struct TotalTestResults {
     /// Number of tests (as defined in the suite)
     tests: usize,
     /// Number of assertions (different combinations of input/output per test)
@@ -20,7 +20,7 @@ pub struct GlobalTestResults {
 }
 
 fn main() {
-    let mut results = GlobalTestResults {
+    let mut results = TotalTestResults {
         tests: 0,
         assertions: 0,
         succeeded: 0,
@@ -67,7 +67,7 @@ fn main() {
     }
 }
 
-fn run_test(test_idx: usize, test: Test, all_results: &mut GlobalTestResults) {
+fn run_test(test_idx: usize, test: Test, all_results: &mut TotalTestResults) {
     #[cfg(feature = "debug_parser_verbose")]
     println!(
         "🧪 Running test #{test_idx}: {}:{}",
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index aa6d50718..003667a84 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -15,7 +15,6 @@ use std::cell::{Ref, RefCell};
 use std::collections::HashMap;
 use std::rc::Rc;
 use crate::html5::node::HTML_NAMESPACE;
-use crate::html5::parser::Html5Parser;
 
 /// Constants that are not directly captured as visible chars
 pub const CHAR_NUL: char = '\u{0000}';

From 46a1c9ebb247a64c13119e06592e175d5a1ac299 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 10:31:38 +0100
Subject: [PATCH 05/14] using default clone derive

---
 src/testing/tree_construction.rs | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/src/testing/tree_construction.rs b/src/testing/tree_construction.rs
index 136e30843..7d10e6212 100644
--- a/src/testing/tree_construction.rs
+++ b/src/testing/tree_construction.rs
@@ -24,7 +24,7 @@ use crate::{
 use result::TestResult;
 
 /// Holds a single parser test
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct Test {
     /// Filename of the test
     pub file_path: String,
@@ -36,17 +36,6 @@ pub struct Test {
     pub document: Vec<String>,
 }
 
-impl Clone for Test {
-    fn clone(&self) -> Self {
-        Self {
-            file_path: self.file_path.clone(),
-            line: self.line,
-            spec: self.spec.clone(),
-            document: self.document.clone(),
-        }
-    }
-}
-
 impl Test {
     /// Returns the script modes that should be tested as an array
     pub fn script_modes(&self) -> &[bool] {

From 9f60c08919c2f09a3b59f79dd3003e0d56993d34 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 10:41:03 +0100
Subject: [PATCH 06/14] benchmark fix

---
 benches/tree_construction.rs | 8 ++++++--
 src/bin/parser-test.rs       | 8 +++-----
 src/html5/tokenizer.rs       | 1 -
 src/testing/tokenizer.rs     | 9 ++-------
 tests/tree_construction.rs   | 4 ++--
 5 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/benches/tree_construction.rs b/benches/tree_construction.rs
index f37f4bbd2..0f8e1f252 100644
--- a/benches/tree_construction.rs
+++ b/benches/tree_construction.rs
@@ -1,5 +1,6 @@
 use criterion::{criterion_group, criterion_main, Criterion};
 use gosub_engine::testing::tree_construction;
+use gosub_engine::testing::tree_construction::Harness;
 
 fn criterion_benchmark(c: &mut Criterion) {
     let mut group = c.benchmark_group("Tree construction");
@@ -7,14 +8,17 @@ fn criterion_benchmark(c: &mut Criterion) {
 
     // Careful about reading files inside the closure
     let filenames = Some(&["tests1.dat"][..]);
-    let fixtures = tree_construction::fixtures(filenames).expect("problem loading fixtures");
+    let fixtures =
+        tree_construction::fixture::read_fixtures(filenames).expect("problem loading fixtures");
+
+    let mut harness = Harness::new();
 
     group.bench_function("fixtures", |b| {
         b.iter(|| {
             for root in &fixtures {
                 for test in &root.tests {
                     for &scripting_enabled in test.script_modes() {
-                        let _ = test.parse(scripting_enabled).unwrap();
+                        let _ = harness.run_test(test.clone(), scripting_enabled);
                     }
                 }
             }
diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 2ea7b9b4e..af63a9ea0 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -178,10 +178,8 @@ fn run_test(test_idx: usize, test: Test, all_results: &mut TotalTestResults) {
 
     // // Display additional data if there a failure is found
     if !result.is_success() {
-        all_results.tests_failed.push((
-            test_idx,
-            test.line,
-            test.document_as_str().to_string(),
-        ));
+        all_results
+            .tests_failed
+            .push((test_idx, test.line, test.document_as_str().to_string()));
     }
 }
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index 003667a84..cea1f5ee5 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -14,7 +14,6 @@ use crate::types::{Error, Result};
 use std::cell::{Ref, RefCell};
 use std::collections::HashMap;
 use std::rc::Rc;
-use crate::html5::node::HTML_NAMESPACE;
 
 /// Constants that are not directly captured as visible chars
 pub const CHAR_NUL: char = '\u{0000}';
diff --git a/src/testing/tokenizer.rs b/src/testing/tokenizer.rs
index 8caa589b8..252930828 100644
--- a/src/testing/tokenizer.rs
+++ b/src/testing/tokenizer.rs
@@ -21,7 +21,6 @@ use std::{
     fs,
     path::{Path, PathBuf},
 };
-use crate::html5::parser::Html5Parser;
 
 pub struct TokenizerBuilder {
     chars: CharIterator,
@@ -206,22 +205,18 @@ impl TestSpec {
     }
 
     pub fn assert_valid(&self) {
-        // Dummy parser, as a tokenizer needs a parser to function correctly (for callbacks)
-        let mut ci = CharIterator::new();
-        let dummy_parser = Html5Parser::new_parser(&mut ci);
-
         for mut builder in self.builders() {
             let mut tokenizer = builder.build();
 
             // If there is no output, still do an (initial) next token so the parser can generate
             // errors.
             if self.output.is_empty() {
-                tokenizer.next_token(&dummy_parser).unwrap();
+                tokenizer.next_token().unwrap();
             }
 
             // There can be multiple tokens to match. Make sure we match all of them
             for expected in self.output.iter() {
-                let actual = tokenizer.next_token(&dummy_parser).unwrap();
+                let actual = tokenizer.next_token().unwrap();
                 assert_eq!(self.escape(&actual), self.escape(expected));
             }
 
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 760f15ad6..1e0774008 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -1,5 +1,5 @@
 use gosub_engine::testing::tree_construction::fixture::{
-    get_fixture_root_path, read_fixture_from_path,
+    fixture_root_path, read_fixture_from_path,
 };
 use gosub_engine::testing::tree_construction::Harness;
 use test_case::test_case;
@@ -68,7 +68,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("webkit02.dat")]
 fn tree_construction(filename: &str) {
     let fixture_file =
-        read_fixture_from_path(&get_fixture_root_path().join(filename)).expect("fixture");
+        read_fixture_from_path(&fixture_root_path().join(filename)).expect("fixture");
     let mut harness = Harness::new();
 
     for test in fixture_file.tests {

From dca49079e695110381e5beba49a2669493a7540a Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 10:57:19 +0100
Subject: [PATCH 07/14] removed obsolete code and refactored loop

---
 src/testing/tree_construction.rs | 310 +------------------------------
 1 file changed, 7 insertions(+), 303 deletions(-)

diff --git a/src/testing/tree_construction.rs b/src/testing/tree_construction.rs
index 7d10e6212..da0911ca6 100644
--- a/src/testing/tree_construction.rs
+++ b/src/testing/tree_construction.rs
@@ -49,6 +49,10 @@ impl Test {
     pub fn document_as_str(&self) -> &str {
         return self.spec.document.as_str();
     }
+
+    pub fn spec_data(&self) -> &str {
+        return self.spec.data.as_str();
+    }
 }
 
 impl Default for Test {
@@ -133,7 +137,7 @@ impl Harness {
         let options = Html5ParserOptions { scripting_enabled };
 
         let mut chars = CharIterator::new();
-        chars.read_from_str(self.test.spec.data.as_str(), None);
+        chars.read_from_str(self.test.spec_data(), None);
 
         let parse_errors = if is_fragment {
             Html5Parser::parse_fragment(
@@ -228,316 +232,16 @@ impl Harness {
         }
 
         // Check if we have additional lines and if so, add as errors
-        loop {
-            let expected_line = self.next_line();
-            if expected_line.is_none() {
-                break;
-            }
-
+        while let Some(expected_line) = self.next_line() {
             result.tree_results.push(TreeLineResult {
                 index: line_idx,
                 result: ResultStatus::Additional,
-                expected: expected_line.expect("").to_string(),
+                expected: expected_line,
                 actual: "".into(),
             });
             line_idx += 1;
         }
 
-        // if ! result.is_success() {
-        //     let actual = generator.generate();
-        //     let expected = self.test.document.clone();
-        //     println!("\n\nactual   : {:?}", actual);
-        //     println!("expected : {:?}\n\n", expected);
-        // }
-
         result
     }
 }
-
-/*
-    /// Verifies that the tree construction code obtains the right result
-    pub fn assert_valid(&self) {
-        let results = self.run().expect("failed to parse");
-
-        fn assert_tree(tree: &SubtreeResult) {
-            match &tree.node {
-                Some(NodeResult::ElementMatchSuccess { .. })
-                | Some(NodeResult::TextMatchSuccess { .. })
-                | None => {}
-
-                Some(NodeResult::TextMatchFailure {
-                         actual, expected, ..
-                     }) => {
-                    panic!("text match failed, wanted: [{expected}], got: [{actual}]");
-                }
-
-                Some(NodeResult::DocTypeMatchFailure {
-                         actual, expected, ..
-                     }) => {
-                    panic!("doctype match failed, wanted: [{expected}], got: [{actual}]");
-                }
-
-                Some(NodeResult::ElementMatchFailure {
-                         actual,
-                         expected,
-                         name,
-                     }) => {
-                    panic!("element [{name}] match failed, wanted: [{expected}], got: [{actual}]");
-                }
-
-                Some(NodeResult::AttributeMatchFailure {
-                         name,
-                         actual,
-                         expected,
-                     }) => {
-                    panic!(
-                        "attribute [{name}] match failed, wanted: [{expected}], got: [{actual}]"
-                    );
-                }
-
-                Some(NodeResult::CommentMatchFailure {
-                         actual, expected, ..
-                     }) => {
-                    panic!("comment match failed, wanted: [{expected}], got: [{actual}]");
-                }
-            }
-
-            tree.children.iter().for_each(assert_tree);
-        }
-
-        for result in results {
-            assert_tree(&result.result);
-            assert!(result.success(), "invalid tree-construction result");
-        }
-    }
-
-
-    /// Returns true if the whole document tree matches the expected result
-    fn match_document_tree(&self, document: &Document) -> SubtreeResult {
-        if self.spec.document_fragment.is_some() {
-            self.match_node(NodeId::from(1), 0, 0, document)
-        } else {
-            self.match_node(NodeId::root(), 0, -1, document)
-        }
-    }
-
-    /// Match a single node and its children
-    fn match_node(
-        &self,
-        node_idx: NodeId,
-        document_offset_id: isize,
-        indent: isize,
-        document: &Document,
-    ) -> SubtreeResult {
-        let mut next_expected_idx = document_offset_id;
-
-        let node = document.get_node_by_id(node_idx).unwrap();
-
-        let node_result = match &node.data {
-            NodeData::DocType(DocTypeData {
-                                  name,
-                                  pub_identifier,
-                                  sys_identifier,
-                              }) => {
-                let doctype_text = if pub_identifier.is_empty() && sys_identifier.is_empty() {
-                    // <!DOCTYPE html>
-                    name.to_string()
-                } else {
-                    // <!DOCTYPE html "pubid" "sysid">
-                    format!(r#"{name} "{pub_identifier}" "{sys_identifier}""#,)
-                };
-
-                let actual = format!(
-                    "|{}<!DOCTYPE {}>",
-                    " ".repeat(indent as usize * 2 + 1),
-                    doctype_text.trim(),
-                );
-
-                let expected = self.get_next_line().expect("line");
-                if actual != expected {
-                    let node = Some(NodeResult::DocTypeMatchFailure {
-                        actual,
-                        expected: "".to_string(),
-                    });
-
-                    return SubtreeResult {
-                        node,
-                        children: vec![],
-                        next_expected_idx: None,
-                    };
-                }
-
-                Some(NodeResult::TextMatchSuccess { expected })
-            }
-            NodeData::Element(element) => {
-                let prefix: String = match &node.namespace {
-                    Some(namespace) => match namespace.as_str() {
-                        HTML_NAMESPACE => "".into(), // HTML elements don't have a prefix
-                        SVG_NAMESPACE => "svg ".into(),
-                        MATHML_NAMESPACE => "math ".into(),
-                        _ => {
-                            panic!("unknown namespace: {}", namespace);
-                        }
-                    },
-                    None => "".into(),
-                };
-
-                let actual = format!(
-                    "|{}<{}{}>",
-                    " ".repeat((indent as usize * 2) + 1),
-                    prefix,
-                    element.name()
-                );
-
-                let expected = self.get_next_line().expect("line");
-
-                if actual != expected {
-                    let node = Some(NodeResult::ElementMatchFailure {
-                        name: element.name().to_owned(),
-                        actual,
-                        expected,
-                    });
-
-                    return SubtreeResult {
-                        node,
-                        children: vec![],
-                        next_expected_idx: None,
-                    };
-                }
-
-                // Check attributes if any
-
-                // Make sure the attributes are sorted
-                let mut sorted_attrs = vec![];
-                for attr in element.attributes.iter() {
-                    sorted_attrs.push(attr);
-                }
-                sorted_attrs.sort_by(|a, b| a.0.cmp(b.0));
-
-                for attr in sorted_attrs {
-                    let expected = self.get_next_line().expect("line");
-
-                    let actual = format!(
-                        "|{}{}=\"{}\"",
-                        " ".repeat((indent as usize * 2) + 3),
-                        attr.0,
-                        attr.1
-                    );
-
-                    if actual != expected {
-                        let node = Some(NodeResult::AttributeMatchFailure {
-                            name: element.name().to_owned(),
-                            actual,
-                            expected,
-                        });
-
-                        return SubtreeResult {
-                            node,
-                            children: vec![],
-                            next_expected_idx: None,
-                        };
-                    }
-                }
-
-                Some(NodeResult::ElementMatchSuccess { actual })
-            }
-            NodeData::Text(text) => {
-                let actual = format!(
-                    "|{}\"{}\"",
-                    " ".repeat(indent as usize * 2 + 1),
-                    text.value()
-                );
-
-                let expected = self.get_next_line().expect("line");
-                if actual != expected {
-                    let node = Some(NodeResult::TextMatchFailure {
-                        actual,
-                        expected,
-                        text: text.value().to_owned(),
-                    });
-
-                    return SubtreeResult {
-                        node,
-                        children: vec![],
-                        next_expected_idx: None,
-                    };
-                }
-
-                Some(NodeResult::TextMatchSuccess { expected })
-            }
-            NodeData::Comment(comment) => {
-                let actual = format!(
-                    "|{}<!-- {} -->",
-                    " ".repeat(indent as usize * 2 + 1),
-                    comment.value()
-                );
-                let expected = self.get_next_line().expect("line");
-
-                if actual != expected {
-                    let node = Some(NodeResult::CommentMatchFailure {
-                        actual,
-                        expected,
-                        comment: comment.value().to_owned(),
-                    });
-
-                    return SubtreeResult {
-                        node,
-                        children: vec![],
-                        next_expected_idx: None,
-                    };
-                }
-
-                Some(NodeResult::TextMatchSuccess { expected })
-            }
-            _ => None,
-        };
-
-        let mut children = vec![];
-
-        for &child_idx in &node.children {
-            let child_result = self.match_node(child_idx, next_expected_idx, indent + 1, document);
-            let next_id = child_result.next_expected_idx;
-            children.push(child_result);
-
-            if let Some(next_id) = next_id {
-                next_expected_idx = next_id as isize;
-                continue;
-            }
-
-            // Child node didn't match, exit early with what we have
-            return SubtreeResult {
-                node: node_result,
-                next_expected_idx: None,
-                children,
-            };
-        }
-
-        SubtreeResult {
-            node: node_result,
-            children,
-            next_expected_idx: Some(next_expected_idx as usize),
-        }
-    }
-
-    #[allow(dead_code)]
-    fn match_error(actual: &TestError, expected: &TestError) -> ErrorResult {
-        if actual == expected {
-            return ErrorResult::Success {
-                actual: actual.to_owned(),
-            };
-        }
-
-        if actual.code != expected.code {
-            return ErrorResult::Failure {
-                actual: actual.to_owned(),
-                expected: expected.to_owned(),
-            };
-        }
-
-        ErrorResult::PositionFailure {
-            expected: expected.to_owned(),
-            actual: actual.to_owned(),
-        }
-    }
-}
- */

From 9bf6eb34eebccae88cd999ed953a447e155cd5cf Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 12:10:56 +0100
Subject: [PATCH 08/14] Added parserdata structure to tokenizer

This parserdata structure is a gateway between the tokenizer and the
parser. In a certain case (just one), the tokenizer needs to know the
state of the parser to generate a correct token. The current setup has
the tokenizer and parser in such a way, that we cannot easily reference
eachother without borrow/check issues.

THerefor we add this "hack", which finds out the data beforehand, and
calls the tokenizer with this data. This means the call is done for each
tokenizer call, instead of only when needed, but it saves a big refactor
of the tokenizer/parser.

In the future, we probably should separate the tokenizer, parser, and
tree builder/sink structure so this is not an issue anymore.
---
 src/bin/parser-test.rs                     |  8 +++---
 src/html5/parser.rs                        | 18 +++++++++++--
 src/html5/tokenizer.rs                     | 30 +++++++++++++++++-----
 src/html5/tokenizer/character_reference.rs |  4 ++-
 src/testing/tokenizer.rs                   |  7 ++---
 tests/tree_construction.rs                 | 14 +++++-----
 6 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index af63a9ea0..5e5f78da3 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -29,7 +29,7 @@ fn main() {
         tests_failed: Vec::new(),
     };
 
-    let filenames = Some(&["tests2.dat"][..]);
+    let filenames = Some(&["plain-text-unsafe.dat"][..]);
     let fixtures = read_fixtures(filenames).expect("fixtures");
 
     for fixture_file in fixtures {
@@ -41,9 +41,9 @@ fn main() {
 
         let mut test_idx = 1;
         for test in fixture_file.tests {
-            // if test_idx == 57 {
-            run_test(test_idx, test, &mut results);
-            // }
+            if test_idx == 11 {
+                run_test(test_idx, test, &mut results);
+            }
             test_idx += 1;
         }
 
diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index 3dc516f1f..43a3606e8 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -17,7 +17,7 @@ use crate::html5::parser::document::{Document, DocumentBuilder, DocumentFragment
 use crate::html5::parser::quirks::QuirksMode;
 use crate::html5::tokenizer::state::State;
 use crate::html5::tokenizer::token::Token;
-use crate::html5::tokenizer::{Tokenizer, CHAR_NUL, CHAR_REPLACEMENT};
+use crate::html5::tokenizer::{ParserData, Tokenizer, CHAR_NUL, CHAR_REPLACEMENT};
 use crate::types::{ParseError, Result};
 use alloc::rc::Rc;
 use core::cell::RefCell;
@@ -3752,13 +3752,27 @@ impl<'chars> Html5Parser<'chars> {
         }
     }
 
+    fn parser_data(&self) -> ParserData {
+        let namespace = self
+            .get_adjusted_current_node()
+            .namespace
+            .unwrap_or_default();
+
+        ParserData {
+            adjusted_node_namespace: namespace,
+        }
+    }
+
     /// Fetches the next token from the tokenizer. However, if the token is a text token AND
     /// it starts with one or more whitespaces, the token is split into 2 tokens: the whitespace part
     /// and the remainder.
     fn fetch_next_token(&mut self) -> Token {
         // If there are no tokens to fetch, fetch the next token from the tokenizer
         if self.token_queue.is_empty() {
-            let token = self.tokenizer.next_token().expect("tokenizer error");
+            let token = self
+                .tokenizer
+                .next_token(self.parser_data())
+                .expect("tokenizer error");
 
             if let Token::Text(value) = token {
                 for c in value.chars() {
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index cea1f5ee5..931370f9e 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -8,6 +8,7 @@ use crate::bytes::Bytes::{self, *};
 use crate::bytes::SeekMode::SeekCur;
 use crate::bytes::{CharIterator, Position};
 use crate::html5::error_logger::{ErrorLogger, ParserError};
+use crate::html5::node::HTML_NAMESPACE;
 use crate::html5::tokenizer::state::State;
 use crate::html5::tokenizer::token::Token;
 use crate::types::{Error, Result};
@@ -50,6 +51,21 @@ pub struct Tokenizer<'stream> {
     pub error_logger: Rc<RefCell<ErrorLogger>>,
 }
 
+/// This struct is a gateway between the parser and the tokenizer. It holds data that can be needed
+/// by the tokenizer in certain cases. See https://github.com/gosub-browser/gosub-engine/issues/230 for
+/// more information and how we should refactor this properly.
+pub struct ParserData {
+    pub adjusted_node_namespace: String,
+}
+
+impl Default for ParserData {
+    fn default() -> Self {
+        ParserData {
+            adjusted_node_namespace: HTML_NAMESPACE.to_string(),
+        }
+    }
+}
+
 /// Options that can be passed to the tokenizer. Mostly needed when dealing with tests.
 pub struct Options {
     /// Sets the initial state of the tokenizer. Normally only needed when dealing with tests
@@ -103,8 +119,8 @@ impl<'stream> Tokenizer<'stream> {
     }
 
     /// Retrieves the next token from the input stream or Token::EOF when the end is reached
-    pub fn next_token(&mut self) -> Result<Token> {
-        self.consume_stream()?;
+    pub fn next_token(&mut self, parser_data: ParserData) -> Result<Token> {
+        self.consume_stream(parser_data)?;
 
         if self.token_queue.is_empty() {
             return Ok(Token::Eof);
@@ -124,7 +140,7 @@ impl<'stream> Tokenizer<'stream> {
     }
 
     /// Consumes the input stream. Continues until the stream is completed or a token has been generated.
-    fn consume_stream(&mut self) -> Result<()> {
+    fn consume_stream(&mut self, parser_data: ParserData) -> Result<()> {
         loop {
             // Something is already in the token buffer, so we can return it.
             if !self.token_queue.is_empty() {
@@ -1210,9 +1226,11 @@ impl<'stream> Tokenizer<'stream> {
                     if self.chars.look_ahead_slice(7) == "[CDATA[" {
                         self.chars.seek(SeekCur, 7);
 
-                        // @TODO: If there is an adjusted current node and it is not an element in the HTML namespace,
-                        // then switch to the CDATA section state. Otherwise, this is a cdata-in-html-content parse error.
-                        // Create a comment token whose data is the "[CDATA[" string. Switch to the bogus comment state.
+                        if parser_data.adjusted_node_namespace != HTML_NAMESPACE {
+                            self.state = State::CDATASection;
+                            continue;
+                        }
+
                         self.parse_error(ParserError::CdataInHtmlContent);
                         self.current_token = Some(Token::Comment("[CDATA[".into()));
 
diff --git a/src/html5/tokenizer/character_reference.rs b/src/html5/tokenizer/character_reference.rs
index 4eb3854a1..5d2796cb0 100644
--- a/src/html5/tokenizer/character_reference.rs
+++ b/src/html5/tokenizer/character_reference.rs
@@ -6,6 +6,7 @@ use crate::bytes::{
 };
 use crate::html5::error_logger::ParserError;
 use crate::html5::tokenizer::replacement_tables::{TOKEN_NAMED_CHARS, TOKEN_REPLACEMENTS};
+
 use crate::html5::tokenizer::{Tokenizer, CHAR_REPLACEMENT};
 use lazy_static::lazy_static;
 
@@ -350,6 +351,7 @@ lazy_static! {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::html5::tokenizer::ParserData;
     use crate::{bytes::CharIterator, html5::error_logger::ErrorLogger};
     use std::cell::RefCell;
     use std::rc::Rc;
@@ -367,7 +369,7 @@ mod tests {
                     let error_logger = Rc::new(RefCell::new(ErrorLogger::new()));
                     let mut tokenizer = Tokenizer::new(&mut chars, None, error_logger.clone());
 
-                    let token = tokenizer.next_token().unwrap();
+                    let token = tokenizer.next_token(ParserData::default()).unwrap();
                     assert_eq!(expected, token.to_string());
                 }
             )*
diff --git a/src/testing/tokenizer.rs b/src/testing/tokenizer.rs
index 252930828..557c0ab75 100644
--- a/src/testing/tokenizer.rs
+++ b/src/testing/tokenizer.rs
@@ -1,5 +1,6 @@
 use super::FIXTURE_ROOT;
 use crate::bytes::CharIterator;
+use crate::html5::tokenizer::ParserData;
 use crate::html5::{
     error_logger::ErrorLogger,
     tokenizer::{
@@ -211,12 +212,12 @@ impl TestSpec {
             // If there is no output, still do an (initial) next token so the parser can generate
             // errors.
             if self.output.is_empty() {
-                tokenizer.next_token().unwrap();
+                tokenizer.next_token(ParserData::default()).unwrap();
             }
 
             // There can be multiple tokens to match. Make sure we match all of them
             for expected in self.output.iter() {
-                let actual = tokenizer.next_token().unwrap();
+                let actual = tokenizer.next_token(ParserData::default()).unwrap();
                 assert_eq!(self.escape(&actual), self.escape(expected));
             }
 
@@ -237,7 +238,7 @@ impl TestSpec {
             let mut tokenizer = builder.build();
 
             for _ in self.output.iter() {
-                tokenizer.next_token().unwrap();
+                tokenizer.next_token(ParserData::default()).unwrap();
             }
         }
     }
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 1e0774008..24fdcfc91 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -15,7 +15,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("tests3.dat")]
 #[test_case("tests4.dat")]
 #[test_case("tests5.dat")]
-// #[test_case("tests6.dat")]
+#[test_case("tests6.dat")]
 #[test_case("tests7.dat")]
 #[test_case("tests8.dat")]
 #[test_case("tests9.dat")]
@@ -27,9 +27,9 @@ const DISABLED_CASES: &[&str] = &[
 // #[test_case("tests16.dat")]
 #[test_case("tests17.dat")]
 #[test_case("tests18.dat")]
-// #[test_case("tests19.dat")]
+#[test_case("tests19.dat")]
 #[test_case("tests20.dat")]
-// #[test_case("tests21.dat")]
+#[test_case("tests21.dat")]
 #[test_case("tests22.dat")]
 #[test_case("tests23.dat")]
 #[test_case("tests24.dat")]
@@ -40,11 +40,11 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("blocks.dat")]
 #[test_case("comments01.dat")]
 #[test_case("doctype01.dat")]
-// #[test_case("domjs-unsafe.dat")]
+#[test_case("domjs-unsafe.dat")]
 #[test_case("entities01.dat")]
 #[test_case("entities02.dat")]
 #[test_case("foreign-fragment.dat")]
-#[test_case("html5test-com.dat")]
+// #[test_case("html5test-com.dat")]
 #[test_case("inbody01.dat")]
 #[test_case("isindex.dat")]
 #[test_case("main-element.dat")]
@@ -54,7 +54,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("noscript01.dat")]
 #[test_case("pending-spec-changes.dat")]
 #[test_case("pending-spec-changes-plain-text-unsafe.dat")]
-// #[test_case("plain-text-unsafe.dat")]
+#[test_case("plain-text-unsafe.dat")]
 #[test_case("quirks01.dat")]
 #[test_case("ruby.dat")]
 #[test_case("scriptdata01.dat")]
@@ -64,7 +64,7 @@ const DISABLED_CASES: &[&str] = &[
 // #[test_case("template.dat")]
 #[test_case("tests_innerHTML_1.dat")]
 #[test_case("tricky01.dat")]
-// #[test_case("webkit01.dat")]
+#[test_case("webkit01.dat")]
 #[test_case("webkit02.dat")]
 fn tree_construction(filename: &str) {
     let fixture_file =

From 6e2ddf0ddfe18856f19a387696a523c7cc4eacd0 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Sun, 5 Nov 2023 20:20:42 +0100
Subject: [PATCH 09/14] fixed suggestions from @charleschen0823

---
 src/bin/parser-test.rs                     |  4 ++--
 src/html5/parser.rs                        | 10 ++++++++--
 src/html5/parser/helper.rs                 |  2 +-
 src/testing/tree_construction/generator.rs |  4 ++--
 src/testing/tree_construction/parser.rs    | 11 ++++++++++-
 tests/tree_construction.rs                 |  8 ++++----
 6 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 5e5f78da3..45ccb7f8e 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -29,7 +29,7 @@ fn main() {
         tests_failed: Vec::new(),
     };
 
-    let filenames = Some(&["plain-text-unsafe.dat"][..]);
+    let filenames = Some(&["html5test-com.dat"][..]);
     let fixtures = read_fixtures(filenames).expect("fixtures");
 
     for fixture_file in fixtures {
@@ -41,7 +41,7 @@ fn main() {
 
         let mut test_idx = 1;
         for test in fixture_file.tests {
-            if test_idx == 11 {
+            if test_idx == 14 {
                 run_test(test_idx, test, &mut results);
             }
             test_idx += 1;
diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index 43a3606e8..ad11b36d4 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -3028,8 +3028,8 @@ impl<'chars> Html5Parser<'chars> {
 
                 self.pop_until_named("template");
                 self.active_formatting_elements_clear_until_marker();
-                self.reset_insertion_mode();
                 self.template_insertion_mode.pop();
+                self.reset_insertion_mode();
             }
             Token::StartTag { name, .. } if name == "head" => {
                 self.parse_error("head tag not allowed in in head insertion mode");
@@ -3753,10 +3753,16 @@ impl<'chars> Html5Parser<'chars> {
     }
 
     fn parser_data(&self) -> ParserData {
+        if self.open_elements.is_empty() {
+            return ParserData {
+                adjusted_node_namespace: HTML_NAMESPACE.to_string(),
+            };
+        }
+
         let namespace = self
             .get_adjusted_current_node()
             .namespace
-            .unwrap_or_default();
+            .expect("namespace not found");
 
         ParserData {
             adjusted_node_namespace: namespace,
diff --git a/src/html5/parser/helper.rs b/src/html5/parser/helper.rs
index 2582614c4..c4b17f2cf 100644
--- a/src/html5/parser/helper.rs
+++ b/src/html5/parser/helper.rs
@@ -256,7 +256,7 @@ impl Html5Parser<'_> {
         if !(self.foster_parenting
             && ["table", "tbody", "thead", "tfoot", "tr"].contains(&target_node.name.as_str()))
         {
-            if target_node.name == "template" {
+            if target_node.name == "template" && target_node.is_namespace(HTML_NAMESPACE) {
                 if let NodeData::Element(element) = target_node.data {
                     if let Some(template_contents) = element.template_contents {
                         return InsertionPositionMode::LastChild {
diff --git a/src/testing/tree_construction/generator.rs b/src/testing/tree_construction/generator.rs
index d5a5db53f..e686e1c2a 100644
--- a/src/testing/tree_construction/generator.rs
+++ b/src/testing/tree_construction/generator.rs
@@ -1,4 +1,4 @@
-use crate::html5::node::{Node, NodeData, NodeTrait, NodeType};
+use crate::html5::node::{Node, NodeData, NodeTrait, NodeType, HTML_NAMESPACE};
 use crate::html5::node::{MATHML_NAMESPACE, SVG_NAMESPACE, XLINK_NAMESPACE, XMLNS_NAMESPACE};
 use crate::html5::parser::document::DocumentHandle;
 
@@ -52,7 +52,7 @@ impl TreeOutputGenerator {
             }
 
             // Template tags have an extra "content" node in the test tree ouput
-            if node.name == "template" {
+            if node.name == "template" && node.is_namespace(HTML_NAMESPACE) {
                 output.push(format!("| {}content", "  ".repeat(indent_level)));
                 indent_level += 1;
             }
diff --git a/src/testing/tree_construction/parser.rs b/src/testing/tree_construction/parser.rs
index 4f50c20ab..a5ba70a28 100644
--- a/src/testing/tree_construction/parser.rs
+++ b/src/testing/tree_construction/parser.rs
@@ -297,7 +297,7 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
 
             TestSpec {
                 position,
-                data: data.to_string().trim_matches(|c| c == '\n').to_string(),
+                data: trim_last_newline(data.to_string()),
                 errors,
                 new_errors: new_errors.unwrap_or_default(),
                 script_mode,
@@ -308,6 +308,15 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
     )(i)
 }
 
+/// Trims only a single newline from the string, even if there are multiple newlines present.
+fn trim_last_newline(s: String) -> String {
+    if s.ends_with('\n') {
+        s[..s.len() - 1].to_owned()
+    } else {
+        s
+    }
+}
+
 pub fn parse_fixture(i: &str) -> Result<Vec<TestSpec>> {
     // Deal with a corner case that makes it hard to parse tricky01.dat.
     let input = i.replace("\"\n\n\"", QUOTED_DOUBLE_NEWLINE).to_owned() + "\n";
diff --git a/tests/tree_construction.rs b/tests/tree_construction.rs
index 24fdcfc91..7d7c80029 100644
--- a/tests/tree_construction.rs
+++ b/tests/tree_construction.rs
@@ -24,7 +24,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("tests12.dat")]
 #[test_case("tests14.dat")]
 #[test_case("tests15.dat")]
-// #[test_case("tests16.dat")]
+#[test_case("tests16.dat")]
 #[test_case("tests17.dat")]
 #[test_case("tests18.dat")]
 #[test_case("tests19.dat")]
@@ -34,7 +34,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("tests23.dat")]
 #[test_case("tests24.dat")]
 #[test_case("tests25.dat")]
-// #[test_case("tests26.dat")]
+#[test_case("tests26.dat")]
 #[test_case("adoption01.dat")]
 #[test_case("adoption02.dat")]
 #[test_case("blocks.dat")]
@@ -44,7 +44,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("entities01.dat")]
 #[test_case("entities02.dat")]
 #[test_case("foreign-fragment.dat")]
-// #[test_case("html5test-com.dat")]
+#[test_case("html5test-com.dat")]
 #[test_case("inbody01.dat")]
 #[test_case("isindex.dat")]
 #[test_case("main-element.dat")]
@@ -61,7 +61,7 @@ const DISABLED_CASES: &[&str] = &[
 #[test_case("search-element.dat")]
 #[test_case("svg.dat")]
 #[test_case("tables01.dat")]
-// #[test_case("template.dat")]
+#[test_case("template.dat")]
 #[test_case("tests_innerHTML_1.dat")]
 #[test_case("tricky01.dat")]
 #[test_case("webkit01.dat")]

From fb881ec596910b38645f47ea909a7b27d6d840af Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Tue, 7 Nov 2023 20:57:13 +0100
Subject: [PATCH 10/14] fixed the tokenizer to run faster

---
 src/bin/parser-test.rs                     |   4 +-
 src/bytes.rs                               | 361 ++-------------------
 src/html5/parser.rs                        | 182 +++++++++--
 src/html5/parser/helper.rs                 |   7 +
 src/html5/tokenizer.rs                     |  29 +-
 src/html5/tokenizer/character_reference.rs |   8 +-
 src/html5/tokenizer/token.rs               |  31 +-
 src/testing/tree_construction/parser.rs    |   4 +-
 8 files changed, 247 insertions(+), 379 deletions(-)

diff --git a/src/bin/parser-test.rs b/src/bin/parser-test.rs
index 45ccb7f8e..9818e146a 100755
--- a/src/bin/parser-test.rs
+++ b/src/bin/parser-test.rs
@@ -29,7 +29,7 @@ fn main() {
         tests_failed: Vec::new(),
     };
 
-    let filenames = Some(&["html5test-com.dat"][..]);
+    let filenames = Some(&["tests15.dat"][..]);
     let fixtures = read_fixtures(filenames).expect("fixtures");
 
     for fixture_file in fixtures {
@@ -41,7 +41,7 @@ fn main() {
 
         let mut test_idx = 1;
         for test in fixture_file.tests {
-            if test_idx == 14 {
+            if test_idx == 10 {
                 run_test(test_idx, test, &mut results);
             }
             test_idx += 1;
diff --git a/src/bytes.rs b/src/bytes.rs
index e63737def..3e9359ec4 100644
--- a/src/bytes.rs
+++ b/src/bytes.rs
@@ -114,15 +114,6 @@ pub struct CharIterator {
     pub has_read_eof: bool, // True when we just read an EOF
 }
 
-pub enum SeekMode {
-    /// Seek from the start of the stream
-    SeekSet,
-    /// Seek from the current stream position
-    SeekCur,
-    /// Seek (backwards) from the end of the stream
-    SeekEnd,
-}
-
 impl Default for CharIterator {
     fn default() -> Self {
         Self::new()
@@ -170,76 +161,30 @@ impl CharIterator {
         self.position.col = 1;
     }
 
-    /// Seek explicit offset in the stream (based on chars)
-    pub fn seek(&mut self, mode: SeekMode, offset: isize) {
-        let abs_offset = match mode {
-            SeekMode::SeekSet => {
-                if offset.is_negative() {
-                    0
-                } else {
-                    offset as usize
-                }
-            }
-            SeekMode::SeekCur => {
-                if offset.is_negative() {
-                    self.position.offset - offset.unsigned_abs()
-                } else {
-                    self.position.offset + offset as usize
-                }
-            }
-            SeekMode::SeekEnd => {
-                // Both -5 and 5 on seek-end do the same thing
-                if offset.abs() > self.length as isize {
-                    0
-                } else {
-                    self.length - offset.unsigned_abs()
-                }
-            }
-        };
+    /// Skip offset characters in the stream (based on chars)
+    pub fn skip(&mut self, offset: usize) {
+        let mut skip_len = offset;
+        if self.position.offset + offset > self.length {
+            skip_len = self.length - self.position.offset;
+        }
 
-        self.position = self.generate_position(abs_offset);
+        for _ in 0..skip_len {
+            self.read_char();
+        }
     }
 
     /// Returns the previous position based on the current position
     pub fn get_previous_position(&mut self) -> Position {
-        // if we are at the begining or the end of the stream, we just return the current position
+        // if we are at the beginning or the end of the stream, we just return the current position
         if self.position.offset == 0 || self.has_read_eof {
             return self.position;
         }
 
-        self.generate_position(self.position.offset - 1)
-    }
+        self.unread();
+        let pos = self.position;
+        self.skip(1);
 
-    /// Generate a new position structure for given offset
-    fn generate_position(&mut self, abs_offset: usize) -> Position {
-        let mut abs_offset = abs_offset;
-
-        // Cap to length if we read past the end of the stream
-        if abs_offset > self.length + 1 {
-            abs_offset = self.length;
-            self.has_read_eof = true;
-        }
-
-        // Detect lines (if needed)
-        self.read_line_endings_until(abs_offset);
-
-        let mut last_line: usize = 0;
-        let mut last_offset = self.line_offsets[last_line];
-        for i in 0..self.line_offsets.len() {
-            if self.line_offsets[i] > abs_offset {
-                break;
-            }
-
-            last_line = i;
-            last_offset = self.line_offsets[last_line];
-        }
-
-        // Set position values
-        Position {
-            offset: abs_offset,
-            line: last_line + 1,
-            col: abs_offset - last_offset + 1,
-        }
+        pos
     }
 
     /// Returns the current offset in the stream
@@ -361,15 +306,26 @@ impl CharIterator {
         // If we still can move forward in the stream, move forwards
         if self.position.offset < self.length {
             let c = self.buffer[self.position.offset];
-            self.seek(SeekMode::SeekCur, 1);
+            if c == Ch('\n') {
+                // Store line offset for the given line
+                if self.line_offsets.len() > self.position.line {
+                    self.line_offsets[self.position.line] = self.position.offset;
+                } else {
+                    self.line_offsets.push(self.position.offset);
+                }
+                // And continue position on the next line
+                self.position.line += 1;
+                self.position.col = 1;
+            } else {
+                self.position.col += 1;
+            }
+            self.position.offset += 1;
             return c;
         }
 
         // otherwise, we have reached the end of the stream
         self.has_read_eof = true;
 
-        self.seek(SeekMode::SeekEnd, 0);
-
         Eof
     }
 
@@ -392,7 +348,14 @@ impl CharIterator {
 
         // If we can track back from the offset, we can do so
         if self.position.offset > 0 {
-            self.seek(SeekMode::SeekCur, -1);
+            self.position.offset -= 1;
+
+            if self.position.col == 1 {
+                self.position.line -= 1;
+                self.position.col = self.line_offsets[self.position.line];
+            } else {
+                self.position.col -= 1;
+            }
         }
     }
 
@@ -414,26 +377,6 @@ impl CharIterator {
 
         self.buffer[self.position.offset + offset]
     }
-
-    /// Populates the line endings by reading the stream until the given length.
-    fn read_line_endings_until(&mut self, abs_offset: usize) {
-        let mut last_offset = *self.line_offsets.last().unwrap();
-
-        while last_offset <= abs_offset {
-            if last_offset >= self.length {
-                self.line_offsets.push(last_offset + 1);
-                break;
-            }
-
-            // Check the next char to see if it's a '\n'
-            let c = self.buffer[last_offset];
-            if c == Ch('\n') {
-                self.line_offsets.push(last_offset + 1);
-            }
-
-            last_offset += 1;
-        }
-    }
 }
 
 #[cfg(test)]
@@ -515,238 +458,6 @@ mod test {
         assert!(!chars.is_certain_encoding());
     }
 
-    #[test]
-    fn test_offsets() {
-        let mut chars = CharIterator::new();
-        chars.read_from_str("abc", Some(Encoding::UTF8));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 0,
-                line: 1,
-                col: 1
-            }
-        );
-        assert_eq!('a', chars.read_char().into());
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 1,
-                line: 1,
-                col: 2
-            }
-        );
-        assert_eq!('b', chars.read_char().into());
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 2,
-                line: 1,
-                col: 3
-            }
-        );
-        assert_eq!('c', chars.read_char().into());
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 3,
-                line: 1,
-                col: 4
-            }
-        );
-        assert!(matches!(chars.read_char(), Eof));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 3,
-                line: 1,
-                col: 4
-            }
-        );
-        assert!(matches!(chars.read_char(), Eof));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 3,
-                line: 1,
-                col: 4
-            }
-        );
-
-        let mut chars = CharIterator::new();
-        chars.read_from_str(
-            "abc\ndefg\n\nhi\njk\nlmno\n\n\npqrst\nu\nv\nw\n\nxy\nz",
-            Some(Encoding::UTF8),
-        );
-        assert_eq!(chars.length, 40);
-
-        chars.seek(SeekMode::SeekSet, 0);
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 0,
-                line: 1,
-                col: 1
-            }
-        );
-        let c = chars.read_char();
-        assert_eq!(c, Ch('a'));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 1,
-                line: 1,
-                col: 2
-            }
-        );
-
-        chars.seek(SeekMode::SeekSet, 7);
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 7,
-                line: 2,
-                col: 4
-            }
-        );
-        assert_eq!(chars.chars_left(), 33);
-
-        let c = chars.read_char();
-        assert_eq!(c, Ch('g'));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 8,
-                line: 2,
-                col: 5
-            }
-        );
-
-        let c = chars.read_char();
-        assert_eq!(c, Ch('\n'));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 9,
-                line: 3,
-                col: 1
-            }
-        );
-
-        let c = chars.read_char();
-        assert_eq!(c, Ch('\n'));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 10,
-                line: 4,
-                col: 1
-            }
-        );
-
-        let c = chars.read_char();
-        assert_eq!(c, Ch('h'));
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 11,
-                line: 4,
-                col: 2
-            }
-        );
-        assert_eq!(chars.chars_left(), 29);
-
-        chars.reset();
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 0,
-                line: 1,
-                col: 1
-            }
-        );
-        assert_eq!(chars.chars_left(), 40);
-
-        chars.seek(SeekMode::SeekSet, 100);
-        assert_eq!(
-            chars.position,
-            Position {
-                offset: 40,
-                line: 15,
-                col: 2
-            }
-        );
-        assert_eq!(chars.chars_left(), 0);
-    }
-
-    #[test]
-    fn test_seek() {
-        let mut chars = CharIterator::new();
-        chars.read_from_str("ab👽cd", Some(Encoding::UTF8));
-        assert_eq!(chars.length, 5);
-        assert_eq!(chars.chars_left(), 5);
-        assert_eq!(chars.read_char(), Ch('a'));
-        assert_eq!(chars.read_char(), Ch('b'));
-        assert_eq!(chars.chars_left(), 3);
-        chars.seek(SeekMode::SeekSet, 0);
-        assert_eq!(chars.chars_left(), 5);
-        assert_eq!(chars.read_char(), Ch('a'));
-        assert_eq!(chars.read_char(), Ch('b'));
-        assert_eq!(chars.chars_left(), 3);
-        chars.seek(SeekMode::SeekSet, 3);
-        assert_eq!(chars.chars_left(), 2);
-        assert_eq!(chars.read_char(), Ch('c'));
-        assert_eq!(chars.read_char(), Ch('d'));
-        assert_eq!(chars.chars_left(), 0);
-        assert!(chars.eof());
-
-        chars.reset();
-        assert_eq!(chars.look_ahead(0), Ch('a'));
-        assert_eq!(chars.look_ahead(3), Ch('c'));
-        assert_eq!(chars.look_ahead(1), Ch('b'));
-        assert!(matches!(chars.look_ahead(100), Eof));
-
-        chars.seek(SeekMode::SeekSet, 0);
-        assert_eq!(chars.look_ahead_slice(1), "a");
-        assert_eq!(chars.look_ahead_slice(2), "ab");
-        assert_eq!(chars.look_ahead_slice(3), "ab👽");
-        assert_eq!(chars.look_ahead_slice(4), "ab👽c");
-        assert_eq!(chars.look_ahead_slice(5), "ab👽cd");
-        assert_eq!(chars.look_ahead_slice(6), "ab👽cd");
-        assert_eq!(chars.look_ahead_slice(100), "ab👽cd");
-
-        chars.seek(SeekMode::SeekSet, 3);
-        assert_eq!(chars.look_ahead_slice(1), "c");
-        assert_eq!(chars.look_ahead_slice(2), "cd");
-
-        chars.seek(SeekMode::SeekSet, 0);
-        assert_eq!(chars.position.offset, 0);
-
-        chars.seek(SeekMode::SeekSet, 3);
-        assert_eq!(chars.position.offset, 3);
-
-        chars.seek(SeekMode::SeekCur, 0);
-        assert_eq!(chars.position.offset, 3);
-
-        chars.seek(SeekMode::SeekCur, 1);
-        assert_eq!(chars.position.offset, 4);
-
-        chars.seek(SeekMode::SeekCur, -2);
-        assert_eq!(chars.position.offset, 2);
-
-        chars.seek(SeekMode::SeekCur, 10);
-        assert_eq!(chars.position.offset, 5);
-
-        chars.seek(SeekMode::SeekSet, 100);
-        assert_eq!(chars.position.offset, 5);
-
-        chars.seek(SeekMode::SeekSet, -100);
-        assert_eq!(chars.position.offset, 0);
-
-        chars.seek(SeekMode::SeekEnd, -100);
-        assert_eq!(chars.position.offset, 0);
-    }
-
     #[test]
     fn test_eof() {
         let mut chars = CharIterator::new();
diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index ad11b36d4..b7d807f1f 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -17,7 +17,7 @@ use crate::html5::parser::document::{Document, DocumentBuilder, DocumentFragment
 use crate::html5::parser::quirks::QuirksMode;
 use crate::html5::tokenizer::state::State;
 use crate::html5::tokenizer::token::Token;
-use crate::html5::tokenizer::{ParserData, Tokenizer, CHAR_NUL, CHAR_REPLACEMENT};
+use crate::html5::tokenizer::{ParserData, Tokenizer, CHAR_REPLACEMENT};
 use crate::types::{ParseError, Result};
 use alloc::rc::Rc;
 use core::cell::RefCell;
@@ -444,6 +444,11 @@ impl<'chars> Html5Parser<'chars> {
         let mut handle_as_script_endtag = false;
 
         match &self.current_token.clone() {
+            Token::Text(value) if self.current_token.is_mixed() => {
+                let tokens = self.split_mixed_token(value);
+                self.tokenizer.insert_tokens_at_queue_start(tokens);
+                return;
+            }
             Token::Text(..) if self.current_token.is_null() => {
                 self.parse_error("null character not allowed in foreign content");
                 self.insert_text_element(&Token::Text(CHAR_REPLACEMENT.to_string()));
@@ -630,23 +635,24 @@ impl<'chars> Html5Parser<'chars> {
     fn process_html_content(&mut self) {
         if self.ignore_lf {
             if let Token::Text(value) = &self.current_token {
-                if value.eq(&"\n".to_string()) {
-                    self.current_token = self.fetch_next_token();
+                if value.starts_with('\n') {
+                    // We don't need to skip 1 char, but we can skip 1 byte, as we just checked for \n
+                    self.current_token = Token::Text(value.chars().skip(1).collect::<String>());
                 }
             }
             self.ignore_lf = false;
         }
 
-        // // Break when we reach the end of the token chars
-        // if self.current_token.is_eof() {
-        //     self.parser_finished = true;
-        // }
-
         match self.insertion_mode {
             InsertionMode::Initial => {
                 let mut anything_else = false;
 
                 match &self.current_token.clone() {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                        return;
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         // ignore token
                     }
@@ -726,6 +732,11 @@ impl<'chars> Html5Parser<'chars> {
                             Some(NodeId::root()),
                         );
                     }
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                        return;
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         // ignore token
                     }
@@ -763,6 +774,11 @@ impl<'chars> Html5Parser<'chars> {
                 let mut anything_else = false;
 
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                        return;
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         // ignore token
                     }
@@ -824,6 +840,11 @@ impl<'chars> Html5Parser<'chars> {
                         self.check_last_element("head");
                         self.insertion_mode = InsertionMode::InHead;
                     }
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                        return;
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.handle_in_head();
                     }
@@ -871,6 +892,11 @@ impl<'chars> Html5Parser<'chars> {
                 let mut anything_else = false;
 
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                        return;
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.insert_text_element(&self.current_token.clone());
                     }
@@ -999,6 +1025,10 @@ impl<'chars> Html5Parser<'chars> {
             InsertionMode::InTable => self.handle_in_table(),
             InsertionMode::InTableText => {
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_null() => {
                         self.parse_error(
                             "null character not allowed in in table text insertion mode",
@@ -1006,18 +1036,10 @@ impl<'chars> Html5Parser<'chars> {
                         // ignore token
                     }
                     Token::Text(value) => {
-                        for c in value.chars() {
-                            if c == CHAR_NUL {
-                                self.parse_error(
-                                    "null character not allowed in in table insertion mode",
-                                );
-                            } else {
-                                self.pending_table_character_tokens.push(c);
-                            }
-                        }
+                        self.pending_table_character_tokens.push_str(value);
                     }
                     _ => {
-                        let tokens = self.pending_table_character_tokens.clone();
+                        let pending_chars = self.pending_table_character_tokens.clone();
 
                         let mut process_as_intable_anything_else = false;
 
@@ -1033,15 +1055,18 @@ impl<'chars> Html5Parser<'chars> {
 
                         if process_as_intable_anything_else {
                             let tmp = self.current_token.clone();
-                            self.current_token = Token::Text(tokens);
-
                             self.foster_parenting = true;
-                            self.handle_in_body();
-                            self.foster_parenting = false;
 
+                            let tokens = self.split_mixed_token(&pending_chars);
+                            for token in tokens {
+                                self.current_token = token;
+                                self.handle_in_body();
+                            }
+
+                            self.foster_parenting = false;
                             self.current_token = tmp;
                         } else {
-                            self.insert_text_element(&Token::Text(tokens));
+                            self.insert_text_element(&Token::Text(pending_chars));
                         }
 
                         self.pending_table_character_tokens.clear();
@@ -1115,6 +1140,10 @@ impl<'chars> Html5Parser<'chars> {
             }
             InsertionMode::InColumnGroup => {
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.insert_text_element(&self.current_token.clone());
                     }
@@ -1468,6 +1497,10 @@ impl<'chars> Html5Parser<'chars> {
             InsertionMode::InTemplate => self.handle_in_template(),
             InsertionMode::AfterBody => {
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.handle_in_body();
                     }
@@ -1508,6 +1541,10 @@ impl<'chars> Html5Parser<'chars> {
             }
             InsertionMode::InFrameset => {
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.insert_text_element(&self.current_token.clone());
                     }
@@ -1566,6 +1603,10 @@ impl<'chars> Html5Parser<'chars> {
             }
             InsertionMode::AfterFrameset => {
                 match &self.current_token {
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.insert_text_element(&self.current_token.clone());
                     }
@@ -1603,6 +1644,10 @@ impl<'chars> Html5Parser<'chars> {
                 Token::DocType { .. } => {
                     self.handle_in_body();
                 }
+                Token::Text(value) if self.current_token.is_mixed() => {
+                    let tokens = self.split_mixed_token(value);
+                    self.tokenizer.insert_tokens_at_queue_start(tokens);
+                }
                 Token::Text(..) if self.current_token.is_empty_or_white() => {
                     self.handle_in_body();
                 }
@@ -1631,6 +1676,10 @@ impl<'chars> Html5Parser<'chars> {
                     Token::DocType { .. } => {
                         self.handle_in_body();
                     }
+                    Token::Text(value) if self.current_token.is_mixed() => {
+                        let tokens = self.split_mixed_token(value);
+                        self.tokenizer.insert_tokens_at_queue_start(tokens);
+                    }
                     Token::Text(..) if self.current_token.is_empty_or_white() => {
                         self.handle_in_body();
                     }
@@ -2067,21 +2116,23 @@ impl<'chars> Html5Parser<'chars> {
     /// Handle insertion mode "in_body"
     fn handle_in_body(&mut self) {
         match &self.current_token.clone() {
+            Token::Text(value) if self.current_token.is_mixed() => {
+                let tokens = self.split_mixed_token(value);
+                self.tokenizer.insert_tokens_at_queue_start(tokens);
+            }
             Token::Text(..) if self.current_token.is_null() => {
                 self.parse_error("null character not allowed in in body insertion mode");
                 // ignore token
             }
-            Token::Text(..) if self.current_token.is_empty_or_white() => {
-                self.reconstruct_formatting();
-
-                self.insert_text_element(&self.current_token.clone());
-            }
             Token::Text(..) => {
                 self.reconstruct_formatting();
 
                 self.insert_text_element(&self.current_token.clone());
 
-                self.frameset_ok = false;
+                // If this mixed token does not have whitespace chars, set frameset_ok to false
+                if !self.current_token.is_empty_or_white() {
+                    self.frameset_ok = false;
+                }
             }
             Token::Comment(..) => {
                 self.insert_comment_element(&self.current_token.clone(), None);
@@ -2906,6 +2957,11 @@ impl<'chars> Html5Parser<'chars> {
         let mut anything_else = false;
 
         match &self.current_token {
+            Token::Text(value) if self.current_token.is_mixed() => {
+                let tokens = self.split_mixed_token(value);
+                self.tokenizer.insert_tokens_at_queue_start(tokens);
+                return;
+            }
             Token::Text(..) if self.current_token.is_empty_or_white() => {
                 self.insert_text_element(&self.current_token.clone());
             }
@@ -3306,6 +3362,10 @@ impl<'chars> Html5Parser<'chars> {
     /// Handle insertion mode "in_select"
     fn handle_in_select(&mut self) {
         match &self.current_token {
+            Token::Text(value) if self.current_token.is_mixed() => {
+                let tokens = self.split_mixed_token(value);
+                self.tokenizer.insert_tokens_at_queue_start(tokens);
+            }
             Token::Text(..) if self.current_token.is_null() => {
                 self.parse_error("null character not allowed in in select insertion mode");
                 // ignore token
@@ -3781,9 +3841,10 @@ impl<'chars> Html5Parser<'chars> {
                 .expect("tokenizer error");
 
             if let Token::Text(value) = token {
-                for c in value.chars() {
-                    self.token_queue.push(Token::Text(c.to_string()));
-                }
+                self.token_queue.push(Token::Text(value));
+                // for c in value.chars() {
+                //     self.token_queue.push(Token::Text(c.to_string()));
+                // }
             } else {
                 // Simply return the token
                 return token;
@@ -3907,6 +3968,63 @@ impl<'chars> Html5Parser<'chars> {
         self.tokenizer
             .set_state(self.find_initial_state_for_context(context_node));
     }
+
+    /// Splits a regular text token with mixed characters into tokens of 3 groups:
+    /// null-characters, (ascii) whitespaces, and regular (rest) characters.
+    /// These tokens are then inserted into the token buffer queue so they can get parsed
+    /// correctly.
+    ///
+    /// example:
+    ///
+    ///   Token::Text("  foo bar\0  ")
+    ///
+    /// is split into 6 tokens:
+    ///
+    ///   Token::Text("  ")  // whitespace
+    ///   Token::Text("foo") // regular
+    ///   Token::Text(" ")   // whitespace
+    ///   Token::Text("bar") // regular
+    ///   Token::Text("\0")  // null
+    ///   Token::Text("  ")  // whitespace
+    ///
+    /// This is needed because the tokenizer does not know about the context of the text it is
+    /// so it will always try to tokenize as greedy as possible. But sometimes we need this split
+    /// to happen where a differentation between whitespaces, null and regular characters are needed.
+    /// Only in those cases, this function is called, and the token will be split into multiple
+    /// tokens.
+    /// The idea is that large blobs of javascript for instance will not be split into separate
+    /// tokens, but still be seen and parsed as a single TextToken.
+    ///
+    fn split_mixed_token(&self, text: &String) -> Vec<Token> {
+        let mut tokens = vec![];
+        let mut last_group = 'x';
+
+        let mut found = String::new();
+
+        for ch in text.chars() {
+            let group = if ch == '\0' {
+                '0'
+            } else if ch.is_ascii_whitespace() {
+                'w'
+            } else {
+                'r'
+            };
+
+            if last_group != group && !found.is_empty() {
+                tokens.push(Token::Text(found.clone()));
+                found.clear();
+            }
+
+            found.push(ch);
+            last_group = group;
+        }
+
+        if !found.is_empty() {
+            tokens.push(Token::Text(found.clone()));
+        }
+
+        tokens
+    }
 }
 
 #[cfg(test)]
diff --git a/src/html5/parser/helper.rs b/src/html5/parser/helper.rs
index c4b17f2cf..8fcbba369 100644
--- a/src/html5/parser/helper.rs
+++ b/src/html5/parser/helper.rs
@@ -240,6 +240,13 @@ impl Html5Parser<'_> {
     }
 
     pub fn insert_text_element(&mut self, token: &Token) {
+        // Skip empty text nodes
+        if let Token::Text(text) = token {
+            if text.is_empty() {
+                return;
+            }
+        }
+
         let insertion_position = self.appropriate_place_insert(None);
         // TODO, for text element, if the insertion_position is Document, should not do next step.
         self.insert_text_helper(insertion_position, token);
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index 931370f9e..b6580f5d4 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -5,7 +5,6 @@ mod character_reference;
 mod replacement_tables;
 
 use crate::bytes::Bytes::{self, *};
-use crate::bytes::SeekMode::SeekCur;
 use crate::bytes::{CharIterator, Position};
 use crate::html5::error_logger::{ErrorLogger, ParserError};
 use crate::html5::node::HTML_NAMESPACE;
@@ -51,6 +50,15 @@ pub struct Tokenizer<'stream> {
     pub error_logger: Rc<RefCell<ErrorLogger>>,
 }
 
+impl<'stream> Tokenizer<'stream> {
+    pub(crate) fn insert_tokens_at_queue_start(&mut self, first_tokens: Vec<Token>) {
+        let mut new_queue = first_tokens.clone();
+        new_queue.extend(self.token_queue.iter().cloned());
+
+        self.token_queue = new_queue;
+    }
+}
+
 /// This struct is a gateway between the parser and the tokenizer. It holds data that can be needed
 /// by the tokenizer in certain cases. See https://github.com/gosub-browser/gosub-engine/issues/230 for
 /// more information and how we should refactor this properly.
@@ -158,7 +166,6 @@ impl<'stream> Tokenizer<'stream> {
                             self.parse_error(ParserError::UnexpectedNullCharacter);
                         }
                         Eof => {
-                            // EOF
                             // if self.has_consumed_data() {
                             //     self.emit_token(Token::TextToken { value: self.get_consumed_str() });
                             //     self.clear_consume_buffer();
@@ -1211,20 +1218,20 @@ impl<'stream> Tokenizer<'stream> {
                         self.current_token = Some(Token::Comment("".into()));
 
                         // Skip the two -- signs
-                        self.chars.seek(SeekCur, 2);
+                        self.chars.skip(2);
 
                         self.state = State::CommentStart;
                         continue;
                     }
 
                     if self.chars.look_ahead_slice(7).to_uppercase() == "DOCTYPE" {
-                        self.chars.seek(SeekCur, 7);
+                        self.chars.skip(7);
                         self.state = State::DOCTYPE;
                         continue;
                     }
 
                     if self.chars.look_ahead_slice(7) == "[CDATA[" {
-                        self.chars.seek(SeekCur, 7);
+                        self.chars.skip(7);
 
                         if parser_data.adjusted_node_namespace != HTML_NAMESPACE {
                             self.state = State::CDATASection;
@@ -1238,9 +1245,9 @@ impl<'stream> Tokenizer<'stream> {
                         continue;
                     }
 
-                    self.chars.seek(SeekCur, 1);
+                    // self.chars.skip(1);
                     self.parse_error(ParserError::IncorrectlyOpenedComment);
-                    self.chars.unread();
+                    // self.chars.unread();
                     self.current_token = Some(Token::Comment("".into()));
 
                     self.state = State::BogusComment;
@@ -1568,20 +1575,20 @@ impl<'stream> Tokenizer<'stream> {
                         _ => {
                             self.chars.unread();
                             if self.chars.look_ahead_slice(6).to_uppercase() == "PUBLIC" {
-                                self.chars.seek(SeekCur, 6);
+                                self.chars.skip(6);
                                 self.state = State::AfterDOCTYPEPublicKeyword;
                                 continue;
                             }
                             if self.chars.look_ahead_slice(6).to_uppercase() == "SYSTEM" {
-                                self.chars.seek(SeekCur, 6);
+                                self.chars.skip(6);
                                 self.state = State::AfterDOCTYPESystemKeyword;
                                 continue;
                             }
                             // Make sure the parser is on the correct position again since we just
                             // unread the character
-                            self.chars.seek(SeekCur, 1);
+                            self.chars.skip(1);
                             self.parse_error(ParserError::InvalidCharacterSequenceAfterDoctypeName);
-                            self.chars.seek(SeekCur, -1);
+                            self.chars.unread();
                             self.set_quirks_mode(true);
                             self.chars.unread();
                             self.state = State::BogusDOCTYPE;
diff --git a/src/html5/tokenizer/character_reference.rs b/src/html5/tokenizer/character_reference.rs
index 5d2796cb0..86bc0a8b8 100644
--- a/src/html5/tokenizer/character_reference.rs
+++ b/src/html5/tokenizer/character_reference.rs
@@ -1,12 +1,8 @@
 extern crate lazy_static;
 
-use crate::bytes::{
-    Bytes::{self, *},
-    SeekMode::SeekCur,
-};
+use crate::bytes::Bytes::{self, *};
 use crate::html5::error_logger::ParserError;
 use crate::html5::tokenizer::replacement_tables::{TOKEN_NAMED_CHARS, TOKEN_REPLACEMENTS};
-
 use crate::html5::tokenizer::{Tokenizer, CHAR_REPLACEMENT};
 use lazy_static::lazy_static;
 
@@ -62,7 +58,7 @@ impl Tokenizer<'_> {
                 }
                 CcrState::NamedCharacterReference => {
                     if let Some(entity) = self.find_entity() {
-                        self.chars.seek(SeekCur, entity.len() as isize);
+                        self.chars.skip(entity.len());
                         let c = self.chars.look_ahead(0);
 
                         if as_attribute
diff --git a/src/html5/tokenizer/token.rs b/src/html5/tokenizer/token.rs
index b48082892..377551d75 100644
--- a/src/html5/tokenizer/token.rs
+++ b/src/html5/tokenizer/token.rs
@@ -30,6 +30,31 @@ pub enum Token {
     Eof,
 }
 
+impl Token {
+    pub(crate) fn is_mixed(&self) -> bool {
+        // Check if there are white characters AND non-white characters in the token
+        if let Token::Text(value) = self {
+            let mut found = 0;
+
+            if value.chars().any(|ch| ch.is_ascii_whitespace()) {
+                found += 1;
+            }
+            if value.chars().any(|ch| ch == '\0') {
+                found += 1;
+            }
+            if value
+                .chars()
+                .any(|ch| !ch.is_ascii_whitespace() && ch != '\0')
+            {
+                found += 1;
+            }
+            found > 1
+        } else {
+            false
+        }
+    }
+}
+
 impl Token {
     /// Returns true when any of the characters in the token are null
     pub fn is_null(&self) -> bool {
@@ -48,7 +73,11 @@ impl Token {
     /// Returns true if the text token is empty or only contains whitespace
     pub fn is_empty_or_white(&self) -> bool {
         if let Token::Text(value) = self {
-            ["\u{0009}", "\u{000a}", "\u{000c}", "\u{000d}", "\u{0020}"].contains(&value.as_str())
+            if value.is_empty() {
+                return true;
+            }
+
+            value.chars().all(|ch| ch.is_ascii_whitespace())
         } else {
             false
         }
diff --git a/src/testing/tree_construction/parser.rs b/src/testing/tree_construction/parser.rs
index a5ba70a28..02b72ba0c 100644
--- a/src/testing/tree_construction/parser.rs
+++ b/src/testing/tree_construction/parser.rs
@@ -310,8 +310,8 @@ fn test(i: Span) -> IResult<Span, TestSpec> {
 
 /// Trims only a single newline from the string, even if there are multiple newlines present.
 fn trim_last_newline(s: String) -> String {
-    if s.ends_with('\n') {
-        s[..s.len() - 1].to_owned()
+    if let Some(s) = s.strip_suffix('\n') {
+        s.to_owned()
     } else {
         s
     }

From d29cccce5ce2b706a4744420c6fc4eb2c4e6a791 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Tue, 7 Nov 2023 22:09:22 +0100
Subject: [PATCH 11/14] fixed clippy issue

---
 src/html5/parser.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index b7d807f1f..93f5f9395 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -3995,7 +3995,7 @@ impl<'chars> Html5Parser<'chars> {
     /// The idea is that large blobs of javascript for instance will not be split into separate
     /// tokens, but still be seen and parsed as a single TextToken.
     ///
-    fn split_mixed_token(&self, text: &String) -> Vec<Token> {
+    fn split_mixed_token(&self, text: &str) -> Vec<Token> {
         let mut tokens = vec![];
         let mut last_group = 'x';
 

From 72b7e4f321dcdfe51a6d7e04e240d6a128483ef2 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Tue, 7 Nov 2023 22:34:49 +0100
Subject: [PATCH 12/14] moved line-offset to line-columns and using hashmap

---
 src/bytes.rs           | 17 ++++++++---------
 src/html5/tokenizer.rs |  5 ++---
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/bytes.rs b/src/bytes.rs
index 3e9359ec4..ae1429274 100644
--- a/src/bytes.rs
+++ b/src/bytes.rs
@@ -1,4 +1,5 @@
 use crate::html5::tokenizer::{CHAR_CR, CHAR_LF};
+use std::collections::HashMap;
 use std::fs::File;
 use std::io::Read;
 use std::{fmt, io};
@@ -105,7 +106,7 @@ pub struct CharIterator {
     /// Length (in chars) of the buffer
     pub length: usize,
     /// Offsets of the given lines
-    line_offsets: Vec<usize>,
+    line_columns: HashMap<usize, usize>,
     /// Reference to the actual buffer stream in characters
     buffer: Vec<Bytes>,
     /// Reference to the actual buffer stream in u8 bytes
@@ -132,7 +133,7 @@ impl CharIterator {
                 col: 1,
             },
             length: 0,
-            line_offsets: vec![0], // first line always starts at 0
+            line_columns: HashMap::new(),
             buffer: Vec::new(),
             u8_buffer: Vec::new(),
             has_read_eof: false,
@@ -164,7 +165,7 @@ impl CharIterator {
     /// Skip offset characters in the stream (based on chars)
     pub fn skip(&mut self, offset: usize) {
         let mut skip_len = offset;
-        if self.position.offset + offset > self.length {
+        if self.position.offset + offset >= self.length {
             skip_len = self.length - self.position.offset;
         }
 
@@ -308,11 +309,8 @@ impl CharIterator {
             let c = self.buffer[self.position.offset];
             if c == Ch('\n') {
                 // Store line offset for the given line
-                if self.line_offsets.len() > self.position.line {
-                    self.line_offsets[self.position.line] = self.position.offset;
-                } else {
-                    self.line_offsets.push(self.position.offset);
-                }
+                self.line_columns
+                    .insert(self.position.line, self.position.col);
                 // And continue position on the next line
                 self.position.line += 1;
                 self.position.col = 1;
@@ -352,7 +350,8 @@ impl CharIterator {
 
             if self.position.col == 1 {
                 self.position.line -= 1;
-                self.position.col = self.line_offsets[self.position.line];
+                let key = self.position.line;
+                self.position.col = *self.line_columns.get(&key).unwrap_or(&1);
             } else {
                 self.position.col -= 1;
             }
diff --git a/src/html5/tokenizer.rs b/src/html5/tokenizer.rs
index b6580f5d4..ec9a3c8df 100644
--- a/src/html5/tokenizer.rs
+++ b/src/html5/tokenizer.rs
@@ -313,7 +313,6 @@ impl<'stream> Tokenizer<'stream> {
                         }
                         _ => {
                             self.parse_error(ParserError::InvalidFirstCharacterOfTagName);
-
                             self.current_token = Some(Token::Comment("".into()));
                             self.chars.unread();
                             self.state = State::BogusComment;
@@ -1245,9 +1244,9 @@ impl<'stream> Tokenizer<'stream> {
                         continue;
                     }
 
-                    // self.chars.skip(1);
+                    self.chars.read_char();
                     self.parse_error(ParserError::IncorrectlyOpenedComment);
-                    // self.chars.unread();
+                    self.chars.unread();
                     self.current_token = Some(Token::Comment("".into()));
 
                     self.state = State::BogusComment;

From 9e1955436cf11d1ee96e0326c4330fbf214fecba Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Wed, 8 Nov 2023 08:15:27 +0100
Subject: [PATCH 13/14] Removing text splitting in body tags

---
 src/html5/parser.rs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index 93f5f9395..ddc835f2f 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -2116,10 +2116,6 @@ impl<'chars> Html5Parser<'chars> {
     /// Handle insertion mode "in_body"
     fn handle_in_body(&mut self) {
         match &self.current_token.clone() {
-            Token::Text(value) if self.current_token.is_mixed() => {
-                let tokens = self.split_mixed_token(value);
-                self.tokenizer.insert_tokens_at_queue_start(tokens);
-            }
             Token::Text(..) if self.current_token.is_null() => {
                 self.parse_error("null character not allowed in in body insertion mode");
                 // ignore token

From d90da26742aa2e018420dae18b80f4f3d0ec7cb4 Mon Sep 17 00:00:00 2001
From: Joshua Thijssen <jthijssen@noxlogic.nl>
Date: Wed, 8 Nov 2023 08:25:31 +0100
Subject: [PATCH 14/14] Added extra splitter for \0 only

---
 src/html5/parser.rs          | 31 +++++++++++++++++++++++++++++++
 src/html5/tokenizer/token.rs | 11 +++++++++++
 2 files changed, 42 insertions(+)

diff --git a/src/html5/parser.rs b/src/html5/parser.rs
index ddc835f2f..511e3f198 100644
--- a/src/html5/parser.rs
+++ b/src/html5/parser.rs
@@ -2116,6 +2116,10 @@ impl<'chars> Html5Parser<'chars> {
     /// Handle insertion mode "in_body"
     fn handle_in_body(&mut self) {
         match &self.current_token.clone() {
+            Token::Text(value) if self.current_token.is_mixed_null() => {
+                let tokens = self.split_mixed_token_null(value);
+                self.tokenizer.insert_tokens_at_queue_start(tokens);
+            }
             Token::Text(..) if self.current_token.is_null() => {
                 self.parse_error("null character not allowed in in body insertion mode");
                 // ignore token
@@ -4021,6 +4025,33 @@ impl<'chars> Html5Parser<'chars> {
 
         tokens
     }
+
+    /// This will split tokens into \0 groups and non-\0 groups.
+    /// @todo: refactor this into split_mixed_token as well, but add a collection of groups callables
+    fn split_mixed_token_null(&self, text: &str) -> Vec<Token> {
+        let mut tokens = vec![];
+        let mut last_group = 'x';
+
+        let mut found = String::new();
+
+        for ch in text.chars() {
+            let group = if ch == '\0' { '0' } else { 'r' };
+
+            if last_group != group && !found.is_empty() {
+                tokens.push(Token::Text(found.clone()));
+                found.clear();
+            }
+
+            found.push(ch);
+            last_group = group;
+        }
+
+        if !found.is_empty() {
+            tokens.push(Token::Text(found.clone()));
+        }
+
+        tokens
+    }
 }
 
 #[cfg(test)]
diff --git a/src/html5/tokenizer/token.rs b/src/html5/tokenizer/token.rs
index 377551d75..f286ceb1b 100644
--- a/src/html5/tokenizer/token.rs
+++ b/src/html5/tokenizer/token.rs
@@ -31,6 +31,7 @@ pub enum Token {
 }
 
 impl Token {
+    /// Returns true when there is a mixture of white and non-white and \0 characters in the token
     pub(crate) fn is_mixed(&self) -> bool {
         // Check if there are white characters AND non-white characters in the token
         if let Token::Text(value) = self {
@@ -53,6 +54,16 @@ impl Token {
             false
         }
     }
+
+    /// Returns true when there is a mixture of \0 and non-\0 characters in the token
+    pub(crate) fn is_mixed_null(&self) -> bool {
+        // Check if there are white characters AND non-white characters in the token
+        if let Token::Text(value) = self {
+            value.chars().any(|ch| ch == '\0') && value.chars().any(|ch| ch != '\0')
+        } else {
+            false
+        }
+    }
 }
 
 impl Token {