Skip to content

Commit

Permalink
fixed tokenizer tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jaytaph
jaytaph committed Aug 27, 2024
1 parent 30e2441 commit 6d04377
Show file tree
Hide file tree
Showing 12 changed files with 733 additions and 155 deletions.
78 changes: 64 additions & 14 deletions crates/gosub_html5/src/node.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ use crate::node::data::text::TextData;
use crate::parser::document::{Document, DocumentHandle};
use core::fmt::Debug;
use derive_more::Display;
use gosub_shared::byte_stream::Location;
use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::Weak;
use gosub_shared::byte_stream::Location;

pub const HTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml";
pub const MATHML_NAMESPACE: &str = "http://www.w3.org/1998/Math/MathML";
Expand Down Expand Up @@ -144,7 +144,7 @@ pub struct Node {
// Returns true when the given node is registered into an arena
pub is_registered: bool,
// Location of the node in the source code
pub location: Location
pub location: Location,
}

impl Node {
Expand Down Expand Up @@ -192,7 +192,7 @@ impl Clone for Node {
data: self.data.clone(),
document: Weak::clone(&self.document),
is_registered: self.is_registered,
location: self.location.clone()
location: self.location.clone(),
}
}
}
Expand All @@ -211,7 +211,7 @@ impl Node {
namespace,
document: document.to_weak(),
is_registered,
location
location,
}
}

Expand All @@ -232,7 +232,7 @@ impl Node {
Self::new(
NodeData::DocType(DocTypeData::new(name, pub_identifier, sys_identifier)),
document,
loc
loc,
)
}

Expand Down Expand Up @@ -263,13 +263,21 @@ impl Node {
/// Creates a new comment node
#[must_use]
pub fn new_comment(document: &DocumentHandle, location: Location, value: &str) -> Self {
Self::new(NodeData::Comment(CommentData::with_value(value)), document, location)
Self::new(
NodeData::Comment(CommentData::with_value(value)),
document,
location,
)
}

/// Creates a new text node
#[must_use]
pub fn new_text(document: &DocumentHandle, location: Location, value: &str) -> Self {
Self::new(NodeData::Text(TextData::with_value(value)), document, location)
Self::new(
NodeData::Text(TextData::with_value(value)),
document,
location,
)
}

/// Returns true if the given node is a "formatting" node
Expand Down Expand Up @@ -570,7 +578,13 @@ mod tests {
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let document = Document::shared(None);
let node = Node::new_element(&document, "div", attributes.clone(), HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
"div",
attributes.clone(),
HTML_NAMESPACE,
Location::default(),
);
assert_eq!(node.id, NodeId::default());
assert_eq!(node.parent, None);
assert!(node.children.is_empty());
Expand Down Expand Up @@ -619,7 +633,13 @@ mod tests {
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let document = Document::shared(None);
let node = Node::new_element(&document, "div", attributes, HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
"div",
attributes,
HTML_NAMESPACE,
Location::default(),
);
assert!(node.is_special());
}

Expand All @@ -634,7 +654,13 @@ mod tests {
assert_eq!(node.type_of(), NodeType::Comment);
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let node = Node::new_element(&document, "div", attributes, HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
"div",
attributes,
HTML_NAMESPACE,
Location::default(),
);
assert_eq!(node.type_of(), NodeType::Element);
}

Expand All @@ -644,7 +670,13 @@ mod tests {
for element in SPECIAL_HTML_ELEMENTS.iter() {
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let node = Node::new_element(&document, element, attributes, HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
element,
attributes,
HTML_NAMESPACE,
Location::default(),
);
assert!(node.is_special());
}
}
Expand All @@ -655,7 +687,13 @@ mod tests {
for element in SPECIAL_MATHML_ELEMENTS.iter() {
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let node = Node::new_element(&document, element, attributes, MATHML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
element,
attributes,
MATHML_NAMESPACE,
Location::default(),
);
assert!(node.is_special());
}
}
Expand All @@ -666,7 +704,13 @@ mod tests {
for element in SPECIAL_SVG_ELEMENTS.iter() {
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let node = Node::new_element(&document, element, attributes, SVG_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
element,
attributes,
SVG_NAMESPACE,
Location::default(),
);
assert!(node.is_special());
}
}
Expand All @@ -682,7 +726,13 @@ mod tests {
assert_eq!(node.type_of(), NodeType::Comment);
let mut attributes = HashMap::new();
attributes.insert("id".to_string(), "test".to_string());
let node = Node::new_element(&document, "div", attributes, HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&document,
"div",
attributes,
HTML_NAMESPACE,
Location::default(),
);
assert_eq!(node.type_of(), NodeType::Element);
}
}
42 changes: 36 additions & 6 deletions crates/gosub_html5/src/node/arena.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ mod tests {
"test",
HashMap::new(),
HTML_NAMESPACE,
Location::default()
Location::default(),
);
let mut document = doc.get_mut();
let id = document.arena.register_node(node);
Expand All @@ -104,7 +104,13 @@ mod tests {
fn register_node_twice() {
let mut doc = Document::shared(None);

let node = Node::new_element(&doc, "test", HashMap::new(), HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&doc,
"test",
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);
let mut document = doc.get_mut();
document.arena.register_node(node);

Expand All @@ -115,7 +121,13 @@ mod tests {
#[test]
fn get_node() {
let mut doc = Document::shared(None);
let node = Node::new_element(&doc, "test", HashMap::new(), HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&doc,
"test",
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);

let mut document = doc.get_mut();
let id = document.arena.register_node(node);
Expand All @@ -127,7 +139,13 @@ mod tests {
#[test]
fn get_node_mut() {
let mut doc = Document::shared(None);
let node = Node::new_element(&doc, "test", HashMap::new(), HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&doc,
"test",
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);

let mut document = doc.get_mut();

Expand All @@ -141,8 +159,20 @@ mod tests {
fn register_node_through_document() {
let mut doc = Document::shared(None);

let parent = Node::new_element(&doc, "parent", HashMap::new(), HTML_NAMESPACE, Location::default());
let child = Node::new_element(&doc, "child", HashMap::new(), HTML_NAMESPACE, Location::default());
let parent = Node::new_element(
&doc,
"parent",
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);
let child = Node::new_element(
&doc,
"child",
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);

let mut document = doc.get_mut();
let parent_id = document.arena.register_node(parent);
Expand Down
59 changes: 46 additions & 13 deletions crates/gosub_html5/src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,11 @@ impl<'chars> Html5Parser<'chars> {
/// Process a token in HTML content
fn process_html_content(&mut self) {
if self.ignore_lf {
if let Token::Text { text: value, location } = &self.current_token {
if let Token::Text {
text: value,
location,
} = &self.current_token
{
if value.starts_with('\n') {
// We don't need to skip 1 char, but we can skip 1 byte, as we just checked for \n
self.current_token = Token::Text {
Expand Down Expand Up @@ -1872,18 +1876,37 @@ impl<'chars> Html5Parser<'chars> {
&name.clone().unwrap_or_default(),
&pub_identifier.clone().unwrap_or_default(),
&sys_identifier.clone().unwrap_or_default(),
location.clone()
location.clone(),
),
Token::StartTag {
name, attributes, location, ..
} => Node::new_element(&self.document, name, attributes.clone(), namespace, location.clone()),
Token::EndTag { name, location, .. } => {
Node::new_element(&self.document, name, HashMap::new(), namespace, location.clone())
}
Token::Comment { comment: value, location, .. } => Node::new_comment(&self.document, location.clone(), value),
Token::Text { text: value, location, .. } => {
Node::new_text(&self.document, location.clone(), value.to_string().as_str())
}
name,
attributes,
location,
..
} => Node::new_element(
&self.document,
name,
attributes.clone(),
namespace,
location.clone(),
),
Token::EndTag { name, location, .. } => Node::new_element(
&self.document,
name,
HashMap::new(),
namespace,
location.clone(),
),
Token::Comment {
comment: value,
location,
..
} => Node::new_comment(&self.document, location.clone(), value),
Token::Text {
text: value,
location,
..
} => Node::new_text(&self.document, location.clone(), value.to_string().as_str()),
Token::Eof { .. } => {
panic!("EOF token not allowed");
}
Expand Down Expand Up @@ -3899,7 +3922,11 @@ impl<'chars> Html5Parser<'chars> {
.next_token(self.parser_data())
.expect("tokenizer error");

if let Token::Text { text: value, location } = token {
if let Token::Text {
text: value,
location,
} = token
{
self.token_queue.push(Token::Text {
text: value,
location: location.clone(),
Expand Down Expand Up @@ -4324,7 +4351,13 @@ mod test {

macro_rules! node_create {
($self:expr, $name:expr) => {{
let node = Node::new_element(&$self.document, $name, HashMap::new(), HTML_NAMESPACE, Location::default());
let node = Node::new_element(
&$self.document,
$name,
HashMap::new(),
HTML_NAMESPACE,
Location::default(),
);
let node_id = $self
.document
.get_mut()
Expand Down
Loading

0 comments on commit 6d04377

Please sign in to comment.