Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Moved tokenizer.rs and tree_construction.rs tests into crates for issue #445 #457

Merged
merged 14 commits into from
May 27, 2024
Merged

Moved tokenizer.rs and tree_construction.rs tests into crates for issue #445 #457

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
67c1ea7
Moved tokenizer.rs from tests to crates
kaigidwani May 3, 2024
9393b48
Deleted tokenizer.rs from tests folder.
kaigidwani May 3, 2024
d290386
Copied tree_construction.rs from tests into a crate.
kaigidwani May 3, 2024
43fca24
Deleted tree_construction.rs from the test folder
kaigidwani May 3, 2024
665c762
Update and rename test.rs to tests.rs
kaigidwani May 5, 2024
2b042a2
Update tokenizer.rs
kaigidwani May 5, 2024
ee44864
Delete tests/lib.rs
kaigidwani May 6, 2024
8fc2ccb
Update Cargo.toml
kaigidwani May 6, 2024
272c205
cargo add test_case --dev -p gosub_html5
MoralCode May 6, 2024
c0c543a
make format
MoralCode May 6, 2024
4a819c3
import gosub testing in html5 crate
MoralCode May 6, 2024
50af07b
move testing data
MoralCode May 6, 2024
c1ee0fa
apply diff from PR to fix final changes
MoralCode May 11, 2024
693eced
remove benchmarks that the other patch didnt get
MoralCode May 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 98 additions & 0 deletions crates/gosub_html5/src/parser/tree_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,101 @@ pub trait TreeBuilder {
/// Insert/update an attribute for an element node.
fn insert_attribute(&mut self, key: &str, value: &str, element_id: NodeId) -> Result<()>;
}

#[cfg(test)]
mod tests {
use gosub_testing::testing::tree_construction::fixture::{
fixture_root_path, read_fixture_from_path,
};
use gosub_testing::testing::tree_construction::Harness;
use test_case::test_case;
Comment on lines +34 to +35
Copy link
Contributor

@MoralCode MoralCode May 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems to not like these imports for some reason when i run make test locally. Where is the test_case macro being defined?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's an external crate. Basically just run

cargo add test_case --dev -p gosub_html5

It also seems like fmt and clippy aren't happy, it can be fixed via make format

I haven't looked really into what is wrong, again I'll do that tomorrow.


const DISABLED_CASES: &[&str] = &[
// tests18.dat
"<!doctype html><template><plaintext>a</template>b",
];

// See tests/data/html5lib-tests/tree-construction/ for other test files.
#[test_case("tests1.dat")]
#[test_case("tests2.dat")]
#[test_case("tests3.dat")]
#[test_case("tests4.dat")]
#[test_case("tests5.dat")]
#[test_case("tests6.dat")]
#[test_case("tests7.dat")]
#[test_case("tests8.dat")]
#[test_case("tests9.dat")]
#[test_case("tests10.dat")]
#[test_case("tests11.dat")]
#[test_case("tests12.dat")]
#[test_case("tests14.dat")]
#[test_case("tests15.dat")]
#[test_case("tests16.dat")]
#[test_case("tests17.dat")]
#[test_case("tests18.dat")]
#[test_case("tests19.dat")]
#[test_case("tests20.dat")]
#[test_case("tests21.dat")]
#[test_case("tests22.dat")]
#[test_case("tests23.dat")]
#[test_case("tests24.dat")]
#[test_case("tests25.dat")]
#[test_case("tests26.dat")]
#[test_case("adoption01.dat")]
#[test_case("adoption02.dat")]
#[test_case("blocks.dat")]
#[test_case("comments01.dat")]
#[test_case("doctype01.dat")]
#[test_case("domjs-unsafe.dat")]
#[test_case("entities01.dat")]
#[test_case("entities02.dat")]
#[test_case("foreign-fragment.dat")]
#[test_case("html5test-com.dat")]
#[test_case("inbody01.dat")]
#[test_case("isindex.dat")]
#[test_case("main-element.dat")]
#[test_case("math.dat")]
#[test_case("menuitem-element.dat")]
#[test_case("namespace-sensitivity.dat")]
#[test_case("noscript01.dat")]
#[test_case("pending-spec-changes.dat")]
#[test_case("pending-spec-changes-plain-text-unsafe.dat")]
#[test_case("plain-text-unsafe.dat")]
#[test_case("quirks01.dat")]
#[test_case("ruby.dat")]
#[test_case("scriptdata01.dat")]
#[test_case("search-element.dat")]
#[test_case("svg.dat")]
#[test_case("tables01.dat")]
#[test_case("template.dat")]
#[test_case("tests_innerHTML_1.dat")]
#[test_case("tricky01.dat")]
#[test_case("webkit01.dat")]
#[test_case("webkit02.dat")]
fn tree_construction(filename: &str) {
let fixture_file = read_fixture_from_path(fixture_root_path().join(filename)).expect("fixture");
let mut harness = Harness::new();

for test in fixture_file.tests {
// skip disabled tests
if DISABLED_CASES.contains(&test.document_as_str()) {
continue;
}

// for each test, run it with and without scripting enabled based on the test file
for &scripting_enabled in test.script_modes() {
let result = harness
.run_test(test.clone(), scripting_enabled)
.expect("problem parsing");

println!(
"tree construction: {}:{} {}",
test.file_path,
test.line,
test.document_as_str()
);
assert!(result.is_success());
}
}
}
}
72 changes: 72 additions & 0 deletions crates/gosub_html5/src/tokenizer/test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
use gosub_testing::testing::tokenizer::{self, FixtureFile};
use lazy_static::lazy_static;
use std::collections::HashSet;
use test_case::test_case;

#[cfg(test)]
mod tests{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think an extra tests module is necessary here. You already made a module by adding the file. So, I guess rename the file to tests.rs and add #[cfg(test) over the module definition in tokenizer.rs (which is missing btw)

const DISABLED_CASES: &[&str] = &[
// TODO: Handle UTF-16 high and low private surrogate characters
// https://www.compart.com/en/unicode/U+DBC0
// https://www.compart.com/en/unicode/U+DC00
";\\uDBC0\\uDC00",
"<!-- -\\uDBC0\\uDC00",
"<!-- \\uDBC0\\uDC00",
"<!----\\uDBC0\\uDC00",
"<!---\\uDBC0\\uDC00",
"<!--\\uDBC0\\uDC00",
"<!DOCTYPE a PUBLIC\"\\uDBC0\\uDC00",
"<!DOCTYPE a PUBLIC'\\uDBC0\\uDC00",
"<!DOCTYPE a SYSTEM\"\\uDBC0\\uDC00",
"<!DOCTYPE a SYSTEM'\\uDBC0\\uDC00",
"<!DOCTYPE a\\uDBC0\\uDC00",
"<!DOCTYPE \\uDBC0\\uDC00",
"<!DOCTYPEa PUBLIC\"\\uDBC0\\uDC00",
"<!DOCTYPEa PUBLIC'\\uDBC0\\uDC00",
"<!DOCTYPEa SYSTEM\"\\uDBC0\\uDC00",
"<!DOCTYPEa SYSTEM'\\uDBC0\\uDC00",
"<!DOCTYPEa\\uDBC0\\uDC00",
"<!DOCTYPE\\uDBC0\\uDC00",
"\\uDBC0\\uDC00",
];

lazy_static! {
static ref DISABLED: HashSet<String> = DISABLED_CASES
.iter()
.map(|s| s.to_string())
.collect::<HashSet<_>>();
}

#[test_case("contentModelFlags.test")]
#[test_case("domjs.test")]
#[test_case("entities.test")]
#[test_case("escapeFlag.test")]
#[test_case("namedEntities.test")]
#[test_case("numericEntities.test")]
#[test_case("pendingSpecChanges.test")]
#[test_case("test1.test")]
#[test_case("test2.test")]
#[test_case("test3.test")]
#[test_case("test4.test")]
// #[test_case("unicodeCharsProblematic.test")]
#[test_case("unicodeChars.test")]
// #[test_case("xmlViolation.test")]
fn tokenization(filename: &str) {
let root = tokenizer::fixture_from_filename(filename).unwrap();

let tests = match root {
FixtureFile::Tests { tests } => tests,
FixtureFile::XmlTests { tests } => tests,
};

for test in tests {
if DISABLED.contains(&test.description) {
// Check that we don't panic
test.tokenize();
continue;
}

test.assert_valid();
}
}
}
69 changes: 0 additions & 69 deletions tests/tokenizer.rs
View file
Open in desktop

This file was deleted.

94 changes: 0 additions & 94 deletions tests/tree_construction.rs
View file
Open in desktop

This file was deleted.