PASS/FAIL is also valid TestOutcome (#138)

Signed-off-by: sagudev <[email protected]> Co-authored-by: Erich Gubler <[email protected]>
erichdongubler-mozilla · Jul 31, 2024 · 52a0fa4 · 52a0fa4
1 parent f362eb8
commit 52a0fa4
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 20 deletions.
diff --git a/moz-webgpu-cts/src/main.rs b/moz-webgpu-cts/src/main.rs
@@ -18,6 +18,7 @@ use std::{
     fs,
     io::{self, BufWriter},
     iter,
+    ops::Add,
     path::{Path, PathBuf},
     process::ExitCode,
     sync::{
@@ -412,6 +413,17 @@ fn run(cli: Cli) -> ExitCode {
                 }
             }
 
+            impl<T: Add<Output = T>> Add for PermaAndIntermittent<T> {
+                type Output = PermaAndIntermittent<T>;
+
+                fn add(self, rhs: Self) -> Self::Output {
+                    Self {
+                        perma: self.perma + rhs.perma,
+                        intermittent: self.intermittent + rhs.intermittent,
+                    }
+                }
+            }
+
             impl<T> PermaAndIntermittent<T> {
                 pub fn as_ref(&self) -> PermaAndIntermittent<&T> {
                     let Self {
@@ -445,6 +457,7 @@ fn run(cli: Cli) -> ExitCode {
                 tests_with_runner_errors: TestSet,
                 tests_with_disabled_or_skip: TestSet,
                 tests_with_crashes: TestSet,
+                tests_with_failures: TestSet,
                 subtests_with_failures_by_test: SubtestByTestSet,
                 subtests_with_timeouts_by_test: SubtestByTestSet,
             }
@@ -606,6 +619,15 @@ fn run(cli: Cli) -> ExitCode {
                                         outcome,
                                     )
                                 }),
+                                TestOutcome::Pass => (),
+                                TestOutcome::Fail => receiver(&mut |analysis| {
+                                    insert_in_test_set(
+                                        &mut analysis.tests_with_failures,
+                                        test_name,
+                                        expected,
+                                        outcome,
+                                    )
+                                }),
                             }
                         }
                     }
@@ -708,6 +730,7 @@ fn run(cli: Cli) -> ExitCode {
                     tests_with_runner_errors,
                     tests_with_disabled_or_skip,
                     tests_with_crashes,
+                    tests_with_failures,
                     subtests_with_failures_by_test,
                     subtests_with_timeouts_by_test,
                 } = analysis;
@@ -774,7 +797,8 @@ fn run(cli: Cli) -> ExitCode {
                     intermittent: num_tests_with_intermittent_failures_somewhere,
                 } = subtests_with_failures_by_test
                     .as_ref()
-                    .map(|tests| tests.len());
+                    .map(|tests| tests.len())
+                    + tests_with_failures.as_ref().map(|tests| tests.len());
                 let PermaAndIntermittent {
                     perma: num_subtests_with_perma_failures_somewhere,
                     intermittent: num_subtests_with_intermittent_failures_somewhere,
@@ -950,7 +974,7 @@ fn run(cli: Cli) -> ExitCode {
                         .iter()
                     {
                         let case = match expected.as_permanent() {
-                            Some(TestOutcome::Ok) => Case::PermaPass,
+                            Some(TestOutcome::Ok | TestOutcome::Pass) => Case::PermaPass,
                             _ => Case::Other,
                         };
                         cases[(platform, build_profile)] = case;

diff --git a/moz-webgpu-cts/src/wpt/metadata.rs b/moz-webgpu-cts/src/wpt/metadata.rs
@@ -1190,6 +1190,8 @@ pub(crate) const ERROR: &str = "ERROR";
 #[serde(rename_all = "UPPERCASE")]
 pub enum TestOutcome {
     Ok,
+    Pass,
+    Fail,
     Timeout,
     Crash,
     Error,
@@ -1209,6 +1211,8 @@ impl Display for TestOutcome {
             "{}",
             match self {
                 Self::Ok => OK,
+                Self::Pass => PASS,
+                Self::Fail => FAIL,
                 Self::Timeout => TIMEOUT,
                 Self::Crash => CRASH,
                 Self::Error => ERROR,
@@ -1227,6 +1231,8 @@ impl<'a> Properties<'a> for TestProps<TestOutcome> {
             helper,
             choice((
                 keyword(OK).to(TestOutcome::Ok),
+                keyword(PASS).to(TestOutcome::Pass),
+                keyword(FAIL).to(TestOutcome::Fail),
                 keyword(CRASH).to(TestOutcome::Crash),
                 keyword(TIMEOUT).to(TestOutcome::Timeout),
                 keyword(ERROR).to(TestOutcome::Error),
@@ -1462,24 +1468,6 @@ r#"
     assert_debug_snapshot!(
         parser().parse(
 r#"
-[asdf]
-  # Incorrect; `PASS` isn't a valid test outcome (but it _is_ a valid subtest outcome).
-  expected: PASS
-"#
-        ),
-        @r###"
-    ParseResult {
-        output: None,
-        errs: [
-            found end of input at 108..112 expected something else,
-        ],
-    }
-    "###
-    );
-
-    assert_debug_snapshot!(
-        parser().parse(
-r#"
 [asdf]
   [blarg]
     expected: [PASS, FAIL]
@@ -1840,4 +1828,66 @@ r#"
     }
     "###
     );
+
+    assert_debug_snapshot!(
+        parser().parse(
+r#"
+[canvas_complex_rgba8unorm_store.https.html]
+  expected: [PASS, FAIL]
+"#
+        ),
+        @r###"
+    ParseResult {
+        output: Some(
+            (
+                "canvas_complex_rgba8unorm_store.https.html",
+                Test {
+                    properties: TestProps {
+                        is_disabled: false,
+                        expected: Some(
+                            ExpandedPropertyValue(
+                                {
+                                    Windows: {
+                                        Debug: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                        Optimized: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                    },
+                                    Linux: {
+                                        Debug: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                        Optimized: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                    },
+                                    MacOs: {
+                                        Debug: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                        Optimized: [
+                                            Pass,
+                                            Fail,
+                                        ],
+                                    },
+                                },
+                            ),
+                        ),
+                        implementation_status: None,
+                    },
+                    subtests: {},
+                },
+            ),
+        ),
+        errs: [],
+    }
+    "###
+    );
 }