Skip to content

Commit

Permalink
WIP: feat(process_reports)!: add opt-out test normalization/relocatio…
Browse files Browse the repository at this point in the history 
…n based on internal CTS paths
  • Loading branch information
@ErichDonGubler
ErichDonGubler committed Dec 27, 2023
1 parent 278e69f commit 0db88f1
Showing 1 changed file with 177 additions and 104 deletions.
281 changes: 177 additions & 104 deletions moz-webgpu-cts/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,10 +324,28 @@ fn run(cli: Cli) -> ExitCode {
files
};

#[derive(Debug, Default)]
struct EntryByCtsPath<'a> {
metadata_path: Option<TestPath<'a>>,
reported_path: Option<TestPath<'a>>,
entry: TestEntry,
}

fn cts_path(test_path: &TestPath<'_>) -> Option<String> {
test_path
.variant
.as_ref()
.filter(|v| v.starts_with("?q=webgpu:"))
.map(|v| v.strip_prefix("?q=").unwrap().to_owned())
.filter(|_q| test_path.path.ends_with("cts.https.html"))
}

let mut file_props_by_file = IndexMap::<Utf8PathBuf, FileProps>::default();
let mut entries_by_cts_path = IndexMap::<String, EntryByCtsPath<'_>>::default();
let mut other_entries_by_test = IndexMap::<TestPath<'_>, TestEntry>::default();

log::info!("loading metadata for comparison to reports…");

for (path, file) in meta_files_by_path {
let File { properties, tests } = file;

Expand All @@ -346,30 +364,47 @@ fn run(cli: Cli) -> ExitCode {

let test_path = TestPath::from_fx_metadata_test(file_rel_path, &name).unwrap();

let cts_path = cts_path(&test_path);

let freak_out_do_nothing = |what: &dyn Display| {
log::error!("hoo boy, not sure what to do yet: {what}")
};

let mut reported_dupe_already = false;
let mut dupe_err = || {
if !reported_dupe_already {
freak_out_do_nothing(&format_args!(
concat!(
"duplicate entry found for {:?}, ",
"discarding previous metadata entries",
),
test_path
))
}
reported_dupe_already = true;
};

let TestEntry {
entry: test_entry,
subtests: subtest_entries,
} = other_entries_by_test
.entry(test_path.clone().into_owned())
.or_default();
} = if let Some(cts_path) = cts_path {
let entry = entries_by_cts_path.entry(cts_path).or_default();
if let Some(_old) =
entry.metadata_path.replace(test_path.clone().into_owned())
{
dupe_err();
}
&mut entry.entry
} else {
other_entries_by_test
.entry(test_path.clone().into_owned())
.or_default()
};

let test_path = &test_path;
let mut reported_dupe_already = false;

if let Some(_old) = test_entry.meta_props.replace(properties) {
freak_out_do_nothing(&format_args!(
concat!(
"duplicate entry for {:?}",
"discarding previous entries with ",
"this and further dupes"
),
test_path
));
reported_dupe_already = true;
dupe_err();
}

for (SectionHeader(subtest_name), subtest) in subtests {
Expand All @@ -380,7 +415,7 @@ fn run(cli: Cli) -> ExitCode {
if !reported_dupe_already {
freak_out_do_nothing(&format_args!(
concat!(
"duplicate subtest in {:?} named {:?}, ",
"found duplicate subtest in {:?} named {:?}, ",
"discarding previous entries with ",
"this and further dupes"
),
Expand Down Expand Up @@ -440,12 +475,31 @@ fn run(cli: Cli) -> ExitCode {
let TestExecutionEntry { test_name, result } = entry;

let test_path = TestPath::from_execution_report(&test_name).unwrap();
let cts_path = cts_path(&test_path);

let TestEntry {
entry: test_entry,
subtests: subtest_entries,
} = other_entries_by_test
.entry(test_path.clone().into_owned())
.or_default();
} = if let Some(cts_path) = cts_path {
let entry = entries_by_cts_path.entry(cts_path).or_default();
if let Some(_old) =
entry.reported_path.replace(test_path.clone().into_owned())
{
log::warn!(
concat!(
"duplicate entry found for {:?}, ",
"discarding previous entries with ",
"this and further dupes"
),
test_path
)
}
&mut entry.entry
} else {
other_entries_by_test
.entry(test_path.clone().into_owned())
.or_default()
};

let (reported_outcome, reported_subtests) = match result {
TestExecutionResult::Complete { outcome, subtests } => (outcome, subtests),
Expand Down Expand Up @@ -510,99 +564,118 @@ fn run(cli: Cli) -> ExitCode {
log::info!("metadata and reports gathered, now reconciling outcomes…");

let mut found_reconciliation_err = false;
let recombined_tests_iter =
other_entries_by_test
.into_iter()
.filter_map(|(test_path, test_entry)| {
fn reconcile<Out>(
entry: Entry<Out>,
preset: ReportProcessingPreset,
) -> TestProps<Out>
where
Out: Debug + Default + EnumSetType,
{
let Entry {
meta_props,
reported,
} = entry;
let normalize = NormalizedExpectationPropertyValue::from_fully_expanded;

let mut meta_props = meta_props.unwrap_or_default();
meta_props.expectations = Some('resolve: {
let resolve = match preset {
ReportProcessingPreset::ResetAll => {
break 'resolve normalize(reported);
}
ReportProcessingPreset::ResetContradictory => {
|meta: Expectation<_>, rep: Option<Expectation<_>>| {
rep.filter(|rep| !meta.is_superset(rep)).unwrap_or(meta)
}
let recombined_tests_iter = entries_by_cts_path
.into_iter()
.map(|(_name, entry)| {
let EntryByCtsPath {
metadata_path,
reported_path,
entry,
} = entry;
if let Some((meta, rep)) = metadata_path
.as_ref()
.zip(reported_path.as_ref())
.filter(|(meta, rep)| meta != rep)
{
log::warn!("relocating {meta:?} to {rep:?}");
}
(
reported_path.or(metadata_path).expect(concat!(
"internal error: CTS path entry created without at least one ",
"report or metadata path specified"
)),
entry,
)
})
.chain(other_entries_by_test.into_iter())

Check failure on line 590 in moz-webgpu-cts/src/main.rs

View workflow job for this annotation

GitHub Actions / Clippy (stable) 

explicit call to `.into_iter()` in function argument accepting `IntoIterator`
.filter_map(|(test_path, test_entry)| {
fn reconcile<Out>(
entry: Entry<Out>,
preset: ReportProcessingPreset,
) -> TestProps<Out>
where
Out: Debug + Default + EnumSetType,
{
let Entry {
meta_props,
reported,
} = entry;
let normalize = NormalizedExpectationPropertyValue::from_fully_expanded;

let mut meta_props = meta_props.unwrap_or_default();
meta_props.expectations = Some('resolve: {
let resolve = match preset {
ReportProcessingPreset::ResetAll => {
break 'resolve normalize(reported);
}
ReportProcessingPreset::ResetContradictory => {
|meta: Expectation<_>, rep: Option<Expectation<_>>| {
rep.filter(|rep| !meta.is_superset(rep)).unwrap_or(meta)
}
ReportProcessingPreset::Merge => |meta, rep| match rep {
Some(rep) => meta | rep,
None => meta,
},
};

normalize(
Platform::iter()
.map(|platform| {
let build_profiles = BuildProfile::iter()
.map(|build_profile| {
(
build_profile,
resolve(
meta_props
.expectations
.as_ref()
.unwrap_or(&Default::default())
.get(platform, build_profile),
reported
.get(&platform)
.and_then(|rep| {
rep.get(&build_profile)
})
.copied(),
),
)
})
.collect();
(platform, build_profiles)
})
.collect(),
)
});
meta_props
}
}
ReportProcessingPreset::Merge => |meta, rep| match rep {
Some(rep) => meta | rep,
None => meta,
},
};

normalize(
Platform::iter()
.map(|platform| {
let build_profiles = BuildProfile::iter()
.map(|build_profile| {
(
build_profile,
resolve(
meta_props
.expectations
.as_ref()
.unwrap_or(&Default::default())
.get(platform, build_profile),
reported
.get(&platform)
.and_then(|rep| rep.get(&build_profile))
.copied(),
),
)
})
.collect();
(platform, build_profiles)
})
.collect(),
)
});
meta_props
}

let TestEntry {
entry: test_entry,
subtests: subtest_entries,
} = test_entry;
let TestEntry {
entry: test_entry,
subtests: subtest_entries,
} = test_entry;

let properties = reconcile(test_entry, preset);
let properties = reconcile(test_entry, preset);

let mut subtests = BTreeMap::new();
for (subtest_name, subtest) in subtest_entries {
let subtest_name = SectionHeader(subtest_name);
if subtests.get(&subtest_name).is_some() {
found_reconciliation_err = true;
log::error!("internal error: duplicate test path {test_path:?}");
}
subtests.insert(
subtest_name,
Subtest {
properties: reconcile(subtest, preset),
},
);
let mut subtests = BTreeMap::new();
for (subtest_name, subtest) in subtest_entries {
let subtest_name = SectionHeader(subtest_name);
if subtests.get(&subtest_name).is_some() {
found_reconciliation_err = true;
log::error!("internal error: duplicate test path {test_path:?}");
}
subtests.insert(
subtest_name,
Subtest {
properties: reconcile(subtest, preset),
},
);
}

if subtests.is_empty() && properties == Default::default() {
None
} else {
Some((test_path, (properties, subtests)))
}
});
if subtests.is_empty() && properties == Default::default() {
None
} else {
Some((test_path, (properties, subtests)))
}
});

log::info!(
"outcome reconciliation complete, gathering tests back into new metadata files…"
Expand Down

0 comments on commit 0db88f1

Please sign in to comment.