From c5eb5d6913b14ca5dbfe42bf9279baff88403c77 Mon Sep 17 00:00:00 2001 From: "Matthieu Baerts (NGI0)" Date: Wed, 30 Oct 2024 17:58:46 +0100 Subject: [PATCH] contest: subcases: handle retry cases In case of failure, some tests are restarted. That's great, but when parsing the subcases (nested tests), the previous results were overridden. Also, when displaying the results, the 'retry' field from the main test was re-used in each subcases: quite confusing for the reader. Now, in case of retry, the previous 'results' list will be modified to add a new 'retry' entry for each test that has been re-validated. If the previous test with the same name cannot be found, that could be because there was major issue before and some subcases have not been executed (or the names are not fixed). In this case, a new entry with a skipped first attempt will be added to the list. When querying the subcases results, the new 'retry' entry will be used, or none if it is not there. Signed-off-by: Matthieu Baerts (NGI0) --- contest/backend/query.py | 3 +++ contest/remote/vmksft-p.py | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/contest/backend/query.py b/contest/backend/query.py index f3d8ded..274b704 100644 --- a/contest/backend/query.py +++ b/contest/backend/query.py @@ -63,6 +63,9 @@ def result_as_l2(raw): del data["results"] if "time" in data: del data["time"] + # in case of retry, the subtest might not have been re-executed + if "retry" in data: + del data["retry"] data |= case data["test"] = l1["test"] + '.' + case["test"] flat.append(data) diff --git a/contest/remote/vmksft-p.py b/contest/remote/vmksft-p.py index ff73e33..f434740 100755 --- a/contest/remote/vmksft-p.py +++ b/contest/remote/vmksft-p.py @@ -69,7 +69,7 @@ def get_prog_list(vm, targets): return [(e.split(":")[0].strip(), e.split(":")[1].strip()) for e in targets] -def _parse_nested_tests(full_run): +def _parse_nested_tests(full_run, prev_results): tests = [] nested_tests = False @@ -110,8 +110,20 @@ def _parse_nested_tests(full_run): r['result'] = result - tests.append(r) + if prev_results is not None: + for entry in prev_results: + if entry['test'] == r['test']: + entry['retry'] = result + break + else: + # the first run didn't validate this test: add it to the list + r['result'] = 'skip' + r['retry'] = result + prev_results.append(r) + else: + tests.append(r) + # return an empty list when there are prev results: no replacement needed return tests def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue): @@ -195,8 +207,13 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue): outcome['crashes'] = crashes if config.getboolean('ksft', 'nested_tests', fallback=False): + if is_retry: + prev_results = outcome['results'] if 'results' in outcome else [] + else: + prev_results = None + # this will only parse nested tests inside the TAP comments - nested_tests = _parse_nested_tests(vm.log_out) + nested_tests = _parse_nested_tests(vm.log_out, prev_results) if nested_tests: outcome['results'] = nested_tests