Skip to content

Commit

Permalink
Addressed additional error where a rule test results in an outcome di…
Browse files Browse the repository at this point in the history
…ctionary with an empty list of results, where no indication of outcome is given. This will now correctly log this error rather than failing, but producing no context on why it fails. Also, added additional check in case some other combination of factors results in a failure without a log describing why it fails.
  • Loading branch information
gonz102 committed Jul 31, 2024
1 parent 9d4113f commit d3225f8
Showing 1 changed file with 20 additions and 6 deletions.
26 changes: 20 additions & 6 deletions rct229/ruletest_engine/ruletest_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,11 @@ def run_section_tests(
if test_error_found:
all_tests_pass = False

if len(test_result_dict["log"]) == 0:
rule_outcome = test_dict["expected_rule_outcome"]
print(
f"FAILURE: {test_id} failed but produced no log errors. Expected '{rule_outcome}'."
)
# Print log of all errors
for test_result_string in test_result_dict["log"]:
print(test_result_string)
Expand Down Expand Up @@ -629,12 +634,21 @@ def evaluate_outcome_object(outcome_dict, test_result_dict, test_dict, test_id):
# If the result key is a list of results (i.e. many elements get tested), keep drilling down until you get single
# dictionary
if isinstance(outcome_dict["result"], list):
# Iterate through each outcome in outcome results recursively until you get down to individual results
for nested_outcome in outcome_dict["result"]:
# Check outcome of each in list recursively until "result" key is not a list, but a dictionary
evaluate_outcome_object(
nested_outcome, test_result_dict, test_dict, test_id
) # , outcome_result_list)

# If the outcome result is an empty list, flag this as a blanket error
if len(outcome_dict["result"]) == 0:

expected_outcome = test_dict["expected_rule_outcome"]
outcome_test = f"FAILURE: Running test {test_id} returned an empty set of results without any reference to 'pass', 'fail', 'undetermined', or 'not_applicable'. Expected '{expected_outcome}'."
test_result_dict["log"].append(outcome_test)

else:
# Iterate through each outcome in outcome results recursively until you get down to individual results
for nested_outcome in outcome_dict["result"]:
# Check outcome of each in list recursively until "result" key is not a list, but a dictionary
evaluate_outcome_object(
nested_outcome, test_result_dict, test_dict, test_id
) # , outcome_result_list)

else:
# Process this tests results
Expand Down

0 comments on commit d3225f8

Please sign in to comment.