diff --git a/bashi/globals.py b/bashi/globals.py new file mode 100644 index 0000000..7e49140 --- /dev/null +++ b/bashi/globals.py @@ -0,0 +1,33 @@ +"""This module contains constants used in the bashi library.""" + +# index positions of the parameter-value +NAME: int = 0 +VERSION: int = 1 + +# parameter key names, whit special meaning +HOST_COMPILER: str = "host_compiler" +DEVICE_COMPILER: str = "device_compiler" + +# name of the used compilers +GCC: str = "gcc" +CLANG: str = "clang" +NVCC: str = "nvcc" +CLANG_CUDA: str = "clang-cuda" +HIPCC: str = "hipcc" +ICPX: str = "icpx" + +# alpaka backend names +ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLE: str = "alpaka_ACC_CPU_B_SEQ_T_SEQ_ENABLE" +ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLE: str = "alpaka_ACC_CPU_B_SEQ_T_THREADS_ENABLE" +ALPAKA_ACC_CPU_B_TBB_T_SEQ_ENABLE: str = "alpaka_ACC_CPU_B_TBB_T_SEQ_ENABLE" +ALPAKA_ACC_CPU_B_OMP2_T_SEQ_ENABLE: str = "alpaka_ACC_CPU_B_OMP2_T_SEQ_ENABLE" +ALPAKA_ACC_CPU_B_SEQ_T_OMP2_ENABLE: str = "alpaka_ACC_CPU_B_SEQ_T_OMP2_ENABLE" +ALPAKA_ACC_GPU_CUDA_ENABLE: str = "alpaka_ACC_GPU_CUDA_ENABLE" +ALPAKA_ACC_GPU_HIP_ENABLE: str = "alpaka_ACC_GPU_HIP_ENABLE" +ALPAKA_ACC_SYCL_ENABLE: str = "alpaka_ACC_SYCL_ENABLE" + +# software dependencies and compiler configurations +UBUNTU: str = "ubuntu" +CMAKE: str = "cmake" +BOOST: str = "boost" +CXX_STANDARD: str = "cxx_standard" diff --git a/bashi/utils.py b/bashi/utils.py index 329d72e..7bb03ec 100644 --- a/bashi/utils.py +++ b/bashi/utils.py @@ -1,10 +1,19 @@ """Different helper functions for bashi""" -from typing import Dict, Tuple, List +from typing import Dict, List, IO from collections import OrderedDict import dataclasses -from packaging.version import Version -from bashi.types import FilterFunction, ParameterValueTuple +import sys +from typeguard import typechecked +from bashi.types import ( + Parameter, + ParameterValue, + ParameterValueTuple, + ParameterValuePair, + ParameterValueMatrix, + CombinationList, + FilterFunction, +) @dataclasses.dataclass @@ -47,10 +56,10 @@ def filter_function(row: OrderedDict[str, Tuple[str, Version]]): function used by allpairspy, see class doc string. """ - param_map: Dict[int, str] + param_map: Dict[int, Parameter] filter_func: FilterFunction - def __call__(self, row: List[Tuple[str, Version]]) -> bool: + def __call__(self, row: List[ParameterValue]) -> bool: """The expected interface of allpairspy filter rule. Transform the type of row from List[Tuple[str, Version]] to [OrderedDict[str, Tuple[str, Version]]]. @@ -65,3 +74,97 @@ def __call__(self, row: List[Tuple[str, Version]]) -> bool: for index, param_name in enumerate(row): ordered_row[self.param_map[index]] = param_name return self.filter_func(ordered_row) + + +@typechecked +def get_expected_parameter_value_pairs( + parameter_matrix: ParameterValueMatrix, +) -> List[ParameterValuePair]: + """Takes parameter-value-matrix an creates a list of all expected parameter-values-pairs. + The pair-wise generator guaranties, that each pair of two parameter-values exist in at least one + combination if no filter rules exist. Therefore the generated the generated list can be used + to verify the output of the pair-wise generator. + + Args: + parameter_matrix (ParameterValueMatrix): matrix of parameter values + + Returns: + List[ParameterValuePair]: list of all possible parameter-value-pairs + """ + expected_pairs: List[ParameterValuePair] = [] + + number_of_keys: int = len(parameter_matrix.keys()) + param_map: Dict[int, str] = {} + for index, param in enumerate(parameter_matrix.keys()): + param_map[index] = param + + for v1_index in range(number_of_keys): + for v2_index in range(v1_index + 1, number_of_keys): + _loop_over_parameter_values( + parameter_matrix, + expected_pairs, + param_map[v1_index], + param_map[v2_index], + ) + + return expected_pairs + + +@typechecked +def _loop_over_parameter_values( + parameters: ParameterValueMatrix, + expected_pairs: List[ParameterValuePair], + v1_parameter: Parameter, + v2_parameter: Parameter, +): + """Creates all parameter-value-pairs for two give parameters. + + Args: + parameters (ParameterValueMatrix): The complete parameter-value-matrix + expected_pairs (List[ParameterValuePair]): Add the generated parameter-values-pairs to the + list. + v1_parameter (Parameter): the first parameter + v2_parameter (Parameter): the second parameter + """ + for v1_name, v1_version in parameters[v1_parameter]: + for v2_name, v2_version in parameters[v2_parameter]: + param_val_pair: ParameterValuePair = OrderedDict() + param_val_pair[v1_parameter] = (v1_name, v1_version) + param_val_pair[v2_parameter] = (v2_name, v2_version) + expected_pairs.append(param_val_pair) + + +def check_parameter_value_pair_in_combination_list( + combination_list: CombinationList, + parameter_value_pairs: List[ParameterValuePair], + output: IO[str] = sys.stdout, +) -> bool: + """Check if all given parameter-values-pairs exist at least in on combination. + + Args: + combination_list (CombinationList): list of given combination + parameter_value_pairs (List[ParameterValuePair]): list of parameter-value-pair to be search + for + output (IO[str], optional): Writes missing parameter-values-pairs to it. Defaults to + sys.stdout. + + Returns: + bool: returns True, if all given parameter-values-pairs was found in the combination-list + """ + missing_expected_param = False + + for ex_param_val_pair in parameter_value_pairs: + param1, param_val1 = list(ex_param_val_pair.items())[0] + param2, param_val2 = list(ex_param_val_pair.items())[1] + found = False + for comb in combination_list: + # comb contains all parameters, therefore a check is not required + if comb[param1] == param_val1 and comb[param2] == param_val2: + found = True + break + + if not found: + print(f"{ex_param_val_pair} is missing in combination list", file=output) + missing_expected_param = True + + return not missing_expected_param diff --git a/tests/test_expected_parameter_value_pairs.py b/tests/test_expected_parameter_value_pairs.py new file mode 100644 index 0000000..1eb0e65 --- /dev/null +++ b/tests/test_expected_parameter_value_pairs.py @@ -0,0 +1,379 @@ +# pylint: disable=missing-docstring +import unittest +from typing import List, Union, Tuple +from collections import OrderedDict +import io +import packaging.version as pkv + +# allpairspy has no type hints +from allpairspy import AllPairs # type: ignore +from bashi.types import ( + Parameter, + ParameterValue, + ValueName, + ParameterValuePair, + ParameterValueMatrix, + Combination, + CombinationList, +) +from bashi.globals import * # pylint: disable=wildcard-import,unused-wildcard-import +from bashi.utils import ( + get_expected_parameter_value_pairs, + check_parameter_value_pair_in_combination_list, +) + + +def parse_param_val(param_val: Tuple[ValueName, Union[str, int, float]]) -> ParameterValue: + val_name, val_version = param_val + return (val_name, pkv.parse(str(val_version))) + + +def parse_param_vals( + param_vals: List[Tuple[ValueName, Union[str, int, float]]] +) -> List[ParameterValue]: + parsed_list: List[ParameterValue] = [] + + for param_val in param_vals: + parsed_list.append(parse_param_val(param_val)) + + return parsed_list + + +def parse_expected_val_pairs( + input_list: List[OrderedDict[Parameter, Tuple[ValueName, Union[str, int, float]]]] +) -> List[ParameterValuePair]: + expected_val_pairs: List[ParameterValuePair] = [] + + for param_val_pair in input_list: + tmp_entry: ParameterValuePair = OrderedDict() + for param in param_val_pair: + tmp_entry[param] = parse_param_val(param_val_pair[param]) + expected_val_pairs.append(tmp_entry) + + return expected_val_pairs + + +class TestExpectedValuePairs(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.param_matrix: ParameterValueMatrix = OrderedDict() + + cls.param_matrix[HOST_COMPILER] = parse_param_vals( + [(GCC, 10), (GCC, 11), (GCC, 12), (CLANG, 16), (CLANG, 17)] + ) + cls.param_matrix[DEVICE_COMPILER] = parse_param_vals( + [(NVCC, 11.2), (NVCC, 12.0), (GCC, 10), (GCC, 11)] + ) + cls.param_matrix[CMAKE] = parse_param_vals([(CMAKE, 3.22), (CMAKE, 3.23)]) + cls.param_matrix[BOOST] = parse_param_vals([(BOOST, 1.81), (BOOST, 1.82), (BOOST, 1.83)]) + + cls.generated_parameter_value_pairs: List[ + ParameterValuePair + ] = get_expected_parameter_value_pairs(cls.param_matrix) + + OD = OrderedDict + + cls.expected_param_val_pairs: List[ParameterValuePair] = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (GCC, 10)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (GCC, 11)}), + OD({HOST_COMPILER: (GCC, 10), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (GCC, 10), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (GCC, 10), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (GCC, 10), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (GCC, 10), BOOST: (BOOST, 1.83)}), + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (GCC, 10)}), + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (GCC, 11)}), + OD({HOST_COMPILER: (GCC, 11), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (GCC, 11), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (GCC, 11), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (GCC, 11), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (GCC, 11), BOOST: (BOOST, 1.83)}), + OD({HOST_COMPILER: (GCC, 12), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 12), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (GCC, 12), DEVICE_COMPILER: (GCC, 10)}), + OD({HOST_COMPILER: (GCC, 12), DEVICE_COMPILER: (GCC, 11)}), + OD({HOST_COMPILER: (GCC, 12), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (GCC, 12), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (GCC, 12), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (GCC, 12), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (GCC, 12), BOOST: (BOOST, 1.83)}), + OD({HOST_COMPILER: (CLANG, 16), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (CLANG, 16), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (CLANG, 16), DEVICE_COMPILER: (GCC, 10)}), + OD({HOST_COMPILER: (CLANG, 16), DEVICE_COMPILER: (GCC, 11)}), + OD({HOST_COMPILER: (CLANG, 16), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (CLANG, 16), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (CLANG, 16), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (CLANG, 16), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (CLANG, 16), BOOST: (BOOST, 1.83)}), + OD({HOST_COMPILER: (CLANG, 17), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (CLANG, 17), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (CLANG, 17), DEVICE_COMPILER: (GCC, 10)}), + OD({HOST_COMPILER: (CLANG, 17), DEVICE_COMPILER: (GCC, 11)}), + OD({HOST_COMPILER: (CLANG, 17), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (CLANG, 17), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (CLANG, 17), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (CLANG, 17), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (CLANG, 17), BOOST: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), CMAKE: (CMAKE, 3.23)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), BOOST: (BOOST, 1.81)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), BOOST: (BOOST, 1.82)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), BOOST: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), CMAKE: (CMAKE, 3.23)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.81)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.82)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (GCC, 10), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (GCC, 10), CMAKE: (CMAKE, 3.23)}), + OD({DEVICE_COMPILER: (GCC, 10), BOOST: (BOOST, 1.81)}), + OD({DEVICE_COMPILER: (GCC, 10), BOOST: (BOOST, 1.82)}), + OD({DEVICE_COMPILER: (GCC, 10), BOOST: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (GCC, 11), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (GCC, 11), CMAKE: (CMAKE, 3.23)}), + OD({DEVICE_COMPILER: (GCC, 11), BOOST: (BOOST, 1.81)}), + OD({DEVICE_COMPILER: (GCC, 11), BOOST: (BOOST, 1.82)}), + OD({DEVICE_COMPILER: (GCC, 11), BOOST: (BOOST, 1.83)}), + OD({CMAKE: (CMAKE, 3.22), BOOST: (BOOST, 1.81)}), + OD({CMAKE: (CMAKE, 3.22), BOOST: (BOOST, 1.82)}), + OD({CMAKE: (CMAKE, 3.22), BOOST: (BOOST, 1.83)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.81)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.82)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ) + + ppv = parse_param_val + cls.handwritten_comb_list: CombinationList = [ + OD( + { + HOST_COMPILER: ppv((GCC, 10)), + DEVICE_COMPILER: ppv((NVCC, 11.2)), + CMAKE: ppv((CMAKE, "3.22")), + BOOST: ppv((BOOST, "1.81")), + } + ), + OD( + { + HOST_COMPILER: ppv((GCC, 10)), + DEVICE_COMPILER: ppv((NVCC, 12.0)), + CMAKE: ppv((CMAKE, "3.22")), + BOOST: ppv((BOOST, "1.82")), + } + ), + OD( + { + HOST_COMPILER: ppv((CLANG, 16)), + DEVICE_COMPILER: ppv((CLANG, 16)), + CMAKE: ppv((CMAKE, "3.23")), + BOOST: ppv((BOOST, "1.83")), + } + ), + ] + + cls.handwritten_all_existing_pairs: List[ParameterValuePair] = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (CLANG, 16), DEVICE_COMPILER: (CLANG, 16)}), + OD({HOST_COMPILER: (GCC, 10), CMAKE: (CMAKE, 3.22)}), + OD({HOST_COMPILER: (CLANG, 16), CMAKE: (CMAKE, 3.23)}), + OD({HOST_COMPILER: (GCC, 10), BOOST: (BOOST, 1.81)}), + OD({HOST_COMPILER: (GCC, 10), BOOST: (BOOST, 1.82)}), + OD({HOST_COMPILER: (CLANG, 16), BOOST: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), CMAKE: (CMAKE, 3.22)}), + OD({DEVICE_COMPILER: (CLANG, 16), CMAKE: (CMAKE, 3.23)}), + OD({DEVICE_COMPILER: (NVCC, 11.2), BOOST: (BOOST, 1.81)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.82)}), + OD({DEVICE_COMPILER: (CLANG, 16), BOOST: (BOOST, 1.83)}), + OD({CMAKE: (CMAKE, 3.22), BOOST: (BOOST, 1.81)}), + OD({CMAKE: (CMAKE, 3.22), BOOST: (BOOST, 1.82)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ) + + def test_expected_value_pairs(self): + self.assertTrue( + len(self.expected_param_val_pairs) == len(self.generated_parameter_value_pairs), + f"\nlen(expected_param_val_pairs): {len(self.expected_param_val_pairs)}\n" + f"len(generated_parameter_value_pairs): {len(self.generated_parameter_value_pairs)}", + ) + + missing_expected_param = False + for ex_param_val_pair in self.expected_param_val_pairs: + try: + self.assertTrue(ex_param_val_pair in self.generated_parameter_value_pairs) + except AssertionError: + missing_expected_param = True + print( + f"{ex_param_val_pair} was not found in the generated parameter-value-pair list" + ) + + self.assertFalse(missing_expected_param) + + missing_generated_param = False + for gen_param_val_pair in self.generated_parameter_value_pairs: + try: + self.assertTrue(gen_param_val_pair in self.expected_param_val_pairs) + except AssertionError: + missing_generated_param = True + print( + f"{gen_param_val_pair} was not found in the expected parameter-value-pair list" + ) + + self.assertFalse(missing_generated_param) + + def test_check_parameter_value_pair_in_combination_list_empty_input(self): + self.assertTrue( + check_parameter_value_pair_in_combination_list(self.handwritten_comb_list, []) + ) + + def test_check_parameter_value_pair_in_combination_list_less_valid_input(self): + OD = OrderedDict + + # all pairs exists in the combination list, but not all pairs are tested + self.assertTrue( + check_parameter_value_pair_in_combination_list( + self.handwritten_comb_list, + parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ), + ) + ) + + def test_check_parameter_value_pair_in_combination_list_complete_valid_input(self): + # test all existing pairs + self.assertTrue( + check_parameter_value_pair_in_combination_list( + self.handwritten_comb_list, self.handwritten_all_existing_pairs + ) + ) + + def test_check_parameter_value_pair_in_combination_list_single_wrong_input(self): + OD = OrderedDict + + single_wrong_pair = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (NVCC, 11.2)}), + ] + ) + + output_wrong_single_pair = io.StringIO() + self.assertFalse( + check_parameter_value_pair_in_combination_list( + self.handwritten_comb_list, + single_wrong_pair, + output_wrong_single_pair, + ) + ) + output_wrong_single_pair_expected_str = ( + str(single_wrong_pair[0]) + " is missing in combination list" + ) + self.assertTrue( + output_wrong_single_pair.getvalue().rstrip() == output_wrong_single_pair_expected_str, + ( + f"\nGet: {output_wrong_single_pair.getvalue().rstrip()}\n" + f"Expected: {output_wrong_single_pair_expected_str}" + ), + ) + + def test_check_parameter_value_pair_in_combination_list_many_wrong_input(self): + OD = OrderedDict + + many_wrong_pairs = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({CMAKE: (CMAKE, 2.23), DEVICE_COMPILER: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.84)}), + ] + ) + + expected_output_many_wrong_pairs_list: List[str] = [] + for pair in many_wrong_pairs: + expected_output_many_wrong_pairs_list.append( + str(pair) + " is missing in combination list" + ) + + expected_output_many_wrong_pairs_list.sort() + + output_wrong_many_pairs = io.StringIO() + self.assertFalse( + check_parameter_value_pair_in_combination_list( + self.handwritten_comb_list, + many_wrong_pairs, + output_wrong_many_pairs, + ) + ) + + output_wrong_many_pairs_list: List[str] = ( + output_wrong_many_pairs.getvalue().rstrip().split("\n") + ) + output_wrong_many_pairs_list.sort() + + self.assertEqual(len(output_wrong_many_pairs_list), 3) + + self.assertEqual(output_wrong_many_pairs_list, expected_output_many_wrong_pairs_list) + + def test_check_parameter_value_pair_in_combination_list_complete_list_plus_wrong_input(self): + OD = OrderedDict + + many_wrong_pairs = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 11), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({CMAKE: (CMAKE, 2.23), DEVICE_COMPILER: (BOOST, 1.83)}), + OD({DEVICE_COMPILER: (NVCC, 12.0), BOOST: (BOOST, 1.84)}), + ] + ) + + all_pairs_plus_wrong_input = self.handwritten_all_existing_pairs + many_wrong_pairs + + expected_output_many_wrong_pairs_list: List[str] = [] + for pair in many_wrong_pairs: + expected_output_many_wrong_pairs_list.append( + str(pair) + " is missing in combination list" + ) + + expected_output_many_wrong_pairs_list.sort() + + output_wrong_many_pairs = io.StringIO() + self.assertFalse( + check_parameter_value_pair_in_combination_list( + self.handwritten_comb_list, + all_pairs_plus_wrong_input, + output_wrong_many_pairs, + ) + ) + + output_wrong_many_pairs_list: List[str] = ( + output_wrong_many_pairs.getvalue().rstrip().split("\n") + ) + output_wrong_many_pairs_list.sort() + + self.assertEqual(len(output_wrong_many_pairs_list), 3) + + self.assertEqual(output_wrong_many_pairs_list, expected_output_many_wrong_pairs_list) + + def test_unrestricted_allpairspy_generator(self): + comb_list: CombinationList = [] + # pylance shows a warning, because it cannot determine the concrete type of a namedtuple, + # which is returned by AllPairs + for all_pair in AllPairs(parameters=self.param_matrix): # type: ignore + comb: Combination = OrderedDict() + for index, param in enumerate(all_pair._fields): # type: ignore + comb[param] = all_pair[index] # type: ignore + comb_list.append(comb) + + self.assertTrue( + check_parameter_value_pair_in_combination_list(comb_list, self.expected_param_val_pairs) + )