From 549042811721801befb76ddc205fb6f83b45f3ce Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Wed, 22 Jan 2025 14:23:44 +0800 Subject: [PATCH 1/2] feat(junit): record app_path in JUnit reports --- pytest-embedded-idf/tests/test_idf.py | 67 +++++++++++++++++++++++ pytest-embedded/pytest_embedded/plugin.py | 57 +++++++++++++++++-- pytest-embedded/pytest_embedded/unity.py | 8 +++ pytest-embedded/tests/test_base.py | 43 +++++++++++++++ 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/pytest-embedded-idf/tests/test_idf.py b/pytest-embedded-idf/tests/test_idf.py index e14c239a..acf08787 100644 --- a/pytest-embedded-idf/tests/test_idf.py +++ b/pytest-embedded-idf/tests/test_idf.py @@ -988,6 +988,73 @@ def test_python_case(dut): for testcase in junit_report[1:]: assert testcase.attrib['is_unity_case'] == '1' # Other test cases + +def test_app_path_in_junit_multi_dut_app(testdir): + testdir.makepyfile(""" + import pytest + + def test_app_path_in_junit_multi_dut_app(app, dut): + assert len(app[0].flash_files) == 3 + assert app[0].target == 'esp32' + + assert len(app[1].flash_files) == 3 + assert app[1].target == 'esp32c3' + + assert getattr(dut[0], 'serial') + with pytest.raises(AttributeError): + assert getattr(dut[1], 'serial') + """) + + testdir.runpytest( + '-s', + '--count', + 2, + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}|{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', + '--embedded-services', + 'esp,idf|idf', + '--junitxml', + 'report.xml', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + testcases = junit_report.findall('.//testcase') + + assert 'app_path' in testcases[0].attrib + assert 'hello_world_esp32' in testcases[0].attrib['app_path'] + assert 'hello_world_esp32c3' in testcases[0].attrib['app_path'] + + +def test_app_path_in_junit_single_dut_app(testdir): + testdir.makepyfile(""" + import pytest + + def test_app_path_in_junit_single_dut_app(app, dut): + assert len(app.flash_files) == 3 + assert app.target == 'esp32c3' + + with pytest.raises(AttributeError): + assert getattr(dut, 'serial') + """) + + testdir.runpytest( + '-s', + '--embedded-services', + 'idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32c3'), + '--junitxml', + 'report.xml', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + testcases = junit_report.findall('.//testcase') + + assert 'app_path' in testcases[0].attrib + # path is relative to pytest root dir (testdir.tmpdir) + assert 'hello_world_esp32c3' == testcases[0].attrib['app_path'] + + def test_esp_bool_parser_returned_values(testdir, copy_mock_esp_idf, monkeypatch): # noqa: ARG001 monkeypatch.setenv('IDF_PATH', str(testdir)) from esp_bool_parser import SOC_HEADERS, SUPPORTED_TARGETS diff --git a/pytest-embedded/pytest_embedded/plugin.py b/pytest-embedded/pytest_embedded/plugin.py index 1c568352..095330b3 100644 --- a/pytest-embedded/pytest_embedded/plugin.py +++ b/pytest-embedded/pytest_embedded/plugin.py @@ -14,6 +14,7 @@ import typing as t import warnings import xml.dom.minidom +import xml.etree.ElementTree as ET from collections import Counter from operator import itemgetter @@ -1167,12 +1168,33 @@ def dut( serial: t.Optional[t.Union['Serial', 'LinuxSerial']], qemu: t.Optional['Qemu'], wokwi: t.Optional['WokwiCLI'], + request: FixtureRequest, ) -> t.Union[Dut, t.List[Dut]]: """ A device under test (DUT) object that could gather output from various sources and redirect them to the pexpect process, and run `expect()` via its pexpect process. """ - return dut_gn(**locals()) + dut_instance = dut_gn( + _fixture_classes_and_options=_fixture_classes_and_options, + openocd=openocd, + gdb=gdb, + app=app, + serial=serial, + qemu=qemu, + wokwi=wokwi, + ) + + if _testcase_app_paths_map_key not in request.config.stash: + request.config.stash[_testcase_app_paths_map_key] = {} + + stash = request.config.stash[_testcase_app_paths_map_key] + + if dut_instance.test_case_name not in stash: + stash[dut_instance.test_case_name] = set() + + stash[dut_instance.test_case_name].add(dut_instance.app.app_path) + + return dut_instance @pytest.fixture @@ -1197,6 +1219,7 @@ def unity_tester(dut: t.Union['IdfDut', t.Tuple['IdfDut']]) -> t.Optional['CaseT _pytest_embedded_key = pytest.StashKey['PytestEmbedded']() _session_tempdir_key = pytest.StashKey['session_tempdir']() _junit_report_path_key = pytest.StashKey[str]() +_testcase_app_paths_map_key = pytest.StashKey[dict]() def pytest_configure(config: Config) -> None: @@ -1273,6 +1296,29 @@ def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: return item.callspec.params.get(key, default) or default + @staticmethod + def add_app_paths_to_testcases(report_path: str, testcase_app_paths_map: dict, rootdir: str) -> None: + with open(report_path, 'r+') as file: + tree = ET.parse(file) + root = tree.getroot() + + for testcase in root.findall('.//testcase'): + pytest_testcase_name = testcase.get('pytest_case_name') + if pytest_testcase_name in testcase_app_paths_map: + # Map the app paths as relative paths and add to the 'app_path' attribute + relative_paths = { + os.path.relpath(path, rootdir) for path in testcase_app_paths_map[pytest_testcase_name] + } + testcase.set('app_path', ';'.join(relative_paths)) + + # Remove the 'pytest_case_name' attribute used for app_path mapping purposes + if 'pytest_case_name' in testcase.attrib: + testcase.attrib.pop('pytest_case_name') + + file.seek(0) + file.truncate() + file.write(escape_illegal_xml_chars(ET.tostring(root, encoding='unicode'))) + @pytest.hookimpl(hookwrapper=True, trylast=True) def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]): # ------ add marker based on target ------ @@ -1382,10 +1428,11 @@ def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: if _stash_junit_report_path: # before we only modified the junit report generated by the unity test cases # now we do it again to check the python test cases - with open(_stash_junit_report_path) as fr: - file_str = fr.read() - with open(_stash_junit_report_path, 'w') as fw: - fw.write(escape_illegal_xml_chars(file_str)) + self.add_app_paths_to_testcases( + _stash_junit_report_path, + testcase_app_paths_map=session.config.stash.get(_testcase_app_paths_map_key, {}), + rootdir=session.config.rootdir, + ) if self.prettify_junit_report: _prettify_xml(_stash_junit_report_path) diff --git a/pytest-embedded/pytest_embedded/unity.py b/pytest-embedded/pytest_embedded/unity.py index 76f16e2d..42d31e57 100644 --- a/pytest-embedded/pytest_embedded/unity.py +++ b/pytest-embedded/pytest_embedded/unity.py @@ -299,11 +299,13 @@ def merge(self, junit_files: List[str]): junit_case_is_fail = junit_case.find('failure') is not None junit_case.attrib['is_unity_case'] = '0' + junit_case.attrib['pytest_case_name'] = junit_case.attrib.get('name') if self.unity_test_report_mode == UnityTestReportMode.REPLACE.value: junit_parent.remove(junit_case) for case in merging_cases: case.attrib['is_unity_case'] = '1' + case.attrib['pytest_case_name'] = junit_case.attrib.get('name') junit_parent.append(case) junit_parent.attrib['errors'] = self._int_add( @@ -324,5 +326,11 @@ def merge(self, junit_files: List[str]): if int(junit_parent.attrib['failures']) > 0: self.failed = True + for testcase in self.junit.findall('.//testcase'): + # Add the 'pytest_case_name' attribute for each test case to enable later extraction + # of the app_path from the pytest_case_name to app_path mapping + if 'pytest_case_name' not in testcase.attrib: + testcase.attrib['pytest_case_name'] = testcase.attrib.get('name') + self.junit.write(self.junit_path) logging.debug(f'Merged junit report dumped to {os.path.realpath(self.junit_path)}') diff --git a/pytest-embedded/tests/test_base.py b/pytest-embedded/tests/test_base.py index d2db85ce..7099cd17 100644 --- a/pytest-embedded/tests/test_base.py +++ b/pytest-embedded/tests/test_base.py @@ -652,6 +652,49 @@ def test_unclosed_file_handler(test_input, dut): result.assert_outcomes(passed=1024) +def test_app_paths_in_junit_report(testdir): + testdir.makepyfile(r""" + import pytest + import inspect + + output = inspect.cleandoc( + ''' + TEST(group, test_case)foo.c:100::FAIL:Expected 2 was 1 + TEST(group, test_case_2)foo.c:101::FAIL:Expected 1 was 2 + TEST(group, test case 3)foo bar.c:102::PASS + TEST(group, test case 4)foo bar.c:103::FAIL:Expected 3 was 4 + ------------------- + 4 Tests 3 Failures 0 Ignored + FAIL + ''') + + @pytest.mark.parametrize('count', [2], indirect=True) + def test_expect_unity_test_output_multi_dut(dut): + dut_0 = dut[0] + dut_1 = dut[1] + + dut_0.write(output) + dut_1.write(output) + dut_0.expect_unity_test_output() + dut_1.expect_unity_test_output() + + @pytest.mark.parametrize('count', [2], indirect=True) + def test_expect_unity_test_output_multi_dut_record_1(dut): + dut_1 = dut[1] + dut_1.write(output) + dut_1.expect_unity_test_output() + """) + + testdir.runpytest('--junitxml', 'report.xml') + + root = ET.parse('report.xml').getroot() + testcases = root.findall('.//testcase') + app_paths = [testcase.get('app_path') for testcase in testcases] + + # should be relative to the pytest root folder + assert app_paths == ['.'] * len(testcases) + + class TestTargetMarkers: def test_add_target_as_marker_simple(self, pytester): pytester.makepyfile(""" From 4191a35517ffa69f615ea60383310897c5289db0 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Thu, 23 Jan 2025 13:57:03 +0800 Subject: [PATCH 2/2] chore: format code with ruff --- pytest-embedded-idf/tests/test_idf.py | 479 ++++++++++++++++---------- 1 file changed, 289 insertions(+), 190 deletions(-) diff --git a/pytest-embedded-idf/tests/test_idf.py b/pytest-embedded-idf/tests/test_idf.py index acf08787..d90f351f 100644 --- a/pytest-embedded-idf/tests/test_idf.py +++ b/pytest-embedded-idf/tests/test_idf.py @@ -28,8 +28,10 @@ def test_idf_serial_flash(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), ) result.assert_outcomes(passed=1) @@ -46,8 +48,10 @@ def test_idf_serial_flash(dut): """) result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), '--esp-flash-force', ) @@ -65,8 +69,10 @@ def test_idf_serial_flash(dut): """) result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), ) result.assert_outcomes(passed=1) @@ -94,8 +100,10 @@ def test_no_matching_word_pass_rest(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), ) result.assert_outcomes(passed=2, failed=2) @@ -122,8 +130,10 @@ def test_no_matching_word_pass_rest(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), ) result.assert_outcomes(passed=2, failed=2) @@ -164,9 +174,12 @@ def test_idf_run_all_single_board_cases(): result = testdir.runpytest( '-s', - '--app-path', p, - '--embedded-services', 'esp,idf', - '--junitxml', 'report.xml', + '--app-path', + p, + '--embedded-services', + 'esp,idf', + '--junitxml', + 'report.xml', ) result.assert_outcomes(passed=4, errors=0) @@ -194,10 +207,14 @@ def test_idf_serial_flash_with_erase_nvs(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), - '--erase-nvs', 'y', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--erase-nvs', + 'y', ) result.assert_outcomes(passed=1) @@ -219,9 +236,12 @@ def test_idf_serial_flash(dut): """) result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), - '--erase-nvs', 'y', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--erase-nvs', + 'y', ) result.assert_outcomes(errors=1) @@ -242,8 +262,10 @@ def test_idf_app(app, dut): result = testdir.runpytest( '-s', - '--embedded-services', 'idf', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32c3'), + '--embedded-services', + 'idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32c3'), ) result.assert_outcomes(passed=1) @@ -267,11 +289,12 @@ def test_multi_dut_app(app, dut): result = testdir.runpytest( '-s', - '--count', 2, - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}' - f'|' - f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', - '--embedded-services', 'esp,idf|idf', + '--count', + 2, + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}|{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', + '--embedded-services', + 'esp,idf|idf', ) result.assert_outcomes(passed=1) @@ -292,13 +315,16 @@ def test_multi_dut_autoflash(app, dut): result = testdir.runpytest( '-s', - '--count', 2, - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}' - f'|' - f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', - '--skip-autoflash', 'y|false', - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--count', + 2, + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}|{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', + '--skip-autoflash', + 'y|false', + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), ) result.assert_outcomes(passed=1) @@ -319,10 +345,14 @@ def test_autoflash_again(app, dut): result = testdir.runpytest( '-s', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), - '--log-cli-level', 'DEBUG', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--log-cli-level', + 'DEBUG', ) result.assert_outcomes(passed=2) @@ -334,7 +364,7 @@ def test_autoflash_again(app, dut): first_index_of_messages( re.compile(r'^hit port-app cache:.+hello_world_esp32[\\/]build$', re.MULTILINE), caplog.messages, - set_app_cache_i + 1 + set_app_cache_i + 1, ) @@ -353,11 +383,16 @@ def test_autoflash_again(app, dut): result = testdir.runpytest( '-s', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), - '--log-cli-level', 'DEBUG', - '--confirm-target-elf-sha256', 'y', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--log-cli-level', + 'DEBUG', + '--confirm-target-elf-sha256', + 'y', ) result.assert_outcomes(passed=2) @@ -369,7 +404,7 @@ def test_autoflash_again(app, dut): hit_app_cache_i = first_index_of_messages( re.compile(r'^hit port-app cache:.+hello_world_esp32[\\/]build$', re.MULTILINE), caplog.messages, - set_app_cache_i + 1 + set_app_cache_i + 1, ) first_index_of_messages( re.compile(r'Confirmed target elf file sha256 the same as your local one\.$', re.MULTILINE), @@ -379,8 +414,10 @@ def test_autoflash_again(app, dut): def test_different_build_dir(testdir): - os.rename(os.path.join(testdir.tmpdir, 'hello_world_esp32', 'build'), - os.path.join(testdir.tmpdir, 'hello_world_esp32', 'test_new_name')) + os.rename( + os.path.join(testdir.tmpdir, 'hello_world_esp32', 'build'), + os.path.join(testdir.tmpdir, 'hello_world_esp32', 'test_new_name'), + ) testdir.makepyfile(""" import pytest @@ -392,18 +429,24 @@ def test_multi_dut_app(app, dut): result = testdir.runpytest( '-s', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), - '--build-dir', 'test_new_name', - '--embedded-services', 'idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--build-dir', + 'test_new_name', + '--embedded-services', + 'idf', ) result.assert_outcomes(passed=1) result = testdir.runpytest( '-s', - '--app-path', os.path.join(testdir.tmpdir, 'hello_world_esp32'), - '--build-dir', os.path.join(testdir.tmpdir, 'hello_world_esp32', 'test_new_name'), - '--embedded-services', 'idf', + '--app-path', + os.path.join(testdir.tmpdir, 'hello_world_esp32'), + '--build-dir', + os.path.join(testdir.tmpdir, 'hello_world_esp32', 'test_new_name'), + '--embedded-services', + 'idf', ) result.assert_outcomes(passed=1) @@ -430,19 +473,22 @@ def test_multi_dut_read_flash(app, serial, dut): result = testdir.runpytest( '-s', - '--count', 2, - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}' - f'|' - f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--count', + 2, + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}|{os.path.join(testdir.tmpdir, "hello_world_esp32c3")}', + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), ) result.assert_outcomes(passed=1) def test_flash_another_app(testdir): - testdir.makepyfile(r""" + testdir.makepyfile( + r""" import pytest import pexpect @@ -452,13 +498,17 @@ def test_flash_another_app(dut): dut.serial.flash(IdfApp('{}')) dut.expect('Hash of data verified.', timeout=5) dut.expect_exact('Hello world!', timeout=5) - """.format(os.path.join(testdir.tmpdir, 'hello_world_esp32'))) + """.format(os.path.join(testdir.tmpdir, 'hello_world_esp32')) + ) result = testdir.runpytest( '-s', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), ) result.assert_outcomes(passed=1) @@ -478,9 +528,12 @@ def test_flash_with_no_elf_file(dut): result = testdir.runpytest( '-s', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--embedded-services', 'esp,idf', - '--part-tool', os.path.join(testdir.tmpdir, 'gen_esp32part.py'), + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--embedded-services', + 'esp,idf', + '--part-tool', + os.path.join(testdir.tmpdir, 'gen_esp32part.py'), ) result.assert_outcomes(passed=1) @@ -497,10 +550,14 @@ def test_detect_port(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--target', 'esp32', - '--erase-all', 'y', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--target', + 'esp32', + '--erase-all', + 'y', ) result.assert_outcomes(passed=1) @@ -517,9 +574,12 @@ def test_detect_port(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--target', 'esp32', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--target', + 'esp32', ) result.assert_outcomes(passed=1) @@ -540,9 +600,12 @@ def test_hello_world_linux(dut): """) result = testdir.runpytest( '-s', - '--embedded-services', 'idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_linux")}', - '--target', 'linux', + '--embedded-services', + 'idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_linux")}', + '--target', + 'linux', ) result.assert_outcomes(passed=1) @@ -554,15 +617,18 @@ def test_unity_tester_with_linux(testdir): def test_unity_tester_with_linux(dut): dut.run_all_single_board_cases() - """ - ) + """) result = testdir.runpytest( '-s', - '--embedded-services', 'idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "unit_test_app_linux")}', - '--target', 'linux', - '--junitxml', 'report.xml', + '--embedded-services', + 'idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "unit_test_app_linux")}', + '--target', + 'linux', + '--junitxml', + 'report.xml', ) result.assert_outcomes(passed=1) @@ -587,11 +653,16 @@ def test_check_coredump(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3_panic")}', - '--target', 'esp32c3', - '--panic-output-decode-script', os.path.join(testdir.tmpdir, 'gdb_panic_server.py'), - '--log-cli-level', 'INFO', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3_panic")}', + '--target', + 'esp32c3', + '--panic-output-decode-script', + os.path.join(testdir.tmpdir, 'gdb_panic_server.py'), + '--log-cli-level', + 'INFO', ) first_index_of_messages( re.compile(r'app_main \(\) at /COMPONENT_MAIN_DIR/hello_world_main.c:17', re.MULTILINE), @@ -613,11 +684,16 @@ def test_skip_check_coredump(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3_panic")}', - '--panic-output-decode-script', os.path.join(testdir.tmpdir, 'gdb_panic_server.py'), - '--skip-check-coredump', 'True', - '--log-cli-level', 'INFO', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32c3_panic")}', + '--panic-output-decode-script', + os.path.join(testdir.tmpdir, 'gdb_panic_server.py'), + '--skip-check-coredump', + 'True', + '--log-cli-level', + 'INFO', ) with pytest.raises(AssertionError): first_index_of_messages( @@ -629,14 +705,14 @@ def test_skip_check_coredump(dut): def test_idf_parse_test_menu(): - s = '''(1)\t"adc1 and i2s work with wifi" [adc][ignore] + s = """(1)\t"adc1 and i2s work with wifi" [adc][ignore] (2)\t"I2C master write slave test" [i2c][test_env=UT_T2_I2C][timeout=150][multi_device] \t(1)\t"i2c_master_write_test" \t(2)\t"i2c_slave_read_test" (3)\t"LEDC continue work after software reset" [ledc][multi_stage] \t(1)\t"ledc_cpu_reset_test_first_stage" \t(2)\t"ledc_cpu_reset_test_second_stage" -''' +""" test_menu = IdfDut._parse_unity_menu_from_str(s) assert len(test_menu) == 3 @@ -674,9 +750,12 @@ def test_idf_hard_reset_and_expect(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--log-cli-level', 'DEBUG', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--log-cli-level', + 'DEBUG', ) result.assert_outcomes(passed=1) @@ -685,65 +764,37 @@ def test_idf_hard_reset_and_expect(dut): def test_select_to_run(): from pytest_embedded_idf.unity_tester import IdfUnityDutMixin - assert IdfUnityDutMixin._select_to_run( - None, None, None, - None, None, None - ) + assert IdfUnityDutMixin._select_to_run(None, None, None, None, None, None) - assert IdfUnityDutMixin._select_to_run( - None, ['name_hello', 'name_world'], None, - None, 'name_hello', None - ) + assert IdfUnityDutMixin._select_to_run(None, ['name_hello', 'name_world'], None, None, 'name_hello', None) - assert not IdfUnityDutMixin._select_to_run( - None, ['name_hello', 'name_world'], None, - None, 'name_hel', None - ) + assert not IdfUnityDutMixin._select_to_run(None, ['name_hello', 'name_world'], None, None, 'name_hel', None) - assert IdfUnityDutMixin._select_to_run( - None, None, {"red": 255}, - None, None, {"red": 255, "green": 10, "blue": 33} - ) + assert IdfUnityDutMixin._select_to_run(None, None, {'red': 255}, None, None, {'red': 255, 'green': 10, 'blue': 33}) assert not IdfUnityDutMixin._select_to_run( - None, None, {"red": 25}, - None, None, {"red": 255, "green": 10, "blue": 33} + None, None, {'red': 25}, None, None, {'red': 255, 'green': 10, 'blue': 33} ) assert IdfUnityDutMixin._select_to_run( - None, None, {"red": 255, "green": 10}, - None, None, {"red": 255, "green": 10, "blue": 33} + None, None, {'red': 255, 'green': 10}, None, None, {'red': 255, 'green': 10, 'blue': 33} ) assert not IdfUnityDutMixin._select_to_run( - None, None, {"red": 255, "green": 0}, - None, None, {"red": 255, "green": 10, "blue": 33} + None, None, {'red': 255, 'green': 0}, None, None, {'red': 255, 'green': 10, 'blue': 33} ) - assert IdfUnityDutMixin._select_to_run( - [['hello']], None, None, - ['hello', 'world'], None, None - ) + assert IdfUnityDutMixin._select_to_run([['hello']], None, None, ['hello', 'world'], None, None) - assert not IdfUnityDutMixin._select_to_run( - [['!hello']], None, None, - ['hello', 'world'], None, None - ) + assert not IdfUnityDutMixin._select_to_run([['!hello']], None, None, ['hello', 'world'], None, None) - assert not IdfUnityDutMixin._select_to_run( - [['hello', '!world']], None, None, - ['hello', 'world'], None, None - ) + assert not IdfUnityDutMixin._select_to_run([['hello', '!world']], None, None, ['hello', 'world'], None, None) assert IdfUnityDutMixin._select_to_run( - [['hello', '!world'], ['sun']], None, None, - ['hello', 'world', 'sun'], None, None + [['hello', '!world'], ['sun']], None, None, ['hello', 'world', 'sun'], None, None ) - assert IdfUnityDutMixin._select_to_run( - [['hello', '!w']], None, None, - ['hello', 'world'], None, None - ) + assert IdfUnityDutMixin._select_to_run([['hello', '!w']], None, None, ['hello', 'world'], None, None) def test_dut_run_all_single_board_cases(testdir): @@ -753,10 +804,14 @@ def test_dut_run_all_single_board_cases(dut): """) testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32c3'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32c3'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -787,10 +842,14 @@ def test_dut_run_all_single_board_cases(dut): """) testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -808,10 +867,14 @@ def test_dut_run_all_single_board_cases(dut): """) testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -829,10 +892,14 @@ def test_dut_run_all_single_board_cases(dut): """) testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -851,13 +918,18 @@ def test_unity_test_case_runner(unity_tester): testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--count', 2, - '--app-path', f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32")}' - f'|' - f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32c3")}', - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml' + '--embedded-services', + 'esp,idf', + '--count', + 2, + '--app-path', + f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32")}' + f'|' + f'{os.path.join(testdir.tmpdir, "unit_test_app_esp32c3")}', + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -890,10 +962,14 @@ def test_erase_all_with_port_cache_case2(dut): result = testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', - '--target', 'esp32', - '--erase-all', 'y', + '--embedded-services', + 'esp,idf', + '--app-path', + f'{os.path.join(testdir.tmpdir, "hello_world_esp32")}', + '--target', + 'esp32', + '--erase-all', + 'y', ) result.assert_outcomes(passed=2) @@ -907,10 +983,14 @@ def test_python_case(dut): testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -919,6 +999,7 @@ def test_python_case(dut): for testcase in junit_report.findall('testcase'): assert testcase.attrib['is_unity_case'] == '1' + def test_preserve_python_tests(testdir): testdir.makepyfile(r""" def test_python_case(dut): @@ -927,11 +1008,16 @@ def test_python_case(dut): testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', - '--unity-test-report-mode', 'merge', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', + '--unity-test-report-mode', + 'merge', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -950,11 +1036,16 @@ def test_python_case(dut): testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', - '--unity-test-report-mode', 'merge', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', + '--unity-test-report-mode', + 'merge', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -975,11 +1066,16 @@ def test_python_case(dut): testdir.runpytest( '-s', - '--embedded-services', 'esp,idf', - '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), - '--log-cli-level', 'DEBUG', - '--junitxml', 'report.xml', - '--unity-test-report-mode', 'merge', + '--embedded-services', + 'esp,idf', + '--app-path', + os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', + 'DEBUG', + '--junitxml', + 'report.xml', + '--unity-test-report-mode', + 'merge', ) junit_report = ET.parse('report.xml').getroot()[0] @@ -1055,9 +1151,10 @@ def test_app_path_in_junit_single_dut_app(app, dut): assert 'hello_world_esp32c3' == testcases[0].attrib['app_path'] -def test_esp_bool_parser_returned_values(testdir, copy_mock_esp_idf, monkeypatch): # noqa: ARG001 +def test_esp_bool_parser_returned_values(testdir, copy_mock_esp_idf, monkeypatch): # noqa: ARG001 monkeypatch.setenv('IDF_PATH', str(testdir)) from esp_bool_parser import SOC_HEADERS, SUPPORTED_TARGETS + assert SOC_HEADERS == { 'esp32': {'SOC_A': 0, 'SOC_B': 1, 'SOC_C': 0}, 'esp32s2': {'SOC_A': 0, 'SOC_B': 0, 'SOC_C': 0}, @@ -1070,12 +1167,12 @@ def test_esp_bool_parser_returned_values(testdir, copy_mock_esp_idf, monkeypatch 'linux': {}, 'esp32c5': {'SOC_A': 1, 'SOC_B': 1, 'SOC_C': 0}, 'esp32c61': {'SOC_A': 0, 'SOC_B': 0, 'SOC_C': 1}, - 'esp32h21': {'SOC_A': 0, 'SOC_B': 0, 'SOC_C': 0} + 'esp32h21': {'SOC_A': 0, 'SOC_B': 0, 'SOC_C': 0}, } assert SUPPORTED_TARGETS == ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6', 'esp32h2', 'esp32p4'] -def test_skip_if_soc(testdir, copy_mock_esp_idf, monkeypatch): # noqa: ARG001 +def test_skip_if_soc(testdir, copy_mock_esp_idf, monkeypatch): # noqa: ARG001 monkeypatch.setenv('IDF_PATH', str(testdir)) from esp_bool_parser import SOC_HEADERS, SUPPORTED_TARGETS @@ -1095,13 +1192,15 @@ def test_skip_if_for_condition(): result = testdir.runpytest('-s', '--embedded-services', 'esp,idf') result.assert_outcomes(passed=to_pass, skipped=to_skip) - for c, cf in [ ('SOC_A == 1', lambda h: h['SOC_A'] == 1), ('SOC_A == 1 or SOC_B == 1', lambda h: h['SOC_A'] == 1 or h['SOC_B'] == 1), ('SOC_A == 1 and SOC_B == 1', lambda h: h['SOC_A'] == 1 and h['SOC_B'] == 1), ('SOC_A == 1 or SOC_B == 1 and SOC_C == 1', lambda h: h['SOC_A'] == 1 or (h['SOC_B'] == 1 and h['SOC_C'] == 1)), - ('SOC_A == 1 and SOC_B == 0 or SOC_C == 1 ', lambda h: (h['SOC_A'] == 1 and h['SOC_B'] == 0) or h['SOC_C'] == 1), # noqa: E501 + ( + 'SOC_A == 1 and SOC_B == 0 or SOC_C == 1 ', + lambda h: (h['SOC_A'] == 1 and h['SOC_B'] == 0) or h['SOC_C'] == 1, + ), ]: run_test_for_condition(c, cf) @@ -1111,7 +1210,7 @@ def test_skip_if_soc_target_in_args(testdir, copy_mock_esp_idf, monkeypatch): # def run_pytest_with_target(target): count = len(target.split('|')) - return testdir.runpytest( '--embedded-services', 'esp,idf', '--target', target, '--count', count) + return testdir.runpytest('--embedded-services', 'esp,idf', '--target', target, '--count', count) testdir.makepyfile(""" import pytest