From c87ca714c228087451220d05c2f1f558b344c3bd Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Fri, 12 Jan 2024 14:48:02 +0100 Subject: [PATCH 1/4] edit --- .github/workflows/compatibility_check.yml | 12 +- requirements.txt | 5 + test/expected.xml | 205 ++++++++++++++++++++++ test/test.py | 87 +++++++++ 4 files changed, 301 insertions(+), 8 deletions(-) create mode 100644 requirements.txt create mode 100644 test/expected.xml create mode 100644 test/test.py diff --git a/.github/workflows/compatibility_check.yml b/.github/workflows/compatibility_check.yml index 06e4297..acbe8d3 100644 --- a/.github/workflows/compatibility_check.yml +++ b/.github/workflows/compatibility_check.yml @@ -19,15 +19,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.12 - - name: Install DSP-TOOLS - run: pip3 install dsp-tools - - name: run the import script - run: python3 import_script.py + - name: Install dependencies + run: pip3 install -r requirements.txt - name: start stack run: dsp-tools start-stack --no-prune - - name: create data model - run: dsp-tools create import_project.json - - name: upload data - run: dsp-tools xmlupload data-processed.xml + - name: run the tests + run: pytest test diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..21d6db1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +dsp-tools +lxml +regex +pandas +pytest diff --git a/test/expected.xml b/test/expected.xml new file mode 100644 index 0000000..b3d95d2 --- /dev/null +++ b/test/expected.xml @@ -0,0 +1,205 @@ + + + + V + V + D + CR + CR + + + M + CR + CR + + + V + V + D + CR + CR + + + M + CR + CR + + + images/Anubis.jpg + + Anubis.jpg + + + + images/Basho_Horohoroto.jpg + + Basho_Horohoroto.jpg + + + + images/BM1888-0601-716.png + + BM1888-0601-716.png + + + + images/GibeonMeteorite.jpg + + GibeonMeteorite.jpg + + + + + Anubis.jpg_37fa98fd-a2e4-4031-bbf6-57da0ff82750 + + + Bengal cat + + + An example of a domesticated cat + + + mammals + humans + + + true + + + #f5f5dc + + + GREGORIAN:CE:2015-01-01:CE:2015-01-01 + + + + + + 4.8 + + + 2661604 + + + https://en.wikipedia.org/wiki/Cat + + + + + GibeonMeteorite.jpg_79eeb9f5-96db-4595-addd-524306d0e586 + + + Gibeon Meteorite + + + This is a piece of the so-called Gibeon Meteroite + + + physics + + + true + + + #808080 + + + GREGORIAN:CE:1908-03-05:CE:1908-03-05 + + + + + + 0.3 + + + 11821111 + + + https://en.wikipedia.org/wiki/Gibeon_(meteorite) + + + + + BM1888-0601-716.png_dbf7cc27-582a-4fc2-a28e-d5b66d29693c + + + Lekythos + + + Attic red-figured Lekythos BM 1888,601.716 + + + artwork + + + true + + + GREGORIAN:CE:1973-12-01:CE:1974-01-06 + + + 0.5 + + + 351274 + + + https://www.britishmuseum.org/collection/object/G_1888-0601-716 + + + + + Basho_Horohoroto.jpg_71a7e332-56cf-4e3b-a717-c34b2463e2c6 + + + Picture and Poem by Matsuo Bashō + + + ほろほろと山吹ちるかたきのおと + + + artwork + + + GREGORIAN:CE:1849:CE:1850 + + + 1.0 + + + https://en.wikipedia.org/wiki/Haiku#/media/File:Basho_Horohoroto.jpg + + + + + Date and time are invented, like for the other resources. + + + Anubis_fda7c834-692e-4e2c-b7c8-82e26a2775d9 + + + + + This is a comment + + + #5d1f1e + + + GibeonMeteorite.jpg_79eeb9f5-96db-4595-addd-524306d0e586 + + + {"type": "rectangle", "lineColor": "#ff3333", "lineWidth": 2, "points": [{"x": 0.08, "y": 0.16}, {"x": 0.73, "y": 0.72}], "original_index": 0} + + + + + This is a comment + + + BM1888-0601-716_97f7982b-eaed-4ef2-8813-cb93f8e2c480 + Horohoroto_5652b411-2c50-4086-896c-23b208ea27a8 + + + diff --git a/test/test.py b/test/test.py new file mode 100644 index 0000000..45deb92 --- /dev/null +++ b/test/test.py @@ -0,0 +1,87 @@ +from collections.abc import Iterator +from pathlib import Path +import re +import subprocess + +from dsp_tools import excel2xml +from lxml import etree +import pytest + +import import_script + + +@pytest.fixture(scope="module") +def generated_xml_file() -> Iterator[Path]: + """Yield the generated XML file as fixture, and delete it afterwards""" + xml_file = Path("data-processed.xml") + yield xml_file + xml_file.unlink(missing_ok=True) + + +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_script_output(generated_xml_file: Path) -> None: + """Execute the import script and compare the generated XML with the expected XML""" + import_script.main() + with open("test/expected.xml", encoding="utf-8") as f: + xml_expected = _derandomize_xsd_id(f.read(), multiple_occurrences=True) + with open(generated_xml_file, encoding="utf-8") as f: + xml_returned = _derandomize_xsd_id(f.read(), multiple_occurrences=True) + assert _sort_xml_by_id(xml_expected) == _sort_xml_by_id(xml_returned) + + +def test_create() -> None: + """Create the project on the DSP server""" + subprocess.run("dsp-tools create import_project.json".split()) + + +def test_upload(generated_xml_file: Path) -> None: + """Upload the created XML to the DSP server""" + subprocess.run(f"dsp-tools xmlupload {generated_xml_file}".split()) + + +def _sort_xml_by_id(xml: str) -> str: + """Sort the elements in the XML by their ID""" + xml_tree = etree.fromstring(xml.encode("utf-8")) + for elem in xml_tree.iter(): + elem[:] = sorted(elem, key=lambda x: x.attrib.get("id", "")) + return etree.tostring(xml_tree).decode("utf-8") + + +def _derandomize_xsd_id( + string: str, + multiple_occurrences: bool = False, +) -> str: + """ + In some contexts, the random component of the output of make_xsd_id_compatible() is a hindrance, + especially for testing. + This method removes the random part, + but leaves the other modifications introduced by make_xsd_id_compatible() in place. + This method's behaviour is defined by the example in the "Examples" section. + + Args: + string: the output of make_xsd_id_compatible() + multiple_occurrences: If true, string can be an entire XML document, and all occurrences will be removed + + Raises: + Exception: if the input cannot be derandomized + + Returns: + the derandomized string + + Examples: + >>> id_1 = make_xsd_id_compatible("Hello!") + >>> id_2 = make_xsd_id_compatible("Hello!") + >>> assert _derandomize_xsd_id(id_1) == _derandomize_xsd_id(id_2) + """ + if not isinstance(string, str) or not excel2xml.check_notna(string): + raise Exception(f"The input '{string}' cannot be derandomized.") + + uuid4_regex = r"[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + if multiple_occurrences: + return re.subn(uuid4_regex, "", string, flags=re.IGNORECASE)[0] + else: + return re.sub(uuid4_regex, "", string, re.IGNORECASE) + + +if __name__ == "__main__": + pytest.main([__file__]) From 7f6ecf7a762649dff61f743bb4961aa21158c98b Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Fri, 12 Jan 2024 14:52:46 +0100 Subject: [PATCH 2/4] renaming of workflows --- .../{compatibility_check.yml => automated-tests.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{compatibility_check.yml => automated-tests.yml} (90%) diff --git a/.github/workflows/compatibility_check.yml b/.github/workflows/automated-tests.yml similarity index 90% rename from .github/workflows/compatibility_check.yml rename to .github/workflows/automated-tests.yml index acbe8d3..3d35711 100644 --- a/.github/workflows/compatibility_check.yml +++ b/.github/workflows/automated-tests.yml @@ -1,6 +1,6 @@ --- -name: test compatibility with DSP-TOOLS +name: automated-tests on: schedule: @@ -10,7 +10,7 @@ on: - main jobs: - compatibility: + automated-tests: runs-on: ubuntu-latest steps: - name: Checkout repo From bb2ead3411741fc9673b53f0519018fda71051b5 Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Fri, 12 Jan 2024 14:53:19 +0100 Subject: [PATCH 3/4] fix test collection --- test/__init__.py | 0 test/{test.py => test_script.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/__init__.py rename test/{test.py => test_script.py} (100%) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test.py b/test/test_script.py similarity index 100% rename from test/test.py rename to test/test_script.py From 5005cfbaff32baccbe6709bdae11a990cb87343c Mon Sep 17 00:00:00 2001 From: Johannes Nussbaum Date: Fri, 12 Jan 2024 14:56:53 +0100 Subject: [PATCH 4/4] add check=True --- .github/workflows/automated-tests.yml | 2 +- test/test_script.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/automated-tests.yml b/.github/workflows/automated-tests.yml index 3d35711..3f9e451 100644 --- a/.github/workflows/automated-tests.yml +++ b/.github/workflows/automated-tests.yml @@ -23,7 +23,7 @@ jobs: run: pip3 install -r requirements.txt - name: start stack run: dsp-tools start-stack --no-prune - - name: run the tests + - name: run tests run: pytest test diff --git a/test/test_script.py b/test/test_script.py index 45deb92..0d9f86f 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -31,12 +31,12 @@ def test_script_output(generated_xml_file: Path) -> None: def test_create() -> None: """Create the project on the DSP server""" - subprocess.run("dsp-tools create import_project.json".split()) + subprocess.run("dsp-tools create import_project.json".split(), check=True) def test_upload(generated_xml_file: Path) -> None: """Upload the created XML to the DSP server""" - subprocess.run(f"dsp-tools xmlupload {generated_xml_file}".split()) + subprocess.run(f"dsp-tools xmlupload {generated_xml_file}".split(), check=True) def _sort_xml_by_id(xml: str) -> str: