From e95f62d6db97e4a2025987c8b1a5fe8316074f09 Mon Sep 17 00:00:00 2001 From: Lincoln Puzey Date: Fri, 6 Dec 2024 16:10:05 +0800 Subject: [PATCH] Remove tests for archived survey occurrence v1 template Add missing test for survey occurrence v2 --- .../templates/test_survey_occurrence_data.py | 292 ------------------ .../test_survey_occurrence_data_v2.py | 41 +++ 2 files changed, 41 insertions(+), 292 deletions(-) delete mode 100644 tests/templates/test_survey_occurrence_data.py diff --git a/tests/templates/test_survey_occurrence_data.py b/tests/templates/test_survey_occurrence_data.py deleted file mode 100644 index 3c1a8da7..00000000 --- a/tests/templates/test_survey_occurrence_data.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Tests for specific for the `survey_occurrence_data` template.""" - -# Third-party -import pytest_mock -import attrs -import pytest -import pandas as pd - -# Standard -import io -import csv -import pathlib - -# Local -from abis_mapping import base -from abis_mapping import models -import abis_mapping.templates.survey_occurrence_data.mapping -from tests import conftest - - -class TestDefaultMap: - @attrs.define(kw_only=True) - class Scenario: - """Dataclass to hold the scenario parameters.""" - - name: str - raws: list[list[str]] - expected_error_codes: set[str] = set() - default_map: dict[str, str] - - # List of scenarios for the apply_validation method tests - scenarios: list[Scenario] = [ - Scenario( - name="valid_with_default_map", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site1", "", "", "", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site3", "-38.94", "115.21", "AGD66", "", "", "", ""], - ["site4", "-38.94", "115.21", "EPSG:4202", "", "", "", ""], - ], - default_map={"site1": "something"}, - ), - Scenario( - name="invalid_missing_from_default_map", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site1", "", "", "", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={"site3": "something"}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_incidental_occurrence_requires_latlong", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["", "", "", "", "", "", "VU", "VIC"], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="valid_incidental_occurrence_requires_latlong", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["", "-38.94", "115.21", "WGS84", "", "", "VU", "VIC"], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - # The following show that non-url safe characters get encoded during mapping. - ["site a", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site/b", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site%20c", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={}, - ), - Scenario( - name="invalid_missing_long", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site1", "-38.94", "", "WGS84", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={"site1": "something"}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_missing_lat", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site1", "", "115.21", "WGS84", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={"site1": "something"}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_incidental_occurrence_missing_lat", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["", "", "115.21", "WGS84", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_incidental_occurrence_missing_long", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["", "-38.94", "", "WGS84", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_missing_geodetic_datum", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["site1", "-38.94", "115.21", "", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={"site1": "something"}, - expected_error_codes={"row-constraint"}, - ), - Scenario( - name="invalid_incidental_occurrence_missing_geodetic_datum", - raws=[ - ["site1", "-38.94", "115.21", "WGS84", "", "", "", ""], - ["", "-38.94", "115.21", "", "", "", "", ""], - ["site2", "-38.94", "115.21", "WGS84", "", "", "", ""], - ], - default_map={}, - expected_error_codes={"row-constraint"}, - ), - ] - - @pytest.mark.parametrize( - argnames="scenario", - argvalues=[scenario for scenario in scenarios], - ids=[scenario.name for scenario in scenarios], - ) - def test_apply_validation(self, scenario: Scenario, mocker: pytest_mock.MockerFixture) -> None: - """Tests the `apply_validation` method with a supplied default map. - - Args: - scenario (Scenario): The parameters of the scenario under test. - mocker (pytest_mock.MockerFixture): The mocker fixture. - """ - # Construct fake data - rawh = [ - "siteID", - "decimalLatitude", - "decimalLongitude", - "geodeticDatum", - "organismQuantity", - "organismQuantityType", - "threatStatus", - "conservationJurisdiction", - ] - all_raw = [{hname: val for hname, val in zip(rawh, ln, strict=True)} for ln in scenario.raws] - - # Get mapper - mapper = abis_mapping.templates.survey_occurrence_data.mapping.SurveyOccurrenceMapper - assert mapper is not None - - # Modify schema to only fields required for test - descriptor = {"fields": [field for field in mapper.schema()["fields"] if field["name"] in rawh]} - descriptor["fields"].sort(key=lambda f: rawh.index(f["name"])) - - # Patch the schema for the test - mocker.patch.object(base.mapper.ABISMapper, "schema").return_value = descriptor - - # Create raw data csv string - with io.StringIO() as output: - csv_writer = csv.DictWriter(output, fieldnames=rawh) - csv_writer.writeheader() - - for row in all_raw: - csv_writer.writerow(row) - - csv_data = output.getvalue().encode("utf-8") - - # Apply validation - report = mapper().apply_validation( - data=csv_data, - site_id_geometry_map=scenario.default_map, - ) - - # Assert - assert report.valid == (scenario.expected_error_codes == set()) - if not report.valid: - error_codes = [code for codes in report.flatten(["type"]) for code in codes] - assert set(error_codes) == scenario.expected_error_codes - - def test_apply_mapping(self) -> None: - """Tests apply_mapping method with default geometry map.""" - # Build a dataframe from an existing csv - df = pd.read_csv("abis_mapping/templates/survey_occurrence_data/examples/organism_qty.csv") - - # Modify and preserve first entry - col_names = ["decimalLongitude", "decimalLatitude", "geodeticDatum"] - s_geo_vals = df[[*col_names, "siteID"]].iloc[0] - df.loc[0] = df.loc[0].drop(col_names) - - # Proving the values null for first row - assert df[col_names].loc[0].isna().all() - - # Write out to memory - with io.StringIO() as output: - # Write dataframe to memory as csv - df.to_csv(output, index=False) - - # Assign csv data to variable - csv_data = output.getvalue().encode("utf-8") - - # Get mapper - mapper = abis_mapping.templates.survey_occurrence_data.mapping.SurveyOccurrenceMapper - assert mapper is not None - - expected = pathlib.Path("abis_mapping/templates/survey_occurrence_data/examples/organism_qty.ttl").read_text() - - # Resulting graph doesn't match expected when no lat/long provided - graphs = list(mapper().apply_mapping(csv_data)) - assert len(graphs) == 1 - assert not conftest.compare_graphs(graphs[0], expected) - - # Make site id geo default map using values extracted previously - val = str( - models.spatial.Geometry( - raw=models.spatial.LatLong(s_geo_vals["decimalLatitude"], s_geo_vals["decimalLongitude"]), - datum=s_geo_vals["geodeticDatum"], - ).to_rdf_literal() - ) - default_map = {s_geo_vals["siteID"]: val} - - # Create graph - graphs = list( - mapper().apply_mapping( - data=csv_data, - site_id_geometry_map=default_map, - ) - ) - assert len(graphs) == 1 - - # Now with the provided default map values the graph should match. - assert conftest.compare_graphs(graphs[0], expected) - assert "None" not in graphs[0].serialize(format="ttl") - - -def test_extract_site_id_keys(mocker: pytest_mock.MockerFixture) -> None: - """Test the extract_site_id_keys method. - - Args: - mocker (pytest_mock.MockerFixture): The mocker fixture. - """ - # Construct a raw data set only using fields relevant to method. - rawh = ["siteID"] - raws = [["site1"], [""], ["site2"], ["site3"], ["site3"]] - - # Amalgamate into a list of dicts - all_raw = [{hname: val for hname, val in zip(rawh, ln, strict=True)} for ln in raws] - - # Get the specific mapper - mapper = abis_mapping.templates.survey_occurrence_data.mapping.SurveyOccurrenceMapper() - - # Modify schema to only include the necessary fields - descriptor = {"fields": [{"name": "siteID", "type": "string"}]} - mocker.patch.object(base.mapper.ABISMapper, "schema").return_value = descriptor - - # Create raw data csv string - with io.StringIO() as output: - csv_writer = csv.DictWriter(output, fieldnames=rawh) - csv_writer.writeheader() - - for row in all_raw: - csv_writer.writerow(row) - - csv_data = output.getvalue().encode("utf-8") - - expected = { - "site1": True, - "site2": True, - "site3": True, - } - - # Invoke method - actual = mapper.extract_site_id_keys(csv_data) - - # Validate - assert actual == expected diff --git a/tests/templates/test_survey_occurrence_data_v2.py b/tests/templates/test_survey_occurrence_data_v2.py index 69d638f6..dbb46614 100644 --- a/tests/templates/test_survey_occurrence_data_v2.py +++ b/tests/templates/test_survey_occurrence_data_v2.py @@ -514,3 +514,44 @@ def test_apply_validation(self, scenario: Scenario, mocker: pytest_mock.MockerFi if not report.valid: error_codes = [code for codes in report.flatten(["type"]) for code in codes] assert set(error_codes) == scenario.expected_error_codes + + +def test_extract_site_id_keys( + mocker: pytest_mock.MockerFixture, + mapper: Mapper, +) -> None: + """Test the extract_site_id_keys method. + + Args: + mocker (pytest_mock.MockerFixture): The mocker fixture. + """ + # Construct a raw data set only using fields relevant to method. + rawh = ["siteID"] + raws = [["site1"], [""], ["site2"], ["site3"], ["site3"]] + + # Amalgamate into a list of dicts + all_raw = [{hname: val for hname, val in zip(rawh, ln, strict=True)} for ln in raws] + + # Modify schema to only include the necessary fields + descriptor = {"fields": [{"name": "siteID", "type": "string"}]} + mocker.patch.object(base.mapper.ABISMapper, "schema").return_value = descriptor + + # Create raw data csv string + output = io.StringIO() + csv_writer = csv.DictWriter(output, fieldnames=rawh) + csv_writer.writeheader() + for row in all_raw: + csv_writer.writerow(row) + csv_data = output.getvalue().encode("utf-8") + + expected = { + "site1": True, + "site2": True, + "site3": True, + } + + # Invoke method + actual = mapper.extract_site_id_keys(csv_data) + + # Validate + assert actual == expected