Fix for JSON unexpected character error

custom_components/knmi/api.py

@@ -1,12 +1,16 @@
 """KnmiApiClient"""
 
 import asyncio
+import json
+import logging
 import socket
 
 import aiohttp
 
 from .const import API_ENDPOINT, API_TIMEOUT
 
+_LOGGER: logging.Logger = logging.getLogger(__package__)
+
 
 class KnmiApiClientError(Exception):
     """Exception to indicate a general API error."""
@@ -27,6 +31,8 @@ class KnmiApiRateLimitError(KnmiApiClientError):
 class KnmiApiClient:
     """KNMI API wrapper"""
 
+    response_text = None
+
     def __init__(
         self,
         api_key: str,
@@ -40,27 +46,39 @@ def __init__(
         self.longitude = longitude
         self._session = session
 
+    async def get_response_text(self) -> str:
+        """Get API response text"""
+        async with asyncio.timeout(API_TIMEOUT):
+            response = await self._session.get(
+                API_ENDPOINT.format(self.api_key, self.latitude, self.longitude)
+            )
+
+            return await response.text()
+
     async def async_get_data(self) -> dict:
         """Get data from the API."""
         try:
-            async with asyncio.timeout(API_TIMEOUT):
-                response = await self._session.get(
-                    API_ENDPOINT.format(self.api_key, self.latitude, self.longitude)
-                )
+            self.response_text = await self.get_response_text()
 
-                response_text = await response.text()
+            # The API has no proper error handling for a wrong API key or rate limit.
+            # Instead a 200 with a message is returned, try to detect that here.
+            if "Vraag eerst een API-key op" in self.response_text:
+                raise KnmiApiClientApiKeyError("The given API key is invalid")
 
-                # The API has no proper error handling for a wrong API key or rate limit.
-                # Instead a 200 with a message is returned, try to detect that here.
-                if "Vraag eerst een API-key op" in response_text:
-                    raise KnmiApiClientApiKeyError("The given API key is invalid")
+            if "Dagelijkse limiet" in self.response_text:
+                raise KnmiApiRateLimitError(
+                    "API key daily limit exceeded, try again tomorrow"
+                )
 
-                if "Dagelijkse limiet" in response_text:
-                    raise KnmiApiRateLimitError(
-                        "API key daily limit exceeded, try again tomorrow"
-                    )
+            # The API has an ongoing issue due ot invalid JSON repsponse.
+            # Where a null value of a number field is set to _ (without quotes).
+            # Here we fix tje JSON by setting the value to null.
+            # More info: https://github.com/golles/ha-knmi/issues/130
+            if '": _,' in self.response_text:
+                _LOGGER.debug("Detected invalid JSON, attempting to fix that...")
+                return json.loads(self.response_text.replace('": _,', '": null,'))
 
-                return await response.json()
+            return json.loads(self.response_text)
 
         except asyncio.TimeoutError as exception:
             raise KnmiApiClientCommunicationError(

custom_components/knmi/diagnostics.py

@@ -20,4 +20,5 @@ async def async_get_config_entry_diagnostics(
     return {
         "config_entry": async_redact_data(config_entry.as_dict(), TO_REDACT),
         "data": coordinator.data,
+        "response_text": coordinator.api.response_text,
     }

tests/conftest.py

@@ -15,7 +15,7 @@
 #
 # See here for more info: https://docs.pytest.org/en/latest/fixture.html (note that
 # pytest includes fixtures OOB which you can use as defined on this page)
-import json
+
 from unittest.mock import PropertyMock, patch
 
 import pytest
@@ -63,31 +63,19 @@ def enable_all_entities():
         yield
 
 
-# This fixture, when used, will have the mocked values from response.json loaded in the integration.
+# This fixture, when used, will have the mocked data from a given json file.
 @pytest.fixture(name="mocked_data")
-def mocked_data_fixture():
-    """Skip calls to get data from API."""
-    data = json.loads(load_fixture(response_json))
-
-    with patch(
-        async_get_data,
-        return_value=data,
-    ):
-        yield
-
-
-# This fixture, when used, will have the mocked values from response.json loaded in the integration.
-# As an addition, the alarm and related values are set.
-@pytest.fixture(name="mocked_data_alarm")
-def mocked_data_alarm_fixture():
-    """Skip calls to get data from API."""
-    data = json.loads(load_fixture(response_json))
-
-    data["liveweer"][0]["alarm"] = 1
+def mocked_data_fixture(request):
+    """Use mocked data in the integration"""
+    json_file = request.node.get_closest_marker("fixture")
+    if json_file is None:
+        json_file = "response.json"
+    else:
+        json_file = json_file.args[0]
 
     with patch(
-        async_get_data,
-        return_value=data,
+        "custom_components.knmi.KnmiApiClient.get_response_text",
+        return_value=load_fixture(json_file),
     ):
         yield