Catch JSON decoding errors when loading forecasts and update wave period

#5wgudf

Changelog.md

@@ -6,10 +6,13 @@ Additions:
 
 Changes:
 
+- Update Bedford wave height period (previously the Wave Watch 3 model was using the wrong wave period from the model).
+
 Fixes
 
 - Adds timeout when a task does not return a response.
 - Catches when a 500 page is returned instead of an NetCDF file.
+- Catch JSON decode errors when trying to load forecasts.
 
 ## 0.1.3 - 2/12/20
 

Readme.md

@@ -120,6 +120,10 @@ CELERY_BROKER_URL=rediss://cache:6379/1
 
 An additional environment variable is currently configured in `docker-compose.yaml` as it is unlikely to need to be changed.
 
+There are additionally two environment variables for changing the timeout when
+querying ERDDAP servers.
+`RETRIEVE_FORECAST_TIMEOUT_SECONDS` and `RETRIEVE_DATAFRAME_TIMEOUT_SECONDS`.
+
 ### Starting Docker
 
 Then you can use `make up` to start the database and Django server.

app/deployments/tasks.py

@@ -24,16 +24,24 @@ def update_values_for_timeseries(timeseries):
 
     except HTTPError as error:
         logger.warning(
-            f"No rows found for {timeseries[0].dataset.name} with constraint {timeseries[0].constraints}", extra={"timeseries": timeseries, "constraints": timeseries[0].constraints}
+            f"No rows found for {timeseries[0].dataset.name} with constraint {timeseries[0].constraints}: {error}",
+            extra={"timeseries": timeseries, "constraints": timeseries[0].constraints},
+            exc_info=True
         )
 
     except Timeout as error:
         logger.warning(
-            f"Timeout when trying to retrieve dataset {timeseries[0].dataset.name} with constraint {timeseries[0].constraints}: {error}", extra={"timeseries": timeseries, "constraints": timeseries[0].constraints}
+            f"Timeout when trying to retrieve dataset {timeseries[0].dataset.name} with constraint {timeseries[0].constraints}: {error}", 
+            extra={"timeseries": timeseries, "constraints": timeseries[0].constraints},
+            exc_info=True
         )
 
     except OSError as error:
-        logger.error(f"Error loading dataset {timeseries[0].dataset.name} with constraints {timeseries[0].constraints}: {error}", extra={"timeseries": timeseries, "constraints": timeseries[0].constraints})
+        logger.error(
+            f"Error loading dataset {timeseries[0].dataset.name} with constraints {timeseries[0].constraints}: {error}", 
+            extra={"timeseries": timeseries, "constraints": timeseries[0].constraints},
+            exc_info=True
+        )
 
     else:
         for series in timeseries:
@@ -94,7 +102,7 @@ def refresh_server(server_id: int, healthcheck: bool = False):
     server = ErddapServer.objects.get(pk=server_id)
 
     if healthcheck:
-        dataset.healthcheck_start()
+        server.healthcheck_start()
 
     for ds in server.erddapdataset_set.all():
         refresh_dataset(ds.id)

app/deployments/views.py

@@ -50,7 +50,6 @@ def refresh(self, request, **kwargs):
         server, dataset = pk.split("-", maxsplit=1)
         dataset = get_object_or_404(self.queryset, name=dataset, server__name=server)
 
-        dataset.healthcheck_start()
         refresh_dataset.delay(dataset.id, healthcheck=True)
 
         serializer = self.serializer_class(dataset, context={"request": request})
@@ -71,7 +70,6 @@ def refresh(self, request, **kwargs):
 
         server = get_object_or_404(self.queryset, pk=pk)
 
-        server.healthcheck_start()
         refresh_server.delay(server.id, healthcheck=True)
 
         serializer = self.serializer_class(server, context={"request": request})

app/forecasts/forecasts/base_erddap_forecast.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import os
 from typing import List, Tuple
 
 from erddapy import ERDDAP
@@ -57,6 +58,16 @@ def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]]
             for row in rows
         ]
 
+    def json(self):
+        """ Returns a dict with standard information about the forecast """
+        return {
+            **super(BaseERDDAPForecast, self).json(),
+            "server": self.server,
+            "dataset": self.dataset,
+            "to_360": self.to_360,
+            "field": self.field,
+        }
+
     def offset_value(self, value: float) -> float:  # pylint: disable=no-self-use
         """ Allows you to override a value to return something more helpful (say Celsius rather than Kelvin) """
         return value
@@ -78,7 +89,10 @@ def request_dataset(self, lat: float, lon: float):
         Returns: 
             Table object from ERDDAP dataset for a given latitude and longitude
          """
-        response = requests.get(self.dataset_url(lat, lon))
+        response = requests.get(
+            self.dataset_url(lat, lon), 
+            timeout=float(os.environ.get("RETRIEVE_FORECAST_TIMEOUT_SECONDS", 60))
+        )
         return response.json()["table"]
 
     def dataset_info_df(self) -> pd.DataFrame:

app/forecasts/forecasts/base_forecast.py

@@ -45,3 +45,17 @@ def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]]
             List of tuples of forecasted times and values
         """
         raise NotImplementedError
+
+    def json(self):
+        """ Returns a dict with standard information about the forecast """
+        return {
+            "slug": self.slug,
+            "forecast_type": self.forecast_type,
+            "name": self.name,
+            "description": self.description,
+            "source_url": self.source_url,
+            "units": self.units
+        }
+
+    def __repr__(self):
+        return f"<Forecast: {self.json()}>"

app/forecasts/forecasts/neracoos_erddap/bedford.py

@@ -28,12 +28,12 @@ class BedfordWavePeriod(BaseBedfordForecast):
     """Bedford wave period forecast"""
 
     slug = "bedford_ww3_wave_period"
-    name = "Bedford Institute Wave Model - Height"
-    description = "Wave Height from the Bedford Institute Wave Model"
+    name = "Bedford Institute Wave Model - Period"
+    description = "Wave Period from the Bedford Institute Wave Model"
     forecast_type = ForecastTypes.WAVE_PERIOD
     units = "seconds"
 
-    field = "fp"
+    field = "t01"
 
 
 class BedfordWaveDirection(BaseBedfordForecast):

app/forecasts/views.py

@@ -1,7 +1,9 @@
 """Viewset for displaying forecasts, and fetching point forecast data is lat,lon are specified"""
+from json import JSONDecodeError
 import logging
+
 from rest_framework import viewsets
-from rest_framework.exceptions import NotFound
+from rest_framework.exceptions import NotFound, APIException
 from rest_framework.response import Response
 
 from forecasts.forecasts import forecast_list
@@ -43,7 +45,16 @@ def retrieve(self, request, pk=None):  # pylint: disable=no-self-use
             data["longitude"] = "`lon` parameter not specified"
 
         if "lat" in request.query_params and "lon" in request.query_params:
-            time_series = forecast.point_forecast(lat, lon)
+            try:
+                time_series = forecast.point_forecast(lat, lon)
+            except JSONDecodeError as error:
+                logger.error(
+                    f"Error retrieving dataset due to a JSON decode error: {error}",
+                    exc_info=True, 
+                )
+                raise APIException(detail=f"Error retrieving dataset for forecast slug: {pk}")
+
+
 
             data["time_series"] = [
                 {"time": reading[0], "reading": reading[1]} for reading in time_series