Clean up and add more documentation

.prospector.yaml

@@ -1,3 +1,19 @@
+ignore-patterns:
+  - "(^|/)migrations(/|$)"
+
+# doc-warnings: true
+
+uses:
+  - django
+
 pylint:
   disable:
     - logging-fstring-interpolation
+pep257:
+  disable:
+    - D203
+    - D213
+# mypy:
+#   run: true
+# vulture:
+#   run: true

Makefile

@@ -58,3 +58,6 @@ fixtures:
 	docker-compose exec web python manage.py dumpdata --format yaml deployments.ErddapServer -o deployments/fixtures/erddapservers.yaml
 	docker-compose exec web python manage.py dumpdata --format yaml deployments.TimeSeries -o deployments/fixtures/TimeSeries.yaml
 	docker-compose exec web python manage.py dumpdata --format yaml deployments.Alert -o deployments/fixtures/Alerts.yaml
+
+lint:
+	docker-compose exec web prospector

Readme.md

@@ -98,6 +98,7 @@ POSTGRES_PASSWORD=secret_string
 POSTGRES_USER=a_user_name
 SECRET_KEY=a_really_long_random_string_that_no_one_should_no_and_should_probably_be_gibberish
 REDIS_CACHE=rediss://cache:6379/0
+DJANGO_DEBUG=True
 ```
 
 ### Starting Docker
@@ -123,8 +124,8 @@ You can use Django fixtures to quickly save models from the database and reload
 - `app/`
   - `account/` Django user account app.
   - `buoy_barn/` Primary Django application.
-  - `deployments/` Database models and API
-  - `forecasts/` Forecast models and API
+  - `deployments/` Database models and API.
+  - `forecasts/` Forecast models and API.
   - `utils/`
     - `wait-for-it.sh` Shell script that can wait until specified services are avaliable before finishing. Helps `make up` launch Django more reliably.
   - `Dockerfile` Django server build steps
@@ -156,6 +157,7 @@ You can use Django fixtures to quickly save models from the database and reload
 - `test` Run unit tests.
 - `requirements-compile` Take the high level requirements files generate a pinned requirements.txt file.
 - `requirements-tree` See the dependency tree of the requirements and any issues.
+- `lint` Run prospector (and associated linting tools).
 
 ## Common Tasks and Problems
 

app/buoy_barn/settings.py

@@ -23,7 +23,11 @@
 SECRET_KEY = os.environ["SECRET_KEY"]
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+try:
+    DEBUG = bool(os.environ["DJANGO_DEBUG"] != "False")
+except KeyError:
+    DEBUG = False
+
 
 ALLOWED_HOSTS = ["*"]
 

app/deployments/migrations/0001_initial.py

@@ -299,4 +299,3 @@ class Migration(migrations.Migration):
             ),
         ),
     ]
-

app/deployments/models.py

@@ -7,7 +7,6 @@
 from django.contrib.postgres.fields import JSONField
 from erddapy import ERDDAP
 from memoize import memoize
-import pandas as pd
 from requests import HTTPError
 
 from deployments.utils.erddap_datasets import filter_dataframe, retrieve_dataframe

app/deployments/utils/erddap_loader.py

@@ -12,11 +12,15 @@
 
 
 def convert_time(time: str) -> datetime:
-    "Convert's from ERDDAP time style to python"
+    """Convert's from ERDDAP time style to python"""
     return datetime.fromisoformat(time.replace("Z", ""))
 
 
-def add_timeseries(platform: Platform, server: str, dataset: str, constraints):
+def add_timeseries(
+    platform: Platform, server: str, dataset: str, constraints
+):  # pylint: disable=too-many-locals
+    """Add datatypes for a new dataset to a platform.
+    See instructions in Readme.md"""
     e = ERDDAP(server)
 
     info = pd.read_csv(e.get_info_url(dataset, response="csv"))

app/deployments/views.py

@@ -16,16 +16,16 @@ class PlatformViewset(viewsets.ReadOnlyModelViewSet):
     queryset = Platform.objects.filter(active=True)
     serializer_class = PlatformSerializer
 
-    def retrieve(self, request, pk=None):
+    def retrieve(self, request, *args, **kwargs):  # pylint: disable=unused-argument
+        pk = kwargs["pk"]
         platform = get_object_or_404(self.queryset, name=pk)
         serializer = self.serializer_class(platform)
         return Response(serializer.data)
 
     @action(detail=False)
-    def refresh(self, request):
+    def refresh(self, request):  # pylint: disable=unused-argument
         # delete_memoized('deployments.models.Platform.latest_erddap_values')
         delete_memoized(Platform.latest_erddap_values)
 
         serializer = self.get_serializer(self.queryset.all(), many=True)
         return Response(serializer.data)
-

app/forecasts/apps.py

@@ -14,7 +14,8 @@ class ForecastsConfig(AppConfig):
 
 
 @register()
-def check_forecasts(app_configs, **kwargs):
+def check_duplicate_forecasts(app_configs, **kwargs):  # pylint: disable=unused-argument
+    """ Return errors for any forecasts with duplicate slugs """
     errors = []
 
     slug_counter = Counter([forecast.slug for forecast in forecast_list])
@@ -46,6 +47,14 @@ def check_forecasts(app_configs, **kwargs):
                 )
             )
 
+    return errors
+
+
+@register()
+def check_forecasts(app_configs, **kwargs):  # pylint: disable=unused-argument
+    """ Check forecast attributes and methods are implemented """
+    errors = []
+
     for forecast in forecast_list:
         forecast_str = str(forecast.__class__)
 

app/forecasts/forecasts/base_erddap_forecast.py

@@ -100,7 +100,7 @@ def dataset_url(self, lat: float, lon: float) -> str:
             lat (float): Latitude in degrees North
             lon (float): Longitude in degrees East
         
-        Returns: 
+        Returns:
             Dataset URL
         """
         return f"{self.server}/griddap/{self.dataset}.json?{self.dataset_query_string(lat, lon)}"
@@ -135,7 +135,7 @@ def coverage_time_str(
     def request_variables(self) -> List[str]:
         """ The variables that should be requested from the dataset.
         Can be overridden for more complicated datasets that require multiple fields
-        
+
         Returns:
             List of ERDDAP variable strings
         """

app/forecasts/forecasts/base_forecast.py

@@ -40,7 +40,7 @@ def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]]
         Args:
             lat (float): Latitude in degrees North
             lon (float): Longitude in degrees East
-        
+
         Returns:
             List of tuples of forecasted times and values
         """

app/forecasts/forecasts/coastwatch_erddap/gfs.py

@@ -104,8 +104,8 @@ class GFSWindSpeed(BaseGFSWindForecast):
     units = "m/s"
 
     def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]]:
-        """ Return a list of tuples for the wind speed
-        
+        """Return a list of tuples for the wind speed
+
         Args:
             lat (float): Latitude in degrees North
             lon (float): Longitude in degrees East

app/forecasts/forecasts/neracoos_erddap/bedford.py

@@ -1,15 +1,20 @@
+"""Bedford Institute forecasts"""
 from forecasts.forecasts.base_forecast import ForecastTypes
 from forecasts.forecasts.neracoos_erddap.base_neracoos_erddap_forecast import (
     BaseNERACOOSERDDAPForecast,
 )
 
 
 class BaseBedfordForecast(BaseNERACOOSERDDAPForecast):
+    """Bedford dataset information"""
+
     dataset = "WW3_72_GulfOfMaine_latest"
     source_url = "http://www.neracoos.org/erddap/griddap/WW3_72_GulfOfMaine_latest.html"
 
 
 class BedfordWaveHeight(BaseBedfordForecast):
+    """Bedford wave height forecast"""
+
     slug = "bedford_ww3_wave_height"
     name = "Bedford Institute Wave Model - Height"
     description = "Wave Height from the Bedford Institute Wave Model"
@@ -20,6 +25,8 @@ class BedfordWaveHeight(BaseBedfordForecast):
 
 
 class BedfordWavePeriod(BaseBedfordForecast):
+    """Bedford wave period forecast"""
+
     slug = "bedford_ww3_wave_period"
     name = "Bedford Institute Wave Model - Height"
     description = "Wave Height from the Bedford Institute Wave Model"
@@ -30,6 +37,8 @@ class BedfordWavePeriod(BaseBedfordForecast):
 
 
 class BedfordWaveDirection(BaseBedfordForecast):
+    """Bedford wave direction forecast"""
+
     slug = "bedford_ww3_wave_direction"
     name = "Bedford Institute Wave Model - Direction"
     description = "Wave Direction from the Bedford Institute Wave Model"

app/forecasts/serializers.py

@@ -1,16 +1,19 @@
+"""Forecast JSON serializer"""
 from rest_framework import serializers
 from rest_framework.reverse import reverse
 
 
 class ForecastSerializer(serializers.BaseSerializer):
-    def to_representation(self, obj):  # pylint: disable=no-self-use
+    """Serialize forecast information to JSON"""
 
+    def to_representation(self, instance):  # pylint: disable=no-self-use
+        """Convert forecast instance to JSON"""
         return {
-            "slug": obj.slug,
-            "forecast_type": obj.forecast_type.value,
-            "name": obj.name,
-            "description": obj.description,
-            "source_url": obj.source_url,
-            "point_forecast": reverse("forecast-detail", kwargs={"pk": obj.slug}),
-            "units": obj.units,
+            "slug": instance.slug,
+            "forecast_type": instance.forecast_type.value,
+            "name": instance.name,
+            "description": instance.description,
+            "source_url": instance.source_url,
+            "point_forecast": reverse("forecast-detail", kwargs={"pk": instance.slug}),
+            "units": instance.units,
         }

app/forecasts/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility functions for forecasts"""

app/forecasts/utils/erddap.py

@@ -1,11 +1,12 @@
+"""ERDDAP dataset interaction utility functions"""
 from datetime import datetime
 from typing import Union
 
 from pandas import DataFrame
 
 
 def attribute_value(info_df: DataFrame, attribute: str) -> Union[float, str, int]:
-    """ Return the value of a single dataset attribute """
+    """Return the value of a single dataset attribute"""
     row = info_df[info_df["Attribute Name"] == attribute].values[0]
     value = row[-1]
     value_type = row[-2]
@@ -17,6 +18,7 @@ def attribute_value(info_df: DataFrame, attribute: str) -> Union[float, str, int
 
 
 def coverage_time_str(info_df: DataFrame) -> str:
+    """Create a coverage time URL string"""
     start = attribute_value(info_df, "time_coverage_start")
     start_dt = parse_time(start)
 
@@ -31,7 +33,7 @@ def coverage_time_str(info_df: DataFrame) -> str:
 
 
 def coordinates_str(info_df: DataFrame, lat: float, lon: float) -> str:
-    """ Return a string with coordinates formatted how ERDDAP expects """
+    """Return a string with coordinates formatted how ERDDAP expects"""
     lat_precision = attribute_value(info_df, "geospatial_lat_resolution")
     lat_value = str(round_to(lat, lat_precision)).split(".")
 
@@ -52,12 +54,15 @@ def coordinates_str(info_df: DataFrame, lat: float, lon: float) -> str:
 # From stack overflow answer https://stackoverflow.com/a/4265592
 # to help with rounding coordinates
 def round_to(n, precision):
-    """ Round a value n to a precision """
+    """Round a value n to a precision"""
     correction = 0.5 if n >= 0 else -0.5
     return int(n / precision + correction) * precision
 
 
 def parse_time(dt: str) -> datetime:
-    """ Return a datetime object for an ERDDAP time 
-    in the format of 2019-01-10T00:00:00Z """
+    """Return a datetime object for an ERDDAP time
+    
+    ERDDAP time is in the format of 2019-01-10T00:00:00Z
+    and it is nicer to use native datetimes for comparisons
+    """
     return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%SZ")

app/forecasts/views.py

@@ -1,3 +1,4 @@
+"""Viewset for displaying forecasts, and fetching point forecast data is lat,lon are specified"""
 from rest_framework import viewsets
 from rest_framework.exceptions import NotFound
 from rest_framework.response import Response
@@ -7,13 +8,15 @@
 
 
 class ForecastViewSet(viewsets.ViewSet):
-    """ A viewset for forecasts """
+    """A viewset for forecasts"""
 
-    def list(self, request):  # pylint: disable=no-self-use
+    def list(self, request):  # pylint: disable=no-self-use,unused-argument
+        """List all forecasts"""
         serializer = ForecastSerializer(forecast_list, many=True)
         return Response(serializer.data)
 
     def retrieve(self, request, pk=None):  # pylint: disable=no-self-use
+        """Display a detail endpoint with point forecast information if lat, lon are given"""
         filtered = [forecast for forecast in forecast_list if forecast.slug == pk]
         try:
             forecast = filtered[0]

app/manage.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+"""Django management module"""
 import os
 import sys
 

app/requirements.in

@@ -13,10 +13,13 @@ netCDF4==1.4.2
 sentry-sdk==0.5.5
 django-cors-headers==2.4.0
 
+# deployment
+uWSGI==2.0.17.1
 
 # development
 ipython==7.0.1
 django-debug-toolbar==1.11
+prospector[with_mypy,with_vulture]==1.1.6.2
 
 # testing
 coverage==4.5.2