From 486edb1bd95931e558b7f1e9a925d22a853c3623 Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Tue, 15 Jan 2019 13:43:37 -0500 Subject: [PATCH] Clean up and add more documentation --- .prospector.yaml | 16 +++++++++++ Makefile | 3 +++ Readme.md | 6 +++-- app/buoy_barn/settings.py | 6 ++++- app/deployments/migrations/0001_initial.py | 1 - app/deployments/models.py | 1 - app/deployments/utils/erddap_loader.py | 8 ++++-- app/deployments/views.py | 6 ++--- app/forecasts/apps.py | 11 +++++++- .../forecasts/base_erddap_forecast.py | 4 +-- app/forecasts/forecasts/base_forecast.py | 2 +- .../forecasts/coastwatch_erddap/gfs.py | 4 +-- .../forecasts/neracoos_erddap/bedford.py | 9 +++++++ app/forecasts/migrations/__init__.py | 0 app/forecasts/serializers.py | 19 +++++++------ app/forecasts/utils/__init__.py | 1 + app/forecasts/utils/erddap.py | 15 +++++++---- app/forecasts/views.py | 7 +++-- app/manage.py | 1 + app/requirements.in | 3 +++ app/requirements.txt | 27 +++++++++++++++++-- docker-compose.yaml | 19 ++++++------- 22 files changed, 127 insertions(+), 42 deletions(-) delete mode 100644 app/forecasts/migrations/__init__.py diff --git a/.prospector.yaml b/.prospector.yaml index 8576383e..1357f4e8 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -1,3 +1,19 @@ +ignore-patterns: + - "(^|/)migrations(/|$)" + +# doc-warnings: true + +uses: + - django + pylint: disable: - logging-fstring-interpolation +pep257: + disable: + - D203 + - D213 +# mypy: +# run: true +# vulture: +# run: true diff --git a/Makefile b/Makefile index 3016caf7..106b5dcb 100644 --- a/Makefile +++ b/Makefile @@ -58,3 +58,6 @@ fixtures: docker-compose exec web python manage.py dumpdata --format yaml deployments.ErddapServer -o deployments/fixtures/erddapservers.yaml docker-compose exec web python manage.py dumpdata --format yaml deployments.TimeSeries -o deployments/fixtures/TimeSeries.yaml docker-compose exec web python manage.py dumpdata --format yaml deployments.Alert -o deployments/fixtures/Alerts.yaml + +lint: + docker-compose exec web prospector diff --git a/Readme.md b/Readme.md index 56573be3..5b98aff8 100644 --- a/Readme.md +++ b/Readme.md @@ -98,6 +98,7 @@ POSTGRES_PASSWORD=secret_string POSTGRES_USER=a_user_name SECRET_KEY=a_really_long_random_string_that_no_one_should_no_and_should_probably_be_gibberish REDIS_CACHE=rediss://cache:6379/0 +DJANGO_DEBUG=True ``` ### Starting Docker @@ -123,8 +124,8 @@ You can use Django fixtures to quickly save models from the database and reload - `app/` - `account/` Django user account app. - `buoy_barn/` Primary Django application. - - `deployments/` Database models and API - - `forecasts/` Forecast models and API + - `deployments/` Database models and API. + - `forecasts/` Forecast models and API. - `utils/` - `wait-for-it.sh` Shell script that can wait until specified services are avaliable before finishing. Helps `make up` launch Django more reliably. - `Dockerfile` Django server build steps @@ -156,6 +157,7 @@ You can use Django fixtures to quickly save models from the database and reload - `test` Run unit tests. - `requirements-compile` Take the high level requirements files generate a pinned requirements.txt file. - `requirements-tree` See the dependency tree of the requirements and any issues. +- `lint` Run prospector (and associated linting tools). ## Common Tasks and Problems diff --git a/app/buoy_barn/settings.py b/app/buoy_barn/settings.py index 97f4ced2..e5569d87 100644 --- a/app/buoy_barn/settings.py +++ b/app/buoy_barn/settings.py @@ -23,7 +23,11 @@ SECRET_KEY = os.environ["SECRET_KEY"] # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +try: + DEBUG = bool(os.environ["DJANGO_DEBUG"] != "False") +except KeyError: + DEBUG = False + ALLOWED_HOSTS = ["*"] diff --git a/app/deployments/migrations/0001_initial.py b/app/deployments/migrations/0001_initial.py index 1eb67fdf..4eec6494 100644 --- a/app/deployments/migrations/0001_initial.py +++ b/app/deployments/migrations/0001_initial.py @@ -299,4 +299,3 @@ class Migration(migrations.Migration): ), ), ] - diff --git a/app/deployments/models.py b/app/deployments/models.py index e0823505..bb0a2d5c 100644 --- a/app/deployments/models.py +++ b/app/deployments/models.py @@ -7,7 +7,6 @@ from django.contrib.postgres.fields import JSONField from erddapy import ERDDAP from memoize import memoize -import pandas as pd from requests import HTTPError from deployments.utils.erddap_datasets import filter_dataframe, retrieve_dataframe diff --git a/app/deployments/utils/erddap_loader.py b/app/deployments/utils/erddap_loader.py index d6a4347b..a087e560 100644 --- a/app/deployments/utils/erddap_loader.py +++ b/app/deployments/utils/erddap_loader.py @@ -12,11 +12,15 @@ def convert_time(time: str) -> datetime: - "Convert's from ERDDAP time style to python" + """Convert's from ERDDAP time style to python""" return datetime.fromisoformat(time.replace("Z", "")) -def add_timeseries(platform: Platform, server: str, dataset: str, constraints): +def add_timeseries( + platform: Platform, server: str, dataset: str, constraints +): # pylint: disable=too-many-locals + """Add datatypes for a new dataset to a platform. + See instructions in Readme.md""" e = ERDDAP(server) info = pd.read_csv(e.get_info_url(dataset, response="csv")) diff --git a/app/deployments/views.py b/app/deployments/views.py index 5acf301b..5f3728d9 100644 --- a/app/deployments/views.py +++ b/app/deployments/views.py @@ -16,16 +16,16 @@ class PlatformViewset(viewsets.ReadOnlyModelViewSet): queryset = Platform.objects.filter(active=True) serializer_class = PlatformSerializer - def retrieve(self, request, pk=None): + def retrieve(self, request, *args, **kwargs): # pylint: disable=unused-argument + pk = kwargs["pk"] platform = get_object_or_404(self.queryset, name=pk) serializer = self.serializer_class(platform) return Response(serializer.data) @action(detail=False) - def refresh(self, request): + def refresh(self, request): # pylint: disable=unused-argument # delete_memoized('deployments.models.Platform.latest_erddap_values') delete_memoized(Platform.latest_erddap_values) serializer = self.get_serializer(self.queryset.all(), many=True) return Response(serializer.data) - diff --git a/app/forecasts/apps.py b/app/forecasts/apps.py index c1429ec6..72447873 100644 --- a/app/forecasts/apps.py +++ b/app/forecasts/apps.py @@ -14,7 +14,8 @@ class ForecastsConfig(AppConfig): @register() -def check_forecasts(app_configs, **kwargs): +def check_duplicate_forecasts(app_configs, **kwargs): # pylint: disable=unused-argument + """ Return errors for any forecasts with duplicate slugs """ errors = [] slug_counter = Counter([forecast.slug for forecast in forecast_list]) @@ -46,6 +47,14 @@ def check_forecasts(app_configs, **kwargs): ) ) + return errors + + +@register() +def check_forecasts(app_configs, **kwargs): # pylint: disable=unused-argument + """ Check forecast attributes and methods are implemented """ + errors = [] + for forecast in forecast_list: forecast_str = str(forecast.__class__) diff --git a/app/forecasts/forecasts/base_erddap_forecast.py b/app/forecasts/forecasts/base_erddap_forecast.py index 32d8ac7f..6124b3f3 100644 --- a/app/forecasts/forecasts/base_erddap_forecast.py +++ b/app/forecasts/forecasts/base_erddap_forecast.py @@ -100,7 +100,7 @@ def dataset_url(self, lat: float, lon: float) -> str: lat (float): Latitude in degrees North lon (float): Longitude in degrees East - Returns: + Returns: Dataset URL """ return f"{self.server}/griddap/{self.dataset}.json?{self.dataset_query_string(lat, lon)}" @@ -135,7 +135,7 @@ def coverage_time_str( def request_variables(self) -> List[str]: """ The variables that should be requested from the dataset. Can be overridden for more complicated datasets that require multiple fields - + Returns: List of ERDDAP variable strings """ diff --git a/app/forecasts/forecasts/base_forecast.py b/app/forecasts/forecasts/base_forecast.py index d53184de..41e60f82 100644 --- a/app/forecasts/forecasts/base_forecast.py +++ b/app/forecasts/forecasts/base_forecast.py @@ -40,7 +40,7 @@ def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]] Args: lat (float): Latitude in degrees North lon (float): Longitude in degrees East - + Returns: List of tuples of forecasted times and values """ diff --git a/app/forecasts/forecasts/coastwatch_erddap/gfs.py b/app/forecasts/forecasts/coastwatch_erddap/gfs.py index 31c1278e..557e5c6d 100644 --- a/app/forecasts/forecasts/coastwatch_erddap/gfs.py +++ b/app/forecasts/forecasts/coastwatch_erddap/gfs.py @@ -104,8 +104,8 @@ class GFSWindSpeed(BaseGFSWindForecast): units = "m/s" def point_forecast(self, lat: float, lon: float) -> List[Tuple[datetime, float]]: - """ Return a list of tuples for the wind speed - + """Return a list of tuples for the wind speed + Args: lat (float): Latitude in degrees North lon (float): Longitude in degrees East diff --git a/app/forecasts/forecasts/neracoos_erddap/bedford.py b/app/forecasts/forecasts/neracoos_erddap/bedford.py index a6c0b4b6..b4d49cfe 100644 --- a/app/forecasts/forecasts/neracoos_erddap/bedford.py +++ b/app/forecasts/forecasts/neracoos_erddap/bedford.py @@ -1,3 +1,4 @@ +"""Bedford Institute forecasts""" from forecasts.forecasts.base_forecast import ForecastTypes from forecasts.forecasts.neracoos_erddap.base_neracoos_erddap_forecast import ( BaseNERACOOSERDDAPForecast, @@ -5,11 +6,15 @@ class BaseBedfordForecast(BaseNERACOOSERDDAPForecast): + """Bedford dataset information""" + dataset = "WW3_72_GulfOfMaine_latest" source_url = "http://www.neracoos.org/erddap/griddap/WW3_72_GulfOfMaine_latest.html" class BedfordWaveHeight(BaseBedfordForecast): + """Bedford wave height forecast""" + slug = "bedford_ww3_wave_height" name = "Bedford Institute Wave Model - Height" description = "Wave Height from the Bedford Institute Wave Model" @@ -20,6 +25,8 @@ class BedfordWaveHeight(BaseBedfordForecast): class BedfordWavePeriod(BaseBedfordForecast): + """Bedford wave period forecast""" + slug = "bedford_ww3_wave_period" name = "Bedford Institute Wave Model - Height" description = "Wave Height from the Bedford Institute Wave Model" @@ -30,6 +37,8 @@ class BedfordWavePeriod(BaseBedfordForecast): class BedfordWaveDirection(BaseBedfordForecast): + """Bedford wave direction forecast""" + slug = "bedford_ww3_wave_direction" name = "Bedford Institute Wave Model - Direction" description = "Wave Direction from the Bedford Institute Wave Model" diff --git a/app/forecasts/migrations/__init__.py b/app/forecasts/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/forecasts/serializers.py b/app/forecasts/serializers.py index 8c696209..bbd55afb 100644 --- a/app/forecasts/serializers.py +++ b/app/forecasts/serializers.py @@ -1,16 +1,19 @@ +"""Forecast JSON serializer""" from rest_framework import serializers from rest_framework.reverse import reverse class ForecastSerializer(serializers.BaseSerializer): - def to_representation(self, obj): # pylint: disable=no-self-use + """Serialize forecast information to JSON""" + def to_representation(self, instance): # pylint: disable=no-self-use + """Convert forecast instance to JSON""" return { - "slug": obj.slug, - "forecast_type": obj.forecast_type.value, - "name": obj.name, - "description": obj.description, - "source_url": obj.source_url, - "point_forecast": reverse("forecast-detail", kwargs={"pk": obj.slug}), - "units": obj.units, + "slug": instance.slug, + "forecast_type": instance.forecast_type.value, + "name": instance.name, + "description": instance.description, + "source_url": instance.source_url, + "point_forecast": reverse("forecast-detail", kwargs={"pk": instance.slug}), + "units": instance.units, } diff --git a/app/forecasts/utils/__init__.py b/app/forecasts/utils/__init__.py index e69de29b..a3b0aa04 100644 --- a/app/forecasts/utils/__init__.py +++ b/app/forecasts/utils/__init__.py @@ -0,0 +1 @@ +"""Utility functions for forecasts""" diff --git a/app/forecasts/utils/erddap.py b/app/forecasts/utils/erddap.py index e7ac8505..534038e6 100644 --- a/app/forecasts/utils/erddap.py +++ b/app/forecasts/utils/erddap.py @@ -1,3 +1,4 @@ +"""ERDDAP dataset interaction utility functions""" from datetime import datetime from typing import Union @@ -5,7 +6,7 @@ def attribute_value(info_df: DataFrame, attribute: str) -> Union[float, str, int]: - """ Return the value of a single dataset attribute """ + """Return the value of a single dataset attribute""" row = info_df[info_df["Attribute Name"] == attribute].values[0] value = row[-1] value_type = row[-2] @@ -17,6 +18,7 @@ def attribute_value(info_df: DataFrame, attribute: str) -> Union[float, str, int def coverage_time_str(info_df: DataFrame) -> str: + """Create a coverage time URL string""" start = attribute_value(info_df, "time_coverage_start") start_dt = parse_time(start) @@ -31,7 +33,7 @@ def coverage_time_str(info_df: DataFrame) -> str: def coordinates_str(info_df: DataFrame, lat: float, lon: float) -> str: - """ Return a string with coordinates formatted how ERDDAP expects """ + """Return a string with coordinates formatted how ERDDAP expects""" lat_precision = attribute_value(info_df, "geospatial_lat_resolution") lat_value = str(round_to(lat, lat_precision)).split(".") @@ -52,12 +54,15 @@ def coordinates_str(info_df: DataFrame, lat: float, lon: float) -> str: # From stack overflow answer https://stackoverflow.com/a/4265592 # to help with rounding coordinates def round_to(n, precision): - """ Round a value n to a precision """ + """Round a value n to a precision""" correction = 0.5 if n >= 0 else -0.5 return int(n / precision + correction) * precision def parse_time(dt: str) -> datetime: - """ Return a datetime object for an ERDDAP time - in the format of 2019-01-10T00:00:00Z """ + """Return a datetime object for an ERDDAP time + + ERDDAP time is in the format of 2019-01-10T00:00:00Z + and it is nicer to use native datetimes for comparisons + """ return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%SZ") diff --git a/app/forecasts/views.py b/app/forecasts/views.py index fea52d92..9fb60797 100644 --- a/app/forecasts/views.py +++ b/app/forecasts/views.py @@ -1,3 +1,4 @@ +"""Viewset for displaying forecasts, and fetching point forecast data is lat,lon are specified""" from rest_framework import viewsets from rest_framework.exceptions import NotFound from rest_framework.response import Response @@ -7,13 +8,15 @@ class ForecastViewSet(viewsets.ViewSet): - """ A viewset for forecasts """ + """A viewset for forecasts""" - def list(self, request): # pylint: disable=no-self-use + def list(self, request): # pylint: disable=no-self-use,unused-argument + """List all forecasts""" serializer = ForecastSerializer(forecast_list, many=True) return Response(serializer.data) def retrieve(self, request, pk=None): # pylint: disable=no-self-use + """Display a detail endpoint with point forecast information if lat, lon are given""" filtered = [forecast for forecast in forecast_list if forecast.slug == pk] try: forecast = filtered[0] diff --git a/app/manage.py b/app/manage.py index a355f7c0..3c01e894 100755 --- a/app/manage.py +++ b/app/manage.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +"""Django management module""" import os import sys diff --git a/app/requirements.in b/app/requirements.in index 0ca40243..e7b6f1c5 100644 --- a/app/requirements.in +++ b/app/requirements.in @@ -13,10 +13,13 @@ netCDF4==1.4.2 sentry-sdk==0.5.5 django-cors-headers==2.4.0 +# deployment +uWSGI==2.0.17.1 # development ipython==7.0.1 django-debug-toolbar==1.11 +prospector[with_mypy,with_vulture]==1.1.6.2 # testing coverage==4.5.2 diff --git a/app/requirements.txt b/app/requirements.txt index 5969688f..a0f78811 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -4,6 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in # +astroid==2.0.4 # via prospector, pylint, pylint-celery, pylint-flask, requirements-detector backcall==0.1.0 # via ipython certifi==2018.11.29 # via requests, sentry-sdk cftime==1.0.3.4 # via netcdf4 @@ -18,6 +19,7 @@ django-redis==4.10.0 django==2.1.2 djangorestframework-gis==0.14 djangorestframework==3.8.2 +dodgy==0.1.9 # via prospector erddapy==0.2.0 freezegun==0.3.11 geojson==2.4.1 @@ -25,33 +27,54 @@ gunicorn==19.9.0 idna==2.8 # via requests, yarl ipython-genutils==0.2.0 # via traitlets ipython==7.0.1 +isort==4.3.4 # via pylint jedi==0.13.2 # via ipython +lazy-object-proxy==1.3.1 # via astroid +mccabe==0.6.1 # via prospector, pylint multidict==4.5.2 # via yarl +mypy-extensions==0.4.1 # via mypy +mypy==0.650 # via prospector netcdf4==1.4.2 numpy==1.15.4 # via cftime, netcdf4, pandas, xarray pandas==0.23.4 # via erddapy, xarray parso==0.3.1 # via jedi +pep8-naming==0.4.1 # via prospector pexpect==4.6.0 # via ipython pickleshare==0.7.5 # via ipython pip_tools==3.2.0 pipdeptree==0.13.1 prompt-toolkit==2.0.7 # via ipython +prospector[with_mypy,with_vulture]==1.1.6.2 psycopg2-binary==2.7.5 ptyprocess==0.6.0 # via pexpect +pycodestyle==2.4.0 # via prospector +pydocstyle==3.0.0 # via prospector +pyflakes==1.6.0 # via prospector pygments==2.3.1 # via ipython +pylint-celery==0.3 # via prospector +pylint-django==2.0.2 # via prospector +pylint-flask==0.5 # via prospector +pylint-plugin-utils==0.4 # via prospector, pylint-celery, pylint-django, pylint-flask +pylint==2.1.1 # via prospector, pylint-celery, pylint-django, pylint-flask, pylint-plugin-utils python-dateutil==2.7.5 # via freezegun, pandas pytz==2018.7 # via django, pandas pyyaml==3.13 redis==3.0.1 # via django-redis requests==2.21.0 # via erddapy +requirements-detector==0.6 # via prospector sentry-sdk==0.5.5 +setoptconf==0.2.0 # via prospector simplegeneric==0.8.1 # via ipython -six==1.12.0 # via freezegun, pip-tools, prompt-toolkit, python-dateutil, traitlets, vcrpy +six==1.12.0 # via astroid, freezegun, pip-tools, prompt-toolkit, pydocstyle, python-dateutil, traitlets, vcrpy +snowballstemmer==1.2.1 # via pydocstyle sqlparse==0.2.4 # via django-debug-toolbar traitlets==4.3.2 # via ipython +typed-ast==1.1.1 # via mypy urllib3==1.24.1 # via requests, sentry-sdk +uwsgi==2.0.17.1 vcrpy==2.0.1 +vulture==0.24 # via prospector wcwidth==0.1.7 # via prompt-toolkit -wrapt==1.10.11 # via vcrpy +wrapt==1.10.11 # via astroid, vcrpy xarray==0.11.0 yarl==1.3.0 # via vcrpy diff --git a/docker-compose.yaml b/docker-compose.yaml index d9899994..a7b8b0a1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,16 +1,16 @@ -version: '3.7' +version: "3.7" services: db: image: mdillon/postgis:10-alpine ports: - - "5432:5432" + - "5432:5432" env_file: - - ./docker-data/secret.env + - ./docker-data/secret.env environment: PGDATA: /var/lib/postgresql/data/PGDATA volumes: - - ./docker-data/postgres:/var/lib/postgresql/data + - ./docker-data/postgres:/var/lib/postgresql/data cache: image: redis:5.0.3-alpine @@ -20,11 +20,12 @@ services: image: buoy_barn command: bash -c "./utils/wait-for-it.sh db:5432 && python manage.py runserver 0:8080" volumes: - - ./app:/app + - ./app:/app + - ./.prospector.yaml:/app/.prospector.yaml ports: - - "8080:8080" + - "8080:8080" env_file: - - ./docker-data/secret.env + - ./docker-data/secret.env depends_on: - - db - - cache + - db + - cache