From 74e41dace43c2f240a985cae1846ac831e436fac Mon Sep 17 00:00:00 2001 From: Sybrand Strauss Date: Fri, 5 Nov 2021 14:25:42 -0700 Subject: [PATCH] Various small tweaks (#1507) Added .env file in root (needed for vscode tests to work correctly), which meant not having a blanket .env ignore rule. Adding launch.json and settings.json configuration for vscode, which meant not having blank .vscode ignore rule, and adding poetry.toml file to force common location for python virtual env. Updated and added info to README files. Adding logging exception in audits. (otherwise, if db is configured incorrectly, it will fail without logging anything) Added a default failback for missing static folder. That's all garbage code that needs to be made redundant anyway by serving up the static content elsewhere. (Without this, an incorrectly configured static folder causes unit tests to fail, which in turn result in unit test discovery failure in vscode) Added example .env Made some changes to s3-backup --- .env | 1 + .gitignore | 15 +++--- .vscode/launch.json | 40 ++++++++++++++ .vscode/settings.json | 54 +++++++++++++++++++ api/README.md | 16 +++++- api/app/db/crud/api_access_audits.py | 16 +++--- api/app/frontend.py | 10 +++- api/app/tests/db/test_api_access_audits.py | 22 ++++++++ api/app/tests/test_frontend.py | 21 ++++++++ api/poetry.toml | 2 + api/pyproject.toml | 2 +- openshift/s3-backup/docker/.env.example | 11 ++++ openshift/s3-backup/docker/.gitignore | 2 + openshift/s3-backup/docker/docker-compose.yml | 4 +- web/README.md | 12 ++++- 15 files changed, 207 insertions(+), 21 deletions(-) create mode 100644 .env create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 api/app/tests/db/test_api_access_audits.py create mode 100644 api/app/tests/test_frontend.py create mode 100644 api/poetry.toml create mode 100644 openshift/s3-backup/docker/.env.example create mode 100644 openshift/s3-backup/docker/.gitignore diff --git a/.env b/.env new file mode 100644 index 000000000..72582669b --- /dev/null +++ b/.env @@ -0,0 +1 @@ +PYTHONPATH=api \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2b1bf8c7b..cda4763c6 100644 --- a/.gitignore +++ b/.gitignore @@ -74,9 +74,10 @@ web/typings/ .yarn-integrity # dotenv environment variables file -.env -.env.test -.env.docker +api/app/.env +api/env/.env.test +api/app/.env.docker +web/.env # parcel-bundler cache (https://parceljs.org/) .cache @@ -112,9 +113,6 @@ dist # Yupyter notebooks .ipynb_checkpoints/ -# VSCode -.vscode/ - # Python __pycache__/ app.egg-info/ @@ -152,4 +150,7 @@ api/static/* .Rhistory .RData .Rprofile -renv \ No newline at end of file +renv + +# Node Version Management +web/.nvmrc \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..9c60dd1f9 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,40 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "api", + "type": "python", + "request": "launch", + "program": "api/app/main.py", + "console": "integratedTerminal" + }, + { + "name": "api - module", + "type": "python", + "request": "launch", + "module": "app.main", + "console": "integratedTerminal" + }, + { + "name": "app.weather_models.env_canada RDPS", + "type": "python", + "request": "launch", + "module": "app.weather_models.env_canada", + "args": [ + "RDPS" + ] + }, + { + "name": "app.weather_models.env_canada HRDPS", + "type": "python", + "request": "launch", + "module": "app.weather_models.env_canada", + "args": [ + "HRDPS" + ] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..5cdf695b4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,54 @@ +{ + "python.pythonPath": "${workspaceFolder}/api/.venv/bin/python", + "python.defaultInterpreterPath": "${workspaceFolder}/api/.venv/bin/python", + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.promptToConfigure": true, + "python.envFile": "${workspaceFolder}/.env", + "python.formatting.autopep8Args": [ + "--max-line-length=110", + "--ignore=E402" + ], + "files.exclude": { + ".github": false, + "**/.pytest_cache": true, + "**/__pycache__**": true, + "web/node_modules": true, + "api/python_cache": true + }, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": true, + "source.fixAll.prettier": true + }, + "editor.formatOnSave": true, + "eslint.format.enable": true, + "eslint.validate": [ + "javascript", + "typescript" + ], + "eslint.run": "onSave", + "[python]": { + "editor.rulers": [110] + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 + }, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 + }, + "[typescriptreact]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 2 + }, + "[yaml]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.insertSpaces": true, + "editor.tabSize": 2, + "editor.formatOnSave": true + }, + "python.testing.pytestArgs": [ + "api" + ] +} \ No newline at end of file diff --git a/api/README.md b/api/README.md index 024b2ebfe..c24c71385 100644 --- a/api/README.md +++ b/api/README.md @@ -56,6 +56,8 @@ tar -xf openjdk-16.0.2_osx-x64_bin.tar.gz sudo mv jdk-16.0.2.jdk /Library/Java/JavaVirtualMachines ``` +Ensure that the CLASSPATH environment variable points to the jar files in api/libs, or unit tests will fail. + ##### Gdal ```bash @@ -89,6 +91,8 @@ poetry install poetry shell # we can't include gdal in poetry as we have little control over the version of gdal available on different platforms - we must match whatever version of gdal is available on the system in question. pip install gdal==$(gdal-config --version) +# on ubuntu, you may have to install pygdal, with the correct version specified. +pip install pygdal==3.0.4.10 ``` **N.B.: If `poetry env use [version]` returns an `EnvCommandError` saying something like "pyenv: python3.8: command not found", but `pyenv versions` shows that 3.8.10 is installed, you must first run `pyenv shell 3.8.10` and then re-run `poetry env use [path to python 3.8.10]`.** @@ -126,9 +130,17 @@ If gdal isn't installing, and you're on a mac, getting errors like "/Application Install system dependancies: ```bash -sudo apt install unixodbc-dev -sudo apt install python3-dev libpq-dev +sudo apt install python3 python3-pip +sudo apt install python-is-python3 +# install osgeo/gdal sudo apt install libgdal-dev +# install R and pre-req for cffdrs +sudo apt install r-base +R +install.packages('rgdal') +install.packages('cffdrs') +# install the jdk (for running tests agains redapp) +sudo apt install default-jdk ``` ##### Fedora diff --git a/api/app/db/crud/api_access_audits.py b/api/app/db/crud/api_access_audits.py index 6eb65c857..c0b1afb4e 100644 --- a/api/app/db/crud/api_access_audits.py +++ b/api/app/db/crud/api_access_audits.py @@ -14,9 +14,13 @@ def create_api_access_audit_log( success: bool, path: str) -> None: """ Create an audit log. """ - with app.db.database.get_write_session_scope() as session: - now = get_utc_now() - audit_log = APIAccessAudit(create_user=username, path=path, success=success, - create_timestamp=now) - session.add(audit_log) - session.commit() + try: + with app.db.database.get_write_session_scope() as session: + now = get_utc_now() + audit_log = APIAccessAudit(create_user=username, path=path, success=success, + create_timestamp=now) + session.add(audit_log) + session.commit() + except Exception as exception: + logger.error(exception, exc_info=True) + raise diff --git a/api/app/frontend.py b/api/app/frontend.py index 31b1666c8..305645efe 100644 --- a/api/app/frontend.py +++ b/api/app/frontend.py @@ -19,8 +19,16 @@ def get_static_foldername(): """ Get the static foldername - it defaults to 'static', but can be changed by setting an environment variable. """ + default_static_folder = 'static' dirname = os.path.dirname(os.path.abspath(__file__)) - static_foldername = config.get('STATIC_FOLDER', os.path.join(dirname, '../', 'static')) + static_foldername = config.get('STATIC_FOLDER', os.path.join(dirname, '../', default_static_folder)) + if not os.path.exists(static_foldername): + # If the specified static folder doesn't exist, an exception is thrown, which breaks unit tests and + # test discovery in visual studio code. It's a pain figuring out why it's failing - so rather just + # switch to a default folder that is know to exist, and log a warning. + logger.warning('static folder "%s" doesn\'t exists, using "%s" instead', + static_foldername, default_static_folder) + static_foldername = os.path.join(dirname, '../', default_static_folder) return static_foldername diff --git a/api/app/tests/db/test_api_access_audits.py b/api/app/tests/db/test_api_access_audits.py new file mode 100644 index 000000000..2bdff66ee --- /dev/null +++ b/api/app/tests/db/test_api_access_audits.py @@ -0,0 +1,22 @@ +""" Unit tests for api_access_audit_log """ +import unittest +from unittest.mock import MagicMock, patch +from app.db.crud.api_access_audits import create_api_access_audit_log + + +class TestApiAccessAudit(unittest.TestCase): + """ Access audit test cases """ + + @patch('app.db.database._get_write_session', return_value=MagicMock()) + @patch('app.db.crud.api_access_audits.logger') + def test_exception_logged(self, mock_logger, mock_get_write_session): + # Create a mock object that raises an exception when commit is called. + mock_session = MagicMock() + mock_session.commit = MagicMock(side_effect=Exception()) + mock_get_write_session.return_value = mock_session + + # We expect an exception to be thrown when we do the audit log + with self.assertRaises(Exception): + create_api_access_audit_log('user', True, '/something') + # But - critically, we expect the exception to be logged. + mock_logger.error.assert_called() diff --git a/api/app/tests/test_frontend.py b/api/app/tests/test_frontend.py new file mode 100644 index 000000000..b1f3b0525 --- /dev/null +++ b/api/app/tests/test_frontend.py @@ -0,0 +1,21 @@ +""" Unit tests for front end """ +import unittest + +from unittest.mock import patch +from app.frontend import get_static_foldername + + +class TestFrontend(unittest.TestCase): + """ Front end unit tests""" + + @patch('app.frontend.config.get') + def test_get_static_foldername_not_exist(self, get_patch): + """ If the configured folder doesn't exist, return the default static folder. """ + get_patch.return_value = 'folder_does_not_exist' + self.assertTrue(get_static_foldername().endswith('static')) + + @patch('app.frontend.config.get') + def test_get_static_foldername_exist(self, get_patch): + """ If the configured folder does exist, we get it back """ + get_patch.return_value = '.' + self.assertEqual(get_static_foldername(), '.') diff --git a/api/poetry.toml b/api/poetry.toml new file mode 100644 index 000000000..ab1033bd3 --- /dev/null +++ b/api/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/api/pyproject.toml b/api/pyproject.toml index 3fd7f4c6a..f39324afb 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -50,5 +50,5 @@ pgadmin4 = "^4" jsonpickle = "^2.0.0" [build-system] -requires = ["poetry>=0.12"] +requires = ["poetry>=1.1.11"] build-backend = "poetry.masonry.api" diff --git a/openshift/s3-backup/docker/.env.example b/openshift/s3-backup/docker/.env.example new file mode 100644 index 000000000..2dff22e01 --- /dev/null +++ b/openshift/s3-backup/docker/.env.example @@ -0,0 +1,11 @@ +PG_HOSTNAME=patroni-wps-pr-1427-leader +PG_DATABASE=wps +PG_PASSWORD=wps +OBJECT_STORE_SERVER=nrs.objectstore.gov.bc.ca +OBJECT_STORE_USER_ID=nr-wps-dev +OBJECT_STORE_SECRET=your secret key +OBJECT_STORE_BUCKET=gpdqha +AWS_HOSTNAME=nrs.objectstore.gov.bc.ca +AWS_ACCESS_KEY=nr-wps-dev +AWS_SECRET_KEY=yoursecretkey +AWS_BUCKET=gpdqha \ No newline at end of file diff --git a/openshift/s3-backup/docker/.gitignore b/openshift/s3-backup/docker/.gitignore new file mode 100644 index 000000000..fc4f2b845 --- /dev/null +++ b/openshift/s3-backup/docker/.gitignore @@ -0,0 +1,2 @@ +.env +.env.docker \ No newline at end of file diff --git a/openshift/s3-backup/docker/docker-compose.yml b/openshift/s3-backup/docker/docker-compose.yml index 49b9f182b..58c58f691 100644 --- a/openshift/s3-backup/docker/docker-compose.yml +++ b/openshift/s3-backup/docker/docker-compose.yml @@ -1,7 +1,7 @@ - +version: "3.7" services: backup: build: context: . env_file: - - .env.docker \ No newline at end of file + - .env.docker diff --git a/web/README.md b/web/README.md index 06e5869a4..3bf54915b 100644 --- a/web/README.md +++ b/web/README.md @@ -8,9 +8,15 @@ Wildfire Predictive Services to support decision making in prevention, preparedn ### Dependencies -- [Node.js](https://nodejs.org/en/) - You’ll need to have Node >= 10.x and npm >= 5.6 on your machine. You can use [nvm](https://github.com/nvm-sh/nvm#installation) (macOS/Linux) or [nvm-windows](https://github.com/coreybutler/nvm-windows#node-version-manager-nvm-for-windows) to switch Node versions between different projects. +#### [Node.js](https://nodejs.org/en/) -Note: We are using Node 10 as a base image on our pipeline. +- You’ll need to have Node >= 10.x and yarn on your machine. You can use [nvm](https://github.com/nvm-sh/nvm#installation) (macOS/Linux) or [nvm-windows](https://github.com/coreybutler/nvm-windows#node-version-manager-nvm-for-windows) to switch Node versions between different projects. +- Note: We are using Node 14 as a base image on our pipeline. +- On ubuntu: `sudo apt install nodejs` + +#### [yarn](https://yarnpkg.com/) + +- `npm install -g yarn` ### Installing @@ -61,7 +67,9 @@ In `openshift/templates/global.config.yaml` there is a template for a global Con This project is licensed under the [Apache License, Version 2.0](https://github.com/bcgov/wps/blob/main/LICENSE). ## Contributing + Frontend changes should follow [MaterialUI](https://material.io) design as closely as possible, leveraging the [Material-UI React implementation library](https://mui.com), unless [decided otherwise](https://github.com/bcgov/wps/wiki/Frontend-Design-Decisions). + ## Acknowledgments Inspiration, code snippets, etc.