Refactor bcgov/api and merge to bcgov/wps (bcgov#432)

* Move all the API files under `api/`
* Update `sonar-project.properties` to narrow down the coverage focus
* Update Openshift scripts (paths and env variables)
* Remove bash scripts and common folder at root
* Update `fireweather_bot.py` to raise more exceptions
* Add more commands in Makefile for docker environment
* Copy all the files at the last step in `dev.Dockerfile`

.gitattributes

@@ -0,0 +1,2 @@
+*.grib2 binary
+*.xlsx binary

.githooks/pre-push

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Based on: https://itnext.io/using-git-hooks-to-enforce-branch-naming-policy-ffd81fa01e5e
+LC_ALL=C
+
+local_branch="$(git rev-parse --abbrev-ref HEAD)"
+
+valid_branch_regex="^(task|story|bug)\/[A-Za-z0-9._-]+(\/[0-9]+)?$"
+
+message="There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex. Your commit will be rejected. You should rename your branch to a valid name and try again."
+
+if [[ ! $local_branch =~ $valid_branch_regex ]]
+then
+    echo "$message"
+    exit 1
+fi
+
+exit 0

.github/workflows/cleanup.yml

@@ -0,0 +1,32 @@
+name: Cleanup
+
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  cleanup:
+    name: Delete all resources associated with pr
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set Variables
+        shell: bash
+        run: |
+          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Clean up api
+        uses: bcgov/openshift-launchpad-deployment@master
+        with:
+          AUTH_TOKEN: ${{ secrets.OC_DEV_TOKEN_EDIT }}
+          SCRIPT: openshift/scripts/oc_cleanup.sh ${SUFFIX} apply
+
+      - name: Clean up database
+        uses: bcgov/openshift-launchpad-deployment@master
+        with:
+          AUTH_TOKEN: ${{ secrets.OC_DEV_TOKEN_EDIT }}
+          SCRIPT: openshift/scripts/oc_cleanup_db.sh ${SUFFIX} apply

.github/workflows/deployment.yml

@@ -0,0 +1,55 @@
+name: Deployment
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  prepare-database:
+    name: Prepare Dev Database
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set Variables
+        shell: bash
+        run: |
+          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Deploy PostGIS instance
+        uses: bcgov/openshift-launchpad-deployment@master
+        with:
+          AUTH_TOKEN: ${{ secrets.OC_DEV_TOKEN_EDIT }}
+          SCRIPT: openshift/scripts/oc_provision_db.sh ${SUFFIX} apply
+
+  build-and-deploy:
+    name: Build, Deploy to Dev & ZAP Baseline Scan
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set Variables
+        shell: bash
+        run: |
+          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Build Image
+        uses: bcgov/openshift-launchpad-deployment@master
+        with:
+          AUTH_TOKEN: ${{ secrets.OC_TOOL_TOKEN_EDIT }}
+          SCRIPT: GIT_BRANCH=${GITHUB_HEAD_REF} openshift/scripts/oc_build.sh ${SUFFIX} apply
+
+      - name: Deploy to Dev
+        uses: bcgov/openshift-launchpad-deployment@master
+        with:
+          AUTH_TOKEN: ${{ secrets.OC_DEV_TOKEN_EDIT }}
+          SCRIPT: openshift/scripts/oc_deploy.sh ${SUFFIX} apply
+
+      # f.y.i.: ZAP Scan must be able to log an issue or it will fail.
+      - name: ZAP Scan
+        uses: zaproxy/[email protected]
+        with:
+          target: "https://wps-api-pr-${{ github.event.number }}.pathfinder.gov.bc.ca"

.github/workflows/integration.yml

@@ -0,0 +1,70 @@
+name: Integration
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint-and-test:
+    name: Lint, Test with Coverage & SonarQube
+    # Ubuntu 18.04 (a.k.a. ubuntu-latest) comes with gdal 2.2.3, which has bugs that cause unit tests to fail
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          # For sonar-scanner to work properly we can't use a shallow fetch.
+          fetch-depth: 0
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install poetry
+        run: |
+          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > /tmp/get-poetry.py
+          cd /tmp
+          POETRY_HOME=/opt/poetry python get-poetry.py --version 1.0.8
+          cd /usr/local/bin
+          /opt/poetry/bin/poetry config virtualenvs.create false
+
+      - name: Install libgdal-dev
+        # The python gdal component relies on libgdal-dev being installed.
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install libgdal-dev
+
+      - name: Install gdal python
+        # We don't have much control over what version of gdal we're getting, it's pretty much whatever is
+        # available to us. As such, gdal is not installed by poetry, since the versio will differ between
+        # platforms.
+        run: CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal python3 -m pip install gdal==$(gdal-config --version)
+
+      - name: Install python dependencies
+        run: |
+          cp api/pyproject.toml api/poetry.lock /tmp/
+          cd /tmp
+          /opt/poetry/bin/poetry install --no-root
+
+      - name: Lint
+        run: python3 -m pylint --rcfile=api/.pylintrc api/app/*.py api/app/**/*.py
+
+      - name: Unit Test (with coverage)
+        run: ORIGINS=testorigin coverage run --source=api/app -m pytest -o log_cli=true --disable-warnings --verbose
+
+      - name: Create coverage report
+        shell: bash
+        run: |
+          coverage report
+          coverage xml -o coverage-reports/coverage-report.xml
+
+      - name: SonarCloud Scan
+        uses: sonarsource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/static_analysis.yml

@@ -0,0 +1,67 @@
+name: Static Analysis
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  coverage:
+    name: SonarQube - (main)
+    # Ubuntu 18.04 (a.k.a. ubuntu-latest) comes with gdal 2.2.3, which has bugs that cause unit tests to fail
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.8]
+    steps:
+      - name: Checkout repo (main)
+        uses: actions/checkout@v2
+        with:
+          # For sonar-scanner to work properly we can't use a shallow fetch.
+          fetch-depth: 0
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install poetry
+        run: |
+          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > /tmp/get-poetry.py
+          cd /tmp
+          POETRY_HOME=/opt/poetry python get-poetry.py --version 1.0.8
+          cd /usr/local/bin
+          /opt/poetry/bin/poetry config virtualenvs.create false
+
+      - name: Install libgdal-dev
+        # The python gdal component relies on libgdal-dev being installed.
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y install libgdal-dev
+
+      - name: Install gdal python
+        # We don't have much control over what version of gdal we're getting, it's pretty much whatever is
+        # available to us. As such, gdal is not installed by poetry, since the versio will differ between
+        # platforms.
+        run: CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal python3 -m pip install gdal==$(gdal-config --version)
+
+      - name: Install dependencies
+        run: |
+          cp api/pyproject.toml api/poetry.lock /tmp/
+          cd /tmp
+          /opt/poetry/bin/poetry install --no-root
+
+      - name: Unit Test (with coverage)
+        run: ORIGINS=testorigin coverage run --source=api/app -m pytest -o log_cli=true --disable-warnings --verbose
+
+      - name: Create coverage report
+        shell: bash
+        run: |
+          coverage report
+          coverage xml -o coverage-reports/coverage-report.xml
+
+      - name: SonarCloud Scan
+        uses: sonarsource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/zap_scan.yml

@@ -0,0 +1,20 @@
+name: Every Sunday at 02:00
+on:
+  schedule:
+    # Scheduled workflows run on the latest commit on the default or base branch.
+    - cron: "0 2 * * 0"
+
+jobs:
+  zap_scan:
+    runs-on: ubuntu-latest
+    name: Scan the webapplication
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: main
+      - name: ZAP Scan
+        # f.y.i.: ZAP Scan must be able to log an issue or it will fail.
+        uses: zaproxy/[email protected]
+        with:
+          target: "https://wps-api-prod.pathfinder.gov.bc.ca/"

.gitignore

@@ -102,7 +102,34 @@ dist
 
 # TernJS port file
 .tern-port
-node_modules/
 
-# IDE
+# Yupyter notebooks
+.ipynb_checkpoints/
+
+# VSCode
 .vscode/
+
+# Python
+__pycache__/
+app.egg-info/
+
+# Apple
+.DS_store
+
+# Access
+*.accdb
+*.laccdb
+*.mdb
+
+# CSV
+csv/
+
+# Coverage folders
+.coverage
+coverage-reports
+
+# Sonar-scanner
+.scannerwork
+
+# Jupyter notebooks
+*.ipynb

COMPLIANCE.yaml

@@ -5,7 +5,7 @@ description: |
 spec:
   - name: PIA
     status: in-progress
-    last-updated: '2020-02-18T20:55:23.313Z'
+    last-updated: "2020-02-18T21:03:09.318Z"
   - name: STRA
     status: in-progress
-    last-updated: '2020-02-18T20:55:23.313Z'
+    last-updated: "2020-02-18T21:03:09.318Z"

api/.dockerignore

@@ -0,0 +1,11 @@
+**/*.zip
+**/*.csv
+**/*.grib2
+grib
+csv
+.vscode
+.github
+.ipynb_checkpoints
+.vscode
+**/__pycache__
+**/*.pyc