MIGRATE: Allow GCP hosting (#53)

* RF: Adjust database connection for GCP deployment

* FIX: Allow connection to Cloud SQL

* WIP: Streamline GCP release

* MAINT: Revert to versioneer

* MAINT: Move heroku deployment path

* MAINT: Avoid sharing .env secrets

* DOCKER: Move to multi-stage build

* MAINT: Improve GCP deployment

* MAINT+DOC: Reduce deployment costs, add docs

* CI: Run pytest --pyargs

* MAINT: Add dist/ folder to dockerignore

* MAINT: Add shutdown script

* MAINT: Quicker shutdown

* TST: Package path

* MAINT: Fix docker build on alpine, distribute tests

* MAINT: Downgrade to cheapest SQL instance

* CI: Update workflows to new deployment

* MAINT: Add script to get version easily

* MAINT: Ensure credentials are not included

* CI: Update deployment script to reflect tag

.dockerignore

@@ -0,0 +1 @@
+dist/

.gcloudignore

@@ -0,0 +1,2 @@
+dist/
+*egg-info

.gitattributes

@@ -0,0 +1 @@
+migas_server/_version.py export-subst

.github/workflows/build.yml

@@ -24,6 +24,6 @@ jobs:
       - name: Start all the things
         run: docker compose up -d
       - name: Test the things
-        run: docker exec migas_server bash -c "pytest /src/migas_server"
+        run: docker exec migas_server bash -c "pytest --pyargs migas_server"
       - name: Stop all the things
         run: docker compose down

.github/workflows/prod-deploy.yml

@@ -7,26 +7,26 @@ on:
 
 jobs:
   deploy:
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Install Heroku CLI
-        run: curl https://cli-assets.heroku.com/install.sh | sh
-      - name: Install builds buildpack
-        run: heroku plugins:install heroku-builds
-      - name: Install python build
-        run: python -m pip install build
-      - name: Package app
-        run: python -m build --sdist
-      - name: Pass in built app to Heroku deployment
-        run: |
-          SOURCE=$(ls dist/migas_server*.tar.gz)
-          heroku builds:create -a migas --source-tar=$SOURCE
-        env:
-          HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }}
-          HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+      # Authenticate and setup SDK
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          workload_identity_provider: 'projects/783007238857/locations/global/workloadIdentityPools/migas-gha/providers/github-actions'
+          service_account: '[email protected]'
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+      # Use provided deployment script
+      - name: Run deployment script
+        run: ./deploy/gcp/release-gcp.sh
 
   release:
     runs-on: ubuntu-latest

.gitignore

@@ -10,3 +10,7 @@ dump.rdb
 mounts/
 .pytest_cache/
 .mypy_cache/
+**/.env
+
+# Ignore generated credentials from google-github-actions/auth
+gha-creds-*.json

.pre-commit-config.yaml

@@ -10,8 +10,10 @@ repos:
     rev: 22.3.0
     hooks:
     -   id: black
+        exclude: migas_server/_version.py
 - repo: https://github.com/pycqa/isort
   rev: 5.10.1
   hooks:
     -   id: isort
+        exclude: migas_server/_version.py
         name: isort (python)

Dockerfile

@@ -1,12 +1,14 @@
-FROM python:3.10.5-slim-bullseye
-
+FROM python:slim AS src
+RUN pip install build
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-COPY . /src
-
-RUN pip install --no-cache-dir /src[test]
+    apt-get install -y --no-install-recommends git
+COPY . /src/migas-server
+RUN python -m build /src/migas-server
+
+FROM python:slim
+COPY --from=src /src/migas-server/dist/*.whl .
+ENV YARL_NO_EXTENSIONS=1 \
+    MULTIDICT_NO_EXTENSIONS=1
+RUN python -m pip install --no-cache-dir $( ls *.whl )[test]
 
 ENTRYPOINT ["migas-server"]

deploy/gcp/example.env

@@ -0,0 +1,5 @@
+MIGAS_REDIS_URI: redis://:c1jPnbTynglGfE3e0xaat1xiILkn23hj@redis-13848.c259.us-central1-2.gce.cloud.redislabs.com:13861
+DATABASE_USER: postgres
+DATABASE_PASSWORD: foobar
+DATABASE_NAME: migas
+GCP_SQL_CONNECTION: migas-12356:us-central1:migas-postgres

deploy/gcp/release-gcp.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# This script uses the `gcloud` SDK to:
+# - Check for required resources
+# - Create required resources if necessary (optional)
+# - Deploy the application
+
+# In addition to `gcloud` commands, this deployment leverages Redis Cloud (https://app.redislabs.com/)
+# to create a small (but free) fully-managed Redis instance (potential hosts: AWS/GCP/Azure)
+# A URI of this connection should be defined as MIGAS_REDIS_URI in the service instance
+
+# See `example.env` for an example expected .env file
+
+set -eux
+
+HERE=$(dirname $(realpath $0))
+ROOT=$(dirname $(dirname $HERE))
+
+VERSION="unknown"
+if [-f ${ROOT}/get_version.py ]; then
+    VERSION=$(python ${ROOT}/get_version.py)
+fi
+
+# These should be available, but will be account specific
+# PROJECT_ID=
+# SQL_INSTANCE_PASSWORD=
+
+# Defaults
+GCP_REGION="us-central1"
+SQL_INSTANCE_NAME="migas-postgres"
+SQL_INSTANCE_TIER="db-g1-micro"
+CLOUD_RUN_SERVICE_NAME="migas-server"
+CLOUD_RUN_ENV_FILE="$HERE/.env"
+
+# Step 0: Install SDK
+# https://github.com/google-github-actions/setup-gcloud
+
+# Step 1: Check for SQL database
+SQL_EXISTS=$(gcloud sql instances list --filter name=$SQL_INSTANCE_NAME --uri)
+
+if [[ -z $SQL_EXISTS ]]; then
+    # Step 1.5: If not, create it
+    gcloud sql instances create $SQL_INSTANCE_NAME \
+        --database-version=POSTGRES_14 \
+        --region=$GCP_REGION \
+        --root-password=$SQL_INSTANCE_PASSWORD \
+        --insights-config-query-insights-enabled \
+        --tier=$SQL_INSTANCE_TIER
+
+    # create migas database
+    gcloud sql databases create migas --instance=migas-postgres
+fi
+
+# Step 2: Build the service image
+GCR_TAG=gcr.io/$PROJECT_ID/$CLOUD_RUN_SERVICE_NAME:$VERSION
+gcloud builds submit \
+    --tag $GCR_TAG
+
+# Step 3: Deploy the service
+gcloud run deploy $CLOUD_RUN_SERVICE_NAME \
+    --region=$GCP_REGION \
+    --image=$GCR_TAG \
+    --platform=managed \
+    --min-instances=1 \
+    --max-instances=3 \
+    --ingress=all \
+    --allow-unauthenticated \
+    --set-cloudsql-instances="$PROJECT_ID:$GCP_REGION:$SQL_INSTANCE_NAME" \
+    --memory=512Mi \
+    --cpu=2 \
+    --args=--port,8080,--proxy-headers,--headers,X-Backend-Server:migas \
+    --env-vars-file=$CLOUD_RUN_ENV_FILE \
+    --cpu-throttling
+
+
+# # Step 4: Map service to custom domain (only needs to be done once)
+# ROOT_DOMAIN=nipreps.org  # The root domain name
+# TARGET_DOMAIN=migas.nipreps.org  # The target domain, including any subdomains
+
+# gcloud domains verify $ROOT_DOMAIN
+# gcloud beta run domain-mappings create --service $CLOUD_RUN_SERVICE_NAME --domain $TARGET_DOMAIN
+# # Generate DNS record
+# gcloud beta run domain-mappings describe --domain $TARGET_DOMAIN

deploy/gcp/shutdown-gcp.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+SQL_INSTANCE_NAME='migas-postgres'
+SERVICE_INSTANCE_NAME='migas-server'
+GCP_REGION="us-central1"
+
+SQL_EXISTS=$(gcloud sql instances list --filter name=$SQL_INSTANCE_NAME --uri)
+if [[ -n $SQL_EXISTS ]]; then
+    echo "Delete database"
+    gcloud sql instances delete $SQL_INSTANCE_NAME --async --quiet
+fi
+
+SERVICE_EXISTS=$(gcloud run services list --filter=SERVICE:$SERVICE_INSTANCE_NAME)
+if [[ -n $SERVICE_EXISTS ]]; then
+    echo "Deleting service"
+    gcloud run services delete $SERVICE_INSTANCE_NAME --region=$GCP_REGION --async --quiet
+fi

docker-compose.yml

@@ -15,7 +15,7 @@ services:
     command: --proxy-headers
     environment:
       MIGAS_REDIS_URI: "redis://cache:6379"
-      DATABASE_URL: "postgres://postgres:crumbs@postgres:5432/migas"
+      DATABASE_URL: "postgresql+asyncpg://postgres:crumbs@postgres:5432/migas"
 
   # Redis in-memory db
   cache:

get_version.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+import sys
+import os.path as op
+
+
+def main():
+    sys.path.insert(0, op.abspath('.'))
+    from migas_server import __version__
+    print(__version__)
+
+
+if __name__ == '__main__':
+    main()

migas_server/__init__.py

@@ -1,3 +1,3 @@
-from importlib.metadata import version
+from . import _version
 
-__version__ = version('migas_server')
+__version__ = _version.get_versions()['version']