From 745a3997abae4ee3042254f530bfaf49821ca1a2 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 01/33] container/build: add basic Containerfile This adds a Containerfile (a technology-agnostic Dockerfile) that can successfully install the app's dependencies into a container image. The image can be built with the following command (note the trailing build context of "."): podman build . The image's contents are controlled via an allowlist in .containerignore, which is expected to change significantly in later commits as more of this repo is used inside the container. The image is not yet set up to be invoked. --- .containerignore | 7 +++++++ Containerfile | 1 + container/build/Containerfile | 15 +++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 .containerignore create mode 120000 Containerfile create mode 100644 container/build/Containerfile diff --git a/.containerignore b/.containerignore new file mode 100644 index 000000000..1ab7dad4d --- /dev/null +++ b/.containerignore @@ -0,0 +1,7 @@ +# Start with a clean slate by ignoring all paths. +* + +# Selectively include paths that need to be part of the image. +!/ynr/ +!/requirements/ +!/requirements.txt diff --git a/Containerfile b/Containerfile new file mode 120000 index 000000000..43de25212 --- /dev/null +++ b/Containerfile @@ -0,0 +1 @@ +container/build/Containerfile \ No newline at end of file diff --git a/container/build/Containerfile b/container/build/Containerfile new file mode 100644 index 000000000..b8b3d02f4 --- /dev/null +++ b/container/build/Containerfile @@ -0,0 +1,15 @@ +from public.ecr.aws/docker/library/python:3.8.15 + +# Set the working directory for the following RUN/COPY/ADD/CMD directives. +workdir /dc/ynr/ + +# Copy the client-side directory "." (i.e. the build context passed to the +# build command) into the container image. +# NB This obeys the rules in the .containerignore file. +copy . code/ + +# Install basic app requirements. Instruct pip not to use a cache directory to +# inprove container-image-level cache effectiveness. +# TODO: run as not-root? +# TODO: run upgrade pip? +run pip install --no-cache-dir -r code/requirements.txt From a11efdf39bb99bcaec6323285b6cdf30db846826 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 02/33] container/run: add postgres compose manifest This adds a postgresql compose manifest as a demonstration that podman-compose can successfully instatiate the service, via: podman compose -f container/run/compose.service.postgres.yml up This is not yet used in any context. --- container/run/compose.service.postgres.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 container/run/compose.service.postgres.yml diff --git a/container/run/compose.service.postgres.yml b/container/run/compose.service.postgres.yml new file mode 100644 index 000000000..ce6fdf4f9 --- /dev/null +++ b/container/run/compose.service.postgres.yml @@ -0,0 +1,13 @@ +version: '3' +volumes: + psql-data: +services: + psql: + image: public.ecr.aws/docker/library/postgres:16 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + volumes: + - psql-data:/var/lib/postgresql/data + ports: + - 5555:5432 From fce811c263f614904e2ebcc432cfa4c50e358c2f Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 03/33] container/build: use Ubuntu base image This changes the approach to building the app image by using an Ubuntu LTS release as the base, and starting by installing the current set of packages explicitly referenced in deploy/vars.yml (modulo some outdated entries such as redis-server). This change is based on discussions with DC personnel with knowledge of the current setup, and influenced by the likely path we'll take to adopting a container-based production setup in the future. The image that's produced is not small, as it contains many, many system packages that are pulled in as dependencies of the package set we explicitly specify. The resulting large package set does, however, permit the app's base Python requirements to be installed successfully, and this can act as a baseline for future size/efficiency improvements. --- container/build/Containerfile | 67 +++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/container/build/Containerfile b/container/build/Containerfile index b8b3d02f4..708d02b02 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -1,15 +1,76 @@ -from public.ecr.aws/docker/library/python:3.8.15 +from public.ecr.aws/lts/ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +run date \ + && apt update \ + && date + +run date \ + && apt install --no-install-suggests --assume-yes \ + build-essential \ + bundler \ + cmake \ + curl \ + gettext \ + git \ + language-pack-en \ + libavcodec-dev \ + libavformat-dev \ + libevent-dev \ + libffi-dev \ + libgtk2.0-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt-dev \ + libyaml-dev \ + npm \ + python3.8-venv \ + opencv-data \ + postgis \ + python-is-python3 \ + python3-dev \ + python3-pip \ + python3-psycopg2 \ + python3-setuptools \ + python3-virtualenv \ + s3cmd \ + unzip \ + yui-compressor \ + && date + +run date \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && date + +# Set a base path for the app install build process. +arg APP_ROOT=/dc/ynr +# Set a path for the app's virtualenv +arg APP_VENV=$APP_ROOT/venv # Set the working directory for the following RUN/COPY/ADD/CMD directives. -workdir /dc/ynr/ +workdir $APP_ROOT # Copy the client-side directory "." (i.e. the build context passed to the # build command) into the container image. # NB This obeys the rules in the .containerignore file. copy . code/ +# Set up a virtualenv to avoid interactions with system packages, and install a +# common pre-req. +run date \ + && python -m venv $APP_VENV \ + && $APP_VENV/bin/pip install wheel \ + && date +# Use the virtualenv without explicit activation. +ENV PATH="$APP_VENV/bin:$PATH" + # Install basic app requirements. Instruct pip not to use a cache directory to # inprove container-image-level cache effectiveness. # TODO: run as not-root? # TODO: run upgrade pip? -run pip install --no-cache-dir -r code/requirements.txt +run date \ + && pip install --no-cache-dir -r code/requirements.txt \ + && date From 669cf2248a76e0544f0bc19e8643071c125e76dd Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 04/33] circleci/config: build container image in CI This adds a "container" workflow to CircleCI. The workflow is unoptimised, and initially serves only to test the buildability of the container image. The .containerignore file is not respected by CircleCI's Docker builders, so it is symlinked into place as .dockerignore. --- .circleci/config.yml | 14 ++++++++++++++ .dockerignore | 1 + Dockerfile | 1 + 3 files changed, 16 insertions(+) create mode 120000 .dockerignore create mode 120000 Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d251b59d..bc05b474a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,6 +147,17 @@ jobs: event: fail template: basic_fail_1 + container_build: + working_directory: ~/repo + docker: + - image: cimg/base:current + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build container image + command: docker build . workflows: test_build_deploy: @@ -157,3 +168,6 @@ workflows: - build_and_test filters: { branches: { only: [ main, master, deployment-upgrades] } } context: [ deployment-production-ynr, slack-secrets ] + container: + jobs: + - container_build diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 000000000..092a75da3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.containerignore \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 000000000..43de25212 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +container/build/Containerfile \ No newline at end of file From 163c1da971996df24c79fb828c1a3e5640cdb6b8 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 05/33] container/build: invoke a django check post-build This adds "python manage.py check" to the end of the container image build process. Whilst this isn't strictly necessary it serves two useful purposes by: - demonstrating that the container *can* run the check; and - pre-generating many .pyc files and including them in the image for faster app initialisation. The Containerfile is also tweaked to: - read the list of system packages from an external file; - install the app's python requirements before copying its code, so the DX happy path doesn't incur the pip-install cost when only modifying code; - install the app's python requirements based on the sopn_parsing.txt file as this transitively includes "pandas", without which the check command fails. --- .containerignore | 5 +++ container/build/Containerfile | 68 ++++++++++++--------------------- container/build/system-packages | 31 +++++++++++++++ 3 files changed, 61 insertions(+), 43 deletions(-) create mode 100644 container/build/system-packages diff --git a/.containerignore b/.containerignore index 1ab7dad4d..a7cb99c6b 100644 --- a/.containerignore +++ b/.containerignore @@ -5,3 +5,8 @@ !/ynr/ !/requirements/ !/requirements.txt +!/manage.py + +# Paths that don't need to be part of the image, but also don't need to be +# excluded when explicitly referenced by COPY and ADD commands. +!/container/build/system-packages diff --git a/container/build/Containerfile b/container/build/Containerfile index 708d02b02..9ed5b218d 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -6,39 +6,9 @@ run date \ && apt update \ && date +copy container/build/system-packages /tmp/apt-packages run date \ - && apt install --no-install-suggests --assume-yes \ - build-essential \ - bundler \ - cmake \ - curl \ - gettext \ - git \ - language-pack-en \ - libavcodec-dev \ - libavformat-dev \ - libevent-dev \ - libffi-dev \ - libgtk2.0-dev \ - libjpeg-dev \ - libpq-dev \ - libssl-dev \ - libxml2-dev \ - libxslt-dev \ - libyaml-dev \ - npm \ - python3.8-venv \ - opencv-data \ - postgis \ - python-is-python3 \ - python3-dev \ - python3-pip \ - python3-psycopg2 \ - python3-setuptools \ - python3-virtualenv \ - s3cmd \ - unzip \ - yui-compressor \ + && Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 06/33] container/build: permit cache invalidation in CI This adds a simplistic mechanism to invalidate CircleCI's Docker Layer Cache (DLC), and force an image rebuild from scratch. Locally, "podman build --no-cache ." should be preferred instead. --- container/build/Containerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/container/build/Containerfile b/container/build/Containerfile index 9ed5b218d..e8094c0f9 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -1,5 +1,9 @@ from public.ecr.aws/lts/ubuntu:20.04 +# Increase this arbitary number to force all image layers to be rebuilt. +# This is designed to invalidate the layer cache inside CI, not locally. +ARG invalidate_all_cached_layers=202411270705 + ARG DEBIAN_FRONTEND=noninteractive run date \ From d3521621724f361e2abae4faae673caed1fcc908 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 07/33] circleci/config: run app tests inside container Invoke a cut-down version of the current CI test command inside the built container. --- .circleci/config.yml | 24 ++++++++++++++++++++++-- container/build/Containerfile.test | 15 +++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 container/build/Containerfile.test diff --git a/.circleci/config.yml b/.circleci/config.yml index bc05b474a..2b4f03eaa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,13 +151,33 @@ jobs: working_directory: ~/repo docker: - image: cimg/base:current + - image: cimg/postgres:12.20 + name: dbpsql + environment: + POSTGRES_USER: ynr + POSTGRES_DB: ynr steps: - checkout - setup_remote_docker: docker_layer_caching: true - run: - name: Build container image - command: docker build . + name: Build base container image + command: docker build -t ynr:base . + - run: + name: Build test container image + command: docker build -t ynr:test -f container/build/Containerfile.test . + - run: + name: Run app tests + command: | + net="$(docker network inspect $(docker network ls -q -f "label=task-network") --format '{{.Name}}')" + docker run -it --rm \ + --net="$net" \ + -e RUN_ENV=test \ + -e PGHOST=dbpsql \ + -e CIRCLECI=true \ + \ + ynr:test \ + pytest --ds=ynr.settings.testing -x workflows: test_build_deploy: diff --git a/container/build/Containerfile.test b/container/build/Containerfile.test new file mode 100644 index 000000000..a27ff9273 --- /dev/null +++ b/container/build/Containerfile.test @@ -0,0 +1,15 @@ +from ynr:base + +# Increase this arbitary number to force all image layers to be rebuilt. +# This is designed to invalidate the layer cache inside CI, not locally. +ARG invalidate_all_cached_layers=202411270705 + +# Base path for the app install build process. +arg APP_ROOT=/dc/ynr +# Path for the app's code. +arg APP_CODE=$APP_ROOT/code + +# Install additional test dependencies. +run date \ + && pip install --no-cache-dir -r $APP_CODE/requirements/testing.txt \ + && date From b7ca49ac26bdf90f304a5ef370ebb5ee8af77a2a Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 08/33] container: add system and pip dependencies This adds some system and pip dependencies that are currently encoded in the CI config. The system dependencies are copied into the container's package file, but a manually fetched system dependency (pandoc) is translated into its convenience pip package equivalent. The change to pandoc's installation mechanism requires the use of an alternative version of the pypandoc library that also provides the underlying pandoc binary. This requires us to use a later version than the currently-used pypandoc version, as the currently-used version doesn't have a binary-including-package counterpart. We bump to the latest pip package version. --- container/build/Containerfile | 8 +------- container/build/system-packages | 6 +++++- requirements/sopn_parsing.txt | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/container/build/Containerfile b/container/build/Containerfile index e8094c0f9..c66696f27 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -6,16 +6,10 @@ ARG invalidate_all_cached_layers=202411270705 ARG DEBIAN_FRONTEND=noninteractive -run date \ - && apt update \ - && date - copy container/build/system-packages /tmp/apt-packages run date \ + && apt update \ && Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 09/33] container/build: merge base & test Containerfiles --- .circleci/config.yml | 6 +++--- container/build/Containerfile | 26 ++++++++++++++++++++++---- container/build/Containerfile.test | 15 --------------- 3 files changed, 25 insertions(+), 22 deletions(-) delete mode 100644 container/build/Containerfile.test diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b4f03eaa..57683895b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,11 +161,11 @@ jobs: - setup_remote_docker: docker_layer_caching: true - run: - name: Build base container image - command: docker build -t ynr:base . + name: Build production container image + command: docker build -t ynr:prod --target prod . - run: name: Build test container image - command: docker build -t ynr:test -f container/build/Containerfile.test . + command: docker build -t ynr:test --target test . - run: name: Run app tests command: | diff --git a/container/build/Containerfile b/container/build/Containerfile index c66696f27..246822785 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -1,10 +1,13 @@ -from public.ecr.aws/lts/ubuntu:20.04 +########################################################################### +## Production image ####################################################### +########################################################################### +from public.ecr.aws/lts/ubuntu:20.04 as prod # Increase this arbitary number to force all image layers to be rebuilt. # This is designed to invalidate the layer cache inside CI, not locally. -ARG invalidate_all_cached_layers=202411270705 +arg invalidate_all_cached_layers=202412030000 -ARG DEBIAN_FRONTEND=noninteractive +arg DEBIAN_FRONTEND=noninteractive copy container/build/system-packages /tmp/apt-packages run date \ @@ -28,7 +31,7 @@ run date \ && $APP_VENV/bin/pip install wheel \ && date # Use the virtualenv without explicit activation. -ENV PATH="$APP_VENV/bin:$PATH" +env PATH="$APP_VENV/bin:$PATH" # Copy dependency manifests into the container image. copy requirements.txt $APP_CODE/ @@ -54,3 +57,18 @@ workdir $APP_CODE # baseline level of correctness, whilst also generating .pyc files for faster # app startup. run python manage.py check --settings=ynr.settings.testing + +########################################################################### +## Testing image ########################################################## +########################################################################### +from prod as test + +# Base path for the app install build process. +arg APP_ROOT=/dc/ynr +# Path for the app's code. +arg APP_CODE=$APP_ROOT/code + +# Install additional test dependencies. +run date \ + && pip install --no-cache-dir -r $APP_CODE/requirements/testing.txt \ + && date diff --git a/container/build/Containerfile.test b/container/build/Containerfile.test deleted file mode 100644 index a27ff9273..000000000 --- a/container/build/Containerfile.test +++ /dev/null @@ -1,15 +0,0 @@ -from ynr:base - -# Increase this arbitary number to force all image layers to be rebuilt. -# This is designed to invalidate the layer cache inside CI, not locally. -ARG invalidate_all_cached_layers=202411270705 - -# Base path for the app install build process. -arg APP_ROOT=/dc/ynr -# Path for the app's code. -arg APP_CODE=$APP_ROOT/code - -# Install additional test dependencies. -run date \ - && pip install --no-cache-dir -r $APP_CODE/requirements/testing.txt \ - && date From db09fc5e5b099f9d1074b585d06310977570c598 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 10/33] compose: add a docker-compose file This adds a docker-compose.yml file which, along with the included changes, permits a developer to run a hot-reloading instance of the app. To make this work, it: - invokes `npm install` early in the image build process - adds gunicorn to requirements/base.txt (as there aren't any environments where this is optional) - makes some (hopefully harmless!) changes to ynr/settings/testing.py that allow the app to start successfully This change also: - adds a convenience script at scripts/container.image.build.bash, intended for use both locally and in CI - increases the commenting/structure in the Containerfile The compose file must be at the repo root (and not inside the defunct container/run/ directory) because relative bind mount sources are resolved relative to the compose file's location, and we need to mount the ynr/ directory from inside the repo root. This Docker (hence also Podman) feature even goes so far (or appears to!) as to resolve the location reading through a symlink, so we can't simply link a compose file sitting at the repo root to a file in a subdirectory. The developer currently needs to manually execute some asset-related commands before starting the compose stack, in order to make app render correctly. These commands place content into the ynr/ directory, which is bind-mounted from the developer's machine. They will be automated in a later change: podman compose run --rm --no-deps -e DJANGO_SETTINGS_MODULE=ynr.settings.testing frontend npm run build podman compose run --rm --no-deps -e DJANGO_SETTINGS_MODULE=ynr.settings.testing frontend python manage.py collectstatic --no-input --- .circleci/config.yml | 9 ++-- .containerignore | 3 ++ Containerfile | 1 - Dockerfile | 1 - container/build/Containerfile | 59 +++++++++++++++------- container/run/compose.service.postgres.yml | 13 ----- docker-compose.yml | 40 +++++++++++++++ requirements/base.txt | 1 + scripts/container.image.build.bash | 20 ++++++++ ynr/settings/testing.py | 3 +- 10 files changed, 111 insertions(+), 39 deletions(-) delete mode 120000 Containerfile delete mode 120000 Dockerfile delete mode 100644 container/run/compose.service.postgres.yml create mode 100644 docker-compose.yml create mode 100755 scripts/container.image.build.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index 57683895b..f3cb3032c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,11 +161,10 @@ jobs: - setup_remote_docker: docker_layer_caching: true - run: - name: Build production container image - command: docker build -t ynr:prod --target prod . - - run: - name: Build test container image - command: docker build -t ynr:test --target test . + name: Build container images + command: | + ./scripts/container.image.build.bash prod + ./scripts/container.image.build.bash test - run: name: Run app tests command: | diff --git a/.containerignore b/.containerignore index a7cb99c6b..cda97bc8c 100644 --- a/.containerignore +++ b/.containerignore @@ -6,7 +6,10 @@ !/requirements/ !/requirements.txt !/manage.py +!/gulpfile.js # Paths that don't need to be part of the image, but also don't need to be # excluded when explicitly referenced by COPY and ADD commands. !/container/build/system-packages +!/package.json +!/package-lock.json diff --git a/Containerfile b/Containerfile deleted file mode 120000 index 43de25212..000000000 --- a/Containerfile +++ /dev/null @@ -1 +0,0 @@ -container/build/Containerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 120000 index 43de25212..000000000 --- a/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -container/build/Containerfile \ No newline at end of file diff --git a/container/build/Containerfile b/container/build/Containerfile index 246822785..4bd7cc2b4 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -3,13 +3,26 @@ ########################################################################### from public.ecr.aws/lts/ubuntu:20.04 as prod +######################### +## Build-time arguments # +######################### # Increase this arbitary number to force all image layers to be rebuilt. # This is designed to invalidate the layer cache inside CI, not locally. arg invalidate_all_cached_layers=202412030000 +# Base path for the app install build process. +arg APP_ROOT=/dc/ynr +# Path for the app's virtualenv. +arg APP_VENV=$APP_ROOT/venv +# Path for the app's code. +arg APP_CODE=$APP_ROOT/code +######################## +## System dependencies # +######################## arg DEBIAN_FRONTEND=noninteractive - +# Copy system dependency manifest into the container image. copy container/build/system-packages /tmp/apt-packages +# Install dependencies. run date \ && apt update \ && /dev/null 2>&1 && pwd )/.." + +# Choose a build tool. +# CircleCI uses docker; others (e.g. developers) use podman. +set +u; if [[ "$CIRCLECI" == "true" ]]; then + builder="docker" +else + builder="podman" +fi; set -u + +# Build the image. +"$builder" build --target "$image" --tag "ynr:$image" -f container/build/Containerfile . diff --git a/ynr/settings/testing.py b/ynr/settings/testing.py index 8cfe82cc3..75b53bcdd 100644 --- a/ynr/settings/testing.py +++ b/ynr/settings/testing.py @@ -17,6 +17,7 @@ def __contains__(self, item): def __getitem__(self, item): return None +ADMINS = [("dummy", "dummy@dummy.example")] MIGRATION_MODULES = DisableMigrations() @@ -29,7 +30,7 @@ def __getitem__(self, item): RUNNING_TESTS = True SECRET_KEY = "just here for testing" -ALLOWED_HOSTS = ["candidates.democracyclub.org.uk"] +ALLOWED_HOSTS = ["candidates.democracyclub.org.uk","*"] SHOW_SOPN_TRACKER = False SHOW_RESULTS_PROGRESS = False From 81513063704825b8d97dd472247a0f0f27555b34 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 11/33] container/build: clean npm cache This cleans the NPM cache after installation, and also uses the more deterministic "npm ci" command instead of "npm install". Also: - help future beginners by defining SOPN in the top-level README - reduce top-level repo clutter by renaming .containerignore to .dockerignore. --- .containerignore | 15 --------------- .dockerignore | 16 +++++++++++++++- README.md | 2 +- container/build/Containerfile | 26 ++++++++++++++++++++++---- 4 files changed, 38 insertions(+), 21 deletions(-) delete mode 100644 .containerignore mode change 120000 => 100644 .dockerignore diff --git a/.containerignore b/.containerignore deleted file mode 100644 index cda97bc8c..000000000 --- a/.containerignore +++ /dev/null @@ -1,15 +0,0 @@ -# Start with a clean slate by ignoring all paths. -* - -# Selectively include paths that need to be part of the image. -!/ynr/ -!/requirements/ -!/requirements.txt -!/manage.py -!/gulpfile.js - -# Paths that don't need to be part of the image, but also don't need to be -# excluded when explicitly referenced by COPY and ADD commands. -!/container/build/system-packages -!/package.json -!/package-lock.json diff --git a/.dockerignore b/.dockerignore deleted file mode 120000 index 092a75da3..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.containerignore \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..cda97bc8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +# Start with a clean slate by ignoring all paths. +* + +# Selectively include paths that need to be part of the image. +!/ynr/ +!/requirements/ +!/requirements.txt +!/manage.py +!/gulpfile.js + +# Paths that don't need to be part of the image, but also don't need to be +# excluded when explicitly referenced by COPY and ADD commands. +!/container/build/system-packages +!/package.json +!/package-lock.json diff --git a/README.md b/README.md index c291099a3..1cddc3d44 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ for their work on the project which we have been able to build on. v0.9 is legacy code and is now frozen. v1.0 is currently in alpha. We plan on publishing a v1 API once we have some more feedback from users and we think it’s stable enough. -# SOPN Parsing +# Statement Of Persons Nominated (SOPN) Parsing YNR uses `pypandoc` (which relies on `pandoc`) to convert SOPN documents to PDF, as needed, to be parsed. diff --git a/container/build/Containerfile b/container/build/Containerfile index 4bd7cc2b4..9a52693d3 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -35,12 +35,22 @@ run date \ ###################### # Copy node dependency manifests into the container image. copy package.json package-lock.json $APP_CODE/ +# "setting NODE_ENV to anything but production is considered an antipattern." +# (https://nodejs.org/en/learn/getting-started/nodejs-the-difference-between-development-and-production) +# However, the version of npm we currently use doesn't appear to offer any way +# to make "npm ci" install the devDependencies (which include the "gulp" +# command that's required later in this build) when NODE_ENV="production". +# Therefore we delay setting NODE_ENV until after "npm ci" has run. +# arg NODE_ENV=production +# env NODE_ENV=$NODE_ENV # Install dependencies. -# TODO: npm equivalent of "apt-get clean && rm -rf /var/lib/apt/lists/*" run date \ && cd $APP_CODE/ \ - && npm install \ + && npm ci \ + && npm cache clean --force \ && date +arg NODE_ENV=production +env NODE_ENV=$NODE_ENV ######################## ## Python dependencies # @@ -67,8 +77,8 @@ run date \ ## App code # ############# # Copy the client-side directory "." (the build context passed to the build -# command) into the container image. -# NB This obeys the rules in the .containerignore file. +# command) into the container image, obeying the inclusions & exclusions +# encoded in the ./.dockerignore file. copy . $APP_CODE/ # Set the working directory for the container entrypoint. workdir $APP_CODE @@ -81,6 +91,14 @@ workdir $APP_CODE # app startup. run python manage.py check --settings=ynr.settings.testing +########### +## Assets # +########### +run date \ + && npm run build \ + && python manage.py collectstatic --no-input --settings=ynr.settings.testing \ + && date + ########################################################################### ## Testing image ########################################################## ########################################################################### From cd375abc6cd08b97f19bf0b722422839dcbc0ae8 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 12/33] container/build: upgrade pip during build --- container/build/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/build/Containerfile b/container/build/Containerfile index 9a52693d3..4a361392c 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -59,7 +59,7 @@ env NODE_ENV=$NODE_ENV # Install a common pre-req. run date \ && python -m venv $APP_VENV \ - && $APP_VENV/bin/pip install wheel \ + && $APP_VENV/bin/pip install --upgrade pip wheel \ && date # Use the virtualenv without explicit activation. env PATH="$APP_VENV/bin:$PATH" From 1025ded454b36076185b5764ad2a0e4c6a64fb66 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 13/33] scripts: run manage.py in container This adds a convenience shim for invoking "python manage.py" inside a freshly instantiated container, connected to the same DB and bind mounts that the composed "frontend" container would be able to access: $ ./scripts/container.compose.manage-py.bash check 2>/dev/null WARNING: no local settings file found. See local.py.example System check identified no issues (0 silenced). Invocations are currently noisy (hence the 2> redirection); this will be improved later. --- .dockerignore | 1 + scripts/container.compose.manage-py.bash | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100755 scripts/container.compose.manage-py.bash diff --git a/.dockerignore b/.dockerignore index cda97bc8c..d808898c6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ !/requirements.txt !/manage.py !/gulpfile.js +!/Makefile # Paths that don't need to be part of the image, but also don't need to be # excluded when explicitly referenced by COPY and ADD commands. diff --git a/scripts/container.compose.manage-py.bash b/scripts/container.compose.manage-py.bash new file mode 100755 index 000000000..134cdcfbd --- /dev/null +++ b/scripts/container.compose.manage-py.bash @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.compose.manage.bash invokes a Django management command in a new +# container, abiding by the invocation setup and bind mounts encoded in +# docker-compose.yml. +command="$1" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +podman compose run --rm --no-deps -e DJANGO_SETTINGS_MODULE=ynr.settings.testing frontend python manage.py $command From 39cbb250b9c7b6590a45e50fec456515b72382cb Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 14/33] compose: align bind mount paths with copied paths This expands the set of file and directory paths which are bind mounted into the frontend container when using compose. The set is aligned with the paths included in the built container image via the .dockerignore file's contents. --- .dockerignore | 22 +++++++++++++++------- docker-compose.yml | 13 ++++++++++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index d808898c6..89bce0722 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,16 +1,24 @@ -# Start with a clean slate by ignoring all paths. +# This file contains paths, .gitignore-style, that will be excluded from being +# copied into a container image during build. +# cf. https://docs.docker.com/build/concepts/context/#dockerignore-files + +# Start with a clean slate by excluding all paths. * -# Selectively include paths that need to be part of the image. +# Selectively re-include paths to copy into the container. This list should +# track and match the bind mounts for the "frontend" service in the +# docker-compose.yml file. !/ynr/ !/requirements/ !/requirements.txt !/manage.py -!/gulpfile.js !/Makefile - -# Paths that don't need to be part of the image, but also don't need to be -# excluded when explicitly referenced by COPY and ADD commands. -!/container/build/system-packages +!/scripts/ +!/gulpfile.js !/package.json !/package-lock.json +!/pyproject.toml + +# Paths that don't need to be part of the image, but also need not to be +# excluded when explicitly referenced by build-time COPY and ADD commands. +!/container/build/system-packages diff --git a/docker-compose.yml b/docker-compose.yml index f3bd34efd..c3eff035f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,9 +18,16 @@ services: DJANGO_SETTINGS_MODULE: ynr.settings.testing PORT: 8080 volumes: - - type: bind - source: ./ynr - target: /dc/ynr/code/ynr + - ./ynr/:/dc/ynr/code/ynr/ + - ./requirements/:/dc/ynr/code/requirements/ + - ./requirements.txt:/dc/ynr/code/requirements.txt + - ./manage.py:/dc/ynr/code/manage.py + - ./Makefile:/dc/ynr/code/Makefile + - ./scripts/:/dc/ynr/code/scripts/ + - ./gulpfile.js:/dc/ynr/code/gulpfile.js + - ./package.json:/dc/ynr/code/package.json + - ./package-lock.json:/dc/ynr/code/package-lock.json + - ./pyproject.toml:/dc/ynr/code/pyproject.toml ports: 8080:8080 depends_on: dbpsql: From 91a495c7f477ad128ac3ff62acae617bf376e302 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 15/33] compose: add build key This permits the container image required by the local compose stack to be built directly using the compose subcommand: podman compose build or podman compose build frontend A test is added to CI that asserts compose is superficially happy. We don't (yet) use compose to run any containers in CI, so this step is necessary trivial. Also: fix a podman/docker inconsistency where podman permitted compose "ports" to be a scalar value, but docker requires a list of values. --- .circleci/config.yml | 6 ++++++ docker-compose.yml | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3cb3032c..b5929ae02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,6 +165,12 @@ jobs: command: | ./scripts/container.image.build.bash prod ./scripts/container.image.build.bash test + - run: + name: Check that compose is happy + command: | + docker compose config + # No-op, due to a shared layer cache with earlier image builds. + docker compose build - run: name: Run app tests command: | diff --git a/docker-compose.yml b/docker-compose.yml index c3eff035f..05bba4d54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,6 @@ services: frontend: image: ynr:test - pull_policy: never command: gunicorn --reload --log-level=debug ynr.wsgi environment: RUN_ENV: test @@ -28,10 +27,17 @@ services: - ./package.json:/dc/ynr/code/package.json - ./package-lock.json:/dc/ynr/code/package-lock.json - ./pyproject.toml:/dc/ynr/code/pyproject.toml - ports: 8080:8080 + ports: + - "8080:8080" depends_on: dbpsql: condition: service_healthy + pull_policy: never + build: + dockerfile: container/build/Containerfile + # Context is relative to this docker-compose.yml file. + context: . + target: test dbpsql: image: public.ecr.aws/docker/library/postgres:12.20 From dbd6977a1a024b5dca61b3de3136de83dab49f04 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 16/33] compose: frontend: use optional envvar file This allows a developer to place a file at the gitignored path env/frontend.env containing VAR=val key/value pairs which the frontend compose container will read on startup and instantiate as environment variables. This mechanism has been chosen as it affects all frontend container invocations, including those that don't start Django as a web server. This consistency will reduce unexpected drift between the primary frontend container started by a developer and any secondary containers. There are complications that currently make it fiddly to get this file *also* bind-mounted as the .env file that ynr/wsgi.py loads. For the moment, this means that environment variable changes require a restart of the frontend container. --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 05bba4d54..fdd9cbd4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,9 @@ services: frontend: image: ynr:test command: gunicorn --reload --log-level=debug ynr.wsgi + env_file: + - path: env/frontend.env + required: false environment: RUN_ENV: test PGHOST: dbpsql From caf9aa281a512f17d07788b3b6abe875fa26a85f Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 17/33] scripts: add podman-compose-exec shim This adds a script that shims "podman compose exec", and changes the renamed shim script that invokes Django management commands to use it. It also removes a couple of envvars from the compose file which are more properly situated in the gitignored, per-developer env/frontend.env file. --- docker-compose.yml | 3 --- scripts/container.compose.manage-py.bash | 12 ------------ scripts/container.exec.bash | 24 ++++++++++++++++++++++++ scripts/container.manage-py.bash | 17 +++++++++++++++++ 4 files changed, 41 insertions(+), 15 deletions(-) delete mode 100755 scripts/container.compose.manage-py.bash create mode 100755 scripts/container.exec.bash create mode 100755 scripts/container.manage-py.bash diff --git a/docker-compose.yml b/docker-compose.yml index fdd9cbd4e..9b73035a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,7 @@ services: - path: env/frontend.env required: false environment: - RUN_ENV: test PGHOST: dbpsql - CIRCLECI: true - DJANGO_SETTINGS_MODULE: ynr.settings.testing PORT: 8080 volumes: - ./ynr/:/dc/ynr/code/ynr/ diff --git a/scripts/container.compose.manage-py.bash b/scripts/container.compose.manage-py.bash deleted file mode 100755 index 134cdcfbd..000000000 --- a/scripts/container.compose.manage-py.bash +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# container.compose.manage.bash invokes a Django management command in a new -# container, abiding by the invocation setup and bind mounts encoded in -# docker-compose.yml. -command="$1" - -# Change to the directory above the directory containing this script. -cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." - -podman compose run --rm --no-deps -e DJANGO_SETTINGS_MODULE=ynr.settings.testing frontend python manage.py $command diff --git a/scripts/container.exec.bash b/scripts/container.exec.bash new file mode 100755 index 000000000..24b070a9f --- /dev/null +++ b/scripts/container.exec.bash @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.exec.bash invokes a command in a "frontend" container. +# The container must be running before running this script (see README.md). +# +# Usage: +# scripts/container.exec.bash echo hello world +# +# This script is a deliberately simple convenience shim, and doesn't include a +# way to set or override environment variables. To do this, invoke "podman" +# directly: +# podman compose exec -e key1=val -e key2=val frontend command param1 param2 +# Updating variables held in env/frontend.env has no effect until the running +# container is restarted. + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +podman compose exec frontend $command diff --git a/scripts/container.manage-py.bash b/scripts/container.manage-py.bash new file mode 100755 index 000000000..d81ee9201 --- /dev/null +++ b/scripts/container.manage-py.bash @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.manage-py.bash invokes a Django management command in a frontend +# container, which must have been started beforehand. +# +# Usage: +# scripts/container.manage-py.bash check + +# The management command does not need to be quoted, unless it contains shell +# meta-characters. Multiple words are fine. +mgmtCommand="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +./scripts/container.exec.bash python manage.py $mgmtCommand From dc446b527d1fe6ec7dce3a5e9de4ac51313fe6b3 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 18/33] container: don't copy ynr/media to container image /ynr/media/ doesn't need to be included inside the frontend container image. --- .dockerignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.dockerignore b/.dockerignore index 89bce0722..1345d8d0f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,6 +19,14 @@ !/package-lock.json !/pyproject.toml +# Paths overriding early re-inclusions. +# The content inside /ynr/media/ is bind-mounted into the container when +# running as part of a compose stack, as part of the bind-mount of its parent, +# /ynr/. However its contents *should not* be statically copied into the +# container image because they're either irrelevant/empty (in non-development +# environments), or potentially very large (in development environments). +/ynr/media/ + # Paths that don't need to be part of the image, but also need not to be # excluded when explicitly referenced by build-time COPY and ADD commands. !/container/build/system-packages From bbb6b3da4b3613427f54dabc78ad359c0b5d656a Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:42 +0000 Subject: [PATCH 19/33] {compose,ci}: bump postgres server to 16.4 YNR's production RDS runs v16.4. Both the local containerised setup and the in-CI DB should mirror this. --- .circleci/config.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5929ae02..c8474ef69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,7 +151,7 @@ jobs: working_directory: ~/repo docker: - image: cimg/base:current - - image: cimg/postgres:12.20 + - image: cimg/postgres:16.4 name: dbpsql environment: POSTGRES_USER: ynr diff --git a/docker-compose.yml b/docker-compose.yml index 9b73035a4..9785af14a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,7 @@ services: target: test dbpsql: - image: public.ecr.aws/docker/library/postgres:12.20 + image: public.ecr.aws/docker/library/postgres:16.4 pull_policy: missing environment: POSTGRES_DB: ynr From 4fd944c1ef26015610ec68bffbdef2802e81f341 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 20/33] scripts: add container.pytest.bash This adds a script that executes pytest inside a running frontend container, passing any parameters through to pytest. --- scripts/container.pytest.bash | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 scripts/container.pytest.bash diff --git a/scripts/container.pytest.bash b/scripts/container.pytest.bash new file mode 100755 index 000000000..b0f6c32b2 --- /dev/null +++ b/scripts/container.pytest.bash @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.test.bash invokes a `pytest` command in a "frontend" container. +# The container and a database server must be running before running this +# script (see README.md). +# +# Usage: +# scripts/container.test.bash # run all tests; continue after failures +# scripts/container.test.bash -x # stop after first failure + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +./scripts/container.exec.bash pytest $command From 267ad0fe352c4808e4041c7fdb4344c788d99a80 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 21/33] compose: add explicit psql healthcheck Docker compose requires an explicit healthcheck, whereas podman appears to infer a healthy container when an exposed port is listening. --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9785af14a..29a6c08fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,3 +51,8 @@ services: - psql-data:/var/lib/postgresql/data ports: - 54321:5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready -d ynr"] + interval: 10s + timeout: 5s + retries: 5 From e8ddac261589ccc073532809c4d8e48cec67fe8c Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 22/33] Revert "compose: add explicit psql healthcheck" This reverts commit 4af80b59f09dcf96f0827e7795181b26b913e87a. This was added as docker required it, so that we could test using docker-compose in CI. This has proven impractical, given the warnings at https://circleci.com/docs/docker-compose/#using-docker-compose-with-docker-executor: The remote docker daemon runs on a different system than the docker CLI and docker compose, so you must move data around to make this work. Mounting can usually be solved by making content available in a docker volume. In order to make this work, we'd have to vary the compose file so that in local dev content was bind-mounted, whereas it would be made available in a volume in CI. This would negate the point of running the compose stack in CI: to prove that local-dev workflows are happy. Because podman appears to have a problem with this setup on certain machines: ERRO[0000] Failed to start transient timer unit: Unit cdb85bbc93bfd59cc38703842f96c12d0cced997a7ee582b2954a6bb87905804.timer was already loaded or has a fragment file. ... our easiest route forward is to revert this change until we genuinely need to solve docker's healthcheck problem. --- docker-compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 29a6c08fd..9785af14a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,8 +51,3 @@ services: - psql-data:/var/lib/postgresql/data ports: - 54321:5432 - healthcheck: - test: ["CMD-SHELL", "pg_isready -d ynr"] - interval: 10s - timeout: 5s - retries: 5 From de5e27ef34548489146ccb1718c41b1001b571a8 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 23/33] scripts: add podman-compose-run shim This adds scripts/container.run.bash, which runs a command in a fresh frontend container. Also: fix trailing whitespace --- scripts/container.exec.bash | 2 +- scripts/container.run.bash | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100755 scripts/container.run.bash diff --git a/scripts/container.exec.bash b/scripts/container.exec.bash index 24b070a9f..b3a4a0696 100755 --- a/scripts/container.exec.bash +++ b/scripts/container.exec.bash @@ -5,7 +5,7 @@ set -euo pipefail # The container must be running before running this script (see README.md). # # Usage: -# scripts/container.exec.bash echo hello world +# scripts/container.exec.bash echo hello world # # This script is a deliberately simple convenience shim, and doesn't include a # way to set or override environment variables. To do this, invoke "podman" diff --git a/scripts/container.run.bash b/scripts/container.run.bash new file mode 100755 index 000000000..5a700424b --- /dev/null +++ b/scripts/container.run.bash @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.run.bash invokes a command in a newly instantiated "frontend" +# container. +# +# The container and its filesystem are removed after the user's command exits. +# Only changes made inside bind-mounted filesystems are persisted (see +# docker-compose.yml for a list of bind mounts). The database container must be +# running before this script can be invoked (see README.md) because the compose +# command sees the database as a required dependency. This is the case even if +# the command being executed doesn't use the database. +# +# Usage: +# scripts/container.run.bash env +# +# Because a new container is instantiated along with its environment each time +# this script is invoked, the contents of the env/frontend.env file are +# respected for each invocation, with environment variables being set as per +# that file. + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +podman compose run --rm --no-deps --name tmp-fe-$$ frontend $command \ + 2> >( grep -v "Error: adding pod to state.*pod already exists" >&2 ) From f39e792cae14515c4ed428cc60765d598e579a56 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 24/33] ynr/settings: add containerised local.py example --- ynr/settings/local.py.container.example | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 ynr/settings/local.py.container.example diff --git a/ynr/settings/local.py.container.example b/ynr/settings/local.py.container.example new file mode 100644 index 000000000..48179889f --- /dev/null +++ b/ynr/settings/local.py.container.example @@ -0,0 +1,26 @@ +# Only set DEBUG to True in development environments. +DEBUG = True + +# This short, known value is insecure. +SECRET_KEY = "development" + +# Certain errors are very noisy (obscuring the real problem) if this list is +# empty. +ADMINS = [("Dummy Admin", "dummy@example.com")] + +# This permits the site to be served at localhost:8080. +ALLOWED_HOSTS = ['*'] + +# This unpleasantness adds the container's internal IP to the list of those IPs +# permitted to access the Django debug toolbar, which allows it to be enabled. +# We believe the container's own IP needs to be in this list because of +# something to do with the container networking, or the HTTP server gunicorn's +# reverse-proxy setup, or both. +# TODO: Replace with a better method, either here or by changing the +# container/gunicorn setup. https://pypi.org/project/netifaces/ also exists, +# but might not be considered "better". +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("8.8.8.8", 80)) +INTERNAL_IPS = [ '127.0.0.1', str(s.getsockname()[0]) ] +s.close() From ae3a373e3807bf8450847f961a9f7d5ce63d115c Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 25/33] scripts: user-specified container build params This allows a developer who invokes scripts/container.image.build.bash to specify arbitrary parameters for the underlying builder. e.g. "--no-cache", which is a better way of avoiding using the local build cache than bumping the CI-focussed build ARG at the top of container/build/Containerfile. --- scripts/container.image.build.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/container.image.build.bash b/scripts/container.image.build.bash index 50b411691..822786b1a 100755 --- a/scripts/container.image.build.bash +++ b/scripts/container.image.build.bash @@ -3,7 +3,8 @@ set -euo pipefail # container.image.build.bash builds an image from a stage defined in # container/build/Containerfile, and tags it as "ynr:$image". -image="$1" +image="$1"; shift +args="$@" # Change to the directory above the directory containing this script. cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." @@ -17,4 +18,4 @@ else fi; set -u # Build the image. -"$builder" build --target "$image" --tag "ynr:$image" -f container/build/Containerfile . +"$builder" build --target "$image" --tag "ynr:$image" -f container/build/Containerfile $args . From b2b4d5c1e1a1e6af8d84b8b47676bfebf673aa78 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Mon, 30 Dec 2024 10:59:43 +0000 Subject: [PATCH 26/33] docs: update for containerised setup This adds some docs to help a developer get started modifying and running YNR inside a container. Also: structure README.md's headings more hierarchically, leaving only a single H1 on the page; some light rewording in README.md to improve its flow, along with a short new section that points a member of the public towards whocanivotefor.co.uk if they happen to come across this repo. --- README.md | 86 +++++---- docs/DEVELOPMENT.md | 242 ++++++++++++++++++++++++ docs/INSTALL.md | 157 +++++++-------- docs/INSTALL.old.md | 91 +++++++++ ynr/apps/sopn_parsing/README.md | 3 + ynr/settings/local.py.container.example | 4 + 6 files changed, 455 insertions(+), 128 deletions(-) create mode 100644 docs/DEVELOPMENT.md create mode 100644 docs/INSTALL.old.md diff --git a/README.md b/README.md index 1cddc3d44..20bd6f366 100644 --- a/README.md +++ b/README.md @@ -3,73 +3,81 @@ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) ![CodeQL](https://github.com/DemocracyClub/yournextrepresentative/workflows/CodeQL/badge.svg) +# A website for crowd-sourcing structured data about election candidates -# A website for crowd-sourcing structured election candidate data +[**candidates.democracyclub.org.uk**](https://candidates.democracyclub.org.uk) -https://candidates.democracyclub.org.uk/ - -YourNextRepresentative is a open source platform for -crowd-sourcing information about candidates for political office +YourNextRepresentative ("**YNR**") is an open source platform +for crowd-sourcing information about candidates for political office, and making it available as open data to anyone. +YNR collects some core data, including: +- who is standing, +- what party they’re standing for, +- their contact details, and +- their social media accounts. + +YNR requires that each change is submitted with a source, so that the collected +information can be verified. + +## Using YNR + +**To find out information** about who you can vote for in upcoming elections, head +over to [whocanivotefor.co.uk](https://whocanivotefor.co.uk) and search for +candidates in your area. -The core data that YourNextRepresentative collects includes who -is standing, what party they’re standing for, their contact -details, their social media accounts etc. The software requires -that each change is submitted with a source, so that the -collected information can be independently checked. +**To contribute information** about candidates, use the YNR application at +[candidates.democracyclub.org.uk](https://candidates.democracyclub.org.uk). -# Installation +## Developing YNR -See [INSTALL.md](https://github.com/DemocracyClub/yournextrepresentative/blob/master/docs/INSTALL.md) +Before you can start modifying the YNR application and website, you'll need to +install its development prerequisites -- as detailed in +[`docs/INSTALL.md`](docs/INSTALL.md). -# Known Bugs +After you've confirmed that the prerequisites are working correctly on your +machine you'll be able to use the workflows detailed in +[`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) to make changes to YNR. + +## Known Bugs You can find a list of known issues to work on here: * https://github.com/DemocracyClub/yournextrepresentative/issues -# Acknowledgements +## Acknowledgements This codebase was originally forked from [mysociety/yournextrepresentative](http://github.com/mysociety/yournextrepresentative) -We no longer track the upstream but we thank [mySociety](http://mysociety.org/) +We no longer track the upstream but we thank [mySociety](https://mysociety.org/) for their work on the project which we have been able to build on. -# API Versions +## API Versions v0.9 is legacy code and is now frozen. v1.0 is currently in alpha. We plan on publishing a v1 API once we have some more feedback from users and we think it’s stable enough. -# Statement Of Persons Nominated (SOPN) Parsing - -YNR uses `pypandoc` (which relies on `pandoc`) to convert SOPN documents to PDF, as needed, to be parsed. +## Statement Of Persons Nominated (SOPN) Parsing -To install `pandoc`, visit this page and follow the instructions for you operating system: -https://pandoc.org/installing.html +See [`ynr/apps/sopn_parsing`](ynr/apps/sopn_parsing#readme). -Once `pandoc` is installed +## Sentry Error Reporting -Install pypandoc (or via `requirements.txt`): +Sentry is used to report errors in production. We have added a url for `sentry-debug` to the [`urls.py`](ynr/urls.py#L42) file. This is to allow us verify that Sentry is configured correctly and working in production. -`pip install pandoc` +## Pre-election Tasks -If `pypandoc` does not install via `pip`, visit https://pypi.org/project/pypandoc/ for further instructions. +### Enable Candidate Leaderboard -# Sentry Error Reporting - -Sentry is used to report errors in production. We have added a url for `sentry-debug` to the `urls.py` file. This is to allow us verify that Sentry is configured correctly and working in production. - -``` - -# Pre-election Tasks - -# Enable Candidate Leaderboard - -The candidate leaderboard is a way of showing the most active candidates on the site. It is a way of encouraging volunteers to add more information about candidates and elections. +The candidate leaderboard shows the most active contributors to the site. +It is a way of encouraging volunteers to add more information about candidates and elections. We take a slice of edits in YNR and assign them to a election leaderboard. - -This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20 +This is defined in [`ynr/apps/candidates/views/mixins.py`](ynr/apps/candidates/views/mixins.py#L20). We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] -If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL` +If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with: +``` +podman compose up -d dbpqsl +./scripts/container.run.bash python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL +podman compose down +``` diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 000000000..06f8e61bd --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,242 @@ +# Developing YNR locally + +This is a guide to the tools and workflows that you'll need to use when +changing and updating YNR on your development machine. + +## Installation + +Install and test the development prerequisites as detailed in +[INSTALL.md](INSTALL.md). This will leave you with a known-good setup that's +able to run the commands described in this guide. + +## Quick start + +### Restoring data + +If you have access to a database dump from a YNR instance you can restore it to +a containerised database as follows: + +1. Start the database container: + `podman compose up -d dbpsql` +1. Restore the database dump: + ``` + cat path/to/database.dump \ + | podman compose exec -T dbpsql pg_restore -d ynr -U ynr --no-owner + ``` +1. Apply any pending migrations: + `./scripts/container.run.bash python manage.py migrate` +1. Shut down the database container: + `podman compose down` + + + +### Running the app + +1. Add or update any environment variables in `env/frontend.env` as required. +1. Incorporate the settings from + [`ynr/settings/local.py.container.example`](../ynr/settings/local.py.container.example) + into your gitignored `ynr/settings/local.py` file. +1. Start the compose stack: + `podman compose up -d` +1. (In a separate terminal) Start tailing the stack's logs: + `podman compose logs --follow` (you can safely CTRL-C this process at any time). +1. Build some required JS resources in the running frontend container: + `./scripts/container.exec.bash npm run build` +1. (**If your active `ynr/settings/...` file does NOT include `DEBUG = True`**) \ + Collect the static assets: + `./scripts/container.manage-py.bash collectstatic --no-input` +1. Browse to [http://localhost:8080](http://localhost:8080) +1. Changes made inside `ynr/` will be immediately available to the app, which + will be auto-reloaded. +1. Remember to shut down the compose stack when you're done: + `podman compose down` + +### Testing your changes + +#### If the app is capable of being started + +1. Start the compose stack: + `podman compose up -d` +1. Run the test suite, stopping on first failure: + `./scripts/container.pytest.bash -x` +1. Stop the compose stack: + `podman compose down` + +#### If the app can't be started, and you need to run the test suite to figure out why + +1. Start the compose stack's database server: + `podman compose up -d dbpsql` +1. Run the entire test suite in a new container (without starting the app): + `./scripts/container.run.bash pytest` +1. Stop the compose stack: + `podman compose down` + +### Running Django management commands + +As detailed [later in this guide](#scripts), there are several different ways +to run a command inside a frontend container. +The method described here uses a +dedicated script to invoke a Django management command inside a frontend +container that's already running the webapp. +(If you need to run a command but don't want to start the webapp, use the more +general `container.run.bash` script instead). + +Run a Django management command: + +1. Add or update any environment variables in `env/frontend.env` as required. +1. Start the compose stack: + `podman compose up -d` +1. Use the `container.manage-py.bash` script to invoke the command: + ``` + ./scripts/container.manage-py.bash command-to-invoke --command-args command params + ``` +1. Stop the compose stack: + `podman compose down` + +After you stop the compose stack, any files added or changed by the management +command inside the `ynr` directory will be persisted directly on your machine. +The same applies to any files mentioned in +[`docker-compose.yml`](../docker-compose.yml), +in the `frontend` container's "`volumes`" section. +**Any changes the management command makes to files *outside* those locations +will be lost when you stop the compose stack**. +Changes to the database are persisted in the database's data volume. + +## Working with Podman + +YNR uses a container runtime called [Podman](https://podman.io/). You can think +of it as like Docker, but able to run without needing a persistent background +daemon and without requiring root access. +Podman provides the `podman` command which is intended to be CLI-compatible +with much of the `docker` command. + +You've also been asked to install `podman-compose` - a separate project that +gives the `podman` command its `podman compose` subcommand. You shouldn't need +to invoke `podman-compose` (with a hyphen) directly. The `podman compose` +command works with the "compose stack" defined in +[docker-compose.yml](../docker-compose.yml), which comprises two services: the +`frontend` webapp, and the Postgres `dbpsql` service. + +### Working on the webapp + +Use `podman compose up -d` to start the compose stack, with the webapp exposed +on [localhost:8080](http://localhost:8080). Changes you make inside the `ynr/` +directory are automatically reflected in the running app. Changes to other +entries in the frontend's [docker-compose.yml](../docker-compose.yml) list of +volumes that are bind-mounted from your local checkout of this repo are also +immediately visible to the running app. + +View the app (and DB) logs with `podman compose logs --follow`. It's a good +idea to run this immediately after starting the stack. + +Shut down the stack with `podman compose down`. This is always safe to run, +even when the stack is already stopped. It deliberately leaves the database's +data behind as a "volume", so that Postgres can access it the next time you +start the stack. If you need to delete the database's contents completely, run +`podman compose down --volumes`. + +`podman compose ...` subcommands do provide the expected `--help` flag, but +some of the docs aren't perfect. Here's a summary of the commands you might +run: + + +| Command                                                   | Purpose | Notes +| :--- | :--- | :--- +| `podman compose up -d` | Start the entire stack. | `-d` forks the action into the background, which is optional but strongly recommended. +| `podman compose up -d dbpsql` | Start only the named container in the stack. | +| `podman compose down` | Stop any running containers in the stack. | +| `podman compose down --volumes` | Stop any running containers in the stack and also destroy their persistent data. | Anything bind-mounted from your local repo into the `frontend` webapp container is left untouched. | +| `podman compose ps` | Display the status of containers in the stack. | +| `podman volume ls` | List the persistent volumes that podman controls on your machine. | +| `podman compose build` | Rebuild the webapp's frontend container image. | +| `podman compose build --no-cache` | Rebuild the webapp's frontend container image from scratch. | Takes several minutes to finish. +| `podman compose logs` | Display the last N stdout/stderr lines emitted by any running containers. | +| `podman compose logs --follow` | Display the last N stdout/stderr lines emitted by any running containers, and then wait for more lines. | +| `podman system reset` | Destroy everything that Podman controls. | "Everything seems to have gone wrong, so I'll just start from scratch". It wipes out all containers, networks, images, volumes, etc ... so **avoid this if possible!** | + +### Scripts + +These executable scripts are available from the [`scripts`](../scripts) directory. + + +| Script | Purpose | Parameters +| :--- | :--- | :--- +| `container.image.build.bash` | Builds the YNR container image | $1 -- The named stage from [`container/build/Containerfile`](../container/build/Containerfile) to build and tag (*required*)
$2, $3, ... -- Any parameters to pass to the underlying builder process (*optional*) +| `container.exec.bash` | Runs a command inside the already-running `frontend` container | The unquoted command to run (*required*) +| `container.manage-py.bash` | Runs a Django management command inside the already-running `frontend` container | The unquoted command to run (*required*) +| `container.pytest.bash` | Runs `pytest` inside the already-running `frontend` container | Any parameters for Pytest (*optional*) +| `container.run.bash` | Runs a command inside a freshly-instantiated, ephemeral `frontend` container | The unquoted command to run (*required*) + +### Rebuilding the application container image + +You will need to rebuild the application's container image if you change any of +the application's dependencies, across any of the packaging ecosystems it +currently relies on: + +- `container/build/system-packages`: System / APT dependendencies +- `package{,-lock}.json`: Node dependencies +- `requirements/*.txt`: Python dependencies +- `.dockerignore`: Container build-time file dependencies + +The above list is presented in descending order of how slow a rebuild will be, +if a particular package ecosystem's dependencies are changed. +Changing a system dependency, for example, forces a longer rebuild than +changing a Python dependency. +**You do not need to rebuild the application's container image if you only +change files in the `ynr/` directory**. Changes to the YNR application are +picked up automatically when using the compose stack locally (as described +elsewhere in this guide). + +The build process for the YNR application is encoded in +[`container/build/Containerfile`](../container/build/Containerfile). +This Docker-compatible file describes two image stages, `prod` and `test`, with +`test` being built on top of `prod`. +Locally, on your development machine, you will need to use the `test` stage. + +#### Build the `test` stage using a build cache + +``` +./scripts/container.image.build.bash test +``` + +#### Build the `test` stage without a build cache + +Avoiding the use of your local build cache significantly increases the time it +takes to build the container image, but is sometimes useful when there's a +problem with external dependencies (e.g. if a important update has been +published for an APT package but it's not visible in the container's package +index). + +``` +./scripts/container.image.build.bash test --no-cache +``` diff --git a/docs/INSTALL.md b/docs/INSTALL.md index cb909a955..a8a4b534e 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,91 +1,70 @@ # Installation -TODO: improve these docs with more detail - -YourNextRepresentative requires python >=3.5 and PostgreSQL - -## Install python dependencies - -``` -pip install -U pip -pip install -r requirements.txt -``` - -## Set up database - -``` -sudo -u postgres createdb ynr -``` - -If using mac-os/homebrew -``` -createdb ynr -``` - -``` -cp ynr/settings/local.py.example ynr/settings/local.py -``` - -Add database credentials to `DATABASES` dict in `local.py` - -``` -brew install libmagic -./manage.py migrate -``` - -To populate the database run from the live site run: - -``` -python manage.py candidates_import_from_live_site -``` - -(Note that this command will take multiple hours to complete.) - -## Build frontend assets - -``` -npm run build -npm install -``` - -## (Optional) Code linting - -A CI will check all code against Black and Flake8. To save pushing commits that don't -pass these tests you can configure pre-commmit hooks. - -Do this by installing `[precommit](https://pre-commit.com/)`: - -``` -pip install pre-commit -pre-commit install -``` - -## (Optional) SOPN parsing - -SOPNs parsing (see `ynr/apps/sopn_parsing/README.md`) is optional -because it depends on various system packages beyond python packages. - -It currently requires [camelot-py](https://camelot-py.readthedocs.io/en/master/user/install.html#install) -and that in turn requires `python-tk` and `ghostscript`. - -Read up on how to install them, and then install the SOPN parsing requirements: - -``` -pip install -r requirements/sopn_parsing.txt -``` - -File conversion relies on `pandoc` to turn non-pdf SOPN files into pdf files. -To install `pandoc`, visit https://pandoc.org/installing.html and follow instructions -for Mac OS and Ubuntu. - -AWS Textract relies on the following packages for viewing image results: - -https://pypi.org/project/pdf2image/ - -To install these packages run: - -``` -brew install poppler -``` - -_If you have omitted SOPN and are having problems getting the project to run, you may need to follow the SOPN steps._ \ No newline at end of file +## Local development + +To develop YNR on your local machine you'll first need to install its +containerisation prerequisites. We use containers in development to isolate the +(non-trivial!) set of *application* prerequisites away from your local machine, +and to get closer to the intended future state of the application's +*production* deployment. + +### Install and test containerisation prerequisites + +1. Clone this repository: + `git clone --branch jcm/wip https://github.com/DemocracyClub/yournextrepresentative` +1. Install the `podman` command: https://podman.io/docs/installation. + These installation mechanisms have been tested: + - System package on Ubuntu 24.04 LTS + - https://podman.io/docs/installation#ubuntu +1. Install the `podman-compose` command: https://pypi.org/project/podman-compose/. + These installation mechanisms have been tested: + - System package on Ubuntu 24.04 LTS + - This version (v1.0.x) emits non-optional verbose debug logs + - https://packages.ubuntu.com/noble/podman-compose + - `apt install podman-compose` + - Manual installation of v1.2.0 APT package on Ubuntu 24.04 LTS + - This version's verbose debug logs are optional + - https://packages.ubuntu.com/oracular/all/podman-compose/download + - `dkpkg -i path/to/debian-package.deb` + - Local `pip` installation of v1.2.0 on Ubuntu 24.04 LTS + - This version's verbose debug logs are optional + - https://pypi.org/project/podman-compose/ + - `pip install podman-compose` + - Either inside a venv, or not, as you prefer +1. Configure `podman` to be less chatty, by placing this configuration in `$HOME/.config/containers/containers.conf`: + ```ini + # Don't emit logs on each invocation of the compose command indicating + # that an external compose provider is being executed. + [engine] + compose_warning_logs=false + ``` +1. Make sure the `bash` shell is available: + `which bash || echo Not found` +1. Build any container images used by the compose stack: + `podman compose build` +1. Pull any 3rd-party container images used by the compose stack: + `podman compose pull` +1. Set up your local/development envvars as needed, by placing keys and values + `env/frontend.env`, like this: + ``` + DJANGO_SETTINGS_MODULE=ynr.settings.testing + ``` +1. Test that the compose stack can be stood up: + ```bash + podman compose up -d # NB Space between "podman" and "compose"! + curl 0:8080 + ``` + Curl **should** report a server error (i.e. a 500) because your database + setup is incomplete. This step tests only that `podman` and `podman-compose` + are able to run successfully on your machine when given YNR's + `docker-compose.yml` file. +1. Test that Django management commands can be invoked: + `./scripts/container.manage-py.bash check` +1. Run the test suite (which only requires that a database server be + *available*, not that it contains any specific data). + This will take a little time to finish: + `./scripts/container.pytest.bash` +1. Shut down the compose stack: + `podman compose stop` + +Now you can use the tools and workflows detailed in [DEVELOPMENT.md](DEVELOPMENT.md). diff --git a/docs/INSTALL.old.md b/docs/INSTALL.old.md new file mode 100644 index 000000000..cb909a955 --- /dev/null +++ b/docs/INSTALL.old.md @@ -0,0 +1,91 @@ +# Installation + +TODO: improve these docs with more detail + +YourNextRepresentative requires python >=3.5 and PostgreSQL + +## Install python dependencies + +``` +pip install -U pip +pip install -r requirements.txt +``` + +## Set up database + +``` +sudo -u postgres createdb ynr +``` + +If using mac-os/homebrew +``` +createdb ynr +``` + +``` +cp ynr/settings/local.py.example ynr/settings/local.py +``` + +Add database credentials to `DATABASES` dict in `local.py` + +``` +brew install libmagic +./manage.py migrate +``` + +To populate the database run from the live site run: + +``` +python manage.py candidates_import_from_live_site +``` + +(Note that this command will take multiple hours to complete.) + +## Build frontend assets + +``` +npm run build +npm install +``` + +## (Optional) Code linting + +A CI will check all code against Black and Flake8. To save pushing commits that don't +pass these tests you can configure pre-commmit hooks. + +Do this by installing `[precommit](https://pre-commit.com/)`: + +``` +pip install pre-commit +pre-commit install +``` + +## (Optional) SOPN parsing + +SOPNs parsing (see `ynr/apps/sopn_parsing/README.md`) is optional +because it depends on various system packages beyond python packages. + +It currently requires [camelot-py](https://camelot-py.readthedocs.io/en/master/user/install.html#install) +and that in turn requires `python-tk` and `ghostscript`. + +Read up on how to install them, and then install the SOPN parsing requirements: + +``` +pip install -r requirements/sopn_parsing.txt +``` + +File conversion relies on `pandoc` to turn non-pdf SOPN files into pdf files. +To install `pandoc`, visit https://pandoc.org/installing.html and follow instructions +for Mac OS and Ubuntu. + +AWS Textract relies on the following packages for viewing image results: + +https://pypi.org/project/pdf2image/ + +To install these packages run: + +``` +brew install poppler +``` + +_If you have omitted SOPN and are having problems getting the project to run, you may need to follow the SOPN steps._ \ No newline at end of file diff --git a/ynr/apps/sopn_parsing/README.md b/ynr/apps/sopn_parsing/README.md index f8cb89e91..875ba63fa 100644 --- a/ynr/apps/sopn_parsing/README.md +++ b/ynr/apps/sopn_parsing/README.md @@ -2,6 +2,9 @@ This app is designed to extract useful information out of UK Statement Of Persons Nominated documents (SOPNs), published before elections. +It uses the `pypandoc_binary` dependency (which both provides an interface to, +and installs the binary of, the upstream [pandoc](https://pandoc.org/) project) +to convert SOPN documents to PDF (as needed) to be parsed. The documents contain information on candidates for a given election, but are published in a wide variety of layouts. diff --git a/ynr/settings/local.py.container.example b/ynr/settings/local.py.container.example index 48179889f..3b9325fd2 100644 --- a/ynr/settings/local.py.container.example +++ b/ynr/settings/local.py.container.example @@ -1,6 +1,10 @@ # Only set DEBUG to True in development environments. DEBUG = True +# These Postgres settings should match docker-compose.yml. +DATABASES["default"]["NAME"] = "ynr" # noqa +DATABASES["default"]["USER"] = "ynr" # noqa + # This short, known value is insecure. SECRET_KEY = "development" From 571f23de38ecb5a9b6c70ca152f23257d3618e4d Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Thu, 9 Jan 2025 10:57:37 +0000 Subject: [PATCH 27/33] docs: record podman-compose 1.2.0 required version Initial development of containerised YNR used podman-compose v1.0.6, but that version is incompatible with the current schema used by the docker-compose.yml file. v1.2.0 is noted as being specifically required as the very recent v1.3.0 has not yet been properly tested with this branch. --- docs/INSTALL.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a8a4b534e..a8cd24f0b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -16,21 +16,15 @@ and to get closer to the intended future state of the application's These installation mechanisms have been tested: - System package on Ubuntu 24.04 LTS - https://podman.io/docs/installation#ubuntu -1. Install the `podman-compose` command: https://pypi.org/project/podman-compose/. +1. Install `podman-compose` v1.2.0: https://pypi.org/project/podman-compose/. These installation mechanisms have been tested: - - System package on Ubuntu 24.04 LTS - - This version (v1.0.x) emits non-optional verbose debug logs - - https://packages.ubuntu.com/noble/podman-compose - - `apt install podman-compose` - - Manual installation of v1.2.0 APT package on Ubuntu 24.04 LTS - - This version's verbose debug logs are optional - - https://packages.ubuntu.com/oracular/all/podman-compose/download - - `dkpkg -i path/to/debian-package.deb` - Local `pip` installation of v1.2.0 on Ubuntu 24.04 LTS - - This version's verbose debug logs are optional - https://pypi.org/project/podman-compose/ - `pip install podman-compose` - Either inside a venv, or not, as you prefer + - Manual installation of v1.2.0 APT package on Ubuntu 24.04 LTS + - https://packages.ubuntu.com/oracular/all/podman-compose/download + - `dkpkg -i path/to/debian-package.deb` 1. Configure `podman` to be less chatty, by placing this configuration in `$HOME/.config/containers/containers.conf`: ```ini # Don't emit logs on each invocation of the compose command indicating From 90e0bedd4cf67c1364ce14c8245ffa75caa9c409 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Thu, 9 Jan 2025 11:05:00 +0000 Subject: [PATCH 28/33] {compose,build}: mount and copy data/ directory Some DB migrations require files that exist in the data directory. This change ensures that it exists in the container image by adding it to the .dockerignore include list, and also bind mounts the path so that developers can update its files without rebuilding the image. --- .dockerignore | 1 + docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 1345d8d0f..d3c14d839 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ # track and match the bind mounts for the "frontend" service in the # docker-compose.yml file. !/ynr/ +!/data/ !/requirements/ !/requirements.txt !/manage.py diff --git a/docker-compose.yml b/docker-compose.yml index 9785af14a..81371d77e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: PORT: 8080 volumes: - ./ynr/:/dc/ynr/code/ynr/ + - ./data/:/dc/ynr/code/data/ - ./requirements/:/dc/ynr/code/requirements/ - ./requirements.txt:/dc/ynr/code/requirements.txt - ./manage.py:/dc/ynr/code/manage.py From 4c0c1a55420f7bce96fa8abf85257d654a4a0202 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Thu, 9 Jan 2025 11:27:43 +0000 Subject: [PATCH 29/33] container/build: uppercase Containerfile Some editors complain if a Containerfile doesn't use uppercase instructions. --- container/build/Containerfile | 50 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/container/build/Containerfile b/container/build/Containerfile index 4a361392c..30e780206 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -1,29 +1,29 @@ ########################################################################### ## Production image ####################################################### ########################################################################### -from public.ecr.aws/lts/ubuntu:20.04 as prod +FROM public.ecr.aws/lts/ubuntu:20.04 AS prod ######################### ## Build-time arguments # ######################### # Increase this arbitary number to force all image layers to be rebuilt. # This is designed to invalidate the layer cache inside CI, not locally. -arg invalidate_all_cached_layers=202412030000 +ARG invalidate_all_cached_layers=202412030000 # Base path for the app install build process. -arg APP_ROOT=/dc/ynr +ARG APP_ROOT=/dc/ynr # Path for the app's virtualenv. -arg APP_VENV=$APP_ROOT/venv +ARG APP_VENV=$APP_ROOT/venv # Path for the app's code. -arg APP_CODE=$APP_ROOT/code +ARG APP_CODE=$APP_ROOT/code ######################## ## System dependencies # ######################## -arg DEBIAN_FRONTEND=noninteractive +ARG DEBIAN_FRONTEND=noninteractive # Copy system dependency manifest into the container image. -copy container/build/system-packages /tmp/apt-packages +COPY container/build/system-packages /tmp/apt-packages # Install dependencies. -run date \ +RUN date \ && apt update \ && Date: Thu, 9 Jan 2025 11:46:19 +0000 Subject: [PATCH 30/33] docs: minor updates This tweaks some docs following PR feedback. --- docs/DEVELOPMENT.md | 7 +++++-- docs/INSTALL.md | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 06f8e61bd..ad47532d3 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -85,8 +85,11 @@ Alternatively, you can populate the database using the public YNR API. 1. Start the compose stack: `podman compose up -d` -1. Run the test suite, stopping on first failure: - `./scripts/container.pytest.bash -x` +1. Run the test suite: + `./scripts/container.pytest.bash` + - We can provide additional pytest options. + For example, to run the test suite and stop on the first failure: + `./scripts/container.pytest.bash -x` 1. Stop the compose stack: `podman compose down` diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a8cd24f0b..a944f1063 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -59,6 +59,6 @@ and to get closer to the intended future state of the application's This will take a little time to finish: `./scripts/container.pytest.bash` 1. Shut down the compose stack: - `podman compose stop` + `podman compose down` Now you can use the tools and workflows detailed in [DEVELOPMENT.md](DEVELOPMENT.md). From 79dfd4029debbb076fc199832ff1c35a7f8b1f78 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Fri, 10 Jan 2025 09:51:13 +0000 Subject: [PATCH 31/33] container/build: change NODE_ENV handling This alters the Containerfile's setting of NODE_ENV so it's less susceptible to future line-order changes having unintended side effects. --- container/build/Containerfile | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/container/build/Containerfile b/container/build/Containerfile index 30e780206..6fd6aa5ce 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -33,24 +33,23 @@ RUN date \ ###################### ## Node dependencies # ###################### +ARG NODE_ENV=production +ENV NODE_ENV=$NODE_ENV # Copy node dependency manifests into the container image. COPY package.json package-lock.json $APP_CODE/ -# "setting NODE_ENV to anything but production is considered an antipattern." -# (https://nodejs.org/en/learn/getting-started/nodejs-the-difference-between-development-and-production) -# However, the version of npm we currently use doesn't appear to offer any way -# to make "npm ci" install the devDependencies (which include the "gulp" -# command that's required later in this build) when NODE_ENV="production". -# Therefore we delay setting NODE_ENV until after "npm ci" has run. -# arg NODE_ENV=production -# env NODE_ENV=$NODE_ENV # Install dependencies. +# The version of npm we currently use doesn't appear to offer any way to make +# "npm ci" install the devDependencies (which include the "gulp" command that's +# required later in this build) when NODE_ENV="production". We fix this by +# overriding NODE_ENV for these npm commands. Whilst "npm cache clean" might +# not require the same value, we keep it aligned in case it has some effect +# (e.g. maybe the cache location is env-specific, etc). RUN date \ + && export NODE_ENV=development \ && cd $APP_CODE/ \ && npm ci \ && npm cache clean --force \ && date -ARG NODE_ENV=production -ENV NODE_ENV=$NODE_ENV ######################## ## Python dependencies # From 196d5f3ffca6f529a661ca5a72b23b04c04b2ee6 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Tue, 14 Jan 2025 10:33:35 +0000 Subject: [PATCH 32/33] container: use DJANGO_SETTINGS_MODULE=ynr.settings This reverts all changes to ynr.settings.testing, and defaults the containerised YNR app to use the base ynr.settings module throughout the build process and the compose invocation. The pytest wrapper script is allowed to use pytest's default settings already configured via pyproject.toml. This change is generally implemented by removing the explicit use of ynr.settings.testing across various scenarios, and opting in to manage.py's default of ynr.settings (which inherits ynr.settings.base). Also: small tweaks to docs. Signed-off-by: Jonathan Matthews --- .circleci/config.yml | 3 +-- .gitignore | 2 +- container/build/Containerfile | 4 ++-- docs/DEVELOPMENT.md | 6 +++--- docs/INSTALL.md | 11 ++++++++--- env/.gitkeep | 0 env/frontend.env.example | 1 + scripts/container.pytest.bash | 6 +++--- ynr/settings/local.py.container.example | 14 +++++++++++--- ynr/settings/testing.py | 3 +-- 10 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 env/.gitkeep create mode 100644 env/frontend.env.example diff --git a/.circleci/config.yml b/.circleci/config.yml index c8474ef69..3b307803a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,12 +177,11 @@ jobs: net="$(docker network inspect $(docker network ls -q -f "label=task-network") --format '{{.Name}}')" docker run -it --rm \ --net="$net" \ - -e RUN_ENV=test \ -e PGHOST=dbpsql \ -e CIRCLECI=true \ \ ynr:test \ - pytest --ds=ynr.settings.testing -x + pytest -x workflows: test_build_deploy: diff --git a/.gitignore b/.gitignore index 35d609aa0..cc1b88b8c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ ynr/assets/* .pytest_cache *.css.map test-results -env/ +/env/*.env node_modules/ .vscode/ /test-env diff --git a/container/build/Containerfile b/container/build/Containerfile index 6fd6aa5ce..1468f5573 100644 --- a/container/build/Containerfile +++ b/container/build/Containerfile @@ -88,14 +88,14 @@ WORKDIR $APP_CODE # Invoke a lightweight, post-build test that proves the container reaches a # baseline level of correctness, whilst also generating .pyc files for faster # app startup. -RUN python manage.py check --settings=ynr.settings.testing +RUN DJANGO_SECRET_KEY=insecure python manage.py check ########### ## Assets # ########### RUN date \ && npm run build \ - && python manage.py collectstatic --no-input --settings=ynr.settings.testing \ + && DJANGO_SECRET_KEY=insecure python manage.py collectstatic --no-input \ && date ########################################################################### diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index ad47532d3..082a26a91 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -119,7 +119,9 @@ Run a Django management command: `podman compose up -d` 1. Use the `container.manage-py.bash` script to invoke the command: ``` - ./scripts/container.manage-py.bash command-to-invoke --command-args command params + # ./scripts/container.manage-py.bash command-to-invoke --command-args command params + # e.g. + ./scripts/container.manage-py.bash check ``` 1. Stop the compose stack: `podman compose down` @@ -190,8 +192,6 @@ GitHub-rendered view so that command strings don't line-wrap. --> These executable scripts are available from the [`scripts`](../scripts) directory. - | Script | Purpose | Parameters | :--- | :--- | :--- | `container.image.build.bash` | Builds the YNR container image | $1 -- The named stage from [`container/build/Containerfile`](../container/build/Containerfile) to build and tag (*required*)
$2, $3, ... -- Any parameters to pass to the underlying builder process (*optional*) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index a944f1063..7ba9f24bb 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -38,11 +38,16 @@ and to get closer to the intended future state of the application's `podman compose build` 1. Pull any 3rd-party container images used by the compose stack: `podman compose pull` -1. Set up your local/development envvars as needed, by placing keys and values - `env/frontend.env`, like this: +1. Set up your development envvars as needed, by placing keys and values in + `env/frontend.env`, using `env/frontend.env.example` as a template. + In general, the only envar you should need is this: ``` - DJANGO_SETTINGS_MODULE=ynr.settings.testing + DJANGO_SETTINGS_MODULE=ynr.settings ``` +1. Copy `ynr/settings/local.py.container.example` to `ynr/settings/local.py`. + If you already have a `ynr/settings/local.py` file, incorporate the example + file's settings. **If you don't use most of the example file's settings, you + *will* experience problems interacting with the app, later**. 1. Test that the compose stack can be stood up: ```bash podman compose up -d # NB Space between "podman" and "compose"! diff --git a/env/.gitkeep b/env/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/env/frontend.env.example b/env/frontend.env.example new file mode 100644 index 000000000..cfa9bcb67 --- /dev/null +++ b/env/frontend.env.example @@ -0,0 +1 @@ +DJANGO_SETTINGS_MODULE=ynr.settings diff --git a/scripts/container.pytest.bash b/scripts/container.pytest.bash index b0f6c32b2..cff57807f 100755 --- a/scripts/container.pytest.bash +++ b/scripts/container.pytest.bash @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -# container.test.bash invokes a `pytest` command in a "frontend" container. +# container.pytest.bash invokes a `pytest` command in a "frontend" container. # The container and a database server must be running before running this # script (see README.md). # # Usage: -# scripts/container.test.bash # run all tests; continue after failures -# scripts/container.test.bash -x # stop after first failure +# scripts/container.pytest.bash # run all tests; continue after failures +# scripts/container.pytest.bash -x # stop after first failure # The command being invoked does not need to be quoted, unless it contains # shell meta-characters or similar. Multiple words are fine, without quotes. diff --git a/ynr/settings/local.py.container.example b/ynr/settings/local.py.container.example index 3b9325fd2..94971eb45 100644 --- a/ynr/settings/local.py.container.example +++ b/ynr/settings/local.py.container.example @@ -2,11 +2,19 @@ DEBUG = True # These Postgres settings should match docker-compose.yml. -DATABASES["default"]["NAME"] = "ynr" # noqa -DATABASES["default"]["USER"] = "ynr" # noqa +# Credentials are not required. +DATABASES = {"default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "dbpsql", "NAME": 'ynr', "USER": "ynr", +}} + +# The containerised setup does not include the same memcache service that +# production currently uses for its cache, so we explicitly set up a dummy +# cache when running locally. +CACHES = {"default": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"}} # This short, known value is insecure. -SECRET_KEY = "development" +SECRET_KEY = "insecure" # Certain errors are very noisy (obscuring the real problem) if this list is # empty. diff --git a/ynr/settings/testing.py b/ynr/settings/testing.py index 75b53bcdd..8cfe82cc3 100644 --- a/ynr/settings/testing.py +++ b/ynr/settings/testing.py @@ -17,7 +17,6 @@ def __contains__(self, item): def __getitem__(self, item): return None -ADMINS = [("dummy", "dummy@dummy.example")] MIGRATION_MODULES = DisableMigrations() @@ -30,7 +29,7 @@ def __getitem__(self, item): RUNNING_TESTS = True SECRET_KEY = "just here for testing" -ALLOWED_HOSTS = ["candidates.democracyclub.org.uk","*"] +ALLOWED_HOSTS = ["candidates.democracyclub.org.uk"] SHOW_SOPN_TRACKER = False SHOW_RESULTS_PROGRESS = False From 7604f5ac717b7657d52947e24659587706e77680 Mon Sep 17 00:00:00 2001 From: Jonathan Matthews Date: Thu, 16 Jan 2025 11:09:10 +0000 Subject: [PATCH 33/33] docs: remove unused DB population method This removes mention of an alternative method for initially populating the DB, via the API, as the fact that it doesn't currently work is a known issue. --- docs/DEVELOPMENT.md | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 082a26a91..510d001a0 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -28,36 +28,6 @@ a containerised database as follows: 1. Shut down the database container: `podman compose down` - - ### Running the app 1. Add or update any environment variables in `env/frontend.env` as required.