Add workflows for local development using containerised YNR

.circleci/config.yml

@@ -147,6 +147,41 @@ jobs:
           event: fail
           template: basic_fail_1
 
+  container_build:
+    working_directory: ~/repo
+    docker:
+      - image: cimg/base:current
+      - image: cimg/postgres:16.4
+        name: dbpsql
+        environment:
+          POSTGRES_USER: ynr
+          POSTGRES_DB: ynr
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Build container images
+          command: |
+            ./scripts/container.image.build.bash prod
+            ./scripts/container.image.build.bash test
+      - run:
+          name: Check that compose is happy
+          command: |
+            docker compose config
+            # No-op, due to a shared layer cache with earlier image builds.
+            docker compose build
+      - run:
+          name: Run app tests
+          command: |
+            net="$(docker network inspect $(docker network ls -q -f "label=task-network") --format '{{.Name}}')"
+            docker run -it --rm \
+              --net="$net" \
+              -e PGHOST=dbpsql \
+              -e CIRCLECI=true \
+              \
+              ynr:test \
+              pytest -x
 
 workflows:
   test_build_deploy:
@@ -157,3 +192,6 @@ workflows:
         - build_and_test
         filters: { branches: { only: [ main, master, deployment-upgrades] } }
         context: [ deployment-production-ynr, slack-secrets ]
+  container:
+    jobs:
+    - container_build

.dockerignore

@@ -0,0 +1,33 @@
+# This file contains paths, .gitignore-style, that will be excluded from being
+# copied into a container image during build.
+# cf. https://docs.docker.com/build/concepts/context/#dockerignore-files
+
+# Start with a clean slate by excluding all paths.
+*
+
+# Selectively re-include paths to copy into the container. This list should
+# track and match the bind mounts for the "frontend" service in the
+# docker-compose.yml file.
+!/ynr/
+!/data/
+!/requirements/
+!/requirements.txt
+!/manage.py
+!/Makefile
+!/scripts/
+!/gulpfile.js
+!/package.json
+!/package-lock.json
+!/pyproject.toml
+
+# Paths overriding early re-inclusions.
+# The content inside /ynr/media/ is bind-mounted into the container when
+# running as part of a compose stack, as part of the bind-mount of its parent,
+# /ynr/. However its contents *should not* be statically copied into the
+# container image because they're either irrelevant/empty (in non-development
+# environments), or potentially very large (in development environments).
+/ynr/media/
+
+# Paths that don't need to be part of the image, but also need not to be
+# excluded when explicitly referenced by build-time COPY and ADD commands.
+!/container/build/system-packages

.gitignore

@@ -20,7 +20,7 @@ ynr/assets/*
 .pytest_cache
 *.css.map
 test-results
-env/
+/env/*.env
 node_modules/
 .vscode/
 /test-env

README.md

@@ -3,73 +3,81 @@
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 ![CodeQL](https://github.com/DemocracyClub/yournextrepresentative/workflows/CodeQL/badge.svg)
 
+# A website for crowd-sourcing structured data about election candidates
 
-# A website for crowd-sourcing structured election candidate data
+[**candidates.democracyclub.org.uk**](https://candidates.democracyclub.org.uk)
 
-https://candidates.democracyclub.org.uk/
-
-YourNextRepresentative is a open source platform for
-crowd-sourcing information about candidates for political office
+YourNextRepresentative ("**YNR**") is an open source platform
+for crowd-sourcing information about candidates for political office,
 and making it available as open data to anyone.
+YNR collects some core data, including:
+- who is standing,
+- what party they’re standing for,
+- their contact details, and
+- their social media accounts.
+
+YNR requires that each change is submitted with a source, so that the collected
+information can be verified.
+
+## Using YNR
+
+**To find out information** about who you can vote for in upcoming elections, head
+over to [whocanivotefor.co.uk](https://whocanivotefor.co.uk) and search for
+candidates in your area.
 
-The core data that YourNextRepresentative collects includes who
-is standing, what party they’re standing for, their contact
-details, their social media accounts etc. The software requires
-that each change is submitted with a source, so that the
-collected information can be independently checked.
+**To contribute information** about candidates, use the YNR application at
+[candidates.democracyclub.org.uk](https://candidates.democracyclub.org.uk).
 
-# Installation
+## Developing YNR
 
-See [INSTALL.md](https://github.com/DemocracyClub/yournextrepresentative/blob/master/docs/INSTALL.md)
+Before you can start modifying the YNR application and website, you'll need to
+install its development prerequisites -- as detailed in
+[`docs/INSTALL.md`](docs/INSTALL.md).
 
-# Known Bugs
+After you've confirmed that the prerequisites are working correctly on your
+machine you'll be able to use the workflows detailed in
+[`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) to make changes to YNR.
+
+## Known Bugs
 
 You can find a list of known issues to work on here:
 
 * https://github.com/DemocracyClub/yournextrepresentative/issues
 
-# Acknowledgements
+## Acknowledgements
 
 This codebase was originally forked from
 [mysociety/yournextrepresentative](http://github.com/mysociety/yournextrepresentative)
-We no longer track the upstream but we thank [mySociety](http://mysociety.org/)
+We no longer track the upstream but we thank [mySociety](https://mysociety.org/)
 for their work on the project which we have been able to build on.
 
-# API Versions
+## API Versions
 
 v0.9 is legacy code and is now frozen. v1.0 is currently in alpha. We plan on publishing a v1 API once we have some more feedback from users and we think it’s stable enough.
 
-# SOPN Parsing
-
-YNR uses `pypandoc` (which relies on `pandoc`) to convert SOPN documents to PDF, as needed, to be parsed.
+## Statement Of Persons Nominated (SOPN) Parsing
 
-To install `pandoc`, visit this page and follow the instructions for you operating system:
-https://pandoc.org/installing.html
+See [`ynr/apps/sopn_parsing`](ynr/apps/sopn_parsing#readme).
 
-Once `pandoc` is installed
+## Sentry Error Reporting
 
-Install pypandoc (or via `requirements.txt`):
+Sentry is used to report errors in production. We have added a url for `sentry-debug` to the [`urls.py`](ynr/urls.py#L42) file. This is to allow us verify that Sentry is configured correctly and working in production.
 
-`pip install pandoc`
+## Pre-election Tasks
 
-If `pypandoc` does not install via `pip`, visit https://pypi.org/project/pypandoc/ for further instructions. 
+### Enable Candidate Leaderboard
 
-# Sentry Error Reporting
-
-Sentry is used to report errors in production. We have added a url for `sentry-debug` to the `urls.py` file. This is to allow us verify that Sentry is configured correctly and working in production.
-
-```
-
-# Pre-election Tasks
-
-# Enable Candidate Leaderboard
-
-The candidate leaderboard is a way of showing the most active candidates on the site. It is a way of encouraging volunteers to add more information about candidates and elections.
+The candidate leaderboard shows the most active contributors to the site.
+It is a way of encouraging volunteers to add more information about candidates and elections.
 
 We take a slice of edits in YNR and assign them to a election leaderboard. 
-
-This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20
+This is defined in [`ynr/apps/candidates/views/mixins.py`](ynr/apps/candidates/views/mixins.py#L20).
 
 We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy]
 
-If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL`
+If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with:
+```
+podman compose up -d dbpqsl
+./scripts/container.run.bash python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL
+podman compose down
+```

container/build/Containerfile

@@ -0,0 +1,114 @@
+###########################################################################
+## Production image #######################################################
+###########################################################################
+FROM public.ecr.aws/lts/ubuntu:20.04 AS prod
+
+#########################
+## Build-time arguments #
+#########################
+# Increase this arbitary number to force all image layers to be rebuilt.
+# This is designed to invalidate the layer cache inside CI, not locally.
+ARG invalidate_all_cached_layers=202412030000
+# Base path for the app install build process.
+ARG APP_ROOT=/dc/ynr
+# Path for the app's virtualenv.
+ARG APP_VENV=$APP_ROOT/venv
+# Path for the app's code.
+ARG APP_CODE=$APP_ROOT/code
+
+########################
+## System dependencies #
+########################
+ARG DEBIAN_FRONTEND=noninteractive
+# Copy system dependency manifest into the container image.
+COPY container/build/system-packages /tmp/apt-packages
+# Install dependencies.
+RUN date \
+ && apt update \
+ && </tmp/apt-packages xargs apt install --no-install-suggests --assume-yes \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && date
+
+######################
+## Node dependencies #
+######################
+ARG NODE_ENV=production
+ENV NODE_ENV=$NODE_ENV
+# Copy node dependency manifests into the container image.
+COPY package.json package-lock.json $APP_CODE/
+# Install dependencies.
+# The version of npm we currently use doesn't appear to offer any way to make
+# "npm ci" install the devDependencies (which include the "gulp" command that's
+# required later in this build) when NODE_ENV="production". We fix this by
+# overriding NODE_ENV for these npm commands. Whilst "npm cache clean" might
+# not require the same value, we keep it aligned in case it has some effect
+# (e.g. maybe the cache location is env-specific, etc).
+RUN date \
+ && export NODE_ENV=development \
+ && cd $APP_CODE/ \
+ && npm ci \
+ && npm cache clean --force \
+ && date
+
+########################
+## Python dependencies #
+########################
+# Set up a virtualenv to avoid interactions with system packages.
+# Install a common pre-req.
+RUN date \
+ && python -m venv $APP_VENV \
+ && $APP_VENV/bin/pip install --upgrade pip wheel \
+ && date
+# Use the virtualenv without explicit activation.
+ENV PATH="$APP_VENV/bin:$PATH"
+# Copy dependency manifests into the container image.
+COPY requirements.txt $APP_CODE/
+COPY requirements/    $APP_CODE/requirements/
+# Install dependencies.
+# Instruct pip not to use a cache directory to inprove container-image-level
+# cache effectiveness.
+RUN date \
+ && pip install --no-cache-dir -r $APP_CODE/requirements/sopn_parsing.txt \
+ && date
+
+#############
+## App code #
+#############
+# Copy the client-side directory "." (the build context passed to the build
+# command) into the container image, obeying the inclusions & exclusions
+# encoded in the ./.dockerignore file.
+COPY . $APP_CODE/
+# Set the working directory for the container entrypoint.
+WORKDIR $APP_CODE
+
+###########
+## Checks #
+###########
+# Invoke a lightweight, post-build test that proves the container reaches a
+# baseline level of correctness, whilst also generating .pyc files for faster
+# app startup.
+RUN DJANGO_SECRET_KEY=insecure python manage.py check
+
+###########
+## Assets #
+###########
+RUN date \
+ && npm run build \
+ && DJANGO_SECRET_KEY=insecure python manage.py collectstatic --no-input \
+ && date
+
+###########################################################################
+## Testing image ##########################################################
+###########################################################################
+FROM prod AS test
+
+# Base path for the app install build process.
+ARG APP_ROOT=/dc/ynr
+# Path for the app's code.
+ARG APP_CODE=$APP_ROOT/code
+
+# Install additional test dependencies.
+RUN date \
+ && pip install --no-cache-dir -r $APP_CODE/requirements/testing.txt \
+ && date

container/build/system-packages

@@ -0,0 +1,35 @@
+build-essential
+bundler
+cmake
+curl
+gettext
+ghostscript
+git
+language-pack-en
+libavcodec-dev
+libavformat-dev
+libevent-dev
+libffi-dev
+libgtk2.0-dev
+libjpeg-dev
+libpq-dev
+libssl-dev
+libxml2-dev
+libxslt-dev
+libyaml-dev
+npm
+opencv-data
+postgis
+python-is-python3
+python-tk
+python3-dev
+python3-pip
+python3-psycopg2
+python3-setuptools
+python3-virtualenv
+python3.8-venv
+s3cmd
+texlive-latex-base
+texlive-latex-extra
+unzip
+yui-compressor