diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d251b59d..3b307803a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,6 +147,41 @@ jobs: event: fail template: basic_fail_1 + container_build: + working_directory: ~/repo + docker: + - image: cimg/base:current + - image: cimg/postgres:16.4 + name: dbpsql + environment: + POSTGRES_USER: ynr + POSTGRES_DB: ynr + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: + name: Build container images + command: | + ./scripts/container.image.build.bash prod + ./scripts/container.image.build.bash test + - run: + name: Check that compose is happy + command: | + docker compose config + # No-op, due to a shared layer cache with earlier image builds. + docker compose build + - run: + name: Run app tests + command: | + net="$(docker network inspect $(docker network ls -q -f "label=task-network") --format '{{.Name}}')" + docker run -it --rm \ + --net="$net" \ + -e PGHOST=dbpsql \ + -e CIRCLECI=true \ + \ + ynr:test \ + pytest -x workflows: test_build_deploy: @@ -157,3 +192,6 @@ workflows: - build_and_test filters: { branches: { only: [ main, master, deployment-upgrades] } } context: [ deployment-production-ynr, slack-secrets ] + container: + jobs: + - container_build diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..d3c14d839 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,33 @@ +# This file contains paths, .gitignore-style, that will be excluded from being +# copied into a container image during build. +# cf. https://docs.docker.com/build/concepts/context/#dockerignore-files + +# Start with a clean slate by excluding all paths. +* + +# Selectively re-include paths to copy into the container. This list should +# track and match the bind mounts for the "frontend" service in the +# docker-compose.yml file. +!/ynr/ +!/data/ +!/requirements/ +!/requirements.txt +!/manage.py +!/Makefile +!/scripts/ +!/gulpfile.js +!/package.json +!/package-lock.json +!/pyproject.toml + +# Paths overriding early re-inclusions. +# The content inside /ynr/media/ is bind-mounted into the container when +# running as part of a compose stack, as part of the bind-mount of its parent, +# /ynr/. However its contents *should not* be statically copied into the +# container image because they're either irrelevant/empty (in non-development +# environments), or potentially very large (in development environments). +/ynr/media/ + +# Paths that don't need to be part of the image, but also need not to be +# excluded when explicitly referenced by build-time COPY and ADD commands. +!/container/build/system-packages diff --git a/.gitignore b/.gitignore index 35d609aa0..cc1b88b8c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ ynr/assets/* .pytest_cache *.css.map test-results -env/ +/env/*.env node_modules/ .vscode/ /test-env diff --git a/README.md b/README.md index c291099a3..20bd6f366 100644 --- a/README.md +++ b/README.md @@ -3,73 +3,81 @@ [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) ![CodeQL](https://github.com/DemocracyClub/yournextrepresentative/workflows/CodeQL/badge.svg) +# A website for crowd-sourcing structured data about election candidates -# A website for crowd-sourcing structured election candidate data +[**candidates.democracyclub.org.uk**](https://candidates.democracyclub.org.uk) -https://candidates.democracyclub.org.uk/ - -YourNextRepresentative is a open source platform for -crowd-sourcing information about candidates for political office +YourNextRepresentative ("**YNR**") is an open source platform +for crowd-sourcing information about candidates for political office, and making it available as open data to anyone. +YNR collects some core data, including: +- who is standing, +- what party they’re standing for, +- their contact details, and +- their social media accounts. + +YNR requires that each change is submitted with a source, so that the collected +information can be verified. + +## Using YNR + +**To find out information** about who you can vote for in upcoming elections, head +over to [whocanivotefor.co.uk](https://whocanivotefor.co.uk) and search for +candidates in your area. -The core data that YourNextRepresentative collects includes who -is standing, what party they’re standing for, their contact -details, their social media accounts etc. The software requires -that each change is submitted with a source, so that the -collected information can be independently checked. +**To contribute information** about candidates, use the YNR application at +[candidates.democracyclub.org.uk](https://candidates.democracyclub.org.uk). -# Installation +## Developing YNR -See [INSTALL.md](https://github.com/DemocracyClub/yournextrepresentative/blob/master/docs/INSTALL.md) +Before you can start modifying the YNR application and website, you'll need to +install its development prerequisites -- as detailed in +[`docs/INSTALL.md`](docs/INSTALL.md). -# Known Bugs +After you've confirmed that the prerequisites are working correctly on your +machine you'll be able to use the workflows detailed in +[`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md) to make changes to YNR. + +## Known Bugs You can find a list of known issues to work on here: * https://github.com/DemocracyClub/yournextrepresentative/issues -# Acknowledgements +## Acknowledgements This codebase was originally forked from [mysociety/yournextrepresentative](http://github.com/mysociety/yournextrepresentative) -We no longer track the upstream but we thank [mySociety](http://mysociety.org/) +We no longer track the upstream but we thank [mySociety](https://mysociety.org/) for their work on the project which we have been able to build on. -# API Versions +## API Versions v0.9 is legacy code and is now frozen. v1.0 is currently in alpha. We plan on publishing a v1 API once we have some more feedback from users and we think it’s stable enough. -# SOPN Parsing - -YNR uses `pypandoc` (which relies on `pandoc`) to convert SOPN documents to PDF, as needed, to be parsed. +## Statement Of Persons Nominated (SOPN) Parsing -To install `pandoc`, visit this page and follow the instructions for you operating system: -https://pandoc.org/installing.html +See [`ynr/apps/sopn_parsing`](ynr/apps/sopn_parsing#readme). -Once `pandoc` is installed +## Sentry Error Reporting -Install pypandoc (or via `requirements.txt`): +Sentry is used to report errors in production. We have added a url for `sentry-debug` to the [`urls.py`](ynr/urls.py#L42) file. This is to allow us verify that Sentry is configured correctly and working in production. -`pip install pandoc` +## Pre-election Tasks -If `pypandoc` does not install via `pip`, visit https://pypi.org/project/pypandoc/ for further instructions. +### Enable Candidate Leaderboard -# Sentry Error Reporting - -Sentry is used to report errors in production. We have added a url for `sentry-debug` to the `urls.py` file. This is to allow us verify that Sentry is configured correctly and working in production. - -``` - -# Pre-election Tasks - -# Enable Candidate Leaderboard - -The candidate leaderboard is a way of showing the most active candidates on the site. It is a way of encouraging volunteers to add more information about candidates and elections. +The candidate leaderboard shows the most active contributors to the site. +It is a way of encouraging volunteers to add more information about candidates and elections. We take a slice of edits in YNR and assign them to a election leaderboard. - -This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20 +This is defined in [`ynr/apps/candidates/views/mixins.py`](ynr/apps/candidates/views/mixins.py#L20). We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] -If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL` +If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with: +``` +podman compose up -d dbpqsl +./scripts/container.run.bash python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL +podman compose down +``` diff --git a/container/build/Containerfile b/container/build/Containerfile new file mode 100644 index 000000000..1468f5573 --- /dev/null +++ b/container/build/Containerfile @@ -0,0 +1,114 @@ +########################################################################### +## Production image ####################################################### +########################################################################### +FROM public.ecr.aws/lts/ubuntu:20.04 AS prod + +######################### +## Build-time arguments # +######################### +# Increase this arbitary number to force all image layers to be rebuilt. +# This is designed to invalidate the layer cache inside CI, not locally. +ARG invalidate_all_cached_layers=202412030000 +# Base path for the app install build process. +ARG APP_ROOT=/dc/ynr +# Path for the app's virtualenv. +ARG APP_VENV=$APP_ROOT/venv +# Path for the app's code. +ARG APP_CODE=$APP_ROOT/code + +######################## +## System dependencies # +######################## +ARG DEBIAN_FRONTEND=noninteractive +# Copy system dependency manifest into the container image. +COPY container/build/system-packages /tmp/apt-packages +# Install dependencies. +RUN date \ + && apt update \ + && +| Command                                                   | Purpose | Notes +| :--- | :--- | :--- +| `podman compose up -d` | Start the entire stack. | `-d` forks the action into the background, which is optional but strongly recommended. +| `podman compose up -d dbpsql` | Start only the named container in the stack. | +| `podman compose down` | Stop any running containers in the stack. | +| `podman compose down --volumes` | Stop any running containers in the stack and also destroy their persistent data. | Anything bind-mounted from your local repo into the `frontend` webapp container is left untouched. | +| `podman compose ps` | Display the status of containers in the stack. | +| `podman volume ls` | List the persistent volumes that podman controls on your machine. | +| `podman compose build` | Rebuild the webapp's frontend container image. | +| `podman compose build --no-cache` | Rebuild the webapp's frontend container image from scratch. | Takes several minutes to finish. +| `podman compose logs` | Display the last N stdout/stderr lines emitted by any running containers. | +| `podman compose logs --follow` | Display the last N stdout/stderr lines emitted by any running containers, and then wait for more lines. | +| `podman system reset` | Destroy everything that Podman controls. | "Everything seems to have gone wrong, so I'll just start from scratch". It wipes out all containers, networks, images, volumes, etc ... so **avoid this if possible!** | + +### Scripts + +These executable scripts are available from the [`scripts`](../scripts) directory. + +| Script | Purpose | Parameters +| :--- | :--- | :--- +| `container.image.build.bash` | Builds the YNR container image | $1 -- The named stage from [`container/build/Containerfile`](../container/build/Containerfile) to build and tag (*required*)
$2, $3, ... -- Any parameters to pass to the underlying builder process (*optional*) +| `container.exec.bash` | Runs a command inside the already-running `frontend` container | The unquoted command to run (*required*) +| `container.manage-py.bash` | Runs a Django management command inside the already-running `frontend` container | The unquoted command to run (*required*) +| `container.pytest.bash` | Runs `pytest` inside the already-running `frontend` container | Any parameters for Pytest (*optional*) +| `container.run.bash` | Runs a command inside a freshly-instantiated, ephemeral `frontend` container | The unquoted command to run (*required*) + +### Rebuilding the application container image + +You will need to rebuild the application's container image if you change any of +the application's dependencies, across any of the packaging ecosystems it +currently relies on: + +- `container/build/system-packages`: System / APT dependendencies +- `package{,-lock}.json`: Node dependencies +- `requirements/*.txt`: Python dependencies +- `.dockerignore`: Container build-time file dependencies + +The above list is presented in descending order of how slow a rebuild will be, +if a particular package ecosystem's dependencies are changed. +Changing a system dependency, for example, forces a longer rebuild than +changing a Python dependency. +**You do not need to rebuild the application's container image if you only +change files in the `ynr/` directory**. Changes to the YNR application are +picked up automatically when using the compose stack locally (as described +elsewhere in this guide). + +The build process for the YNR application is encoded in +[`container/build/Containerfile`](../container/build/Containerfile). +This Docker-compatible file describes two image stages, `prod` and `test`, with +`test` being built on top of `prod`. +Locally, on your development machine, you will need to use the `test` stage. + +#### Build the `test` stage using a build cache + +``` +./scripts/container.image.build.bash test +``` + +#### Build the `test` stage without a build cache + +Avoiding the use of your local build cache significantly increases the time it +takes to build the container image, but is sometimes useful when there's a +problem with external dependencies (e.g. if a important update has been +published for an APT package but it's not visible in the container's package +index). + +``` +./scripts/container.image.build.bash test --no-cache +``` diff --git a/docs/INSTALL.md b/docs/INSTALL.md index cb909a955..7ba9f24bb 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,91 +1,69 @@ # Installation -TODO: improve these docs with more detail - -YourNextRepresentative requires python >=3.5 and PostgreSQL - -## Install python dependencies - -``` -pip install -U pip -pip install -r requirements.txt -``` - -## Set up database - -``` -sudo -u postgres createdb ynr -``` - -If using mac-os/homebrew -``` -createdb ynr -``` - -``` -cp ynr/settings/local.py.example ynr/settings/local.py -``` - -Add database credentials to `DATABASES` dict in `local.py` - -``` -brew install libmagic -./manage.py migrate -``` - -To populate the database run from the live site run: - -``` -python manage.py candidates_import_from_live_site -``` - -(Note that this command will take multiple hours to complete.) - -## Build frontend assets - -``` -npm run build -npm install -``` - -## (Optional) Code linting - -A CI will check all code against Black and Flake8. To save pushing commits that don't -pass these tests you can configure pre-commmit hooks. - -Do this by installing `[precommit](https://pre-commit.com/)`: - -``` -pip install pre-commit -pre-commit install -``` - -## (Optional) SOPN parsing - -SOPNs parsing (see `ynr/apps/sopn_parsing/README.md`) is optional -because it depends on various system packages beyond python packages. - -It currently requires [camelot-py](https://camelot-py.readthedocs.io/en/master/user/install.html#install) -and that in turn requires `python-tk` and `ghostscript`. - -Read up on how to install them, and then install the SOPN parsing requirements: - -``` -pip install -r requirements/sopn_parsing.txt -``` - -File conversion relies on `pandoc` to turn non-pdf SOPN files into pdf files. -To install `pandoc`, visit https://pandoc.org/installing.html and follow instructions -for Mac OS and Ubuntu. - -AWS Textract relies on the following packages for viewing image results: - -https://pypi.org/project/pdf2image/ - -To install these packages run: - -``` -brew install poppler -``` - -_If you have omitted SOPN and are having problems getting the project to run, you may need to follow the SOPN steps._ \ No newline at end of file +## Local development + +To develop YNR on your local machine you'll first need to install its +containerisation prerequisites. We use containers in development to isolate the +(non-trivial!) set of *application* prerequisites away from your local machine, +and to get closer to the intended future state of the application's +*production* deployment. + +### Install and test containerisation prerequisites + +1. Clone this repository: + `git clone --branch jcm/wip https://github.com/DemocracyClub/yournextrepresentative` +1. Install the `podman` command: https://podman.io/docs/installation. + These installation mechanisms have been tested: + - System package on Ubuntu 24.04 LTS + - https://podman.io/docs/installation#ubuntu +1. Install `podman-compose` v1.2.0: https://pypi.org/project/podman-compose/. + These installation mechanisms have been tested: + - Local `pip` installation of v1.2.0 on Ubuntu 24.04 LTS + - https://pypi.org/project/podman-compose/ + - `pip install podman-compose` + - Either inside a venv, or not, as you prefer + - Manual installation of v1.2.0 APT package on Ubuntu 24.04 LTS + - https://packages.ubuntu.com/oracular/all/podman-compose/download + - `dkpkg -i path/to/debian-package.deb` +1. Configure `podman` to be less chatty, by placing this configuration in `$HOME/.config/containers/containers.conf`: + ```ini + # Don't emit logs on each invocation of the compose command indicating + # that an external compose provider is being executed. + [engine] + compose_warning_logs=false + ``` +1. Make sure the `bash` shell is available: + `which bash || echo Not found` +1. Build any container images used by the compose stack: + `podman compose build` +1. Pull any 3rd-party container images used by the compose stack: + `podman compose pull` +1. Set up your development envvars as needed, by placing keys and values in + `env/frontend.env`, using `env/frontend.env.example` as a template. + In general, the only envar you should need is this: + ``` + DJANGO_SETTINGS_MODULE=ynr.settings + ``` +1. Copy `ynr/settings/local.py.container.example` to `ynr/settings/local.py`. + If you already have a `ynr/settings/local.py` file, incorporate the example + file's settings. **If you don't use most of the example file's settings, you + *will* experience problems interacting with the app, later**. +1. Test that the compose stack can be stood up: + ```bash + podman compose up -d # NB Space between "podman" and "compose"! + curl 0:8080 + ``` + Curl **should** report a server error (i.e. a 500) because your database + setup is incomplete. This step tests only that `podman` and `podman-compose` + are able to run successfully on your machine when given YNR's + `docker-compose.yml` file. +1. Test that Django management commands can be invoked: + `./scripts/container.manage-py.bash check` +1. Run the test suite (which only requires that a database server be + *available*, not that it contains any specific data). + This will take a little time to finish: + `./scripts/container.pytest.bash` +1. Shut down the compose stack: + `podman compose down` + +Now you can use the tools and workflows detailed in [DEVELOPMENT.md](DEVELOPMENT.md). diff --git a/docs/INSTALL.old.md b/docs/INSTALL.old.md new file mode 100644 index 000000000..cb909a955 --- /dev/null +++ b/docs/INSTALL.old.md @@ -0,0 +1,91 @@ +# Installation + +TODO: improve these docs with more detail + +YourNextRepresentative requires python >=3.5 and PostgreSQL + +## Install python dependencies + +``` +pip install -U pip +pip install -r requirements.txt +``` + +## Set up database + +``` +sudo -u postgres createdb ynr +``` + +If using mac-os/homebrew +``` +createdb ynr +``` + +``` +cp ynr/settings/local.py.example ynr/settings/local.py +``` + +Add database credentials to `DATABASES` dict in `local.py` + +``` +brew install libmagic +./manage.py migrate +``` + +To populate the database run from the live site run: + +``` +python manage.py candidates_import_from_live_site +``` + +(Note that this command will take multiple hours to complete.) + +## Build frontend assets + +``` +npm run build +npm install +``` + +## (Optional) Code linting + +A CI will check all code against Black and Flake8. To save pushing commits that don't +pass these tests you can configure pre-commmit hooks. + +Do this by installing `[precommit](https://pre-commit.com/)`: + +``` +pip install pre-commit +pre-commit install +``` + +## (Optional) SOPN parsing + +SOPNs parsing (see `ynr/apps/sopn_parsing/README.md`) is optional +because it depends on various system packages beyond python packages. + +It currently requires [camelot-py](https://camelot-py.readthedocs.io/en/master/user/install.html#install) +and that in turn requires `python-tk` and `ghostscript`. + +Read up on how to install them, and then install the SOPN parsing requirements: + +``` +pip install -r requirements/sopn_parsing.txt +``` + +File conversion relies on `pandoc` to turn non-pdf SOPN files into pdf files. +To install `pandoc`, visit https://pandoc.org/installing.html and follow instructions +for Mac OS and Ubuntu. + +AWS Textract relies on the following packages for viewing image results: + +https://pypi.org/project/pdf2image/ + +To install these packages run: + +``` +brew install poppler +``` + +_If you have omitted SOPN and are having problems getting the project to run, you may need to follow the SOPN steps._ \ No newline at end of file diff --git a/env/.gitkeep b/env/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/env/frontend.env.example b/env/frontend.env.example new file mode 100644 index 000000000..cfa9bcb67 --- /dev/null +++ b/env/frontend.env.example @@ -0,0 +1 @@ +DJANGO_SETTINGS_MODULE=ynr.settings diff --git a/requirements/base.txt b/requirements/base.txt index 4aa84244d..7ea70b495 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -40,6 +40,7 @@ freezegun==1.2.2 futures==3.1.1 gender-detector==0.1.0 git+https://github.com/dessibelle/sorl-thumbnail-serializer-field.git#egg=sorl-thumbnail-serializer-field +gunicorn==23.0.0 html5lib==1.1 idna==3.4 ipaddress==1.0.23 diff --git a/requirements/sopn_parsing.txt b/requirements/sopn_parsing.txt index 587271dfb..56ae97e05 100644 --- a/requirements/sopn_parsing.txt +++ b/requirements/sopn_parsing.txt @@ -2,7 +2,7 @@ pdfminer.six==20201018 camelot-py[cv]==0.8.2 -pypandoc==1.7.2 +pypandoc_binary==1.14 PyPDF2<3.0 amazon-textract-response-parser -amazon-textract-helper \ No newline at end of file +amazon-textract-helper diff --git a/scripts/container.exec.bash b/scripts/container.exec.bash new file mode 100755 index 000000000..b3a4a0696 --- /dev/null +++ b/scripts/container.exec.bash @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.exec.bash invokes a command in a "frontend" container. +# The container must be running before running this script (see README.md). +# +# Usage: +# scripts/container.exec.bash echo hello world +# +# This script is a deliberately simple convenience shim, and doesn't include a +# way to set or override environment variables. To do this, invoke "podman" +# directly: +# podman compose exec -e key1=val -e key2=val frontend command param1 param2 +# Updating variables held in env/frontend.env has no effect until the running +# container is restarted. + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +podman compose exec frontend $command diff --git a/scripts/container.image.build.bash b/scripts/container.image.build.bash new file mode 100755 index 000000000..822786b1a --- /dev/null +++ b/scripts/container.image.build.bash @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.image.build.bash builds an image from a stage defined in +# container/build/Containerfile, and tags it as "ynr:$image". +image="$1"; shift +args="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +# Choose a build tool. +# CircleCI uses docker; others (e.g. developers) use podman. +set +u; if [[ "$CIRCLECI" == "true" ]]; then + builder="docker" +else + builder="podman" +fi; set -u + +# Build the image. +"$builder" build --target "$image" --tag "ynr:$image" -f container/build/Containerfile $args . diff --git a/scripts/container.manage-py.bash b/scripts/container.manage-py.bash new file mode 100755 index 000000000..d81ee9201 --- /dev/null +++ b/scripts/container.manage-py.bash @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.manage-py.bash invokes a Django management command in a frontend +# container, which must have been started beforehand. +# +# Usage: +# scripts/container.manage-py.bash check + +# The management command does not need to be quoted, unless it contains shell +# meta-characters. Multiple words are fine. +mgmtCommand="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +./scripts/container.exec.bash python manage.py $mgmtCommand diff --git a/scripts/container.pytest.bash b/scripts/container.pytest.bash new file mode 100755 index 000000000..cff57807f --- /dev/null +++ b/scripts/container.pytest.bash @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.pytest.bash invokes a `pytest` command in a "frontend" container. +# The container and a database server must be running before running this +# script (see README.md). +# +# Usage: +# scripts/container.pytest.bash # run all tests; continue after failures +# scripts/container.pytest.bash -x # stop after first failure + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +./scripts/container.exec.bash pytest $command diff --git a/scripts/container.run.bash b/scripts/container.run.bash new file mode 100755 index 000000000..5a700424b --- /dev/null +++ b/scripts/container.run.bash @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# container.run.bash invokes a command in a newly instantiated "frontend" +# container. +# +# The container and its filesystem are removed after the user's command exits. +# Only changes made inside bind-mounted filesystems are persisted (see +# docker-compose.yml for a list of bind mounts). The database container must be +# running before this script can be invoked (see README.md) because the compose +# command sees the database as a required dependency. This is the case even if +# the command being executed doesn't use the database. +# +# Usage: +# scripts/container.run.bash env +# +# Because a new container is instantiated along with its environment each time +# this script is invoked, the contents of the env/frontend.env file are +# respected for each invocation, with environment variables being set as per +# that file. + +# The command being invoked does not need to be quoted, unless it contains +# shell meta-characters or similar. Multiple words are fine, without quotes. +command="$@" + +# Change to the directory above the directory containing this script. +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/.." + +podman compose run --rm --no-deps --name tmp-fe-$$ frontend $command \ + 2> >( grep -v "Error: adding pod to state.*pod already exists" >&2 ) diff --git a/ynr/apps/sopn_parsing/README.md b/ynr/apps/sopn_parsing/README.md index f8cb89e91..875ba63fa 100644 --- a/ynr/apps/sopn_parsing/README.md +++ b/ynr/apps/sopn_parsing/README.md @@ -2,6 +2,9 @@ This app is designed to extract useful information out of UK Statement Of Persons Nominated documents (SOPNs), published before elections. +It uses the `pypandoc_binary` dependency (which both provides an interface to, +and installs the binary of, the upstream [pandoc](https://pandoc.org/) project) +to convert SOPN documents to PDF (as needed) to be parsed. The documents contain information on candidates for a given election, but are published in a wide variety of layouts. diff --git a/ynr/settings/local.py.container.example b/ynr/settings/local.py.container.example new file mode 100644 index 000000000..94971eb45 --- /dev/null +++ b/ynr/settings/local.py.container.example @@ -0,0 +1,38 @@ +# Only set DEBUG to True in development environments. +DEBUG = True + +# These Postgres settings should match docker-compose.yml. +# Credentials are not required. +DATABASES = {"default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "dbpsql", "NAME": 'ynr', "USER": "ynr", +}} + +# The containerised setup does not include the same memcache service that +# production currently uses for its cache, so we explicitly set up a dummy +# cache when running locally. +CACHES = {"default": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"}} + +# This short, known value is insecure. +SECRET_KEY = "insecure" + +# Certain errors are very noisy (obscuring the real problem) if this list is +# empty. +ADMINS = [("Dummy Admin", "dummy@example.com")] + +# This permits the site to be served at localhost:8080. +ALLOWED_HOSTS = ['*'] + +# This unpleasantness adds the container's internal IP to the list of those IPs +# permitted to access the Django debug toolbar, which allows it to be enabled. +# We believe the container's own IP needs to be in this list because of +# something to do with the container networking, or the HTTP server gunicorn's +# reverse-proxy setup, or both. +# TODO: Replace with a better method, either here or by changing the +# container/gunicorn setup. https://pypi.org/project/netifaces/ also exists, +# but might not be considered "better". +import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("8.8.8.8", 80)) +INTERNAL_IPS = [ '127.0.0.1', str(s.getsockname()[0]) ] +s.close()