Support Multiple Python Versions

.github/workflows/deploy.yaml

@@ -22,6 +22,12 @@ jobs:
         with:
           name: ${{ inputs.artifact }}
 
+      - name: Set up Python
+        id: python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
       # Load the image to make use of common layers during the final build.
       - name: Load image from archive
         run: docker load -i ${{ inputs.artifact }}.tar
@@ -106,3 +112,24 @@ jobs:
           tags: |
             ghcr.io/python-discord/snekbox-venv:latest
             ghcr.io/python-discord/snekbox-venv:${{ inputs.version }}
+
+      # Build a lightweight image with a singular python version
+      - name: Regenerate dockerfile
+        run: python scripts/set_versions.py
+        env:
+          VERSIONS_CONFIG: config/versions-light.json
+
+      - name: Build & push slim image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          cache-from: |
+            ghcr.io/python-discord/snekbox-base:latest
+            ghcr.io/python-discord/snekbox-venv:latest
+            ghcr.io/python-discord/snekbox:latest
+          cache-to: type=inline
+          tags: |
+            ghcr.io/python-discord/snekbox:latest-slim
+            ghcr.io/python-discord/snekbox:${{ inputs.version }}-slim

.gitignore

@@ -106,3 +106,6 @@ venv.bak/
 
 # mypy
 .mypy_cache/
+
+# Project
+config/versions-dev.json

.pre-commit-config.yaml

@@ -44,3 +44,14 @@ repos:
           --format,
           "::error file=%(path)s,line=%(row)d,col=%(col)d::[flake8] %(code)s: %(text)s",
         ]
+  - repo: local
+    hooks:
+      - id: python-version-script
+        name: check py versions
+        entry: python scripts/set_versions.py
+        language: system
+        always_run: true
+        pass_filenames: false
+        description: Check the Python versions around the project are up to date. If this fails, you most likely need to re-run the set_versions script.
+        args:
+          - --error-modified

Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1
+# THIS FILE IS AUTOGENERATED, DO NOT MODIFY! #
 FROM python:3.11-slim-buster as builder
 
 WORKDIR /nsjail
@@ -19,9 +19,20 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
 RUN make
 
+# ------------------------------------------------------------------------------
+FROM python:3.11-slim-buster as base-first
+
+FROM python:3.7-slim-buster as base-3-7
+COPY --from=base-first / /
+
+FROM python:3.9-slim-buster as base-3-9
+COPY --from=base-3-7 / /
+
 # ------------------------------------------------------------------------------
 FROM python:3.11-slim-buster as base
 
+COPY --from=base-3-9 / /
+
 # Everything will be a user install to allow snekbox's dependencies to be kept
 # separate from the packages exposed during eval.
 ENV PATH=/root/.local/bin:$PATH \

Makefile

@@ -34,8 +34,12 @@ test:
 report: setup
 	coverage report
 
+.PHONY: prepare-versions
+prepare-versions:
+	python scripts/set_versions.py
+
 .PHONY: build
-build:
+build: prepare-versions
 	docker build -t ghcr.io/python-discord/snekbox:latest .
 
 .PHONY: devsh

README.md

@@ -55,6 +55,16 @@ The above command will make the API accessible on the host via `http://localhost
 
 Configuration files can be edited directly. However, this requires rebuilding the image. Alternatively, a Docker volume or bind mounts can be used to override the configuration files at their default locations.
 
+### Python Versions
+You can configure python versions in [`versions.json`](config/versions.json).
+The `image_tag` is the name of the image in the docker registry, `version_name` is the
+name used by python on the system (for instance a version_name of "3.11" would correspond to a binary of /python3.11).
+The `display_name` is only decorative, and is used to distinguish the different versions in a human-readable fashion.
+
+Exactly one of the python versions should be set as "main", which is the default eval version,
+and the version used to run the server. It must be a version which can support the features used in the codebase.
+If the versions file has been updated, the `prepare-versions` make target must be re-run.
+
 ### NsJail
 
 The main features of the default configuration are:

config/snekbox.cfg

@@ -14,7 +14,6 @@ envar: "OPENBLAS_NUM_THREADS=5"
 envar: "MKL_NUM_THREADS=5"
 envar: "VECLIB_MAXIMUM_THREADS=5"
 envar: "NUMEXPR_NUM_THREADS=5"
-envar: "PYTHONPATH=/snekbox/user_base/lib/python3.11/site-packages"
 envar: "PYTHONIOENCODING=utf-8:strict"
 envar: "HOME=home"
 
@@ -99,13 +98,6 @@ mount {
     rw: false
 }
 
-mount {
-    src: "/usr/local/bin/python3.11"
-    dst: "/usr/local/bin/python3.11"
-    is_bind: true
-    rw: false
-}
-
 cgroup_mem_max: 52428800
 cgroup_mem_swap_max: 0
 cgroup_mem_mount: "/sys/fs/cgroup/memory"
@@ -114,8 +106,3 @@ cgroup_pids_max: 6
 cgroup_pids_mount: "/sys/fs/cgroup/pids"
 
 iface_no_lo: true
-
-exec_bin {
-    path: "/usr/local/bin/python"
-    arg: "-BSqu"
-}

config/versions-light.json

@@ -0,0 +1,8 @@
+[
+  {
+    "image_tag": "3.11-slim-buster",
+    "version_name": "3.11",
+    "display_name": "CPython 3.11",
+    "is_main": true
+  }
+]

config/versions.json

@@ -0,0 +1,20 @@
+[
+  {
+    "image_tag": "3.7-slim-buster",
+    "version_name": "3.7",
+    "display_name": "CPython 3.7",
+    "is_main": false
+  },
+  {
+    "image_tag": "3.9-slim-buster",
+    "version_name": "3.9",
+    "display_name": "CPython 3.9",
+    "is_main": false
+  },
+  {
+    "image_tag": "3.11-slim-buster",
+    "version_name": "3.11",
+    "display_name": "CPython 3.11",
+    "is_main": true
+  }
+]

pyproject.toml

@@ -70,5 +70,5 @@ force-exclude = "snekbox/config_pb2.py"
 line_length = 100
 profile = "black"
 skip_gitignore = true
-src_paths = ["snekbox"]
+src_paths = ["snekbox", "scripts"]
 extend_skip = ["snekbox/config_pb2.py"]

scripts/in.Dockerfile

@@ -0,0 +1,81 @@
+FROM python:{main_version_tag} as builder
+
+WORKDIR /nsjail
+
+RUN apt-get -y update \
+    && apt-get install -y \
+        bison=2:3.3.* \
+        flex=2.6.* \
+        g++=4:8.3.* \
+        gcc=4:8.3.* \
+        git=1:2.20.* \
+        libprotobuf-dev=3.6.* \
+        libnl-route-3-dev=3.4.* \
+        make=4.2.* \
+        pkg-config=0.29-6 \
+        protobuf-compiler=3.6.*
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+RUN make
+
+# ------------------------------------------------------------------------------
+{python_install_commands}
+# ------------------------------------------------------------------------------
+FROM python:{main_version_tag} as base
+
+COPY --from=base-{final_base} / /
+
+# Everything will be a user install to allow snekbox's dependencies to be kept
+# separate from the packages exposed during eval.
+ENV PATH=/root/.local/bin:$PATH \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_CACHE_DIR=false \
+    PIP_USER=1
+
+RUN apt-get -y update \
+    && apt-get install -y \
+        gcc=4:8.3.* \
+        git=1:2.20.* \
+        libnl-route-3-200=3.4.* \
+        libprotobuf17=3.6.* \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /nsjail/nsjail /usr/sbin/
+RUN chmod +x /usr/sbin/nsjail
+
+# ------------------------------------------------------------------------------
+FROM base as venv
+
+COPY requirements/ /snekbox/requirements/
+WORKDIR /snekbox
+
+# pip installs to the default user site since PIP_USER is set.
+RUN pip install -U -r requirements/requirements.pip
+
+# This must come after the first pip command! From the docs:
+# All RUN instructions following an ARG instruction use the ARG variable
+# implicitly (as an environment variable), thus can cause a cache miss.
+ARG DEV
+
+# Install numpy when in dev mode; one of the unit tests needs it.
+RUN if [ -n "${DEV}" ]; \
+    then \
+        pip install -U -r requirements/coverage.pip \
+        && PYTHONUSERBASE=/snekbox/user_base pip install numpy~=1.19; \
+    fi
+
+# At the end to avoid re-installing dependencies when only a config changes.
+COPY config/ /snekbox/config/
+
+ENTRYPOINT ["gunicorn"]
+CMD ["-c", "config/gunicorn.conf.py"]
+
+# ------------------------------------------------------------------------------
+FROM venv
+
+# Use a separate directory to avoid importing the source over the installed pkg.
+# The venv already installed dependencies, so nothing besides snekbox itself
+# will be installed. Note requirements.pip cannot be used as a constraint file
+# because it contains extras, which pip disallows.
+RUN --mount=source=.,target=/snekbox_src,rw \
+    pip install /snekbox_src[gunicorn,sentry]

scripts/python_version.py

@@ -0,0 +1,43 @@
+"""
+Parse and return python version information from the versions file.
+
+The version file is read from the environment variable VERSIONS_CONFIG,
+and defaults to config/versions.json otherwise.
+"""
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+VERSIONS_FILE = Path(os.getenv("VERSIONS_CONFIG", "config/versions.json"))
+
+
+@dataclass(frozen=True)
+class Version:
+    """A python image available for eval."""
+
+    image_tag: str
+    version_name: str
+    display_name: str
+    is_main: bool
+
+
+ALL_VERSIONS: list[Version] = []
+"""A list of all versions available for eval."""
+MAIN_VERSION: Version = None
+"""The default eval version, and the version used by the server."""
+VERSION_DISPLAY_NAMES: list[str] = []
+"""The display names for all available eval versions."""
+
+if MAIN_VERSION is None:
+    # Set the constants' values the first time the file is imported
+    for version_json in json.loads(VERSIONS_FILE.read_text("utf-8")):
+        version = Version(**version_json)
+        if version.is_main:
+            MAIN_VERSION = version
+        ALL_VERSIONS.append(version)
+        VERSION_DISPLAY_NAMES.append(version.display_name)
+
+    if MAIN_VERSION is None:
+        raise Exception("Exactly one version must be configured as the main version.")

scripts/set_versions.py

@@ -0,0 +1,67 @@
+"""
+Generate a Dockerfile from in.Dockerfile and a version JSON file, and write version info.
+
+If the argument --error-modified is passed, the script will not write any output
+and will exit with status code 1 if the output files were modified.
+"""
+
+import difflib
+import sys
+from pathlib import Path
+from textwrap import dedent
+
+try:
+    from scripts.python_version import ALL_VERSIONS, MAIN_VERSION
+except ModuleNotFoundError:
+    sys.path.insert(0, Path(__file__).parent.parent.absolute().as_posix())
+    from scripts.python_version import ALL_VERSIONS, MAIN_VERSION
+
+DOCKERFILE_TEMPLATE = Path("scripts/in.Dockerfile").read_text("utf-8")
+DOCKERFILE = Path("Dockerfile")
+ERROR_MODIFIED = "--error-modified" in sys.argv
+
+# Download and copy multiple python images into one layer
+python_build = ""
+previous_layer = "first"
+
+for version in ALL_VERSIONS:
+    if version.is_main:
+        # Main is handled separately later
+        continue
+
+    # Add the current version to the Dockerfile
+    layer_name = version.version_name.replace(".", "-")  # Dots aren't valid in layer names
+    python_build += dedent(
+        f"""
+        FROM python:{version.image_tag} as base-{layer_name}
+        COPY --from=base-{previous_layer} / /
+    """
+    )
+    previous_layer = layer_name
+
+# Main version is installed twice, once at the very beginning to make sure
+# its files aren't overwritten, and once at the end which actually makes use of the version
+python_build = f"FROM python:{MAIN_VERSION.image_tag} as base-first\n" + python_build
+
+# Write new dockerfile
+# fmt: off
+# Black makes the following block much less readable
+dockerfile_out = (
+    "# THIS FILE IS AUTOGENERATED, DO NOT MODIFY! #\n"
+    + DOCKERFILE_TEMPLATE
+    .replace("{python_install_commands}", python_build)
+    .replace("{final_base}", previous_layer).replace("{main_version_tag}", MAIN_VERSION.image_tag)
+)
+# fmt: on
+
+if ERROR_MODIFIED:
+    if (original := DOCKERFILE.read_text("utf-8")) != dockerfile_out:
+        print("Dockerfile modified:")
+        print("\n".join(difflib.unified_diff(dockerfile_out.splitlines(), original.splitlines())))
+        raise SystemExit(1)
+    else:
+        exit(0)
+else:
+    DOCKERFILE.write_text(dockerfile_out, "utf-8")
+
+print("Finished!")

snekbox/api/resources/__init__.py

@@ -1,3 +1,7 @@
 from .eval import EvalResource
+from .info import InformationResource
 
-__all__ = ("EvalResource",)
+__all__ = (
+    "EvalResource",
+    "InformationResource",
+)