Merge branch 'main' into tom/517-separate-dbs

SAFEHR-data · Jan 27, 2025 · 2feae68 · 2feae68
2 parents 9611dec + 4af6ed0
commit 2feae68
Show file tree

Hide file tree

Showing 34 changed files with 2,868 additions and 258 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,7 @@
+**/build
+.git
+**/.mypy_cache
+.venv*
+**/.pytest_cache
+**/.ruff_cache
+**/__pycache__
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -56,22 +56,23 @@ jobs:
         steps:
             - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
 
-            - name: Init Python
-              uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+            - name: Install uv
+              uses: astral-sh/setup-uv@v4
               with:
+                  version: "0.5.10"
                   python-version: "3.11"
-                  cache: "pip"
+                  enable-cache: true
+
 
             - name: Install Python dependencies
               run: |
-                  python -m pip install --upgrade pip
-                  pip install -e pytest-pixl/
-                  pip install -e pixl_core/[test]
-                  pip install -e ${{ matrix.package_dir }}/[test]
+                  uv sync
 
             - name: Run tests and generate coverage report
               working-directory: ${{ matrix.package_dir }}
-              run: COV_CORE_SOURCE=src COV_CORE_CONFIG=.coveragerc COV_CORE_DATAFILE=.coverage.eager pytest --cov=src --cov-append --cov-report=xml --cov-report=term-missing
+              run: |
+                  source ../.venv/bin/activate
+                  COV_CORE_SOURCE=src COV_CORE_CONFIG=.coveragerc COV_CORE_DATAFILE=.coverage.eager pytest --cov=src --cov-append --cov-report=xml --cov-report=term-missing
               env:
                   ENV: test
                   AZURE_KEY_VAULT_NAME: test
@@ -91,19 +92,19 @@ jobs:
         steps:
             - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
             - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3
-            - name: Init Python
-              uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+
+            - name: Install uv
+              uses: astral-sh/setup-uv@v4
               with:
+                  version: "0.5.10"
                   python-version: "3.11"
-                  cache: "pip"
+                  enable-cache: true
 
             - name: Install Python dependencies
-              # The -e option is required here due to the way the
+              # Editable install required here due to the way the
               # code determines the export directory. See issue #318.
               run: |
-                  pip install -e pytest-pixl/
-                  pip install -e pixl_core/[test]
-                  pip install -e cli/[test]
+                  uv sync
 
             - name: Create .secrets.env
               run: touch test/.secrets.env
@@ -129,7 +130,7 @@ jobs:
                   HASHER_API_AZ_TENANT_ID: ${{ secrets.EXPORT_AZ_TENANT_ID }}
                   HASHER_API_AZ_KEY_VAULT_NAME: ${{ secrets.EXPORT_AZ_KEY_VAULT_NAME }}
               run: |
-                  ./run-system-test.sh coverage
+                  uv run ./run-system-test.sh coverage
                   echo FINISHED SYSTEM TEST SCRIPT
 
             - name: Dump queue docker logs for debugging
@@ -183,4 +184,4 @@ jobs:
               with:
                 directory: test
               env:
-                CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+                CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -53,3 +53,9 @@ repos:
             language: script
             types_or: [python, shell, yaml, dockerfile]
             entry: bin/linters/check_headers_exist.sh
+
+    # avoid forgetting to update the lock file after pyproject.toml changes
+    - repo: https://github.com/astral-sh/uv-pre-commit
+      rev: 0.5.15
+      hooks:
+        - id: uv-lock
diff --git a/README.md b/README.md
@@ -14,18 +14,17 @@ several services orchestrated by [Docker Compose](https://docs.docker.com/compos
 To get access to the GAE, [see the documentation on Slab](https://uclh.slab.com/posts/gae-access-7hkddxap). 
 Please request access to Slab and add further details in a [new blank issue](https://github.com/SAFEHR-data/PIXL/issues/new).
 
-## Installation
+## Installation in production
 
 Install the PIXL Python modules by running the following commands from the top-level `PIXL/` directory:
 
 ```shell
-python -m pip install -e pixl_core/
-python -m pip install -e cli/
+uv sync --package pixl-cli
 ```
 
 Note, the CLI currently [needs to be installed in editable mode](https://github.com/SAFEHR-data/PIXL/issues/318).
 
-## Development
+## Developer setup
 
 [Follow the developer setup instructions](./docs/setup/developer.md).
 

diff --git a/cli/README.md b/cli/README.md
@@ -9,15 +9,13 @@ stopped cleanly.
 * Python version 3.11 (matching python versions in [pixl-ci](../.github/workflows/main.yml) and [dev](../docs/setup/developer.md#installation-of-pixl-modules)).
 * [Docker](https://docs.docker.com/get-docker/) with version `>=27.0.3`
 * [Docker Compose](https://docs.docker.com/compose/install/#installation-scenarios) with version `>=v2.28.1-desktop.1`
-* We recommend installing PIXL project in specific virtual environment using a environment
-management tool such as [conda](https://docs.conda.io/en/latest/) or
-[virtualenv](https://virtualenv.pypa.io/en/latest/). 
+* [uv](https://docs.astral.sh/uv/) to install PIXL in a virtual environment
 See detailed instructions [here](../docs/setup/developer.md#setting-up-python-virtual-environment)
 
 ## Installation
 Activate your python virtual environment and install `PIXL` project in editable mode by running
 ```bash
-python -m pip install -e ../pixl_core -e .
+uv sync
 ```
 
 ## Usage
@@ -130,7 +128,7 @@ are currently available, you can use the `pixl --help` command:
 ### Local installation
 Activate your python environment and install project locally in editable mode with the development and testing dependencies by running
 ```bash
-python -m pip install -e ../pixl_core -e ../pytest-pixl -e ".[test]" -e ".[dev]"
+uv sync
 ```
 
 ### Running tests

diff --git a/cli/pyproject.toml b/cli/pyproject.toml
@@ -4,7 +4,7 @@ version = "0.2.0rc0"
 authors = [{ name = "PIXL authors" }]
 description = "PIXL command line interface"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = ["Programming Language :: Python :: 3"]
 dependencies = [
     "core==0.2.0rc0",
@@ -15,7 +15,6 @@ dependencies = [
 [project.optional-dependencies]
 test = [
     "core[test]==0.2.0rc0",
-    "pytest-mock==3.14.*",
 ]
 dev = [
     "core[dev]==0.2.0rc0",
@@ -25,8 +24,13 @@ dev = [
 pixl = "pixl_cli.main:cli"
 
 [build-system]
-requires = ["setuptools>=61.0"]
-build-backend = "setuptools.build_meta"
+requires = ["hatchling>=1.0.0"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+dev-mode-dirs = [
+    "src"
+]
 
 [tool.coverage.report]
 exclude_also = [

diff --git a/docker/.dockerignore b/docker/.dockerignore
diff --git a/docker/orthanc/Dockerfile b/docker/orthanc/Dockerfile
@@ -11,54 +11,56 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-FROM orthancteam/orthanc:24.7.3@sha256:57a3d037729897331027ddc00c12695b50f1effbbf805f855396f3d0248d2d5f AS pixl_orthanc_apt
+FROM orthancteam/orthanc:24.7.3@sha256:57a3d037729897331027ddc00c12695b50f1effbbf805f855396f3d0248d2d5f AS pixl_orthanc_uv
 SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
 
 # Create a virtual environment, recommended since python 3.11 and Debian bookworm based images
 # where you get a warning when installing system-wide packages.
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install --yes --no-install-recommends python3-venv tzdata
-RUN python3 -m venv /.venv
-ENV PYTHONPATH=/.venv/lib64/python3.11/site-packages/
+    apt-get install --yes --no-install-recommends curl tzdata && \
+    apt-get autoremove --yes && apt-get clean --yes && rm -rf /var/lib/apt/lists/*
 
-# Install curl for now, but try to remove this dependency
-RUN apt-get --assume-yes install curl
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:0.5.10 /uv /uvx /bin/
+ENV PYTHONPATH=/.venv/lib64/python3.11/site-packages/
 
-FROM pixl_orthanc_apt AS pixl_orthanc_with_spec
+FROM pixl_orthanc_uv AS pixl_orthanc_with_spec
 # This part changes rarely, so do it nice and early to avoid redoing it every time we change our code.
 # It does have a dependency though, which would normally be fulfilled by our project files, so install that
 # manually.
 # Do it in dead end build stage to discard this environment afterwards,
 # and because the spec is only needed in orthanc-anon.
-RUN /.venv/bin/pip install dicom-validator
+RUN uv venv
+RUN uv pip install dicom-validator
 COPY ./orthanc/orthanc-anon/plugin/download_dicom_spec.py /etc/orthanc/download_dicom_spec.py
-RUN --mount=type=cache,target=/root/.cache \
+RUN --mount=type=cache,target=/root/.cache,id=dlspec \
     python3 /etc/orthanc/download_dicom_spec.py
 
 
-FROM pixl_orthanc_apt AS pixl_orthanc_base
-
-# Install requirements before copying modules
-COPY ./pixl_core/pyproject.toml /pixl_core/pyproject.toml
-COPY ./pixl_dcmd/pyproject.toml /pixl_dcmd/pyproject.toml
-
-RUN --mount=type=cache,target=/root/.cache \
-    /.venv/bin/pip install pixl_core/ \
-    && /.venv/bin/pip install pixl_dcmd/
-
-COPY ./pixl_core/ /pixl_core
-RUN --mount=type=cache,target=/root/.cache \
-    /.venv/bin/pip install --no-cache-dir --force-reinstall --no-deps ./pixl_core
-
-COPY ./pixl_dcmd/ /pixl_dcmd
-RUN --mount=type=cache,target=/root/.cache \
-    /.venv/bin/pip install --no-cache-dir --force-reinstall --no-deps ./pixl_dcmd
+FROM pixl_orthanc_uv AS pixl_orthanc_base
+# Install our code
+# The workspace (root-level) project file
+COPY pyproject.toml .
+# All pyproject.toml files referenced by the root one must exist or uv will error
+COPY ./pixl_imaging/pyproject.toml pixl_imaging/pyproject.toml
+COPY ./pixl_export/pyproject.toml pixl_export/pyproject.toml
+COPY ./cli/pyproject.toml cli/pyproject.toml
+COPY ./hasher/pyproject.toml hasher/pyproject.toml
+# prereqs for dcmd
+COPY ./pytest-pixl/ pytest-pixl/
+COPY ./pixl_core/ pixl_core/
+COPY ./pixl_dcmd/ pixl_dcmd
+RUN uv venv
+# Need --no-editable so that our packages are in
+# site-packages, so orthanc plugin manager can find them
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked uv sync --no-editable --package pixl-dcmd
 
 ARG ORTHANC_DIR
 COPY ./orthanc/${ORTHANC_DIR}/plugin/pixl.py /etc/orthanc/pixl.py
 COPY ./orthanc/${ORTHANC_DIR}/config /run/secrets
 
+
 # Orthanc can't substitute environment veriables as integers so copy and replace before running
 ARG ORTHANC_CONCURRENT_JOBS
 RUN sed -i "s/\${ORTHANC_CONCURRENT_JOBS}/${ORTHANC_CONCURRENT_JOBS:-5}/g" /run/secrets/orthanc.json

diff --git a/docker/pixl-python/Dockerfile b/docker/pixl-python/Dockerfile
@@ -16,8 +16,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
 
 ARG TEST="false"
 
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update && \
+RUN <<EOF
+    export DEBIAN_FRONTEND=noninteractive &&
+    apt-get update &&
     apt-get install --yes --no-install-recommends \
         procps \
         ca-certificates \
@@ -26,30 +27,44 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         curl \
         gnupg \
         locales \
-        tzdata
-
-RUN sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
-RUN  apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
+        tzdata &&
+    sed -i '/en_GB.UTF-8/s/^# //g' /etc/locale.gen && locale-gen &&
+    apt-get autoremove --yes && apt-get clean --yes && rm -rf /var/lib/apt/lists/*
+EOF
 
 HEALTHCHECK CMD /usr/bin/curl -f http://0.0.0.0:8000/heart-beat || exit 1
 
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:0.5.10 /uv /uvx /bin/
+
 WORKDIR /app
 # specify what we're installing using build time arg
 ARG PIXL_PACKAGE_DIR
 
-# Install requirements before copying modules
-COPY ./pixl_core/pyproject.toml ./pixl_core/pyproject.toml
-COPY ./$PIXL_PACKAGE_DIR/pyproject.toml ./$PIXL_PACKAGE_DIR/pyproject.toml
-RUN pip3 install --no-cache-dir pixl_core/ \
-    && pip3 install --no-cache-dir $PIXL_PACKAGE_DIR/
+# Install our code:
+
+# COPY doesn't support a wildcard copy which also preserves directory structure,
+# so we have to specify everything separately. We could have copied everything, but
+# then every change would trigger a rebuild of all containers.
 
-# Install our code
+# The workspace project file
+COPY pyproject.toml .
+# All pyproject.toml files referenced by the root one must exist or uv will error
+COPY ./pixl_imaging/pyproject.toml pixl_imaging/pyproject.toml
+COPY ./pixl_export/pyproject.toml pixl_export/pyproject.toml
+COPY ./pixl_dcmd/pyproject.toml pixl_dcmd/pyproject.toml
+COPY ./cli/pyproject.toml cli/pyproject.toml
+COPY ./hasher/pyproject.toml hasher/pyproject.toml
+# prereqs for all projects
+COPY ./pytest-pixl/ pytest-pixl/
 COPY ./pixl_core/ pixl_core/
-COPY ./$PIXL_PACKAGE_DIR/ .
-RUN pip install --no-cache-dir --force-reinstall --no-deps pixl_core/ \
-    --no-cache-dir --force-reinstall --no-deps . && \
-    if [ "$TEST" = "true" ]; then pip install --no-cache-dir pixl_core/[test] .[test]; fi
+# the actual thing we are building
+COPY ./$PIXL_PACKAGE_DIR/ $PIXL_PACKAGE_DIR/
+RUN uv venv
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked uv sync --package "$PIXL_PACKAGE_DIR"
 
+# make uvicorn etc available
+ENV PATH="/app/.venv/bin:$PATH"
 
 # Each container should be run with a different entry point
 FROM pixl_python_base AS export_api
@@ -59,4 +74,4 @@ FROM pixl_python_base AS hasher_api
 ENTRYPOINT ["uvicorn", "hasher.main:app", "--host", "0.0.0.0", "--port", "8000"]
 
 FROM pixl_python_base AS imaging_api
-ENTRYPOINT ["/app/scripts/migrate_and_run.sh"]
+ENTRYPOINT ["/app/pixl_imaging/scripts/migrate_and_run.sh"]