mock Google Cloud Storage: tickets/DM-42231D

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yaml

@@ -0,0 +1,117 @@
+name: CI
+
+"on":
+  push:
+    branches-ignore:
+      # These should always correspond to pull requests, so ignore them for
+      # the push trigger and let them be triggered by the pull_request
+      # trigger, avoiding running the workflow twice.  This is a minor
+      # optimization so there's no need to ensure this is comprehensive.
+      - "dependabot/**"
+      - "renovate/**"
+      - "tickets/**"
+      - "u/**"
+    tags:
+      - "*"
+  pull_request: {}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]
+
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run tox
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: ${{ matrix.python }}
+          tox-envs: "py,coverage-report,typing"
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [lint, test]
+    timeout-minutes: 10
+
+    # Only do Docker builds of tagged releases and pull requests from ticket
+    # branches.  This will still trigger on pull requests from untrusted
+    # repositories whose branch names match our tickets/* branch convention,
+    # but in this case the build will fail with an error since the secret
+    # won't be set.
+    if: >
+      startsWith(github.ref, 'refs/tags/')
+      || startsWith(github.head_ref, 'tickets/')
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: lsst-sqre/build-and-push-to-ghcr@v1
+        id: build
+        with:
+          image: ${{ github.repository }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  test-packaging:
+
+    name: Test packaging
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # full history for setuptools_scm
+
+      - name: Build and publish
+        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        with:
+          python-version: "3.11"
+          upload: false
+
+  pypi:
+
+    # This job requires set up:
+    # 1. Set up a trusted publisher for PyPI
+    # 2. Set up a "pypi" environment in the repository
+    # See https://github.com/lsst-sqre/build-and-publish-to-pypi
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    needs: [lint, test, test-packaging]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/giftless
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # full history for setuptools_scm
+
+      - name: Build and publish
+        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        with:
+          python-version: "3.11"

.github/workflows/periodic.yaml

@@ -0,0 +1,45 @@
+# This is a separate run of the Python test suite that doesn't cache the tox
+# environment and runs from a schedule. The purpose is to test whether
+# updating pinned dependencies would cause any tests to fail.
+
+name: Periodic CI
+
+"on":
+  schedule:
+    - cron: "0 12 * * 1"
+  workflow_dispatch: {}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run neophile
+        uses: lsst-sqre/run-neophile@v1
+        with:
+          python-version: ${{ matrix.python }}
+          mode: update
+
+      - name: Run tox
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: ${{ matrix.python }}
+          tox-envs: "lint,typing,py"
+
+      - name: Report status
+        if: always()
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "Periodic test for {repo} failed"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}

Makefile

@@ -28,25 +28,25 @@ VERSION := $(shell $(PYTHON) -c "from importlib.metadata import version;print(ve
 
 default: help
 
-## Install packages necessary for make to work
-init:
-	pip install --upgrade pip pre-commit pip-tools tox
-
 ## Regenerate requirements files
 requirements: requirements/dev.txt requirements/dev.in requirements/main.txt requirements/main.in
 
 ## Set up the development environment
 dev-setup: $(SENTINELS)/dev-setup
 
+## Run all linting checks
+lint: $(SENTINELS)
+	pre-commit run --all-files
+
 ## Run all tests
-test: $(SENTINELS)/dev-setup
+test: $(SENTINELS)
 	$(PYTEST) $(PYTEST_EXTRA_ARGS) $(PACKAGE_DIRS) $(TESTS_DIR)
 
 ## Build a local Docker image
 docker: requirements/main.txt
 	$(DOCKER) build --cache-from "$(DOCKER_CACHE_FROM)" -t $(DOCKER_HOST)/$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) .
 
-## Tag and push a release (disabled)
+## Tag and push a release (disabled; done via GitHub Actions now)
 release:
 	@echo "Package '$(PACKAGE_NAME)' releases are managed via GitHub"
 
@@ -77,13 +77,14 @@ $(SENTINELS):
 	mkdir $@
 
 $(SENTINELS)/dist-setup: | $(SENTINELS)
-	$(PIP) install -U pip wheel twine pre-commit
+	$(PIP) install -U wheel twine
 	@touch $@
 
 $(SENTINELS)/dist: $(SENTINELS)/dist-setup $(DIST_DIR)/$(PACKAGE_NAME)-$(VERSION).tar.gz $(DIST_DIR)/$(PACKAGE_NAME)-$(VERSION)-py3-none-any.whl | $(SENTINELS)
 	@touch $@
 
-$(SENTINELS)/dev-setup: requirements/main.txt requirements/dev.txt | $(SENTINELS)
+$(SENTINELS)/dev-setup: init requirements/main.txt requirements/dev.txt | $(SENTINELS)
+	$(PIP) install -U pip pip-tools pre-commit tox
 	$(PIP) install -r requirements/main.txt
 	$(PIP) install -e .
 	$(PIP) install -r requirements/dev.txt

giftless/storage/google_cloud.py

@@ -9,8 +9,8 @@
 from typing import Any, BinaryIO, cast
 
 import google.auth
+import google.cloud
 from google.auth import impersonated_credentials
-from google.cloud import storage
 from google.oauth2 import service_account
 
 from giftless.storage import ExternalStorage, StreamingStorage
@@ -40,7 +40,7 @@ def __init__(
             | impersonated_credentials.Credentials
             | None
         ) = self._load_credentials(account_key_file, account_key_base64)
-        self.storage_client = storage.Client(
+        self.storage_client = google.cloud.storage.Client(
             project=project_name, credentials=self.credentials
         )
         if not self.credentials:

requirements/dev.in

@@ -10,6 +10,8 @@ pytest-env
 pytest-cov
 pytest-vcr
 
+cloud-storage-mocker
+
 pytz
 types-pytz
 types-jwt

requirements/dev.txt

@@ -12,9 +12,9 @@ babel==2.14.0
     # via sphinx
 beautifulsoup4==4.12.2
     # via furo
-boto3-stubs==1.34.16
+boto3-stubs==1.34.19
     # via -r requirements/dev.in
-botocore-stubs==1.34.16
+botocore-stubs==1.34.19
     # via
     #   -r requirements/dev.in
     #   boto3-stubs
@@ -39,6 +39,8 @@ click==8.1.7
     # via
     #   -c requirements/main.txt
     #   pip-tools
+cloud-storage-mocker==0.3.1
+    # via -r requirements/dev.in
 colorama==0.4.6
     # via tox
 commonmark==0.9.1
@@ -62,12 +64,41 @@ flake8==7.0.0
     # via -r requirements/dev.in
 furo==2023.9.10
     # via -r requirements/dev.in
-google-auth==2.26.1
+google-api-core==2.15.0
+    # via
+    #   -c requirements/main.txt
+    #   google-cloud-core
+    #   google-cloud-storage
+google-auth==2.26.2
     # via
     #   -c requirements/main.txt
+    #   google-api-core
     #   google-auth-stubs
+    #   google-cloud-core
+    #   google-cloud-storage
 google-auth-stubs==0.2.0
     # via -r requirements/dev.in
+google-cloud-core==2.4.1
+    # via
+    #   -c requirements/main.txt
+    #   google-cloud-storage
+google-cloud-storage==2.14.0
+    # via
+    #   -c requirements/main.txt
+    #   cloud-storage-mocker
+google-crc32c==1.5.0
+    # via
+    #   -c requirements/main.txt
+    #   google-cloud-storage
+    #   google-resumable-media
+google-resumable-media==2.7.0
+    # via
+    #   -c requirements/main.txt
+    #   google-cloud-storage
+googleapis-common-protos==1.62.0
+    # via
+    #   -c requirements/main.txt
+    #   google-api-core
 grpc-stubs==1.53.0.5
     # via google-auth-stubs
 grpcio==1.60.0
@@ -117,6 +148,11 @@ pluggy==1.3.0
     # via
     #   pytest
     #   tox
+protobuf==4.25.2
+    # via
+    #   -c requirements/main.txt
+    #   google-api-core
+    #   googleapis-common-protos
 pyasn1==0.5.1
     # via
     #   -c requirements/main.txt
@@ -167,6 +203,8 @@ recommonmark==0.7.1
 requests==2.31.0
     # via
     #   -c requirements/main.txt
+    #   google-api-core
+    #   google-cloud-storage
     #   sphinx
 rsa==4.9
     # via
@@ -183,28 +221,23 @@ sphinx==7.2.6
     #   recommonmark
     #   sphinx-autodoc-typehints
     #   sphinx-basic-ng
-    #   sphinxcontrib-applehelp
-    #   sphinxcontrib-devhelp
-    #   sphinxcontrib-htmlhelp
-    #   sphinxcontrib-qthelp
-    #   sphinxcontrib-serializinghtml
 sphinx-autodoc-typehints==1.25.2
     # via -r requirements/dev.in
 sphinx-basic-ng==1.0.0b2
     # via furo
-sphinxcontrib-applehelp==1.0.7
+sphinxcontrib-applehelp==1.0.8
     # via sphinx
-sphinxcontrib-devhelp==1.0.5
+sphinxcontrib-devhelp==1.0.6
     # via sphinx
-sphinxcontrib-htmlhelp==2.0.4
+sphinxcontrib-htmlhelp==2.0.5
     # via sphinx
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
-sphinxcontrib-qthelp==1.0.6
+sphinxcontrib-qthelp==1.0.7
     # via sphinx
-sphinxcontrib-serializinghtml==1.1.9
+sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
-tox==4.11.4
+tox==4.12.0
     # via -r requirements/dev.in
 types-awscrt==0.20.0
     # via botocore-stubs

requirements/main.txt

@@ -12,9 +12,9 @@ azure-storage-blob==12.19.0
     # via -r requirements/main.in
 blinker==1.7.0
     # via flask
-boto3==1.34.16
+boto3==1.34.19
     # via -r requirements/main.in
-botocore==1.34.16
+botocore==1.34.19
     # via
     #   boto3
     #   s3transfer
@@ -47,7 +47,7 @@ google-api-core==2.15.0
     # via
     #   google-cloud-core
     #   google-cloud-storage
-google-auth==2.26.1
+google-auth==2.26.2
     # via
     #   google-api-core
     #   google-cloud-core