DM-44907: Add boto3 and update for USDF endpoints

.github/workflows/test.yml

@@ -12,11 +12,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.11
 
       - name: Install package
         run: |
-          pip install pip-tools>=6.2.0
+          pip install pip-tools>=7.4.0
           pip-sync requirements.txt dev-requirements.txt
           pip install .
 
@@ -33,51 +33,3 @@ jobs:
           # Run pip-compile - but only if requirements.in or
           # dev-requirements.in have changed in this pull request's diff.
           pre-commit run pip-compile --from-ref=$GITHUB_BASE_REF --to-ref=$GITHUB_HEAD_REF
-
-  unit-test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install package
-        run: |
-          pip install pip-tools>=6.2.0
-          pip-sync requirements.txt dev-requirements.txt
-          pip install .
-
-      - name: Run unit tests
-        run: pytest
-
-  integration-test:
-    runs-on: ubuntu-latest
-    concurrency: integration-test
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Set up cloud SDK
-        uses: google-github-actions/setup-gcloud@master
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_SA_KEY }}
-          export_default_credentials: true
-
-      - name: Install package
-        run: |
-          pip install pip-tools>=6.2.0
-          pip-sync requirements.txt dev-requirements.txt
-          pip install .
-
-      - name: Run integration tests
-        run: pytest
-        env:
-          ALERTDB_TEST_GCP_PROJECT: ${{ secrets.GCP_PROJECT_ID }}

.gitignore

@@ -0,0 +1,64 @@
+
+.idea/alert_database_server.iml
+
+.idea/inspectionProfiles/profiles_settings.xml
+
+.idea/inspectionProfiles/Project_Default.xml
+
+.idea/misc.xml
+
+.idea/modules.xml
+
+.idea/vcs.xml
+
+.idea/workspace.xml
+
+alertdb/__pycache__/__init__.cpython-311.pyc
+
+alertdb/__pycache__/__init__.cpython-39.pyc
+
+alertdb/__pycache__/server.cpython-311.pyc
+
+alertdb/__pycache__/server.cpython-39.pyc
+
+alertdb/__pycache__/storage.cpython-311.pyc
+
+alertdb/__pycache__/storage.cpython-39.pyc
+
+alertdb/bin/__pycache__/__init__.cpython-311.pyc
+
+alertdb/bin/__pycache__/__init__.cpython-39.pyc
+
+alertdb/bin/__pycache__/alertdb.cpython-311.pyc
+
+alertdb/bin/__pycache__/alertdb.cpython-39.pyc
+
+build/lib/alertdb/__init__.py
+
+build/lib/alertdb/bin/__init__.py
+
+build/lib/alertdb/bin/alertdb.py
+
+build/lib/alertdb/server.py
+
+build/lib/alertdb/storage.py
+
+lsst_alert_database_server.egg-info/dependency_links.txt
+
+lsst_alert_database_server.egg-info/entry_points.txt
+
+lsst_alert_database_server.egg-info/PKG-INFO
+
+lsst_alert_database_server.egg-info/requires.txt
+
+lsst_alert_database_server.egg-info/SOURCES.txt
+
+lsst_alert_database_server.egg-info/top_level.txt
+
+tests/__pycache__/test_integration.cpython-311-pytest-8.3.2.pyc
+
+tests/__pycache__/test_integration.cpython-39-pytest-7.1.1.pyc
+
+tests/__pycache__/test_server.cpython-311-pytest-8.3.2.pyc
+
+tests/__pycache__/test_server.cpython-39-pytest-7.1.1.pyc

.pre-commit-config.yaml

@@ -2,38 +2,38 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.0.1
+  rev: v5.0.0
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
   - id: check-yaml
 
 - repo: https://github.com/PyCQA/flake8
-  rev: 3.9.2
+  rev: 7.1.1
   hooks:
   - id: flake8
 
 - repo: https://github.com/PyCQA/isort
-  rev: 5.8.0
+  rev: 5.13.2
   hooks:
   - id: isort
     additional_dependencies:
       - toml
 
 - repo: https://github.com/psf/black
-  rev: 21.7b0
+  rev: 24.10.0
   hooks:
   - id: black
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.910
+  rev: v1.13.0
   hooks:
   - id: mypy
     additional_dependencies:
       - types-requests
 
 - repo: https://github.com/jazzband/pip-tools
-  rev: 6.2.0
+  rev: 7.4.1
   hooks:
   - id: pip-compile
     name: pip-compile dev-requirements.in

alertdb/bin/alertdb.py

@@ -4,7 +4,7 @@
 import uvicorn
 
 from alertdb.server import create_server
-from alertdb.storage import FileBackend, GoogleObjectStorageBackend
+from alertdb.storage import FileBackend, USDFObjectStorageBackend
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +30,7 @@ def main():
     parser.add_argument(
         "--backend",
         type=str,
-        choices=("local-files", "google-cloud"),
+        choices=("local-files", "s3-remote"),
         default="local-files",
         help="backend to use to source alerts",
     )
@@ -41,22 +41,22 @@ def main():
         help="when using the local-files backend, the root directory where alerts should be found",
     )
     parser.add_argument(
-        "--gcp-project",
+        "--endpoint-url",
         type=str,
         default=None,
-        help="when using the google-cloud backend, the name of the GCP project",
+        help="when using the s3-remote backend, the endpoint url of the project",
     )
     parser.add_argument(
-        "--gcp-bucket-alerts",
+        "--bucket-alerts",
         type=str,
         default="alert-packets",
-        help="when using the google-cloud backend, the name of the GCS bucket for alert packets",
+        help="when using the google-cloud backend, the name of the boto3 bucket for alert packets",
     )
     parser.add_argument(
-        "--gcp-bucket-schemas",
+        "--bucket-schemas",
         type=str,
         default="alert-schemas",
-        help="when using the google-cloud backend, the name of the GCS bucket for alert schemas",
+        help="when using the google-cloud backend, the name of the boto3 bucket for alert schemas",
     )
     parser.add_argument("--verbose", action="store_true", help="log a bunch")
     parser.add_argument("--debug", action="store_true", help="log even more")
@@ -85,22 +85,20 @@ def main():
 
         backend = FileBackend(args.local_file_root)
 
-    elif args.backend == "google-cloud":
-        logger.info("using google-cloud backend")
+    elif args.backend == "s3-remote":
+        logger.info("using s3-remote backend")
 
-        if args.gcp_project is None:
-            parser.error("--backend=google-cloud requires --gcp-project be set")
-        logger.info("gcp_project: %s", args.gcp_project)
+        logger.info("endpoint-url: %s", args.endpoint_url)
 
-        if args.gcp_bucket_alerts is None or args.gcp_bucket_schemas is None:
+        if args.bucket_alerts is None or args.bucket_schemas is None:
             parser.error(
-                "--backend=google-cloud requires --gcp-bucket-alerts and --gcp-bucket-schemas be set"
+                "--backend=s3-remote requires --bucket-alerts and --bucket-schemas be set"
             )
-        logger.info("gcp_bucket_alerts: %s", args.gcp_bucket_alerts)
-        logger.info("gcp_bucket_schemas: %s", args.gcp_bucket_schemas)
+        logger.info("bucket_alerts: %s", args.bucket_alerts)
+        logger.info("bucket_schemas: %s", args.bucket_schemas)
 
-        backend = GoogleObjectStorageBackend(
-            args.gcp_project, args.gcp_bucket_alerts, args.gcp_bucket_schemas
+        backend = USDFObjectStorageBackend(
+            args.endpoint_url, args.bucket_alerts, args.bucket_schemas
         )
 
     else:

alertdb/storage.py

@@ -6,8 +6,7 @@
 import logging
 import os.path
 
-import google.api_core.exceptions
-import google.cloud.storage as gcs
+import boto3
 
 logger = logging.getLogger(__name__)
 
@@ -131,38 +130,38 @@ def get_schema(self, schema_id: str) -> bytes:
             raise NotFoundError("schema not found") from file_not_found
 
 
-class GoogleObjectStorageBackend(AlertDatabaseBackend):
-    """
-    Retrieves alerts and schemas from a Google Cloud Storage bucket.
-
-    The path for alert and schema objects follows the scheme in DMTN-183.
-    """
-
+class USDFObjectStorageBackend(AlertDatabaseBackend):
     def __init__(
-        self, gcp_project: str, packet_bucket_name: str, schema_bucket_name: str
+        self, endpoint_url: str, packet_bucket_name: str, schema_bucket_name: str
     ):
-        self.object_store_client = gcs.Client(project=gcp_project)
-        self.packet_bucket = self.object_store_client.bucket(packet_bucket_name)
-        self.schema_bucket = self.object_store_client.bucket(schema_bucket_name)
+        self.object_store_client = boto3.client(
+            "s3", endpoint_url=endpoint_url
+        )  # Default way of getting a boto3 client that an talk to s3
+        self.packet_bucket = packet_bucket_name
+        self.schema_bucket = schema_bucket_name
 
     def get_alert(self, alert_id: str) -> bytes:
         logger.info("retrieving alert id=%s", alert_id)
         try:
-            blob = self.packet_bucket.blob(
-                f"/alert_archive/v1/alerts/{alert_id}.avro.gz"
+            alert_key = f"/alert_archive/v1/alerts/{alert_id}.avro.gz"
+            # boto3 terminology for objects, objects live in prefixes inside
+            # of buckets
+            blob = self.object_store_client.get_object(
+                Bucket=self.packet_bucket, Key=alert_key
             )
-            return blob.download_as_bytes()
-        except google.api_core.exceptions.NotFound as not_found:
+            return blob["Body"].read()
+        except self.object_store_client.exceptions.NoSuchKey as not_found:
             raise NotFoundError("alert not found") from not_found
 
     def get_schema(self, schema_id: str) -> bytes:
         logger.info("retrieving schema id=%s", schema_id)
         try:
-            blob = self.schema_bucket.blob(
-                f"/alert_archive/v1/schemas/{schema_id}.json"
+            schema_key = f"/alert_archive/v1/schemas/{schema_id}.json"
+            blob = self.object_store_client.get_object(
+                Bucket=self.schema_bucket, Key=schema_key
             )
-            return blob.download_as_bytes()
-        except google.api_core.exceptions.NotFound as not_found:
+            return blob["Body"].read()
+        except self.object_store_client.exceptions.NoSuchKey as not_found:
             raise NotFoundError("alert not found") from not_found