Blobwrites

.devcontainer/docker-compose.extend.yml

@@ -14,6 +14,10 @@ services:
             # - datatracker-vscode-ext:/root/.vscode-server/extensions
         # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
         network_mode: service:db
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
 
 volumes:
     datatracker-vscode-ext:

README.md

@@ -106,6 +106,23 @@ Nightly database dumps of the datatracker are available as Docker images: `ghcr.
 
 > Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
 
+### Blob storage for dev/test
+
+The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port.
+
+```
+$ docker compose port blobstore 9001
+0.0.0.0:<some ephemeral port>
+
+$ curl -I http://localhost:<some ephemeral port>
+HTTP/1.1 200 OK
+...
+```
+
+
+The minio container exposes the minio api at port 9000 and the minio console at port 9001
+
+
 ### Frontend Development
 
 #### Intro

dev/deploy-to-container/settings_local.py

@@ -79,3 +79,17 @@
 
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }

dev/diff/settings_local.py

@@ -66,3 +66,17 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }

dev/tests/settings_local.py

@@ -65,3 +65,17 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }

docker-compose.yml

@@ -15,6 +15,7 @@ services:
         depends_on:
             - db
             - mq
+            - blobstore
 
         ipc: host
 
@@ -83,6 +84,19 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    blobstore:
+        image: quay.io/minio/minio:latest
+        restart: unless-stopped
+        volumes:
+            - "minio-data:/data"
+        environment:
+            - MINIO_ROOT_USER=minio_root
+            - MINIO_ROOT_PASSWORD=minio_pass
+            - MINIO_DEFAULT_BUCKETS=defaultbucket
+        command: server --console-address ":9001" /data
+
+
+
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
 #
@@ -105,3 +119,4 @@ services:
 volumes:
     postgresdb-data:
     app-assets:
+    minio-data:

docker/app.Dockerfile

@@ -43,8 +43,8 @@ RUN rm -rf /tmp/library-scripts
 # Copy the startup file
 COPY docker/scripts/app-init.sh /docker-init.sh
 COPY docker/scripts/app-start.sh /docker-start.sh
-RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh
-RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh
+RUN sed -i 's/\r$//' /docker-init.sh && chmod +rx /docker-init.sh
+RUN sed -i 's/\r$//' /docker-start.sh && chmod +rx /docker-start.sh
 
 # Fix user UID / GID to match host
 RUN groupmod --gid $USER_GID $USERNAME \

docker/configs/settings_local.py

@@ -1,11 +1,12 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+import boto3
 
 ALLOWED_HOSTS = ['*']
 
-from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
+from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -37,6 +38,20 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+            bucket_name=storagename,
+        ),
+    }
+
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'

docker/docker-compose.extend.yml

@@ -16,6 +16,10 @@ services:
     pgadmin:
         ports:
             - '5433'
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
     celery:
         volumes:
             - .:/workspace

docker/scripts/app-configure-blobstore.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+import boto3
+import sys
+
+from ietf.settings import MORE_STORAGE_NAMES
+
+
+def init_blobstore():
+    blobstore = boto3.resource(
+        "s3",
+        endpoint_url="http://blobstore:9000",
+        aws_access_key_id="minio_root",
+        aws_secret_access_key="minio_pass",
+        aws_session_token=None,
+        config=boto3.session.Config(signature_version="s3v4"),
+        verify=False,
+    )
+    for bucketname in MORE_STORAGE_NAMES:
+        blobstore.create_bucket(Bucket=bucketname)
+
+
+if __name__ == "__main__":
+    sys.exit(init_blobstore())

docker/scripts/app-init.sh

@@ -73,6 +73,11 @@ echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
 
+# Configure the development blobstore
+
+echo "Configuring blobstore..."
+PYTHONPATH=/workspace python ./docker/scripts/app-configure-blobstore.py
+
 # Download latest coverage results file
 
 echo "Downloading latest coverage results file..."

ietf/doc/storage_utils.py

@@ -0,0 +1,71 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+import debug  # pyflakes ignore
+
+from django.core.files.base import ContentFile
+from django.core.files.storage import storages, Storage
+
+from ietf.utils.log import log
+
+
+def _get_storage(kind: str) -> Storage:
+    if kind in [
+        "bofreq",
+        "charter",
+        "conflrev",
+        "draft",
+        "draft",
+        "draft",
+    ]:
+        return storages[kind]
+    else:
+        debug.say(f"Got into not-implemented looking for {kind}")
+        raise NotImplementedError(f"Don't know how to store {kind}")
+
+
+def store_bytes(
+    kind: str, name: str, content: bytes, allow_overwrite: bool = False
+) -> None:
+    store = _get_storage(kind)
+    if not allow_overwrite:
+        try:
+            new_name = store.save(name, ContentFile(content))
+        except Exception as e:
+            # Log and then swallow the exception while we're learning.
+            # Don't let failure pass so quietly when these are the autoritative bits.
+            log(f"Failed to save {kind}:{name}", e)
+            debug.show("e")
+            return None
+        if new_name != name:
+            log(
+                f"Conflict encountered saving {name} - results stored in {new_name} instead."
+            )
+    else:
+        try:
+            with store.open(name) as f:
+                f.write(content)
+        except Exception as e:
+            # Log and then swallow the exception while we're learning.
+            # Don't let failure pass so quietly when these are the autoritative bits.
+            log(f"Failed to save {kind}:{name}", e)
+            return None
+        raise NotImplementedError()
+
+
+def retrieve_bytes(kind: str, name: str) -> bytes:
+    store = _get_storage(kind)
+    with store.open(name) as f:
+        content = f.read()
+    return content
+
+
+def store_str(
+    kind: str, name: str, content: str, allow_overwrite: bool = False
+) -> None:
+    content_bytes = content.encode("utf-8")
+    store_bytes(kind, name, content_bytes, allow_overwrite)
+
+
+def retrieve_str(kind: str, name: str) -> str:
+    content_bytes = retrieve_bytes(kind, name)
+    return content_bytes.decode("utf-8")

ietf/doc/tests_bofreq.py

@@ -16,6 +16,7 @@
 from django.template.loader import render_to_string
 from django.utils import timezone
 
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.factories import RoleFactory
 from ietf.doc.factories import BofreqFactory, NewRevisionDocEventFactory
 from ietf.doc.models import State, Document, NewRevisionDocEvent
@@ -340,6 +341,7 @@ def test_submit(self):
                         doc = reload_db_objects(doc)
                         self.assertEqual('%02d'%(int(rev)+1) ,doc.rev)
                         self.assertEqual(f'# {username}', doc.text())
+                        self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name()))
                         self.assertEqual(docevent_count+1, doc.docevent_set.count())
                         self.assertEqual(1, len(outbox))
                         rev = doc.rev
@@ -379,6 +381,7 @@ def test_start_new_bofreq(self):
                     self.assertEqual(list(bofreq_editors(bofreq)), [nobody])
                     self.assertEqual(bofreq.latest_event(NewRevisionDocEvent).rev, '00')
                     self.assertEqual(bofreq.text_or_error(), 'some stuff')
+                    self.assertEqual(retrieve_str('bofreq',bofreq.get_base_name()), 'some stuff')
                     self.assertEqual(len(outbox),1)
         finally:
             os.unlink(file.name)

ietf/doc/tests_charter.py

@@ -16,6 +16,7 @@
 from ietf.doc.factories import CharterFactory, NewRevisionDocEventFactory, TelechatDocEventFactory
 from ietf.doc.models import ( Document, State, BallotDocEvent, BallotType, NewRevisionDocEvent,
     TelechatDocEvent, WriteupDocEvent )
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import ( next_revision, default_review_text, default_action_text,
     charter_name_for_group )
 from ietf.doc.utils import close_open_ballots
@@ -519,6 +520,11 @@ def test_submit_charter(self):
         ftp_charter_path = Path(settings.FTP_DIR) / "charter" / charter_path.name
         self.assertTrue(ftp_charter_path.exists())
         self.assertTrue(charter_path.samefile(ftp_charter_path))
+        blobstore_contents = retrieve_str("charter", charter.get_base_name())
+        self.assertEqual(
+            blobstore_contents,
+            "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode("utf-8"),
+        )        
 
 
     def test_submit_initial_charter(self):

ietf/doc/tests_conflict_review.py

@@ -16,6 +16,7 @@
 
 from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory, RgDraftFactory
 from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State, DocTagName
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_conflict_review import default_approval_text
 from ietf.group.models import Person
@@ -422,6 +423,7 @@ def test_initial_submission(self):
             f.close()
         self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
+        self.assertEqual(retrieve_str("conflrev",basename), "Some initial review text\n")
 
     def test_subsequent_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')

ietf/doc/views_bofreq.py

@@ -17,6 +17,7 @@
     email_bofreq_new_revision, email_bofreq_responsible_changed)
 from ietf.doc.models import (Document, DocEvent, NewRevisionDocEvent, 
     BofreqEditorDocEvent, BofreqResponsibleDocEvent, State)
+from ietf.doc.storage_utils import store_str
 from ietf.doc.utils import add_state_change_event
 from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
 from ietf.ietfauth.utils import has_role, role_required
@@ -101,6 +102,7 @@ def submit(request, name):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            store_str("bofreq", bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)
 
@@ -175,6 +177,7 @@ def new_bof_request(request):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            store_str("bofreq", bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)