feat: beginning of blob writes

docker/configs/settings_local.py

@@ -1,11 +1,12 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+import boto3
 
 ALLOWED_HOSTS = ['*']
 
-from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
+from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -37,6 +38,19 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
+STORAGES["ietfdata"] = {
+    "BACKEND": "storages.backends.s3.S3Storage",
+    "OPTIONS": dict(
+          endpoint_url="http://blobstore:9000",
+          access_key="minio_root",
+          secret_key="minio_pass",
+          security_token=None,
+          client_config=boto3.session.Config(signature_version="s3v4"),
+          verify=False,
+          bucket_name="ietfdata",
+    ),
+}
+
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'

docker/scripts/app-configure-blobstore.py

@@ -13,7 +13,38 @@ def init_blobstore():
         config=boto3.session.Config(signature_version="s3v4"),
         verify=False
     )
-    blobstore.create_bucket(Bucket="ietfdata")
+    for bucketname in [
+        "agenda",
+        "bluesheets",
+        "bofreq",
+        "charter",
+        "chatlog",
+        "conflrev",
+        "draft-xml",
+        "draft-txt",
+        "draft-html",
+        "draft-htmlized",
+        "draft-pdf",
+        "draft-pdfized",
+        "liai-att",
+        "liaison",
+        "minutes",
+        "narrativeminutes",
+        "polls",
+        "procmaterials",
+        "recording",
+        "review",
+        "rfc-txt",
+        "rfc-html",
+        "rfc-pdf",
+        "rfc-htmlized",
+        "rfc-pdfized",
+        "slides",
+        "statchg",
+        "statement",
+    ]:
+        blobstore.create_bucket(Bucket=bucketname)
+
 
 if __name__ == "__main__":
     sys.exit(init_blobstore())

ietf/doc/tests_bofreq.py

@@ -16,6 +16,7 @@
 from django.template.loader import render_to_string
 from django.utils import timezone
 
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.factories import RoleFactory
 from ietf.doc.factories import BofreqFactory, NewRevisionDocEventFactory
 from ietf.doc.models import State, Document, NewRevisionDocEvent
@@ -340,6 +341,7 @@ def test_submit(self):
                         doc = reload_db_objects(doc)
                         self.assertEqual('%02d'%(int(rev)+1) ,doc.rev)
                         self.assertEqual(f'# {username}', doc.text())
+                        self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name()))
                         self.assertEqual(docevent_count+1, doc.docevent_set.count())
                         self.assertEqual(1, len(outbox))
                         rev = doc.rev
@@ -379,6 +381,7 @@ def test_start_new_bofreq(self):
                     self.assertEqual(list(bofreq_editors(bofreq)), [nobody])
                     self.assertEqual(bofreq.latest_event(NewRevisionDocEvent).rev, '00')
                     self.assertEqual(bofreq.text_or_error(), 'some stuff')
+                    self.assertEqual(retrieve_str('bofreq',bofreq.get_base_name()), 'some stuff')
                     self.assertEqual(len(outbox),1)
         finally:
             os.unlink(file.name)

ietf/doc/tests_charter.py

@@ -16,6 +16,7 @@
 from ietf.doc.factories import CharterFactory, NewRevisionDocEventFactory, TelechatDocEventFactory
 from ietf.doc.models import ( Document, State, BallotDocEvent, BallotType, NewRevisionDocEvent,
     TelechatDocEvent, WriteupDocEvent )
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import ( next_revision, default_review_text, default_action_text,
     charter_name_for_group )
 from ietf.doc.utils import close_open_ballots
@@ -519,6 +520,11 @@ def test_submit_charter(self):
         ftp_charter_path = Path(settings.FTP_DIR) / "charter" / charter_path.name
         self.assertTrue(ftp_charter_path.exists())
         self.assertTrue(charter_path.samefile(ftp_charter_path))
+        blobstore_contents = retrieve_str("charter", charter.get_base_name())
+        self.assertEqual(
+            blobstore_contents,
+            "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode("utf-8"),
+        )        
 
 
     def test_submit_initial_charter(self):

ietf/doc/tests_conflict_review.py

@@ -16,6 +16,7 @@
 
 from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory, RgDraftFactory
 from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State, DocTagName
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_conflict_review import default_approval_text
 from ietf.group.models import Person
@@ -422,6 +423,7 @@ def test_initial_submission(self):
             f.close()
         self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
+        self.assertEqual(retrieve_str("conflrev",basename), "Some initial review text\n")
 
     def test_subsequent_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')

ietf/doc/views_bofreq.py

@@ -17,6 +17,7 @@
     email_bofreq_new_revision, email_bofreq_responsible_changed)
 from ietf.doc.models import (Document, DocEvent, NewRevisionDocEvent, 
     BofreqEditorDocEvent, BofreqResponsibleDocEvent, State)
+from ietf.doc.storage_utils import store_str
 from ietf.doc.utils import add_state_change_event
 from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
 from ietf.ietfauth.utils import has_role, role_required
@@ -101,6 +102,7 @@ def submit(request, name):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            store_str("bofreq", bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)
 
@@ -175,6 +177,7 @@ def new_bof_request(request):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            store_str("bofreq", bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)
 

ietf/doc/views_charter.py

@@ -26,6 +26,7 @@
 from ietf.doc.models import ( Document, DocHistory, State, DocEvent,
     BallotDocEvent, BallotPositionDocEvent, InitialReviewDocEvent, NewRevisionDocEvent,
     WriteupDocEvent, TelechatDocEvent )
+from ietf.doc.storage_utils import store_str
 from ietf.doc.utils import ( add_state_change_event, close_open_ballots,
     create_ballot, get_chartering_type )
 from ietf.doc.utils_charter import ( historic_milestones_for_charter,
@@ -441,9 +442,10 @@ def submit(request, name, option=None):
             )  # update rev
             with charter_filename.open("w", encoding="utf-8") as destination:
                 if form.cleaned_data["txt"]:
-                    destination.write(form.cleaned_data["txt"])
+                    content=form.cleaned_data["txt"]
                 else:
-                    destination.write(form.cleaned_data["content"])
+                    content=form.cleaned_data["content"]
+                destination.write(content)
             # Also provide a copy to the legacy ftp source directory, which is served by rsync
             # This replaces the hardlink copy that ghostlink has made in the past
             # Still using a hardlink as long as these are on the same filesystem.
@@ -454,7 +456,8 @@ def submit(request, name, option=None):
                 log(
                     "There was an error creating a hardlink at %s pointing to %s"
                     % (ftp_filename, charter_filename)
-                )     
+                )
+            store_str("charter", charter_filename.name, content)     
 
 
             if option in ["initcharter", "recharter"] and charter.ad == None:

ietf/doc/views_conflict_review.py

@@ -19,6 +19,7 @@
 
 from ietf.doc.models import ( BallotDocEvent, BallotPositionDocEvent, DocEvent,
     Document, NewRevisionDocEvent, State )
+from ietf.doc.storage_utils import store_str
 from ietf.doc.utils import ( add_state_change_event, close_open_ballots,
     create_ballot_if_not_open, update_telechat )
 from ietf.doc.mails import email_iana, email_ad_approved_conflict_review
@@ -186,9 +187,10 @@ def save(self, review):
         filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename
         with filepath.open('w', encoding='utf-8') as destination:
             if self.cleaned_data['txt']:
-                destination.write(self.cleaned_data['txt'])
+                content = self.cleaned_data['txt']
             else:
-                destination.write(self.cleaned_data['content'])
+                content = self.cleaned_data['content']
+            destination.write(content)
         ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename
         try:
             os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10
@@ -197,6 +199,7 @@ def save(self, review):
                 "There was an error creating a hardlink at %s pointing to %s: %s"
                 % (ftp_filepath, filepath, e)
             )
+        store_str("conflrev", basename, content)
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')

ietf/settings.py

@@ -6,6 +6,7 @@
 # BASE_DIR and "settings_local" are from
 # http://code.djangoproject.com/wiki/SplitSettings
 
+import boto3  # pyflakes:ignore
 import os
 import sys
 import datetime
@@ -735,6 +736,11 @@ def skip_unreadable_post(record):
     "schedule_name": r"(?P<name>[A-Za-z0-9-:_]+)",
 }
 
+STORAGES = {
+    "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
+    "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
+}
+
 # Override this in settings_local.py if needed
 # *_PATH variables ends with a slash/ .
 

ietf/settings_test.py

@@ -14,7 +14,7 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER
+from ietf.settings import boto3, STORAGES, TEST_CODE_COVERAGE_CHECKER
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -105,3 +105,24 @@ def tempdir_with_cleanup(**kwargs):
         'level': 'INFO',
     },
 }
+
+for bucketname in [
+    "bofreq",
+    "charter",
+    "conflrev",
+    "draft-xml",
+    "draft-txt",
+    "draft-html",
+]:
+    STORAGES[bucketname] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=boto3.session.Config(signature_version="s3v4"),
+            verify=False,
+            bucket_name=bucketname,
+        ),
+    }

ietf/submit/tests.py

@@ -31,6 +31,7 @@
                                 ReviewFactory, WgRfcFactory)
 from ietf.doc.models import ( Document, DocEvent, State,
     BallotPositionDocEvent, DocumentAuthor, SubmissionDocEvent )
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import create_ballot_if_not_open, can_edit_docextresources, update_action_holders
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group
@@ -428,6 +429,7 @@ def submit_new_wg(self, formats):
         self.assertTrue(draft.latest_event(type="added_suggested_replaces"))
         self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
         self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev))))
+        self.assertTrue(len(retrieve_str("draft-txt",f"{name}-{rev}.txt"))>0)
         self.assertEqual(draft.type_id, "draft")
         self.assertEqual(draft.stream_id, "ietf")
         self.assertTrue(draft.expires >= timezone.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1))

ietf/submit/utils.py

@@ -36,6 +36,7 @@
     DocumentAuthor, AddedMessageEvent )
 from ietf.doc.models import NewRevisionDocEvent
 from ietf.doc.models import RelatedDocument, DocRelationshipName, DocExtResource
+from ietf.doc.storage_utils import store_bytes
 from ietf.doc.utils import (add_state_change_event, rebuild_reference_relations,
     set_replaces_for_document, prettify_std_name, update_doc_extresources, 
     can_edit_docextresources, update_documentauthors, update_action_holders,
@@ -665,6 +666,9 @@ def move_files_to_repository(submission):
             ftp_dest = Path(settings.FTP_DIR) / "internet-drafts" / dest.name
             os.link(dest, all_archive_dest)
             os.link(dest, ftp_dest)
+            with open(dest,"rb") as f:
+                content_bytes = f.read()
+                store_bytes(f"draft-{ext}", fname, content_bytes)
         elif dest.exists():
             log.log("Intended to move '%s' to '%s', but found source missing while destination exists.")
         elif f".{ext}" in submission.file_types.split(','):

requirements.txt

@@ -21,6 +21,7 @@ django-markup>=1.5    # Limited use - need to reconcile against direct use of ma
 django-oidc-provider==0.8.2    # 0.8.3 changes logout flow and claim return
 django-referrer-policy>=1.0
 django-simple-history>=3.0.0
+django-storages>=1.14.4
 django-stubs>=4.2.7,<5    # The django-stubs version used determines the the mypy version indicated below
 django-tastypie>=0.14.7,<0.15.0    # Version must be locked in sync with version of Django
 django-vite>=2.0.2,<3