ci: merge main to relase (#8487)

dev/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250117T1516
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250128T1728
 LABEL maintainer="IETF Tools Team <[email protected]>"
 
 ENV DEBIAN_FRONTEND=noninteractive

dev/build/TARGET_BASE

@@ -1 +1 @@
-20250117T1516
+20250128T1728

ietf/doc/templatetags/ietf_filters.py

@@ -26,10 +26,11 @@
 from ietf.doc.models import BallotDocEvent, Document
 from ietf.doc.models import ConsensusDocEvent
 from ietf.ietfauth.utils import can_request_rfc_publication as utils_can_request_rfc_publication
-from ietf.utils.html import sanitize_fragment
 from ietf.utils import log
 from ietf.doc.utils import prettify_std_name
-from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner, validate_url
+from ietf.utils.html import clean_html
+from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, linkify
+from ietf.utils.validators import validate_url
 
 register = template.Library()
 
@@ -98,7 +99,7 @@ def sanitize(value):
     attributes to those deemed acceptable.  See ietf/utils/html.py
     for the details.
     """
-    return mark_safe(sanitize_fragment(value))
+    return mark_safe(clean_html(value))
 
 
 # For use with ballot view
@@ -446,16 +447,16 @@ def ad_area(user):
 @register.filter
 def format_history_text(text, trunc_words=25):
     """Run history text through some cleaning and add ellipsis if it's too long."""
-    full = mark_safe(bleach_cleaner.clean(text))
-    full = bleach_linker.linkify(urlize_ietf_docs(full))
+    full = mark_safe(clean_html(text))
+    full = linkify(urlize_ietf_docs(full))
 
     return format_snippet(full, trunc_words)
 
 @register.filter
 def format_snippet(text, trunc_words=25): 
     # urlize if there aren't already links present
-    text = bleach_linker.linkify(text)
-    full = keep_spacing(collapsebr(linebreaksbr(mark_safe(sanitize_fragment(text)))))
+    text = linkify(text)
+    full = keep_spacing(collapsebr(linebreaksbr(mark_safe(clean_html(text)))))
     snippet = truncatewords_html(full, trunc_words)
     if snippet != full:
         return mark_safe('<div class="snippet">%s<button type="button" aria-label="Expand" class="btn btn-sm btn-primary show-all"><i class="bi bi-caret-down"></i></button></div><div class="d-none full">%s</div>' % (snippet, full))

ietf/doc/tests.py

@@ -5,6 +5,8 @@
 import os
 import datetime
 import io
+from hashlib import sha384
+
 from django.http import HttpRequest
 import lxml
 import bibtexparser
@@ -3280,6 +3282,41 @@ def test_investigate_fragment(self):
             "draft-this-should-not-be-possible-00.txt",
         )
 
+    @mock.patch("ietf.doc.utils.caches")
+    def test_investigate_fragment_cache(self, mock_caches):
+        """investigate_fragment should cache its result"""
+        mock_default_cache = mock_caches["default"]
+        mock_default_cache.get.return_value = None  # disable cache
+        result = investigate_fragment("this-is-active")
+        self.assertEqual(len(result["can_verify"]), 1)
+        self.assertEqual(len(result["unverifiable_collections"]), 0)
+        self.assertEqual(len(result["unexpected"]), 0)
+        self.assertEqual(
+            list(result["can_verify"])[0].name, "draft-this-is-active-00.txt"
+        )
+        self.assertTrue(mock_default_cache.get.called)
+        self.assertTrue(mock_default_cache.set.called)
+        expected_key = f"investigate_fragment:{sha384(b'this-is-active').hexdigest()}"
+        self.assertEqual(mock_default_cache.set.call_args.kwargs["key"], expected_key)
+        cached_value = mock_default_cache.set.call_args.kwargs["value"]  # hang on to this
+        mock_default_cache.reset_mock()
+
+        # Check that a cached value is used
+        mock_default_cache.get.return_value = cached_value
+        with mock.patch("ietf.doc.utils.Path") as mock_path:
+            result = investigate_fragment("this-is-active")
+        # Check that we got the same results
+        self.assertEqual(len(result["can_verify"]), 1)
+        self.assertEqual(len(result["unverifiable_collections"]), 0)
+        self.assertEqual(len(result["unexpected"]), 0)
+        self.assertEqual(
+            list(result["can_verify"])[0].name, "draft-this-is-active-00.txt"
+        )
+        # And that we used the cache
+        self.assertFalse(mock_path.called)  # a proxy for "did the method do any real work"
+        self.assertTrue(mock_default_cache.get.called)
+        self.assertEqual(mock_default_cache.get.call_args, mock.call(expected_key))
+
     def test_investigate_get(self):
         """GET with no querystring should retrieve the investigate UI"""
         url = urlreverse("ietf.doc.views_doc.investigate")

ietf/doc/utils.py

@@ -11,12 +11,14 @@
 
 from collections import defaultdict, namedtuple, Counter
 from dataclasses import dataclass
+from hashlib import sha384
 from pathlib import Path
 from typing import Iterator, Optional, Union
 from zoneinfo import ZoneInfo
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.cache import caches
 from django.db.models import OuterRef
 from django.forms import ValidationError
 from django.http import Http404
@@ -1459,35 +1461,43 @@ def get_doc_email_aliases(name: Optional[str] = None):
     return sorted(aliases, key=lambda a: (a["doc_name"]))
 
 
-def investigate_fragment(name_fragment):
-    can_verify = set()
-    for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:
-        can_verify.update(list(Path(root).glob(f"*{name_fragment}*")))
-    archive_verifiable_names = set([p.name for p in can_verify])
-    # Can also verify drafts in proceedings directories
-    can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*")))
-
-    # N.B. This reflects the assumption that the internet draft archive dir is in the
-    # a directory with other collections (at /a/ietfdata/draft/collections as this is written)
-    unverifiable_collections = set([
-        p for p in
-        Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*")
-        if p.name not in archive_verifiable_names
-    ])
+def investigate_fragment(name_fragment: str):
+    cache = caches["default"]
+    # Ensure name_fragment does not interact badly with the cache key handling
+    name_digest = sha384(name_fragment.encode("utf8")).hexdigest()
+    cache_key = f"investigate_fragment:{name_digest}"
+    result = cache.get(cache_key)
+    if result is None:
+        can_verify = set()
+        for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:
+            can_verify.update(list(Path(root).glob(f"*{name_fragment}*")))
+        archive_verifiable_names = set([p.name for p in can_verify])
+        # Can also verify drafts in proceedings directories
+        can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*")))
 
-    unverifiable_collections.difference_update(can_verify)
-
-    expected_names = set([p.name for p in can_verify.union(unverifiable_collections)])
-    maybe_unexpected = list(
-        Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR).glob(f"*{name_fragment}*")
-    )
-    unexpected = [p for p in maybe_unexpected if p.name not in expected_names]
-
-    return dict(
-        can_verify=can_verify,
-        unverifiable_collections=unverifiable_collections,
-        unexpected=unexpected,
-    )
+        # N.B. This reflects the assumption that the internet draft archive dir is in the
+        # a directory with other collections (at /a/ietfdata/draft/collections as this is written)
+        unverifiable_collections = set([
+            p for p in
+            Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*")
+            if p.name not in archive_verifiable_names
+        ])
+
+        unverifiable_collections.difference_update(can_verify)
+
+        expected_names = set([p.name for p in can_verify.union(unverifiable_collections)])
+        maybe_unexpected = list(
+            Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR).glob(f"*{name_fragment}*")
+        )
+        unexpected = [p for p in maybe_unexpected if p.name not in expected_names]
+        result = dict(
+            can_verify=can_verify,
+            unverifiable_collections=unverifiable_collections,
+            unexpected=unexpected,
+        )
+        # 1 hour caching
+        cache.set(key=cache_key, timeout=3600, value=result)
+    return result
 
 
 def update_or_create_draft_bibxml_file(doc, rev):

ietf/iesg/utils.py

@@ -33,7 +33,7 @@ def telechat_page_count(date=None, docs=None, ad=None):
             ballot = draft.active_ballot()
             if ballot:
                 positions = ballot.active_balloter_positions()
-                ad_position = positions[ad]
+                ad_position = positions.get(ad, None)
                 if ad_position is None or ad_position.pos_id == "norecord":
                     ad_pages_left_to_ballot_on += draft.pages or 0
 

ietf/meeting/tests_views.py

@@ -6423,8 +6423,7 @@ def test_upload_minutes_agenda(self):
             text = doc.text()
             self.assertIn('Some text', text)
             self.assertNotIn('<section>', text)
-            self.assertIn('charset="utf-8"', text)
-
+
             # txt upload
             test_file = BytesIO(b'This is some text for a test, with the word\nvirtual at the beginning of a line.')
             test_file.name = "some.txt"

ietf/meeting/utils.py

@@ -30,7 +30,7 @@
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
 from ietf.stats.models import MeetingRegistration
-from ietf.utils.html import sanitize_document
+from ietf.utils.html import clean_html
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
 
@@ -738,25 +738,12 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
 
     This function takes a _binary mode_ file object, a filename and a meeting object and subdir as string.
     It saves the file to the appropriate directory, get_materials_path() + subdir.
-    If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
-    zip file and unzips the file in the new directory.
     """
     filename = Path(filename)
-    is_zipfile = filename.suffix == '.zip'
 
     path = Path(meeting.get_materials_path()) / subdir
-    if is_zipfile:
-        path = path / filename.stem
     path.mkdir(parents=True, exist_ok=True)
 
-    # agendas and minutes can only have one file instance so delete file if it already exists
-    if subdir in ('agenda', 'minutes'):
-        for f in path.glob(f'{filename.stem}.*'):
-            try:
-                f.unlink()
-            except FileNotFoundError:
-                pass  # if the file is already gone, so be it
-
     with (path / filename).open('wb+') as destination:
         # prep file for reading
         if hasattr(file, "chunks"):
@@ -786,8 +773,8 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
                     return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
             # Whole file sanitization; add back what's missing from a complete
             # document (sanitize will remove these).
-            clean = sanitize_document(text)
-            destination.write(clean.encode('utf8'))
+            clean = clean_html(text)
+            destination.write(clean.encode("utf8"))
             if request and clean != text:
                 messages.warning(request,
                                  (
@@ -799,10 +786,6 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
             for chunk in chunks:
                 destination.write(chunk)
 
-    # unzip zipfile
-    if is_zipfile:
-        subprocess.call(['unzip', filename], cwd=path)
-
     return None
 
 def new_doc_for_session(type_id, session):

ietf/static/js/complete-review.js

@@ -24,6 +24,8 @@ $(document)
             .before(mailArchiveSearchTemplate);
 
         var mailArchiveSearch = form.find(".mail-archive-search");
+        const isReviewer = mailArchiveSearch.data('isReviewer');
+        const searchMailArchiveUrl = mailArchiveSearch.data('searchMailArchiveUrl');
 
         var retrievingData = null;
 
@@ -190,4 +192,4 @@ $(document)
             form.find("[name=review_submission][value=link]")
                 .trigger("click");
         }
-    });
+    });

ietf/static/js/investigate.js

@@ -18,13 +18,13 @@ document.addEventListener('DOMContentLoaded', () => {
             loadResultsFromTask('bogus-task-id') // bad task id will generate an error from Django
         }
         const taskId = (await response.json()).id
-        // Poll for completion of the investigation up to 18*10 = 180 seconds 
-        waitForResults(taskId, 18)
+        // Poll for completion of the investigation up to 60*10 = 600 seconds 
+        waitForResults(taskId, 60)
     }
 
     const waitForResults = async (taskId, retries) => {
         // indicate that investigation is in progress
-        document.getElementById('spinner').classList.remove('d-none')
+        document.querySelectorAll('.investigation-indicator').forEach(elt => elt.classList.remove('d-none'))
         document.getElementById('investigate-button').disabled = true
         investigateForm.elements['id_name_fragment'].disabled = true
 

ietf/templates/doc/investigate.html

@@ -13,13 +13,15 @@ <h1>Investigate</h1>
             {% csrf_token %}
             {% bootstrap_form form %}
             <button class="btn btn-primary" type="submit" id="investigate-button">
-            <span id="spinner"
-                  class="spinner-border spinner-border-sm d-none"
-                  role="status"
-                  aria-hidden="true">
-            </span>
+                <span class="spinner-border spinner-border-sm investigation-indicator d-none"
+                      role="status"
+                      aria-hidden="true">
+                </span>
                 Investigate
             </button>
+            <div class="alert alert-info mt-3 d-none investigation-indicator">
+                Please be patient, processing may take several minutes.
+            </div>
         </form>
     </div>
     {% if results %}

ietf/templates/doc/review/complete_review.html

@@ -94,7 +94,13 @@ <h1>
         {% endif %}
         <div class="template d-none">
             {% if mail_archive_query_urls %}
-                <div class="mail-archive-search">
+                <div class="mail-archive-search"
+                    {% if assignment %}
+                     data-search-mail-archive-url="{% url "ietf.doc.views_review.search_mail_archive" name=doc.name assignment_id=assignment.pk %}"
+                    {% else %}
+                     data-search-mail-archive-url="{% url "ietf.doc.views_review.search_mail_archive" name=doc.name acronym=team.acronym %}"
+                    {% endif %}
+                     data-is-reviewer="{{ is_reviewer|yesno:"true,false" }}">
                     <div class="offset-md-2 col-md-10">
                         <label for="mail-archive-subjects" class="form-label">Search {{team.list_email}} mail archive subjects for:</label>
                         <div class="input-group mb-3">
@@ -144,13 +150,5 @@ <h1>
 {% endblock %}
 {% block js %}
     <script src="{% static 'ietf/js/datepicker.js' %}"></script>
-    <script>
-        {% if assignment %}
-            var searchMailArchiveUrl = "{% url "ietf.doc.views_review.search_mail_archive" name=doc.name assignment_id=assignment.pk %}";
-        {% else %}
-            var searchMailArchiveUrl = "{% url "ietf.doc.views_review.search_mail_archive" name=doc.name acronym=team.acronym %}";
-        {% endif %}
-        var isReviewer = {{ is_reviewer|yesno:'true,false' }};
-    </script>
     <script src="{% static 'ietf/js/complete-review.js' %}"></script>
 {% endblock %}