fix(advisory): adjust advisory publishing to new mailman hyperkitty s…

…etup

Extract the advisory generation functions to the advisory helper module
instead of the flask view. This allows an easier usage across different
views.

Instead of actually pulling the published content from the mailinglist,
lets store the current security tracker content instead.

config/00-default.conf

@@ -3,7 +3,7 @@ advisory_url = https://security.archlinux.org/AVG-{1}
 group_url = https://security.archlinux.org/AVG-{0}
 issue_url = https://security.archlinux.org/{0}
 bugtracker_url = https://bugs.archlinux.org/task/{0}
-mailman_url = https://lists.archlinux.org/pipermail/arch-security/
+mailman_url = https://lists.archlinux.org/archives/list/arch-security@lists.archlinux.org/
 password_length_min = 16
 password_length_max = 64
 summary_length_max = 200

tracker/advisory.py

@@ -1,30 +1,119 @@
-from datetime import date
 from datetime import datetime
 from html import unescape
+from os.path import join
 from re import IGNORECASE
 from re import escape
 from re import search
 from re import sub
+from urllib.parse import urlparse
 
+from flask import render_template
 from markupsafe import escape as html_escape
 from requests import get
 
+from config import TRACKER_ADVISORY_URL
+from config import TRACKER_BUGTRACKER_URL
+from config import TRACKER_GROUP_URL
+from config import TRACKER_ISSUE_URL
 from config import TRACKER_MAILMAN_URL
+from tracker import db
+from tracker.model import CVE
+from tracker.model import Advisory
+from tracker.model import CVEGroup
+from tracker.model import CVEGroupEntry
+from tracker.model import CVEGroupPackage
+from tracker.model import Package
+from tracker.model.enum import Publication
+from tracker.model.enum import Remote
+from tracker.user import user_can_handle_advisory
 from tracker.util import chunks
 from tracker.util import issue_to_numeric
+from tracker.util import multiline_to_list
+
+
+def generate_advisory(advisory_id, with_subject=True, raw=True):
+    entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE)
+               .filter(Advisory.id == advisory_id)
+               .join(CVEGroupPackage, Advisory.group_package)
+               .join(CVEGroup, CVEGroupPackage.group)
+               .join(CVEGroupEntry, CVEGroup.issues)
+               .join(CVE, CVEGroupEntry.cve)
+               .order_by(CVE.id)
+               ).all()
+    if not entries:
+        return None
+
+    advisory = entries[0][0]
+    group = entries[0][1]
+    package = entries[0][2]
+    issues = sorted([issue for (advisory, group, package, issue) in entries])
+    severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type)
+    severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity)
+    remote = any([issue.remote is Remote.remote for issue in issues])
+    issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues])
+
+    link = TRACKER_ADVISORY_URL.format(advisory.id, group.id)
+    upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0]
+    upstream_version = group.fixed.split('-')[0].split('+')[0]
+    if ':' in upstream_version:
+        upstream_version = upstream_version[upstream_version.index(':') + 1:]
+    unique_issue_types = []
+    for issue in severity_sorted_issues:
+        if issue.issue_type not in unique_issue_types:
+            unique_issue_types.append(issue.issue_type)
+
+    references = []
+    if group.bug_ticket:
+        references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket))
+    references.extend([ref for ref in multiline_to_list(group.reference)
+                       if ref not in references])
+    list(map(lambda issue: references.extend(
+        [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues))
+
+    raw_asa = render_template('advisory.txt',
+                              advisory=advisory,
+                              group=group,
+                              package=package,
+                              issues=issues,
+                              remote=remote,
+                              issue_listing_formatted=issue_listing_formatted,
+                              link=link,
+                              workaround=advisory.workaround,
+                              impact=advisory.impact,
+                              upstream_released=upstream_released,
+                              upstream_version=upstream_version,
+                              unique_issue_types=unique_issue_types,
+                              references=references,
+                              with_subject=with_subject,
+                              TRACKER_ISSUE_URL=TRACKER_ISSUE_URL,
+                              TRACKER_GROUP_URL=TRACKER_GROUP_URL)
+    if raw:
+        return raw_asa
+
+    raw_asa = '\n'.join(raw_asa.split('\n')[2:])
+    raw_asa = str(html_escape(raw_asa))
+    raw_asa = advisory_extend_html(raw_asa, issues, package)
+    return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True)
+
+
+def render_html_advisory(advisory, package, group, raw_asa, generated):
+    return render_template('advisory.html',
+                           title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type),
+                           advisory=advisory,
+                           package=package,
+                           raw_asa=raw_asa,
+                           generated=generated,
+                           can_handle_advisory=user_can_handle_advisory(),
+                           Publication=Publication)
 
 
 def advisory_fetch_from_mailman(url):
     try:
         response = get(url)
         if 200 != response.status_code:
             return None
-        asa = unescape(sub('</?A[^<]*?>', '', response.text))
-        start = '<PRE>'
-        start_marker = '{}Arch Linux Security Advisory'.format(start)
-        end = '\n-------------- next part --------------'
-        asa = asa[asa.index(start_marker) + len(start):asa.index(end)]
-        return asa.strip()
+
+        return response.text
     except Exception:
         return None
 
@@ -33,18 +122,29 @@ def advisory_fetch_reference_url_from_mailman(advisory):
     try:
         year = advisory.id[4:8]
         month = advisory.id[8:10]
-        publish_date = date(int(year), int(month), 1)
-        mailman_url = '{}{}-{}/'.format(TRACKER_MAILMAN_URL, year, publish_date.strftime('%B'))
-        response = get(mailman_url)
+        mailman_monthly = '{}{}/{}/?count=100'.format(TRACKER_MAILMAN_URL, year, month)
+
+        response = get(mailman_monthly)
         if 200 != response.status_code:
             return None
+
+        mailman_url = urlparse(TRACKER_MAILMAN_URL)
+        thread_url_base = join(mailman_url.path, 'thread')
+
+        message_url = None
         for line in response.text.split('\n'):
+            if thread_url_base in line:
+                match = search(r'href="{}/([/a-zA-Z0-9]+)"'.format(thread_url_base), line)
+                if not match:
+                    continue
+
+                thread = match.group(1)
+                message_url = join(TRACKER_MAILMAN_URL, 'message', thread)
+
             if not '[{}]'.format(advisory.id) in line:
                 continue
-            match = search(r'HREF="(\d+.html)"', line)
-            if not match:
-                continue
-            return '{}{}'.format(mailman_url, match.group(1))
+
+            return message_url
         return None
     except Exception:
         return None
@@ -96,14 +196,6 @@ def advisory_extend_html(advisory, issues, package):
     return advisory
 
 
-def advisory_extend_model_from_advisory_text(advisory):
-    if not advisory.content:
-        return advisory
-    advisory.impact = advisory_get_impact_from_text(advisory.content)
-    advisory.workaround = advisory_get_workaround_from_text(advisory.content)
-    return advisory
-
-
 def advisory_get_date_label(utctimetuple=None):
     now = utctimetuple if utctimetuple else datetime.utcnow().utctimetuple()
     return '{}{:02}'.format(now.tm_year, now.tm_mon)

tracker/form/validators.py

@@ -6,6 +6,7 @@
 
 from tracker import db
 from tracker.advisory import advisory_fetch_from_mailman
+from tracker.advisory import generate_advisory
 from tracker.model import Package
 from tracker.model.advisory import advisory_regex
 from tracker.model.cve import cve_id_regex
@@ -21,18 +22,20 @@ def __call__(self, form, field):
         if not field.data:
             return
 
-        form.advisory_content = advisory_fetch_from_mailman(field.data)
-        if not form.advisory_content:
+        mailman_content = advisory_fetch_from_mailman(field.data)
+        if not mailman_content:
             raise ValidationError('Failed to fetch advisory')
 
-        m = search(advisory_regex[1:-1], form.advisory_content)
+        m = search(advisory_regex[1:-1], mailman_content)
         if not m:
             raise ValidationError('Failed to fetch advisory')
 
         found = m.group(1)
         if found != form.advisory_id:
             raise ValidationError('Advisory mismatched: {}'.format(found))
 
+        form.advisory_content = generate_advisory(advisory_id=form.advisory_id, with_subject=False, raw=True)
+
 
 class ValidPackageName(object):
     def __init__(self):

tracker/templates/advisory.txt

@@ -1,5 +1,6 @@
-Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
+{%- if with_subject %}Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
 
+{% endif -%}
 {% set asa_title = 'Arch Linux Security Advisory ' + advisory.id %}
 {{- asa_title }}
 {% set asa_title_separator = '=' * asa_title|length %}

tracker/view/advisory.py

@@ -13,7 +13,6 @@
 from config import TRACKER_ISSUE_URL
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.advisory import advisory_get_date_label
 from tracker.advisory import advisory_get_label
@@ -219,7 +218,6 @@ def publish_advisory(asa):
 
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data
     advisory.publication = Publication.published
     db.session.commit()

tracker/view/edit.py

@@ -13,7 +13,6 @@
 
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.form import CVEForm
 from tracker.form import GroupForm
@@ -114,7 +113,6 @@ def edit_advisory(advisory_id):
     advisory.workaround = form.workaround.data or None
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data or None
 
     # update changed date on modification

tracker/view/show.py

@@ -20,6 +20,8 @@
 from tracker.advisory import advisory_escape_html
 from tracker.advisory import advisory_extend_html
 from tracker.advisory import advisory_format_issue_listing
+from tracker.advisory import generate_advisory
+from tracker.advisory import render_html_advisory
 from tracker.form.advisory import AdvisoryForm
 from tracker.model import CVE
 from tracker.model import Advisory
@@ -488,17 +490,6 @@ def show_package(pkgname):
                            package=data)
 
 
-def render_html_advisory(advisory, package, group, raw_asa, generated):
-    return render_template('advisory.html',
-                           title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type),
-                           advisory=advisory,
-                           package=package,
-                           raw_asa=raw_asa,
-                           generated=generated,
-                           can_handle_advisory=user_can_handle_advisory(),
-                           Publication=Publication)
-
-
 @tracker.route('/advisory/<regex("{}"):advisory_id>/raw'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('/<regex("{}"):advisory_id>/raw'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_advisory_raw(advisory_id):
@@ -554,67 +545,10 @@ def show_advisory(advisory_id, raw=False):
 @tracker.route('/advisory/<regex("{}"):advisory_id>/generate'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('/<regex("{}"):advisory_id>/generate'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_generated_advisory(advisory_id, raw=False):
-    entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE)
-               .filter(Advisory.id == advisory_id)
-               .join(CVEGroupPackage, Advisory.group_package)
-               .join(CVEGroup, CVEGroupPackage.group)
-               .join(CVEGroupEntry, CVEGroup.issues)
-               .join(CVE, CVEGroupEntry.cve)
-               .order_by(CVE.id)
-               ).all()
-    if not entries:
+    advisory = generate_advisory(advisory_id, with_subject=True, raw=raw)
+    if not advisory:
         return not_found()
-
-    advisory = entries[0][0]
-    group = entries[0][1]
-    package = entries[0][2]
-    issues = sorted([issue for (advisory, group, package, issue) in entries])
-    severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type)
-    severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity)
-    remote = any([issue.remote is Remote.remote for issue in issues])
-    issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues])
-
-    link = TRACKER_ADVISORY_URL.format(advisory.id, group.id)
-    upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0]
-    upstream_version = group.fixed.split('-')[0].split('+')[0]
-    if ':' in upstream_version:
-        upstream_version = upstream_version[upstream_version.index(':') + 1:]
-    unique_issue_types = []
-    for issue in severity_sorted_issues:
-        if issue.issue_type not in unique_issue_types:
-            unique_issue_types.append(issue.issue_type)
-
-    references = []
-    if group.bug_ticket:
-        references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket))
-    references.extend([ref for ref in multiline_to_list(group.reference)
-                       if ref not in references])
-    list(map(lambda issue: references.extend(
-        [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues))
-
-    raw_asa = render_template('advisory.txt',
-                              advisory=advisory,
-                              group=group,
-                              package=package,
-                              issues=issues,
-                              remote=remote,
-                              issue_listing_formatted=issue_listing_formatted,
-                              link=link,
-                              workaround=advisory.workaround,
-                              impact=advisory.impact,
-                              upstream_released=upstream_released,
-                              upstream_version=upstream_version,
-                              unique_issue_types=unique_issue_types,
-                              references=references,
-                              TRACKER_ISSUE_URL=TRACKER_ISSUE_URL,
-                              TRACKER_GROUP_URL=TRACKER_GROUP_URL)
-    if raw:
-        return raw_asa
-
-    raw_asa = '\n'.join(raw_asa.split('\n')[2:])
-    raw_asa = str(escape(raw_asa))
-    raw_asa = advisory_extend_html(raw_asa, issues, package)
-    return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True)
+    return advisory
 
 
 @tracker.route('/advisory/<regex("{}"):advisory_id>/log'.format(advisory_regex[1:-1]), methods=['GET'])