From c17735ea023246e33137cc443484f15aaa2b3aee Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 25 Oct 2022 00:51:42 +0200 Subject: [PATCH] fix(advisory): adjust advisory publishing to new mailman hyperkitty setup Extract the advisory generation functions to the advisory helper module instead of the flask view. This allows an easier usage across different views. Instead of actually pulling the published content from the mailinglist, lets store the current security tracker content instead. --- config/00-default.conf | 2 +- tracker/advisory.py | 136 +++++++++++++++++++++++++++------ tracker/form/validators.py | 9 ++- tracker/templates/advisory.txt | 3 +- tracker/view/advisory.py | 2 - tracker/view/edit.py | 2 - tracker/view/show.py | 76 ++---------------- 7 files changed, 128 insertions(+), 102 deletions(-) diff --git a/config/00-default.conf b/config/00-default.conf index ae01afc2..8849a419 100644 --- a/config/00-default.conf +++ b/config/00-default.conf @@ -3,7 +3,7 @@ advisory_url = https://security.archlinux.org/AVG-{1} group_url = https://security.archlinux.org/AVG-{0} issue_url = https://security.archlinux.org/{0} bugtracker_url = https://bugs.archlinux.org/task/{0} -mailman_url = https://lists.archlinux.org/pipermail/arch-security/ +mailman_url = https://lists.archlinux.org/archives/list/arch-security@lists.archlinux.org/ password_length_min = 16 password_length_max = 64 summary_length_max = 200 diff --git a/tracker/advisory.py b/tracker/advisory.py index 7ffa0fbb..0207d9e4 100644 --- a/tracker/advisory.py +++ b/tracker/advisory.py @@ -1,17 +1,110 @@ -from datetime import date from datetime import datetime from html import unescape +from os.path import join from re import IGNORECASE from re import escape from re import search from re import sub +from urllib.parse import urlparse +from flask import render_template from markupsafe import escape as html_escape from requests import get +from config import TRACKER_ADVISORY_URL +from config import TRACKER_BUGTRACKER_URL +from config import TRACKER_GROUP_URL +from config import TRACKER_ISSUE_URL from config import TRACKER_MAILMAN_URL +from tracker import db +from tracker.model import CVE +from tracker.model import Advisory +from tracker.model import CVEGroup +from tracker.model import CVEGroupEntry +from tracker.model import CVEGroupPackage +from tracker.model import Package +from tracker.model.enum import Publication +from tracker.model.enum import Remote +from tracker.user import user_can_handle_advisory from tracker.util import chunks from tracker.util import issue_to_numeric +from tracker.util import multiline_to_list + + +def generate_advisory(advisory_id, with_subject=True, raw=True): + entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE) + .filter(Advisory.id == advisory_id) + .join(CVEGroupPackage, Advisory.group_package) + .join(CVEGroup, CVEGroupPackage.group) + .join(CVEGroupEntry, CVEGroup.issues) + .join(CVE, CVEGroupEntry.cve) + .order_by(CVE.id) + ).all() + if not entries: + return None + + advisory = entries[0][0] + group = entries[0][1] + package = entries[0][2] + issues = sorted([issue for (advisory, group, package, issue) in entries]) + severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type) + severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity) + remote = any([issue.remote is Remote.remote for issue in issues]) + issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues]) + + link = TRACKER_ADVISORY_URL.format(advisory.id, group.id) + upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0] + upstream_version = group.fixed.split('-')[0].split('+')[0] + if ':' in upstream_version: + upstream_version = upstream_version[upstream_version.index(':') + 1:] + unique_issue_types = [] + for issue in severity_sorted_issues: + if issue.issue_type not in unique_issue_types: + unique_issue_types.append(issue.issue_type) + + references = [] + if group.bug_ticket: + references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket)) + references.extend([ref for ref in multiline_to_list(group.reference) + if ref not in references]) + list(map(lambda issue: references.extend( + [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues)) + + raw_asa = render_template('advisory.txt', + advisory=advisory, + group=group, + package=package, + issues=issues, + remote=remote, + issue_listing_formatted=issue_listing_formatted, + link=link, + workaround=advisory.workaround, + impact=advisory.impact, + upstream_released=upstream_released, + upstream_version=upstream_version, + unique_issue_types=unique_issue_types, + references=references, + with_subject=with_subject, + TRACKER_ISSUE_URL=TRACKER_ISSUE_URL, + TRACKER_GROUP_URL=TRACKER_GROUP_URL) + if raw: + return raw_asa + + raw_asa = '\n'.join(raw_asa.split('\n')[2:]) + raw_asa = str(html_escape(raw_asa)) + raw_asa = advisory_extend_html(raw_asa, issues, package) + return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True) + + +def render_html_advisory(advisory, package, group, raw_asa, generated): + return render_template('advisory.html', + title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type), + advisory=advisory, + package=package, + raw_asa=raw_asa, + generated=generated, + can_handle_advisory=user_can_handle_advisory(), + Publication=Publication) def advisory_fetch_from_mailman(url): @@ -19,12 +112,8 @@ def advisory_fetch_from_mailman(url): response = get(url) if 200 != response.status_code: return None - asa = unescape(sub('', '', response.text)) - start = '
'
-        start_marker = '{}Arch Linux Security Advisory'.format(start)
-        end = '\n-------------- next part --------------'
-        asa = asa[asa.index(start_marker) + len(start):asa.index(end)]
-        return asa.strip()
+
+        return response.text
     except Exception:
         return None
 
@@ -33,18 +122,29 @@ def advisory_fetch_reference_url_from_mailman(advisory):
     try:
         year = advisory.id[4:8]
         month = advisory.id[8:10]
-        publish_date = date(int(year), int(month), 1)
-        mailman_url = '{}{}-{}/'.format(TRACKER_MAILMAN_URL, year, publish_date.strftime('%B'))
-        response = get(mailman_url)
+        mailman_monthly = '{}{}/{}/?count=100'.format(TRACKER_MAILMAN_URL, year, month)
+
+        response = get(mailman_monthly)
         if 200 != response.status_code:
             return None
+
+        mailman_url = urlparse(TRACKER_MAILMAN_URL)
+        thread_url_base = join(mailman_url.path, 'thread')
+
+        message_url = None
         for line in response.text.split('\n'):
+            if thread_url_base in line:
+                match = search(r'href="{}/([/a-zA-Z0-9]+)"'.format(thread_url_base), line)
+                if not match:
+                    continue
+
+                thread = match.group(1)
+                message_url = join(TRACKER_MAILMAN_URL, 'message', thread)
+
             if not '[{}]'.format(advisory.id) in line:
                 continue
-            match = search(r'HREF="(\d+.html)"', line)
-            if not match:
-                continue
-            return '{}{}'.format(mailman_url, match.group(1))
+
+            return message_url
         return None
     except Exception:
         return None
@@ -96,14 +196,6 @@ def advisory_extend_html(advisory, issues, package):
     return advisory
 
 
-def advisory_extend_model_from_advisory_text(advisory):
-    if not advisory.content:
-        return advisory
-    advisory.impact = advisory_get_impact_from_text(advisory.content)
-    advisory.workaround = advisory_get_workaround_from_text(advisory.content)
-    return advisory
-
-
 def advisory_get_date_label(utctimetuple=None):
     now = utctimetuple if utctimetuple else datetime.utcnow().utctimetuple()
     return '{}{:02}'.format(now.tm_year, now.tm_mon)
diff --git a/tracker/form/validators.py b/tracker/form/validators.py
index ee29c192..4b6855c1 100644
--- a/tracker/form/validators.py
+++ b/tracker/form/validators.py
@@ -6,6 +6,7 @@
 
 from tracker import db
 from tracker.advisory import advisory_fetch_from_mailman
+from tracker.advisory import generate_advisory
 from tracker.model import Package
 from tracker.model.advisory import advisory_regex
 from tracker.model.cve import cve_id_regex
@@ -21,11 +22,11 @@ def __call__(self, form, field):
         if not field.data:
             return
 
-        form.advisory_content = advisory_fetch_from_mailman(field.data)
-        if not form.advisory_content:
+        mailman_content = advisory_fetch_from_mailman(field.data)
+        if not mailman_content:
             raise ValidationError('Failed to fetch advisory')
 
-        m = search(advisory_regex[1:-1], form.advisory_content)
+        m = search(advisory_regex[1:-1], mailman_content)
         if not m:
             raise ValidationError('Failed to fetch advisory')
 
@@ -33,6 +34,8 @@ def __call__(self, form, field):
         if found != form.advisory_id:
             raise ValidationError('Advisory mismatched: {}'.format(found))
 
+        form.advisory_content = generate_advisory(advisory_id=form.advisory_id, with_subject=False, raw=True)
+
 
 class ValidPackageName(object):
     def __init__(self):
diff --git a/tracker/templates/advisory.txt b/tracker/templates/advisory.txt
index af3c2ef1..811145bd 100644
--- a/tracker/templates/advisory.txt
+++ b/tracker/templates/advisory.txt
@@ -1,5 +1,6 @@
-Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
+{%- if with_subject %}Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
 
+{% endif -%}
 {% set asa_title = 'Arch Linux Security Advisory ' + advisory.id %}
 {{- asa_title }}
 {% set asa_title_separator = '=' * asa_title|length %}
diff --git a/tracker/view/advisory.py b/tracker/view/advisory.py
index 640642c7..8eb8c16a 100644
--- a/tracker/view/advisory.py
+++ b/tracker/view/advisory.py
@@ -13,7 +13,6 @@
 from config import TRACKER_ISSUE_URL
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.advisory import advisory_get_date_label
 from tracker.advisory import advisory_get_label
@@ -219,7 +218,6 @@ def publish_advisory(asa):
 
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data
     advisory.publication = Publication.published
     db.session.commit()
diff --git a/tracker/view/edit.py b/tracker/view/edit.py
index 8de17472..31ba617a 100644
--- a/tracker/view/edit.py
+++ b/tracker/view/edit.py
@@ -13,7 +13,6 @@
 
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.form import CVEForm
 from tracker.form import GroupForm
@@ -114,7 +113,6 @@ def edit_advisory(advisory_id):
     advisory.workaround = form.workaround.data or None
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data or None
 
     # update changed date on modification
diff --git a/tracker/view/show.py b/tracker/view/show.py
index 26e5b85b..fd555df8 100644
--- a/tracker/view/show.py
+++ b/tracker/view/show.py
@@ -20,6 +20,8 @@
 from tracker.advisory import advisory_escape_html
 from tracker.advisory import advisory_extend_html
 from tracker.advisory import advisory_format_issue_listing
+from tracker.advisory import generate_advisory
+from tracker.advisory import render_html_advisory
 from tracker.form.advisory import AdvisoryForm
 from tracker.model import CVE
 from tracker.model import Advisory
@@ -488,17 +490,6 @@ def show_package(pkgname):
                            package=data)
 
 
-def render_html_advisory(advisory, package, group, raw_asa, generated):
-    return render_template('advisory.html',
-                           title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type),
-                           advisory=advisory,
-                           package=package,
-                           raw_asa=raw_asa,
-                           generated=generated,
-                           can_handle_advisory=user_can_handle_advisory(),
-                           Publication=Publication)
-
-
 @tracker.route('/advisory//raw'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('//raw'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_advisory_raw(advisory_id):
@@ -554,67 +545,10 @@ def show_advisory(advisory_id, raw=False):
 @tracker.route('/advisory//generate'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('//generate'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_generated_advisory(advisory_id, raw=False):
-    entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE)
-               .filter(Advisory.id == advisory_id)
-               .join(CVEGroupPackage, Advisory.group_package)
-               .join(CVEGroup, CVEGroupPackage.group)
-               .join(CVEGroupEntry, CVEGroup.issues)
-               .join(CVE, CVEGroupEntry.cve)
-               .order_by(CVE.id)
-               ).all()
-    if not entries:
+    advisory = generate_advisory(advisory_id, with_subject=True, raw=raw)
+    if not advisory:
         return not_found()
-
-    advisory = entries[0][0]
-    group = entries[0][1]
-    package = entries[0][2]
-    issues = sorted([issue for (advisory, group, package, issue) in entries])
-    severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type)
-    severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity)
-    remote = any([issue.remote is Remote.remote for issue in issues])
-    issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues])
-
-    link = TRACKER_ADVISORY_URL.format(advisory.id, group.id)
-    upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0]
-    upstream_version = group.fixed.split('-')[0].split('+')[0]
-    if ':' in upstream_version:
-        upstream_version = upstream_version[upstream_version.index(':') + 1:]
-    unique_issue_types = []
-    for issue in severity_sorted_issues:
-        if issue.issue_type not in unique_issue_types:
-            unique_issue_types.append(issue.issue_type)
-
-    references = []
-    if group.bug_ticket:
-        references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket))
-    references.extend([ref for ref in multiline_to_list(group.reference)
-                       if ref not in references])
-    list(map(lambda issue: references.extend(
-        [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues))
-
-    raw_asa = render_template('advisory.txt',
-                              advisory=advisory,
-                              group=group,
-                              package=package,
-                              issues=issues,
-                              remote=remote,
-                              issue_listing_formatted=issue_listing_formatted,
-                              link=link,
-                              workaround=advisory.workaround,
-                              impact=advisory.impact,
-                              upstream_released=upstream_released,
-                              upstream_version=upstream_version,
-                              unique_issue_types=unique_issue_types,
-                              references=references,
-                              TRACKER_ISSUE_URL=TRACKER_ISSUE_URL,
-                              TRACKER_GROUP_URL=TRACKER_GROUP_URL)
-    if raw:
-        return raw_asa
-
-    raw_asa = '\n'.join(raw_asa.split('\n')[2:])
-    raw_asa = str(escape(raw_asa))
-    raw_asa = advisory_extend_html(raw_asa, issues, package)
-    return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True)
+    return advisory
 
 
 @tracker.route('/advisory//log'.format(advisory_regex[1:-1]), methods=['GET'])