diff --git a/config/00-default.conf b/config/00-default.conf index ae01afc..8849a41 100644 --- a/config/00-default.conf +++ b/config/00-default.conf @@ -3,7 +3,7 @@ advisory_url = https://security.archlinux.org/AVG-{1} group_url = https://security.archlinux.org/AVG-{0} issue_url = https://security.archlinux.org/{0} bugtracker_url = https://bugs.archlinux.org/task/{0} -mailman_url = https://lists.archlinux.org/pipermail/arch-security/ +mailman_url = https://lists.archlinux.org/archives/list/arch-security@lists.archlinux.org/ password_length_min = 16 password_length_max = 64 summary_length_max = 200 diff --git a/tracker/advisory.py b/tracker/advisory.py index 7ffa0fb..0207d9e 100644 --- a/tracker/advisory.py +++ b/tracker/advisory.py @@ -1,17 +1,110 @@ -from datetime import date from datetime import datetime from html import unescape +from os.path import join from re import IGNORECASE from re import escape from re import search from re import sub +from urllib.parse import urlparse +from flask import render_template from markupsafe import escape as html_escape from requests import get +from config import TRACKER_ADVISORY_URL +from config import TRACKER_BUGTRACKER_URL +from config import TRACKER_GROUP_URL +from config import TRACKER_ISSUE_URL from config import TRACKER_MAILMAN_URL +from tracker import db +from tracker.model import CVE +from tracker.model import Advisory +from tracker.model import CVEGroup +from tracker.model import CVEGroupEntry +from tracker.model import CVEGroupPackage +from tracker.model import Package +from tracker.model.enum import Publication +from tracker.model.enum import Remote +from tracker.user import user_can_handle_advisory from tracker.util import chunks from tracker.util import issue_to_numeric +from tracker.util import multiline_to_list + + +def generate_advisory(advisory_id, with_subject=True, raw=True): + entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE) + .filter(Advisory.id == advisory_id) + .join(CVEGroupPackage, Advisory.group_package) + .join(CVEGroup, CVEGroupPackage.group) + .join(CVEGroupEntry, CVEGroup.issues) + .join(CVE, CVEGroupEntry.cve) + .order_by(CVE.id) + ).all() + if not entries: + return None + + advisory = entries[0][0] + group = entries[0][1] + package = entries[0][2] + issues = sorted([issue for (advisory, group, package, issue) in entries]) + severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type) + severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity) + remote = any([issue.remote is Remote.remote for issue in issues]) + issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues]) + + link = TRACKER_ADVISORY_URL.format(advisory.id, group.id) + upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0] + upstream_version = group.fixed.split('-')[0].split('+')[0] + if ':' in upstream_version: + upstream_version = upstream_version[upstream_version.index(':') + 1:] + unique_issue_types = [] + for issue in severity_sorted_issues: + if issue.issue_type not in unique_issue_types: + unique_issue_types.append(issue.issue_type) + + references = [] + if group.bug_ticket: + references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket)) + references.extend([ref for ref in multiline_to_list(group.reference) + if ref not in references]) + list(map(lambda issue: references.extend( + [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues)) + + raw_asa = render_template('advisory.txt', + advisory=advisory, + group=group, + package=package, + issues=issues, + remote=remote, + issue_listing_formatted=issue_listing_formatted, + link=link, + workaround=advisory.workaround, + impact=advisory.impact, + upstream_released=upstream_released, + upstream_version=upstream_version, + unique_issue_types=unique_issue_types, + references=references, + with_subject=with_subject, + TRACKER_ISSUE_URL=TRACKER_ISSUE_URL, + TRACKER_GROUP_URL=TRACKER_GROUP_URL) + if raw: + return raw_asa + + raw_asa = '\n'.join(raw_asa.split('\n')[2:]) + raw_asa = str(html_escape(raw_asa)) + raw_asa = advisory_extend_html(raw_asa, issues, package) + return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True) + + +def render_html_advisory(advisory, package, group, raw_asa, generated): + return render_template('advisory.html', + title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type), + advisory=advisory, + package=package, + raw_asa=raw_asa, + generated=generated, + can_handle_advisory=user_can_handle_advisory(), + Publication=Publication) def advisory_fetch_from_mailman(url): @@ -19,12 +112,8 @@ def advisory_fetch_from_mailman(url): response = get(url) if 200 != response.status_code: return None - asa = unescape(sub('', '', response.text)) - start = '
'
-        start_marker = '{}Arch Linux Security Advisory'.format(start)
-        end = '\n-------------- next part --------------'
-        asa = asa[asa.index(start_marker) + len(start):asa.index(end)]
-        return asa.strip()
+
+        return response.text
     except Exception:
         return None
 
@@ -33,18 +122,29 @@ def advisory_fetch_reference_url_from_mailman(advisory):
     try:
         year = advisory.id[4:8]
         month = advisory.id[8:10]
-        publish_date = date(int(year), int(month), 1)
-        mailman_url = '{}{}-{}/'.format(TRACKER_MAILMAN_URL, year, publish_date.strftime('%B'))
-        response = get(mailman_url)
+        mailman_monthly = '{}{}/{}/?count=100'.format(TRACKER_MAILMAN_URL, year, month)
+
+        response = get(mailman_monthly)
         if 200 != response.status_code:
             return None
+
+        mailman_url = urlparse(TRACKER_MAILMAN_URL)
+        thread_url_base = join(mailman_url.path, 'thread')
+
+        message_url = None
         for line in response.text.split('\n'):
+            if thread_url_base in line:
+                match = search(r'href="{}/([/a-zA-Z0-9]+)"'.format(thread_url_base), line)
+                if not match:
+                    continue
+
+                thread = match.group(1)
+                message_url = join(TRACKER_MAILMAN_URL, 'message', thread)
+
             if not '[{}]'.format(advisory.id) in line:
                 continue
-            match = search(r'HREF="(\d+.html)"', line)
-            if not match:
-                continue
-            return '{}{}'.format(mailman_url, match.group(1))
+
+            return message_url
         return None
     except Exception:
         return None
@@ -96,14 +196,6 @@ def advisory_extend_html(advisory, issues, package):
     return advisory
 
 
-def advisory_extend_model_from_advisory_text(advisory):
-    if not advisory.content:
-        return advisory
-    advisory.impact = advisory_get_impact_from_text(advisory.content)
-    advisory.workaround = advisory_get_workaround_from_text(advisory.content)
-    return advisory
-
-
 def advisory_get_date_label(utctimetuple=None):
     now = utctimetuple if utctimetuple else datetime.utcnow().utctimetuple()
     return '{}{:02}'.format(now.tm_year, now.tm_mon)
diff --git a/tracker/form/validators.py b/tracker/form/validators.py
index ee29c19..4b6855c 100644
--- a/tracker/form/validators.py
+++ b/tracker/form/validators.py
@@ -6,6 +6,7 @@
 
 from tracker import db
 from tracker.advisory import advisory_fetch_from_mailman
+from tracker.advisory import generate_advisory
 from tracker.model import Package
 from tracker.model.advisory import advisory_regex
 from tracker.model.cve import cve_id_regex
@@ -21,11 +22,11 @@ def __call__(self, form, field):
         if not field.data:
             return
 
-        form.advisory_content = advisory_fetch_from_mailman(field.data)
-        if not form.advisory_content:
+        mailman_content = advisory_fetch_from_mailman(field.data)
+        if not mailman_content:
             raise ValidationError('Failed to fetch advisory')
 
-        m = search(advisory_regex[1:-1], form.advisory_content)
+        m = search(advisory_regex[1:-1], mailman_content)
         if not m:
             raise ValidationError('Failed to fetch advisory')
 
@@ -33,6 +34,8 @@ def __call__(self, form, field):
         if found != form.advisory_id:
             raise ValidationError('Advisory mismatched: {}'.format(found))
 
+        form.advisory_content = generate_advisory(advisory_id=form.advisory_id, with_subject=False, raw=True)
+
 
 class ValidPackageName(object):
     def __init__(self):
diff --git a/tracker/templates/advisory.txt b/tracker/templates/advisory.txt
index af3c2ef..811145b 100644
--- a/tracker/templates/advisory.txt
+++ b/tracker/templates/advisory.txt
@@ -1,5 +1,6 @@
-Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
+{%- if with_subject %}Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }}
 
+{% endif -%}
 {% set asa_title = 'Arch Linux Security Advisory ' + advisory.id %}
 {{- asa_title }}
 {% set asa_title_separator = '=' * asa_title|length %}
diff --git a/tracker/view/advisory.py b/tracker/view/advisory.py
index 640642c..8eb8c16 100644
--- a/tracker/view/advisory.py
+++ b/tracker/view/advisory.py
@@ -13,7 +13,6 @@
 from config import TRACKER_ISSUE_URL
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.advisory import advisory_get_date_label
 from tracker.advisory import advisory_get_label
@@ -219,7 +218,6 @@ def publish_advisory(asa):
 
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data
     advisory.publication = Publication.published
     db.session.commit()
diff --git a/tracker/view/edit.py b/tracker/view/edit.py
index 8de1747..31ba617 100644
--- a/tracker/view/edit.py
+++ b/tracker/view/edit.py
@@ -13,7 +13,6 @@
 
 from tracker import db
 from tracker import tracker
-from tracker.advisory import advisory_extend_model_from_advisory_text
 from tracker.advisory import advisory_fetch_reference_url_from_mailman
 from tracker.form import CVEForm
 from tracker.form import GroupForm
@@ -114,7 +113,6 @@ def edit_advisory(advisory_id):
     advisory.workaround = form.workaround.data or None
     if advisory.reference != form.reference.data:
         advisory.content = form.advisory_content
-        advisory_extend_model_from_advisory_text(advisory)
     advisory.reference = form.reference.data or None
 
     # update changed date on modification
diff --git a/tracker/view/show.py b/tracker/view/show.py
index 26e5b85..fd555df 100644
--- a/tracker/view/show.py
+++ b/tracker/view/show.py
@@ -20,6 +20,8 @@
 from tracker.advisory import advisory_escape_html
 from tracker.advisory import advisory_extend_html
 from tracker.advisory import advisory_format_issue_listing
+from tracker.advisory import generate_advisory
+from tracker.advisory import render_html_advisory
 from tracker.form.advisory import AdvisoryForm
 from tracker.model import CVE
 from tracker.model import Advisory
@@ -488,17 +490,6 @@ def show_package(pkgname):
                            package=data)
 
 
-def render_html_advisory(advisory, package, group, raw_asa, generated):
-    return render_template('advisory.html',
-                           title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type),
-                           advisory=advisory,
-                           package=package,
-                           raw_asa=raw_asa,
-                           generated=generated,
-                           can_handle_advisory=user_can_handle_advisory(),
-                           Publication=Publication)
-
-
 @tracker.route('/advisory//raw'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('//raw'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_advisory_raw(advisory_id):
@@ -554,67 +545,10 @@ def show_advisory(advisory_id, raw=False):
 @tracker.route('/advisory//generate'.format(advisory_regex[1:-1]), methods=['GET'])
 @tracker.route('//generate'.format(advisory_regex[1:-1]), methods=['GET'])
 def show_generated_advisory(advisory_id, raw=False):
-    entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE)
-               .filter(Advisory.id == advisory_id)
-               .join(CVEGroupPackage, Advisory.group_package)
-               .join(CVEGroup, CVEGroupPackage.group)
-               .join(CVEGroupEntry, CVEGroup.issues)
-               .join(CVE, CVEGroupEntry.cve)
-               .order_by(CVE.id)
-               ).all()
-    if not entries:
+    advisory = generate_advisory(advisory_id, with_subject=True, raw=raw)
+    if not advisory:
         return not_found()
-
-    advisory = entries[0][0]
-    group = entries[0][1]
-    package = entries[0][2]
-    issues = sorted([issue for (advisory, group, package, issue) in entries])
-    severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type)
-    severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity)
-    remote = any([issue.remote is Remote.remote for issue in issues])
-    issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues])
-
-    link = TRACKER_ADVISORY_URL.format(advisory.id, group.id)
-    upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0]
-    upstream_version = group.fixed.split('-')[0].split('+')[0]
-    if ':' in upstream_version:
-        upstream_version = upstream_version[upstream_version.index(':') + 1:]
-    unique_issue_types = []
-    for issue in severity_sorted_issues:
-        if issue.issue_type not in unique_issue_types:
-            unique_issue_types.append(issue.issue_type)
-
-    references = []
-    if group.bug_ticket:
-        references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket))
-    references.extend([ref for ref in multiline_to_list(group.reference)
-                       if ref not in references])
-    list(map(lambda issue: references.extend(
-        [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues))
-
-    raw_asa = render_template('advisory.txt',
-                              advisory=advisory,
-                              group=group,
-                              package=package,
-                              issues=issues,
-                              remote=remote,
-                              issue_listing_formatted=issue_listing_formatted,
-                              link=link,
-                              workaround=advisory.workaround,
-                              impact=advisory.impact,
-                              upstream_released=upstream_released,
-                              upstream_version=upstream_version,
-                              unique_issue_types=unique_issue_types,
-                              references=references,
-                              TRACKER_ISSUE_URL=TRACKER_ISSUE_URL,
-                              TRACKER_GROUP_URL=TRACKER_GROUP_URL)
-    if raw:
-        return raw_asa
-
-    raw_asa = '\n'.join(raw_asa.split('\n')[2:])
-    raw_asa = str(escape(raw_asa))
-    raw_asa = advisory_extend_html(raw_asa, issues, package)
-    return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True)
+    return advisory
 
 
 @tracker.route('/advisory//log'.format(advisory_regex[1:-1]), methods=['GET'])