diff --git a/config/00-default.conf b/config/00-default.conf index ae01afc..8849a41 100644 --- a/config/00-default.conf +++ b/config/00-default.conf @@ -3,7 +3,7 @@ advisory_url = https://security.archlinux.org/AVG-{1} group_url = https://security.archlinux.org/AVG-{0} issue_url = https://security.archlinux.org/{0} bugtracker_url = https://bugs.archlinux.org/task/{0} -mailman_url = https://lists.archlinux.org/pipermail/arch-security/ +mailman_url = https://lists.archlinux.org/archives/list/arch-security@lists.archlinux.org/ password_length_min = 16 password_length_max = 64 summary_length_max = 200 diff --git a/tracker/advisory.py b/tracker/advisory.py index 7ffa0fb..0207d9e 100644 --- a/tracker/advisory.py +++ b/tracker/advisory.py @@ -1,17 +1,110 @@ -from datetime import date from datetime import datetime from html import unescape +from os.path import join from re import IGNORECASE from re import escape from re import search from re import sub +from urllib.parse import urlparse +from flask import render_template from markupsafe import escape as html_escape from requests import get +from config import TRACKER_ADVISORY_URL +from config import TRACKER_BUGTRACKER_URL +from config import TRACKER_GROUP_URL +from config import TRACKER_ISSUE_URL from config import TRACKER_MAILMAN_URL +from tracker import db +from tracker.model import CVE +from tracker.model import Advisory +from tracker.model import CVEGroup +from tracker.model import CVEGroupEntry +from tracker.model import CVEGroupPackage +from tracker.model import Package +from tracker.model.enum import Publication +from tracker.model.enum import Remote +from tracker.user import user_can_handle_advisory from tracker.util import chunks from tracker.util import issue_to_numeric +from tracker.util import multiline_to_list + + +def generate_advisory(advisory_id, with_subject=True, raw=True): + entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE) + .filter(Advisory.id == advisory_id) + .join(CVEGroupPackage, Advisory.group_package) + .join(CVEGroup, CVEGroupPackage.group) + .join(CVEGroupEntry, CVEGroup.issues) + .join(CVE, CVEGroupEntry.cve) + .order_by(CVE.id) + ).all() + if not entries: + return None + + advisory = entries[0][0] + group = entries[0][1] + package = entries[0][2] + issues = sorted([issue for (advisory, group, package, issue) in entries]) + severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type) + severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity) + remote = any([issue.remote is Remote.remote for issue in issues]) + issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues]) + + link = TRACKER_ADVISORY_URL.format(advisory.id, group.id) + upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0] + upstream_version = group.fixed.split('-')[0].split('+')[0] + if ':' in upstream_version: + upstream_version = upstream_version[upstream_version.index(':') + 1:] + unique_issue_types = [] + for issue in severity_sorted_issues: + if issue.issue_type not in unique_issue_types: + unique_issue_types.append(issue.issue_type) + + references = [] + if group.bug_ticket: + references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket)) + references.extend([ref for ref in multiline_to_list(group.reference) + if ref not in references]) + list(map(lambda issue: references.extend( + [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues)) + + raw_asa = render_template('advisory.txt', + advisory=advisory, + group=group, + package=package, + issues=issues, + remote=remote, + issue_listing_formatted=issue_listing_formatted, + link=link, + workaround=advisory.workaround, + impact=advisory.impact, + upstream_released=upstream_released, + upstream_version=upstream_version, + unique_issue_types=unique_issue_types, + references=references, + with_subject=with_subject, + TRACKER_ISSUE_URL=TRACKER_ISSUE_URL, + TRACKER_GROUP_URL=TRACKER_GROUP_URL) + if raw: + return raw_asa + + raw_asa = '\n'.join(raw_asa.split('\n')[2:]) + raw_asa = str(html_escape(raw_asa)) + raw_asa = advisory_extend_html(raw_asa, issues, package) + return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True) + + +def render_html_advisory(advisory, package, group, raw_asa, generated): + return render_template('advisory.html', + title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type), + advisory=advisory, + package=package, + raw_asa=raw_asa, + generated=generated, + can_handle_advisory=user_can_handle_advisory(), + Publication=Publication) def advisory_fetch_from_mailman(url): @@ -19,12 +112,8 @@ def advisory_fetch_from_mailman(url): response = get(url) if 200 != response.status_code: return None - asa = unescape(sub('?A[^<]*?>', '', response.text)) - start = '
' - start_marker = '{}Arch Linux Security Advisory'.format(start) - end = '\n-------------- next part --------------' - asa = asa[asa.index(start_marker) + len(start):asa.index(end)] - return asa.strip() + + return response.text except Exception: return None @@ -33,18 +122,29 @@ def advisory_fetch_reference_url_from_mailman(advisory): try: year = advisory.id[4:8] month = advisory.id[8:10] - publish_date = date(int(year), int(month), 1) - mailman_url = '{}{}-{}/'.format(TRACKER_MAILMAN_URL, year, publish_date.strftime('%B')) - response = get(mailman_url) + mailman_monthly = '{}{}/{}/?count=100'.format(TRACKER_MAILMAN_URL, year, month) + + response = get(mailman_monthly) if 200 != response.status_code: return None + + mailman_url = urlparse(TRACKER_MAILMAN_URL) + thread_url_base = join(mailman_url.path, 'thread') + + message_url = None for line in response.text.split('\n'): + if thread_url_base in line: + match = search(r'href="{}/([/a-zA-Z0-9]+)"'.format(thread_url_base), line) + if not match: + continue + + thread = match.group(1) + message_url = join(TRACKER_MAILMAN_URL, 'message', thread) + if not '[{}]'.format(advisory.id) in line: continue - match = search(r'HREF="(\d+.html)"', line) - if not match: - continue - return '{}{}'.format(mailman_url, match.group(1)) + + return message_url return None except Exception: return None @@ -96,14 +196,6 @@ def advisory_extend_html(advisory, issues, package): return advisory -def advisory_extend_model_from_advisory_text(advisory): - if not advisory.content: - return advisory - advisory.impact = advisory_get_impact_from_text(advisory.content) - advisory.workaround = advisory_get_workaround_from_text(advisory.content) - return advisory - - def advisory_get_date_label(utctimetuple=None): now = utctimetuple if utctimetuple else datetime.utcnow().utctimetuple() return '{}{:02}'.format(now.tm_year, now.tm_mon) diff --git a/tracker/form/validators.py b/tracker/form/validators.py index ee29c19..4b6855c 100644 --- a/tracker/form/validators.py +++ b/tracker/form/validators.py @@ -6,6 +6,7 @@ from tracker import db from tracker.advisory import advisory_fetch_from_mailman +from tracker.advisory import generate_advisory from tracker.model import Package from tracker.model.advisory import advisory_regex from tracker.model.cve import cve_id_regex @@ -21,11 +22,11 @@ def __call__(self, form, field): if not field.data: return - form.advisory_content = advisory_fetch_from_mailman(field.data) - if not form.advisory_content: + mailman_content = advisory_fetch_from_mailman(field.data) + if not mailman_content: raise ValidationError('Failed to fetch advisory') - m = search(advisory_regex[1:-1], form.advisory_content) + m = search(advisory_regex[1:-1], mailman_content) if not m: raise ValidationError('Failed to fetch advisory') @@ -33,6 +34,8 @@ def __call__(self, form, field): if found != form.advisory_id: raise ValidationError('Advisory mismatched: {}'.format(found)) + form.advisory_content = generate_advisory(advisory_id=form.advisory_id, with_subject=False, raw=True) + class ValidPackageName(object): def __init__(self): diff --git a/tracker/templates/advisory.txt b/tracker/templates/advisory.txt index af3c2ef..811145b 100644 --- a/tracker/templates/advisory.txt +++ b/tracker/templates/advisory.txt @@ -1,5 +1,6 @@ -Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }} +{%- if with_subject %}Subject: [{{ advisory.id }}] {{ package.pkgname }}: {{ advisory.advisory_type }} +{% endif -%} {% set asa_title = 'Arch Linux Security Advisory ' + advisory.id %} {{- asa_title }} {% set asa_title_separator = '=' * asa_title|length %} diff --git a/tracker/view/advisory.py b/tracker/view/advisory.py index 640642c..8eb8c16 100644 --- a/tracker/view/advisory.py +++ b/tracker/view/advisory.py @@ -13,7 +13,6 @@ from config import TRACKER_ISSUE_URL from tracker import db from tracker import tracker -from tracker.advisory import advisory_extend_model_from_advisory_text from tracker.advisory import advisory_fetch_reference_url_from_mailman from tracker.advisory import advisory_get_date_label from tracker.advisory import advisory_get_label @@ -219,7 +218,6 @@ def publish_advisory(asa): if advisory.reference != form.reference.data: advisory.content = form.advisory_content - advisory_extend_model_from_advisory_text(advisory) advisory.reference = form.reference.data advisory.publication = Publication.published db.session.commit() diff --git a/tracker/view/edit.py b/tracker/view/edit.py index 8de1747..31ba617 100644 --- a/tracker/view/edit.py +++ b/tracker/view/edit.py @@ -13,7 +13,6 @@ from tracker import db from tracker import tracker -from tracker.advisory import advisory_extend_model_from_advisory_text from tracker.advisory import advisory_fetch_reference_url_from_mailman from tracker.form import CVEForm from tracker.form import GroupForm @@ -114,7 +113,6 @@ def edit_advisory(advisory_id): advisory.workaround = form.workaround.data or None if advisory.reference != form.reference.data: advisory.content = form.advisory_content - advisory_extend_model_from_advisory_text(advisory) advisory.reference = form.reference.data or None # update changed date on modification diff --git a/tracker/view/show.py b/tracker/view/show.py index 26e5b85..fd555df 100644 --- a/tracker/view/show.py +++ b/tracker/view/show.py @@ -20,6 +20,8 @@ from tracker.advisory import advisory_escape_html from tracker.advisory import advisory_extend_html from tracker.advisory import advisory_format_issue_listing +from tracker.advisory import generate_advisory +from tracker.advisory import render_html_advisory from tracker.form.advisory import AdvisoryForm from tracker.model import CVE from tracker.model import Advisory @@ -488,17 +490,6 @@ def show_package(pkgname): package=data) -def render_html_advisory(advisory, package, group, raw_asa, generated): - return render_template('advisory.html', - title='[{}] {}: {}'.format(advisory.id, package.pkgname, advisory.advisory_type), - advisory=advisory, - package=package, - raw_asa=raw_asa, - generated=generated, - can_handle_advisory=user_can_handle_advisory(), - Publication=Publication) - - @tracker.route('/advisory//raw'.format(advisory_regex[1:-1]), methods=['GET']) @tracker.route('/ /raw'.format(advisory_regex[1:-1]), methods=['GET']) def show_advisory_raw(advisory_id): @@ -554,67 +545,10 @@ def show_advisory(advisory_id, raw=False): @tracker.route('/advisory/ /generate'.format(advisory_regex[1:-1]), methods=['GET']) @tracker.route('/ /generate'.format(advisory_regex[1:-1]), methods=['GET']) def show_generated_advisory(advisory_id, raw=False): - entries = (db.session.query(Advisory, CVEGroup, CVEGroupPackage, CVE) - .filter(Advisory.id == advisory_id) - .join(CVEGroupPackage, Advisory.group_package) - .join(CVEGroup, CVEGroupPackage.group) - .join(CVEGroupEntry, CVEGroup.issues) - .join(CVE, CVEGroupEntry.cve) - .order_by(CVE.id) - ).all() - if not entries: + advisory = generate_advisory(advisory_id, with_subject=True, raw=raw) + if not advisory: return not_found() - - advisory = entries[0][0] - group = entries[0][1] - package = entries[0][2] - issues = sorted([issue for (advisory, group, package, issue) in entries]) - severity_sorted_issues = sorted(issues, key=lambda issue: issue.issue_type) - severity_sorted_issues = sorted(severity_sorted_issues, key=lambda issue: issue.severity) - remote = any([issue.remote is Remote.remote for issue in issues]) - issue_listing_formatted = advisory_format_issue_listing([issue.id for issue in issues]) - - link = TRACKER_ADVISORY_URL.format(advisory.id, group.id) - upstream_released = group.affected.split('-')[0].split('+')[0] != group.fixed.split('-')[0].split('+')[0] - upstream_version = group.fixed.split('-')[0].split('+')[0] - if ':' in upstream_version: - upstream_version = upstream_version[upstream_version.index(':') + 1:] - unique_issue_types = [] - for issue in severity_sorted_issues: - if issue.issue_type not in unique_issue_types: - unique_issue_types.append(issue.issue_type) - - references = [] - if group.bug_ticket: - references.append(TRACKER_BUGTRACKER_URL.format(group.bug_ticket)) - references.extend([ref for ref in multiline_to_list(group.reference) - if ref not in references]) - list(map(lambda issue: references.extend( - [ref for ref in multiline_to_list(issue.reference) if ref not in references]), issues)) - - raw_asa = render_template('advisory.txt', - advisory=advisory, - group=group, - package=package, - issues=issues, - remote=remote, - issue_listing_formatted=issue_listing_formatted, - link=link, - workaround=advisory.workaround, - impact=advisory.impact, - upstream_released=upstream_released, - upstream_version=upstream_version, - unique_issue_types=unique_issue_types, - references=references, - TRACKER_ISSUE_URL=TRACKER_ISSUE_URL, - TRACKER_GROUP_URL=TRACKER_GROUP_URL) - if raw: - return raw_asa - - raw_asa = '\n'.join(raw_asa.split('\n')[2:]) - raw_asa = str(escape(raw_asa)) - raw_asa = advisory_extend_html(raw_asa, issues, package) - return render_html_advisory(advisory=advisory, package=package, group=group, raw_asa=raw_asa, generated=True) + return advisory @tracker.route('/advisory/ /log'.format(advisory_regex[1:-1]), methods=['GET'])