From 729a1a85b7b6d6dd2f6ddc89c7cf5998b1c2f01d Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Thu, 29 Aug 2024 08:56:14 -0600 Subject: [PATCH] cli: link to rule names to capa rules website (#2338) * web: rules: redirect from various rule names to canonical rule URL closes #2319 Update index.html Co-authored-by: Moritz * cli: link to rule names to capa rules website * just: make `just lint` run all steps, not fail on first error --------- Co-authored-by: Moritz --- .justfile | 9 +- CHANGELOG.md | 2 +- capa/render/default.py | 228 +++++++++++------- tests/test_render.py | 9 +- .../src/components/RuleMatchesTable.vue | 2 +- web/explorer/src/utils/urlHelpers.js | 3 +- web/public/index.html | 4 +- web/rules/.gitignore | 1 + web/rules/scripts/build_root.py | 3 +- web/rules/scripts/build_rules.py | 71 ++++-- 10 files changed, 219 insertions(+), 113 deletions(-) diff --git a/.justfile b/.justfile index 90dc01e25..91953dd1f 100644 --- a/.justfile +++ b/.justfile @@ -16,5 +16,10 @@ @deptry: pre-commit run deptry --hook-stage manual --all-files -lint: isort black ruff flake8 mypy deptry - +@lint: + -just isort + -just black + -just ruff + -just flake8 + -just mypy + -just deptry diff --git a/CHANGELOG.md b/CHANGELOG.md index 954deda3f..22a599a52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,11 @@ Unlock powerful malware analysis with capa's new [VMRay sandbox](https://www.vmr ### New Features - regenerate ruleset cache automatically on source change (only in dev mode) #2133 @s-ff - - add landing page https://mandiant.github.io/capa/ @williballenthin #2310 - add rules website https://mandiant.github.io/capa/rules @DeeyaSingh #2310 - add .justfile @williballenthin #2325 - dynamic: add support for VMRay dynamic sandbox traces #2208 @mike-hunhoff @r-sm2024 @mr-tz +- cli: use modern terminal features to hyperlink to the rules website #2337 @williballenthin ### Breaking Changes diff --git a/capa/render/default.py b/capa/render/default.py index e49a31e3c..6629d6a78 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -6,18 +6,43 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import io import collections +import urllib.parse -import tabulate +import rich +import rich.table +import rich.console +from rich.console import Console import capa.render.utils as rutils import capa.render.result_document as rd import capa.features.freeze.features as frzf from capa.rules import RuleSet from capa.engine import MatchResults -from capa.render.utils import StringIO -tabulate.PRESERVE_WHITESPACE = True + +def bold_markup(s) -> str: + """ + Generate Rich markup in a bold style. + + The resulting string should be passed to a Rich renderable + and/or printed via Rich or the markup will be visible to the user. + """ + return f"[cyan]{s}[/cyan]" + + +def link_markup(s: str, href: str) -> str: + """ + Generate Rich markup for a clickable hyperlink. + This works in many modern terminals. + When it doesn't work, the fallback is just to show the link name (s), + as if it was not a link. + + The resulting string should be passed to a Rich renderable + and/or printed via Rich or the markup will be visible to the user. + """ + return f"[link={href}]{s}[/link]" def width(s: str, character_count: int) -> str: @@ -28,11 +53,16 @@ def width(s: str, character_count: int) -> str: return s -def render_meta(doc: rd.ResultDocument, ostream: StringIO): +def render_sample_link(hash: str) -> str: + url = "https://www.virustotal.com/gui/file/" + hash + return link_markup(hash, url) + + +def render_meta(doc: rd.ResultDocument, console: Console): rows = [ - (width("md5", 22), width(doc.meta.sample.md5, 82)), - ("sha1", doc.meta.sample.sha1), - ("sha256", doc.meta.sample.sha256), + ("md5", render_sample_link(doc.meta.sample.md5)), + ("sha1", render_sample_link(doc.meta.sample.sha1)), + ("sha256", render_sample_link(doc.meta.sample.sha256)), ("analysis", doc.meta.flavor.value), ("os", doc.meta.analysis.os), ("format", doc.meta.analysis.format), @@ -40,8 +70,14 @@ def render_meta(doc: rd.ResultDocument, ostream: StringIO): ("path", doc.meta.sample.path), ] - ostream.write(tabulate.tabulate(rows, tablefmt="mixed_outline")) - ostream.write("\n") + table = rich.table.Table(show_header=False, min_width=100) + table.add_column() + table.add_column() + + for row in rows: + table.add_row(*row) + + console.print(table) def find_subrule_matches(doc: rd.ResultDocument): @@ -71,7 +107,12 @@ def rec(match: rd.Match): return matches -def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +def render_rule_name(name: str) -> str: + url = f"https://mandiant.github.io/capa/rules/{urllib.parse.quote(name)}/" + return bold_markup(link_markup(name, url)) + + +def render_capabilities(doc: rd.ResultDocument, console: Console): """ example:: @@ -95,25 +136,30 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): count = len(rule.matches) if count == 1: - capability = rutils.bold(rule.meta.name) + capability = render_rule_name(rule.meta.name) else: - capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" + capability = render_rule_name(rule.meta.name) + f" ({count} matches)" rows.append((capability, rule.meta.namespace)) if rows: - ostream.write( - tabulate.tabulate( - rows, - headers=[width("Capability", 50), width("Namespace", 50)], - tablefmt="mixed_outline", - ) - ) - ostream.write("\n") + table = rich.table.Table(min_width=100) + table.add_column(width("Capability", 20)) + table.add_column("Namespace") + + for row in rows: + table.add_row(*row) + + console.print(table) else: - ostream.writeln(rutils.bold("no capabilities found")) + console.print(bold_markup("no capabilities found")) + + +def render_attack_link(id: str) -> str: + url = f"https://attack.mitre.org/techniques/{id.replace('.', '/')}/" + return rf"\[{link_markup(id, url)}]" -def render_attack(doc: rd.ResultDocument, ostream: StringIO): +def render_attack(doc: rd.ResultDocument, console: Console): """ example:: @@ -132,35 +178,36 @@ def render_attack(doc: rd.ResultDocument, ostream: StringIO): tactics = collections.defaultdict(set) for rule in rutils.capability_rules(doc): for attack in rule.meta.attack: - tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id)) + tactics[attack.tactic].add((attack.technique, attack.subtechnique, attack.id.strip("[").strip("]"))) rows = [] for tactic, techniques in sorted(tactics.items()): inner_rows = [] for technique, subtechnique, id in sorted(techniques): if not subtechnique: - inner_rows.append(f"{rutils.bold(technique)} {id}") + # example: File and Directory Discovery [T1083] + inner_rows.append(f"{bold_markup(technique)} {render_attack_link(id)}") else: - inner_rows.append(f"{rutils.bold(technique)}::{subtechnique} {id}") - rows.append( - ( - rutils.bold(tactic.upper()), - "\n".join(inner_rows), - ) - ) + # example: Code Discovery::Enumerate PE Sections [T1084.001] + inner_rows.append(f"{bold_markup(technique)}::{subtechnique} {render_attack_link(id)}") + + tactic = bold_markup(tactic.upper()) + technique = "\n".join(inner_rows) + + rows.append((tactic, technique)) if rows: - ostream.write( - tabulate.tabulate( - rows, - headers=[width("ATT&CK Tactic", 20), width("ATT&CK Technique", 80)], - tablefmt="mixed_grid", - ) - ) - ostream.write("\n") + table = rich.table.Table(min_width=100) + table.add_column(width("ATT&CK Tactic", 20)) + table.add_column("ATT&CK Technique") + + for row in rows: + table.add_row(*row) + console.print(table) -def render_maec(doc: rd.ResultDocument, ostream: StringIO): + +def render_maec(doc: rd.ResultDocument, console: Console): """ example:: @@ -193,20 +240,37 @@ def render_maec(doc: rd.ResultDocument, ostream: StringIO): for category in sorted(maec_categories): values = maec_table.get(category, set()) if values: - rows.append((rutils.bold(category.replace("_", "-")), "\n".join(sorted(values)))) + rows.append((bold_markup(category.replace("_", "-")), "\n".join(sorted(values)))) if rows: - ostream.write( - tabulate.tabulate( - rows, - headers=[width("MAEC Category", 25), width("MAEC Value", 75)], - tablefmt="mixed_grid", - ) - ) - ostream.write("\n") + table = rich.table.Table(min_width=100) + table.add_column(width("MAEC Category", 20)) + table.add_column("MAEC Value") + + for row in rows: + table.add_row(*row) + + console.print(table) + + +def render_mbc_link(id: str, objective: str, behavior: str) -> str: + if id[0] in {"B", "T", "E", "F"}: + # behavior + base_url = "https://github.com/MBCProject/mbc-markdown/blob/main" + elif id[0] == "C": + # micro-behavior + base_url = "https://github.com/MBCProject/mbc-markdown/blob/main/micro-behaviors" + else: + raise ValueError("unexpected MBC prefix") + objective_fragment = objective.lower().replace(" ", "-") + behavior_fragment = behavior.lower().replace(" ", "-") -def render_mbc(doc: rd.ResultDocument, ostream: StringIO): + url = f"{base_url}/{objective_fragment}/{behavior_fragment}.md" + return rf"\[{link_markup(id, url)}]" + + +def render_mbc(doc: rd.ResultDocument, console: Console): """ example:: @@ -223,48 +287,48 @@ def render_mbc(doc: rd.ResultDocument, ostream: StringIO): objectives = collections.defaultdict(set) for rule in rutils.capability_rules(doc): for mbc in rule.meta.mbc: - objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id)) + objectives[mbc.objective].add((mbc.behavior, mbc.method, mbc.id.strip("[").strip("]"))) rows = [] for objective, behaviors in sorted(objectives.items()): inner_rows = [] - for behavior, method, id in sorted(behaviors): - if not method: - inner_rows.append(f"{rutils.bold(behavior)} [{id}]") + for technique, subtechnique, id in sorted(behaviors): + if not subtechnique: + # example: File and Directory Discovery [T1083] + inner_rows.append(f"{bold_markup(technique)} {render_mbc_link(id, objective, technique)}") else: - inner_rows.append(f"{rutils.bold(behavior)}::{method} [{id}]") - rows.append( - ( - rutils.bold(objective.upper()), - "\n".join(inner_rows), - ) - ) + # example: Code Discovery::Enumerate PE Sections [T1084.001] + inner_rows.append( + f"{bold_markup(technique)}::{subtechnique} {render_mbc_link(id, objective, technique)}" + ) + + objective = bold_markup(objective.upper()) + technique = "\n".join(inner_rows) + + rows.append((objective, technique)) if rows: - ostream.write( - tabulate.tabulate( - rows, - headers=[width("MBC Objective", 25), width("MBC Behavior", 75)], - tablefmt="mixed_grid", - ) - ) - ostream.write("\n") + table = rich.table.Table(min_width=100) + table.add_column(width("MBC Objective", 20)) + table.add_column("MBC Behavior") + + for row in rows: + table.add_row(*row) + + console.print(table) def render_default(doc: rd.ResultDocument): - ostream = rutils.StringIO() - - render_meta(doc, ostream) - ostream.write("\n") - render_attack(doc, ostream) - ostream.write("\n") - render_maec(doc, ostream) - ostream.write("\n") - render_mbc(doc, ostream) - ostream.write("\n") - render_capabilities(doc, ostream) - - return ostream.getvalue() + f = io.StringIO() + console = rich.console.Console() + + render_meta(doc, console) + render_attack(doc, console) + render_maec(doc, console) + render_mbc(doc, console) + render_capabilities(doc, console) + + return f.getvalue() def render(meta, rules: RuleSet, capabilities: MatchResults) -> str: diff --git a/tests/test_render.py b/tests/test_render.py index e56d013c5..7f896949c 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -5,10 +5,12 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import io import textwrap from unittest.mock import Mock import fixtures +import rich.console import capa.rules import capa.render.utils @@ -151,9 +153,10 @@ def test_render_meta_maec(): mock_rd.rules = {"test rule": rm} # capture the output of render_maec - output_stream = capa.render.utils.StringIO() - capa.render.default.render_maec(mock_rd, output_stream) - output = output_stream.getvalue() + f = io.StringIO() + console = rich.console.Console(file=f) + capa.render.default.render_maec(mock_rd, console) + output = f.getvalue() assert "analysis-conclusion" in output assert analysis_conclusion in output diff --git a/web/explorer/src/components/RuleMatchesTable.vue b/web/explorer/src/components/RuleMatchesTable.vue index 61c176e09..91718c0a5 100644 --- a/web/explorer/src/components/RuleMatchesTable.vue +++ b/web/explorer/src/components/RuleMatchesTable.vue @@ -235,7 +235,7 @@ const contextMenuItems = computed(() => [ label: "View rule in capa-rules", icon: "pi pi-external-link", target: "_blank", - url: createCapaRulesUrl(selectedNode.value, props.data.meta.version) + url: createCapaRulesUrl(selectedNode.value) }, { label: "Lookup rule in VirusTotal", diff --git a/web/explorer/src/utils/urlHelpers.js b/web/explorer/src/utils/urlHelpers.js index 6ab0717b5..42639c9ff 100644 --- a/web/explorer/src/utils/urlHelpers.js +++ b/web/explorer/src/utils/urlHelpers.js @@ -62,9 +62,8 @@ export function createATTACKHref(attack) { */ export function createCapaRulesUrl(node, tag) { if (!node || !node.data || !tag) return null; - const namespace = node.data.namespace || "lib"; const ruleName = node.data.name.toLowerCase().replace(/\s+/g, "-"); - return `https://github.com/mandiant/capa-rules/blob/v${tag}/${namespace}/${ruleName}.yml`; + return `https://mandiant.github.io/capa/rules/${ruleName}/`; } /** diff --git a/web/public/index.html b/web/public/index.html index 68fa4dfef..1f7362e46 100644 --- a/web/public/index.html +++ b/web/public/index.html @@ -200,14 +200,14 @@

Rule Updates

  • added: - + overwrite DLL .text section to remove hooks
  • added: - + attach BPF to socket on Linux
    • diff --git a/web/rules/.gitignore b/web/rules/.gitignore index af0b0688f..075de1382 100644 --- a/web/rules/.gitignore +++ b/web/rules/.gitignore @@ -10,3 +10,4 @@ file_modification_dates.txt public/*.html public/pagefind/ public/index.html +public/ diff --git a/web/rules/scripts/build_root.py b/web/rules/scripts/build_root.py index 67f1f6f3b..aefd8d882 100644 --- a/web/rules/scripts/build_root.py +++ b/web/rules/scripts/build_root.py @@ -259,7 +259,6 @@ def generate_html(categories_data, color_map): for card in cards_data: first_word = get_first_word(card["namespace"]) rectangle_color = color_map[first_word] - file_name = card["filename"].rpartition(".yml")[0] card_html = f"""
    @@ -267,7 +266,7 @@ def generate_html(categories_data, color_map):
    {card['namespace']}
    -
    {card['name']}
    +
    {card['name']}
    {', '.join(card['authors'])}
    diff --git a/web/rules/scripts/build_rules.py b/web/rules/scripts/build_rules.py index 4aaff906a..32316ad73 100644 --- a/web/rules/scripts/build_rules.py +++ b/web/rules/scripts/build_rules.py @@ -10,6 +10,7 @@ import os import sys +import logging import urllib.parse from glob import glob from pathlib import Path @@ -20,6 +21,9 @@ import capa.rules +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + input_directory = Path(sys.argv[1]) txt_file_path = Path(sys.argv[2]) output_directory = Path(sys.argv[3]) @@ -29,13 +33,13 @@ assert output_directory.exists(), "output directory must exist" -def convert_yaml_to_html(timestamps, yaml_file: Path, output_dir: Path): - rule_content = yaml_file.read_text(encoding="utf-8") +def render_rule(timestamps, path: Path) -> str: + rule_content = path.read_text(encoding="utf-8") rule = capa.rules.Rule.from_yaml(rule_content, use_ruamel=True) - filename = os.path.basename(yaml_file).rpartition(".yml")[0] + filename = path.with_suffix("").name namespace = rule.meta.get("namespace", "") - timestamp = timestamps[yaml_file.as_posix()] + timestamp = timestamps[path.as_posix()] rendered_rule = pygments.highlight( rule_content, @@ -53,7 +57,7 @@ def convert_yaml_to_html(timestamps, yaml_file: Path, output_dir: Path): vt_fragment = urllib.parse.quote(urllib.parse.quote(vt_query)) vt_link = f"https://www.virustotal.com/gui/search/{vt_fragment}/files" ns_query = f'"namespace: {namespace} "' - ns_link = f"./?{urllib.parse.urlencode({'q': ns_query})}" + ns_link = f"../?{urllib.parse.urlencode({'q': ns_query})}" html_content = f""" @@ -62,12 +66,12 @@ def convert_yaml_to_html(timestamps, yaml_file: Path, output_dir: Path): {rule.name} - - - - - - + + + + + +