From 175417fcdae49562c7149f045c9c7aec19dff2a3 Mon Sep 17 00:00:00 2001 From: msm Date: Tue, 27 Dec 2022 23:04:38 +0100 Subject: [PATCH 1/8] Disallow degenerate queries --- src/app.py | 15 ++++++++++++++- src/mqueryfront/src/query/QueryParseStatus.js | 18 +++++++++++------- src/mqueryfront/src/status/BackendStatus.js | 2 +- src/schema.py | 1 + 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/app.py b/src/app.py index 13aa4183..1cb73ef4 100644 --- a/src/app.py +++ b/src/app.py @@ -326,7 +326,7 @@ def query( ) if not rules: - raise HTTPException(status_code=400, detail=f"No rule was specified.") + raise HTTPException(status_code=400, detail="No rule was specified.") if data.method == RequestQueryMethod.parse: return [ @@ -335,11 +335,24 @@ def query( rule_author=rule.author, is_global=rule.is_global, is_private=rule.is_private, + is_degenerate=rule.parse().is_degenerate, parsed=rule.parse().query, ) for rule in rules ] + degenerate_rules = [r.name for r in rules if r.parse().is_degenerate] + if degenerate_rules: + degenerate_rule_names = ", ".join(degenerate_rules) + doc_url = "https://cert-polska.github.io/mquery/docs/yara.html" + raise HTTPException(status_code=400, detail=( + "Invalid query. " + "Some of the rules would require a Yara scan of every indexed " + "file, and this is not allowed by this instance. " + f"Problematic rules: {degenerate_rule_names}. " + f"Read {doc_url} for more details." + )) + active_agents = db.get_active_agents() for agent, agent_spec in active_agents.items(): diff --git a/src/mqueryfront/src/query/QueryParseStatus.js b/src/mqueryfront/src/query/QueryParseStatus.js index 7566f5f6..bfb2533c 100644 --- a/src/mqueryfront/src/query/QueryParseStatus.js +++ b/src/mqueryfront/src/query/QueryParseStatus.js @@ -6,14 +6,18 @@ const QueryParseStatus = (props) => { if (!queryPlan) return null; const parseResult = queryPlan.map((rule) => { - const { is_private, is_global, rule_name, parsed } = rule; + const { is_private, is_global, is_degenerate, rule_name, parsed } = rule; - const badge = - is_private || is_global ? ( - - {is_private ? "private" : "global"} - + const private_badge =is_private ?( + private ) : null; + const global_badge =is_global?( + private + ) : null; + const degenerate_badge =is_degenerate?( + degenerate + ) : null; + const badges = <>{private_badge} {global_badge} {degenerate_badge}; return (
@@ -22,7 +26,7 @@ const QueryParseStatus = (props) => { {rule_name} - {badge} + {badges}
{parsed} diff --git a/src/mqueryfront/src/status/BackendStatus.js b/src/mqueryfront/src/status/BackendStatus.js index c64ec3af..76696465 100644 --- a/src/mqueryfront/src/status/BackendStatus.js +++ b/src/mqueryfront/src/status/BackendStatus.js @@ -32,7 +32,7 @@ class AgentStatus extends Component { let badge = null; if (!this.props.alive) { badge = ( - offline + offline ); } diff --git a/src/schema.py b/src/schema.py index b3de7c57..0f4a33a2 100644 --- a/src/schema.py +++ b/src/schema.py @@ -76,6 +76,7 @@ class ParseResponseSchema(BaseModel): rule_author: str is_global: bool is_private: bool + is_degenerate: bool parsed: str From 527aacd1043fc17df562e1445d693fd69d927651 Mon Sep 17 00:00:00 2001 From: msm Date: Thu, 29 Dec 2022 01:19:33 +0100 Subject: [PATCH 2/8] Disallow degenerate queries --- src/app.py | 20 +++++++----- src/db.py | 2 ++ src/mqueryfront/src/config/ConfigEntries.js | 1 + src/mqueryfront/src/query/QueryParseStatus.js | 32 ++++++++++++------- 4 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/app.py b/src/app.py index 1cb73ef4..d8e3f98a 100644 --- a/src/app.py +++ b/src/app.py @@ -342,16 +342,20 @@ def query( ] degenerate_rules = [r.name for r in rules if r.parse().is_degenerate] - if degenerate_rules: + disallow_degenerate = db.get_mquery_config_key("query_disallow_degenerate") + if degenerate_rules and (disallow_degenerate == "true"): degenerate_rule_names = ", ".join(degenerate_rules) doc_url = "https://cert-polska.github.io/mquery/docs/yara.html" - raise HTTPException(status_code=400, detail=( - "Invalid query. " - "Some of the rules would require a Yara scan of every indexed " - "file, and this is not allowed by this instance. " - f"Problematic rules: {degenerate_rule_names}. " - f"Read {doc_url} for more details." - )) + raise HTTPException( + status_code=400, + detail=( + "Invalid query. " + "Some of the rules would require a Yara scan of every indexed " + "file, and this is not allowed by this instance. " + f"Problematic rules: {degenerate_rule_names}. " + f"Read {doc_url} for more details." + ), + ) active_agents = db.get_active_agents() diff --git a/src/db.py b/src/db.py index e8760a84..39456d82 100644 --- a/src/db.py +++ b/src/db.py @@ -331,6 +331,8 @@ def get_core_config(self) -> Dict[str, str]: "openid_url": "OpenID Connect base url", "openid_client_id": "OpenID client ID", "openid_secret": "Secret used for JWT token verification", + # Query and performance config + "query_disallow_degenerate": "Reject any queries that will end up scanning the whole malware collection", } def get_config(self) -> List[ConfigSchema]: diff --git a/src/mqueryfront/src/config/ConfigEntries.js b/src/mqueryfront/src/config/ConfigEntries.js index beb6d551..0bda2607 100644 --- a/src/mqueryfront/src/config/ConfigEntries.js +++ b/src/mqueryfront/src/config/ConfigEntries.js @@ -10,6 +10,7 @@ const KNOWN_RULES = { openid_url: R_URL, auth_enabled: R_BOOL, auth_default_roles: R_ROLES, + query_disallow_degenerate: R_BOOL, }; class ConfigRow extends Component { diff --git a/src/mqueryfront/src/query/QueryParseStatus.js b/src/mqueryfront/src/query/QueryParseStatus.js index bfb2533c..cc9b7340 100644 --- a/src/mqueryfront/src/query/QueryParseStatus.js +++ b/src/mqueryfront/src/query/QueryParseStatus.js @@ -6,18 +6,28 @@ const QueryParseStatus = (props) => { if (!queryPlan) return null; const parseResult = queryPlan.map((rule) => { - const { is_private, is_global, is_degenerate, rule_name, parsed } = rule; + const { + is_private, + is_global, + is_degenerate, + rule_name, + parsed, + } = rule; - const private_badge =is_private ?( - private - ) : null; - const global_badge =is_global?( - private - ) : null; - const degenerate_badge =is_degenerate?( - degenerate - ) : null; - const badges = <>{private_badge} {global_badge} {degenerate_badge}; + const private_badge = is_private ? ( + private + ) : null; + const global_badge = is_global ? ( + private + ) : null; + const degenerate_badge = is_degenerate ? ( + degenerate + ) : null; + const badges = ( + <> + {private_badge} {global_badge} {degenerate_badge} + + ); return (
From fcb9088b8128b4b7e8e925f610a1bc27bfd8d812 Mon Sep 17 00:00:00 2001 From: msm-code Date: Thu, 29 Dec 2022 19:17:06 +0100 Subject: [PATCH 3/8] Update src/app.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: MichaƂ Praszmo --- src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index d8e3f98a..6ab1e1c4 100644 --- a/src/app.py +++ b/src/app.py @@ -343,7 +343,7 @@ def query( degenerate_rules = [r.name for r in rules if r.parse().is_degenerate] disallow_degenerate = db.get_mquery_config_key("query_disallow_degenerate") - if degenerate_rules and (disallow_degenerate == "true"): + if degenerate_rules and disallow_degenerate == "true": degenerate_rule_names = ", ".join(degenerate_rules) doc_url = "https://cert-polska.github.io/mquery/docs/yara.html" raise HTTPException( From b23a95d943b0f1e2db5389277a7214a7f7f51e3b Mon Sep 17 00:00:00 2001 From: msm Date: Wed, 4 Jan 2023 19:33:29 +0100 Subject: [PATCH 4/8] WIP allow force-running a slow query (and merge) --- src/app.py | 22 +++++++++++++------ src/mqueryfront/src/query/QueryField.js | 1 + .../src/query/QueryLayoutManager.js | 22 +++++++++++-------- src/mqueryfront/src/query/QueryNavigation.js | 6 ++++- src/mqueryfront/src/query/QueryPage.js | 11 ++++++++++ src/mqueryfront/src/query/QuerySubmitNav.js | 16 ++++++++------ src/schema.py | 1 + 7 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/app.py b/src/app.py index 6ab1e1c4..1b7a1853 100644 --- a/src/app.py +++ b/src/app.py @@ -18,7 +18,7 @@ from starlette.staticfiles import StaticFiles # type: ignore from zmq import Again -from lib.yaraparse import parse_yara +from lib.yaraparse import YaraRuleData, parse_yara from util import mquery_version from db import Database, JobId @@ -342,17 +342,25 @@ def query( ] degenerate_rules = [r.name for r in rules if r.parse().is_degenerate] - disallow_degenerate = db.get_mquery_config_key("query_disallow_degenerate") - if degenerate_rules and disallow_degenerate == "true": + if degenerate_rules and not data.force_slow_queries: + allow_degenerate = ( + db.get_mquery_config_key("query_disallow_degenerate") != "true" + ) + if allow_degenerate: + # Warning: "You can force a slow query" literal is used to + # pattern match on the error message in the frontend. + help_message = "You can force a slow query if you want." + else: + help_message = "This is not allowed by this server." degenerate_rule_names = ", ".join(degenerate_rules) doc_url = "https://cert-polska.github.io/mquery/docs/yara.html" raise HTTPException( status_code=400, detail=( - "Invalid query. " - "Some of the rules would require a Yara scan of every indexed " - "file, and this is not allowed by this instance. " - f"Problematic rules: {degenerate_rule_names}. " + "Invalid query. Some of the rules require a full Yara scan of" + "every indexed file. " + f"{help_message} " + f"Slow rules: {degenerate_rule_names}. " f"Read {doc_url} for more details." ), ) diff --git a/src/mqueryfront/src/query/QueryField.js b/src/mqueryfront/src/query/QueryField.js index 7ece5438..d45e24de 100644 --- a/src/mqueryfront/src/query/QueryField.js +++ b/src/mqueryfront/src/query/QueryField.js @@ -12,6 +12,7 @@ const QueryField = (props) => ( onTaintSelect={props.onTaintSelect} availableTaints={props.availableTaints} selectedTaints={props.selectedTaints} + forceSlowQueries={props.forceSlowQueries} />
{ onYaraUpdate, parsedError, selectedTaints, + forceSlowQueries, } = props; - const queryParse = queryError ? ( - - ) : queryPlan ? ( - - ) : null; - const queryResults = job ? (
diff --git a/src/mqueryfront/src/query/QueryNavigation.js b/src/mqueryfront/src/query/QueryNavigation.js index a8fe60e3..d4f87aa2 100644 --- a/src/mqueryfront/src/query/QueryNavigation.js +++ b/src/mqueryfront/src/query/QueryNavigation.js @@ -12,11 +12,15 @@ const QueryNavigation = (props) => { isEditActive, availableTaints, selectedTaints, + forceSlowQueries, } = props; return (
- + ); diff --git a/src/mqueryfront/src/query/QuerySubmitNav.js b/src/mqueryfront/src/query/QuerySubmitNav.js index c1fadb19..63b7659c 100644 --- a/src/mqueryfront/src/query/QuerySubmitNav.js +++ b/src/mqueryfront/src/query/QuerySubmitNav.js @@ -1,23 +1,25 @@ import React from "react"; const QuerySubmitNav = (props) => { - const { onClick } = props; + const { onClick, forceMode } = props; + + const label = forceMode ? "Force query (may be very slow!)" : "Query"; + const style = forceMode ? "btn-danger" : "btn-success"; return ( -
+