From de3dd51342485cfa66b36e264906150dcfb86d15 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Wed, 4 Sep 2024 19:00:47 -0700 Subject: [PATCH 01/10] Fix snuba admin's query tracing to connect to right storage and query nodes --- snuba/admin/views.py | 75 +++++++++++++++++++++++++++++++++-------- tests/admin/test_api.py | 1 + 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index 0b68ed96c4..01007ac097 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -2,10 +2,11 @@ import io import sys +import time from contextlib import redirect_stdout from dataclasses import asdict from datetime import datetime -from typing import Any, List, Mapping, Optional, Sequence, Tuple, cast +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, cast import sentry_sdk import simplejson as json @@ -501,11 +502,43 @@ def clickhouse_trace_query() -> Response: for query_trace_data in parse_trace_for_query_ids(query_trace, storage): sql = profile_events_raw_sql.format(query_trace_data.query_id) logger.info( - "Profile event gathering host: {}, port = {}, storage = {}, sql = {}, g.user = {}".format( + "Gathering profile event using host: {}, port = {}, storage = {}, sql = {}, g.user = {}".format( query_trace_data.host, query_trace_data.port, storage, sql, g.user ) ) - # TODO: Onkar to add the profile event logic later. + system_query_result, counter = None, 0 + while counter < 60: + # There is a race between the trace query and the 'SELECT ProfileEvents...' query. ClickHouse does not immediately + # return the rows for 'SELECT ProfileEvents...' query. To make it return rows, sleep between the query executions. + system_query_result = run_system_query_on_host_with_sql( + query_trace_data.host, + int(query_trace_data.port), + storage, + sql, + False, + g.user, + ) + if not system_query_result.results: + time.sleep(1) + counter += 1 + else: + break + + if system_query_result is not None and len(system_query_result.results) > 0: + query_trace.profile_events_meta.append(system_query_result.meta) + query_trace.profile_events_profile = cast( + Dict[str, int], system_query_result.profile + ) + columns = system_query_result.meta + if columns: + res = {} + res["column_names"] = [name for name, _ in columns] + res["rows"] = [] + for query_result in system_query_result.results: + if query_result[0]: + res["rows"].append(json.dumps(query_result[0])) + query_trace.profile_events_results[query_trace_data.node_name] = res + return make_response(jsonify(asdict(query_trace)), 200) except InvalidCustomQuery as err: return make_response( @@ -520,6 +553,7 @@ def clickhouse_trace_query() -> Response: 400, ) except ClickhouseError as err: + logger.error(err, exc_info=True) details = { "type": "clickhouse", "message": str(err), @@ -527,6 +561,7 @@ def clickhouse_trace_query() -> Response: } return make_response(jsonify({"error": details}), 400) except Exception as err: + logger.error(err, exc_info=True) return make_response( jsonify({"error": {"type": "unknown", "message": str(err)}}), 500, @@ -536,7 +571,7 @@ def clickhouse_trace_query() -> Response: def parse_trace_for_query_ids( trace_output: TraceOutput, storage_key: str ) -> List[QueryTraceData]: - result = [] + traces = [] summarized_trace_output = trace_output.summarized_trace_output storage_info = get_storage_info() matched = next( @@ -545,16 +580,28 @@ def parse_trace_for_query_ids( if matched is not None: local_nodes = matched.get("local_nodes", []) query_node = matched.get("query_node", None) - result = [ - QueryTraceData( - host=local_nodes[0].get("host") if local_nodes else query_node.get("host"), # type: ignore - port=local_nodes[0].get("port") if local_nodes else query_node.get("port"), # type: ignore - query_id=query_summary.query_id, - node_name=node_name, - ) - for node_name, query_summary in summarized_trace_output.query_summaries.items() - ] - return result + for node_name, query_summary in summarized_trace_output.query_summaries.items(): + if local_nodes: + for local_node in local_nodes: + traces.append( + QueryTraceData( + host=local_node.get("host"), # type: ignore + port=local_node.get("port"), # type: ignore + query_id=query_summary.query_id, + node_name=node_name, + ) + ) + if query_node: + traces.append( + QueryTraceData( + host=query_node.get("host"), # type: ignore + port=query_node.get("port"), # type: ignore + query_id=query_summary.query_id, + node_name=node_name, + ) + ) + + return traces @application.route("/clickhouse_querylog_query", methods=["POST"]) diff --git a/tests/admin/test_api.py b/tests/admin/test_api.py index 014c302fc7..ca04de2f27 100644 --- a/tests/admin/test_api.py +++ b/tests/admin/test_api.py @@ -259,6 +259,7 @@ def test_query_trace(admin_api: FlaskClient) -> None: data = json.loads(response.data) assert " executeQuery" in data["trace_output"] assert "summarized_trace_output" in data + assert "profile_events_results" in data @pytest.mark.redis_db From f338a1ac7255250de715db5c4a671e919495ed56 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Thu, 5 Sep 2024 20:05:09 -0700 Subject: [PATCH 02/10] Fix parsing logic --- snuba/admin/views.py | 57 +++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index 01007ac097..c215ad09f2 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import re import sys import time from contextlib import redirect_stdout @@ -499,7 +500,7 @@ def clickhouse_trace_query() -> Response: profile_events_raw_sql = "SELECT ProfileEvents FROM system.query_log WHERE query_id = '{}' AND type = 'QueryFinish'" - for query_trace_data in parse_trace_for_query_ids(query_trace, storage): + for query_trace_data in parse_trace_for_query_ids(query_trace): sql = profile_events_raw_sql.format(query_trace_data.query_id) logger.info( "Gathering profile event using host: {}, port = {}, storage = {}, sql = {}, g.user = {}".format( @@ -568,40 +569,30 @@ def clickhouse_trace_query() -> Response: ) -def parse_trace_for_query_ids( - trace_output: TraceOutput, storage_key: str -) -> List[QueryTraceData]: - traces = [] +def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData]: summarized_trace_output = trace_output.summarized_trace_output - storage_info = get_storage_info() - matched = next( - (info for info in storage_info if info["storage_name"] == storage_key), None - ) - if matched is not None: - local_nodes = matched.get("local_nodes", []) - query_node = matched.get("query_node", None) - for node_name, query_summary in summarized_trace_output.query_summaries.items(): - if local_nodes: - for local_node in local_nodes: - traces.append( - QueryTraceData( - host=local_node.get("host"), # type: ignore - port=local_node.get("port"), # type: ignore - query_id=query_summary.query_id, - node_name=node_name, - ) - ) - if query_node: - traces.append( - QueryTraceData( - host=query_node.get("host"), # type: ignore - port=query_node.get("port"), # type: ignore - query_id=query_summary.query_id, - node_name=node_name, - ) - ) + node_name_to_query_id = { + node_name: query_summary.query_id + for node_name, query_summary in summarized_trace_output.query_summaries.items() + } + return [ + QueryTraceData( + host="127.0.0.1" if is_local_container_id(node_name) else node_name, + port=9000, + query_id=query_id, + node_name=node_name, + ) + for node_name, query_id in node_name_to_query_id.items() + ] + - return traces +def is_local_container_id(container_id: str) -> bool: + # Local Clickhouse docker container ids are like e51ba1535fb1. They can be either 12 charcters or 64 characters long + # and must be hexadecimal. Regular expression fullmatch() makes sure that entire string matches the pattern. Thus, + # strings like "clickhouse" or "snuba-event-analytics-platform-query-1-1" are not matched. + # This is to make sure that snuba admin running locally on laptop has host set as "127.0.0.1" instead of "e51ba1535fb1". + pattern = r"^[a-f0-9]{12}$|^[a-f0-9]{64}$" + return bool(re.fullmatch(pattern, container_id)) @application.route("/clickhouse_querylog_query", methods=["POST"]) From 8e40fc7c9f361f456babef233549e44d2a0321c4 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Thu, 5 Sep 2024 20:32:36 -0700 Subject: [PATCH 03/10] Debug --- snuba/admin/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index c215ad09f2..e4b59d5630 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -575,6 +575,7 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] node_name: query_summary.query_id for node_name, query_summary in summarized_trace_output.query_summaries.items() } + logger.info("node to query id mapping: {}".format(node_name_to_query_id)) return [ QueryTraceData( host="127.0.0.1" if is_local_container_id(node_name) else node_name, From f3b127ddcb0b3b5150b299ff364a94e533999f5b Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Thu, 5 Sep 2024 20:53:05 -0700 Subject: [PATCH 04/10] Debug --- snuba/admin/views.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index e4b59d5630..ce73619ee4 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -1,7 +1,6 @@ from __future__ import annotations import io -import re import sys import time from contextlib import redirect_stdout @@ -578,7 +577,8 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] logger.info("node to query id mapping: {}".format(node_name_to_query_id)) return [ QueryTraceData( - host="127.0.0.1" if is_local_container_id(node_name) else node_name, + # host="127.0.0.1" if is_local_container_id(node_name) else node_name, + host=node_name, port=9000, query_id=query_id, node_name=node_name, @@ -587,13 +587,13 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] ] -def is_local_container_id(container_id: str) -> bool: - # Local Clickhouse docker container ids are like e51ba1535fb1. They can be either 12 charcters or 64 characters long - # and must be hexadecimal. Regular expression fullmatch() makes sure that entire string matches the pattern. Thus, - # strings like "clickhouse" or "snuba-event-analytics-platform-query-1-1" are not matched. - # This is to make sure that snuba admin running locally on laptop has host set as "127.0.0.1" instead of "e51ba1535fb1". - pattern = r"^[a-f0-9]{12}$|^[a-f0-9]{64}$" - return bool(re.fullmatch(pattern, container_id)) +# def is_local_container_id(container_id: str) -> bool: +# # Local Clickhouse docker container ids are like e51ba1535fb1. They can be either 12 charcters or 64 characters long +# # and must be hexadecimal. Regular expression fullmatch() makes sure that entire string matches the pattern. Thus, +# # strings like "clickhouse" or "snuba-event-analytics-platform-query-1-1" are not matched. +# # This is to make sure that snuba admin running locally on laptop has host set as "127.0.0.1" instead of "e51ba1535fb1". +# pattern = r"^[a-f0-9]{12}$|^[a-f0-9]{64}$" +# return bool(re.fullmatch(pattern, container_id)) @application.route("/clickhouse_querylog_query", methods=["POST"]) From da7e0ca93bc4820c0a769189290a812173164120 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Fri, 6 Sep 2024 18:20:43 -0700 Subject: [PATCH 05/10] Attempt to fix hostname resolution --- docker-compose.gcb.yml | 42 ++++++++++++++++++++++++++++++++++++++ snuba/admin/views.py | 10 --------- snuba/settings/__init__.py | 2 +- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/docker-compose.gcb.yml b/docker-compose.gcb.yml index 8c73632b65..c0a3318d6a 100644 --- a/docker-compose.gcb.yml +++ b/docker-compose.gcb.yml @@ -71,6 +71,13 @@ services: KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" clickhouse: image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse.dev.ci + extra_hosts: + - "clickhouse.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" volumes: - ./config/clickhouse/macros.xml:/etc/clickhouse-server/config.d/macros.xml - ./config/clickhouse/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml @@ -85,6 +92,13 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse-query.dev.ci + extra_hosts: + - "clickhouse-query.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml @@ -97,6 +111,13 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse-01.dev.ci + extra_hosts: + - "clickhouse-01.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml @@ -110,6 +131,13 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse-02.dev.ci + extra_hosts: + - "clickhouse-02.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml @@ -123,6 +151,13 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse-03.dev.ci + extra_hosts: + - "clickhouse-03.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-03.xml:/etc/clickhouse-server/config.d/macros.xml @@ -136,6 +171,13 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" + hostname: clickhouse-04.dev.ci + extra_hosts: + - "clickhouse-04.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-04.xml:/etc/clickhouse-server/config.d/macros.xml diff --git a/snuba/admin/views.py b/snuba/admin/views.py index ce73619ee4..b65ba90b63 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -577,7 +577,6 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] logger.info("node to query id mapping: {}".format(node_name_to_query_id)) return [ QueryTraceData( - # host="127.0.0.1" if is_local_container_id(node_name) else node_name, host=node_name, port=9000, query_id=query_id, @@ -587,15 +586,6 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] ] -# def is_local_container_id(container_id: str) -> bool: -# # Local Clickhouse docker container ids are like e51ba1535fb1. They can be either 12 charcters or 64 characters long -# # and must be hexadecimal. Regular expression fullmatch() makes sure that entire string matches the pattern. Thus, -# # strings like "clickhouse" or "snuba-event-analytics-platform-query-1-1" are not matched. -# # This is to make sure that snuba admin running locally on laptop has host set as "127.0.0.1" instead of "e51ba1535fb1". -# pattern = r"^[a-f0-9]{12}$|^[a-f0-9]{64}$" -# return bool(re.fullmatch(pattern, container_id)) - - @application.route("/clickhouse_querylog_query", methods=["POST"]) @check_tool_perms(tools=[AdminTools.QUERYLOG]) def clickhouse_querylog_query() -> Response: diff --git a/snuba/settings/__init__.py b/snuba/settings/__init__.py index 0d23df207e..d26951574d 100644 --- a/snuba/settings/__init__.py +++ b/snuba/settings/__init__.py @@ -82,7 +82,7 @@ CLUSTERS: Sequence[Mapping[str, Any]] = [ { - "host": os.environ.get("CLICKHOUSE_HOST", "127.0.0.1"), + "host": os.environ.get("CLICKHOUSE_HOST", "clickhouse.dev.local"), "port": int(os.environ.get("CLICKHOUSE_PORT", 9000)), "max_connections": int(os.environ.get("CLICKHOUSE_MAX_CONNECTIONS", 1)), "block_connections": bool( From ed14df79278f040c156d446c93e3c3537484fd79 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Fri, 6 Sep 2024 18:37:49 -0700 Subject: [PATCH 06/10] Remove ports because they are conflicting --- docker-compose.gcb.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docker-compose.gcb.yml b/docker-compose.gcb.yml index c0a3318d6a..c4eff81770 100644 --- a/docker-compose.gcb.yml +++ b/docker-compose.gcb.yml @@ -95,10 +95,6 @@ services: hostname: clickhouse-query.dev.ci extra_hosts: - "clickhouse-query.dev.ci:127.0.0.1" # Add entry to /etc/hosts file - ports: - - "8123:8123" - - "9000:9000" - - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml @@ -114,10 +110,6 @@ services: hostname: clickhouse-01.dev.ci extra_hosts: - "clickhouse-01.dev.ci:127.0.0.1" # Add entry to /etc/hosts file - ports: - - "8123:8123" - - "9000:9000" - - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml @@ -134,10 +126,6 @@ services: hostname: clickhouse-02.dev.ci extra_hosts: - "clickhouse-02.dev.ci:127.0.0.1" # Add entry to /etc/hosts file - ports: - - "8123:8123" - - "9000:9000" - - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml @@ -154,10 +142,6 @@ services: hostname: clickhouse-03.dev.ci extra_hosts: - "clickhouse-03.dev.ci:127.0.0.1" # Add entry to /etc/hosts file - ports: - - "8123:8123" - - "9000:9000" - - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-03.xml:/etc/clickhouse-server/config.d/macros.xml @@ -174,10 +158,6 @@ services: hostname: clickhouse-04.dev.ci extra_hosts: - "clickhouse-04.dev.ci:127.0.0.1" # Add entry to /etc/hosts file - ports: - - "8123:8123" - - "9000:9000" - - "9009:9009" profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-04.xml:/etc/clickhouse-server/config.d/macros.xml From a93ad19f329b6c880da4649dae12a7d532e9651f Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Fri, 6 Sep 2024 22:09:19 -0700 Subject: [PATCH 07/10] Attempt - 1 --- docker-compose.gcb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.gcb.yml b/docker-compose.gcb.yml index c4eff81770..e5ff6127c3 100644 --- a/docker-compose.gcb.yml +++ b/docker-compose.gcb.yml @@ -17,7 +17,7 @@ x-test-common: &test-common - ".artifacts:/.artifacts" environment: SNUBA_SETTINGS: "$SNUBA_SETTINGS" - CLICKHOUSE_HOST: clickhouse + CLICKHOUSE_HOST: clickhouse.dev.ci USE_REDIS_CLUSTER: "1" REDIS_HOST: "redis-cluster" REDIS_PORT: 7000 From a7fcaa9dd1ab6d42c98ba088bb7f6cadcf8ff5cc Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Mon, 9 Sep 2024 14:48:31 -0700 Subject: [PATCH 08/10] Try using localhost instead of clickhouse.dev.local --- docker-compose.gcb.yml | 26 +++++++++++++------------- snuba/settings/__init__.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker-compose.gcb.yml b/docker-compose.gcb.yml index e5ff6127c3..5653a1f807 100644 --- a/docker-compose.gcb.yml +++ b/docker-compose.gcb.yml @@ -17,7 +17,7 @@ x-test-common: &test-common - ".artifacts:/.artifacts" environment: SNUBA_SETTINGS: "$SNUBA_SETTINGS" - CLICKHOUSE_HOST: clickhouse.dev.ci + CLICKHOUSE_HOST: clickhouse.local USE_REDIS_CLUSTER: "1" REDIS_HOST: "redis-cluster" REDIS_PORT: 7000 @@ -71,9 +71,9 @@ services: KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" clickhouse: image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse.dev.ci + hostname: clickhouse.local extra_hosts: - - "clickhouse.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse.local:127.0.0.1" # Add entry to /etc/hosts file ports: - "8123:8123" - "9000:9000" @@ -92,9 +92,9 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse-query.dev.ci + hostname: clickhouse-query.local extra_hosts: - - "clickhouse-query.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse-query.local:127.0.0.1" # Add entry to /etc/hosts file profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/zookeeper.xml:/etc/clickhouse-server/config.d/zookeeper.xml @@ -107,9 +107,9 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse-01.dev.ci + hostname: clickhouse-01.local extra_hosts: - - "clickhouse-01.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse-01.local:127.0.0.1" # Add entry to /etc/hosts file profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-01.xml:/etc/clickhouse-server/config.d/macros.xml @@ -123,9 +123,9 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse-02.dev.ci + hostname: clickhouse-02.local extra_hosts: - - "clickhouse-02.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse-02.local:127.0.0.1" # Add entry to /etc/hosts file profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-02.xml:/etc/clickhouse-server/config.d/macros.xml @@ -139,9 +139,9 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-altinity/clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse-03.dev.ci + hostname: clickhouse-03.local extra_hosts: - - "clickhouse-03.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse-03.local:127.0.0.1" # Add entry to /etc/hosts file profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-03.xml:/etc/clickhouse-server/config.d/macros.xml @@ -155,9 +155,9 @@ services: depends_on: - zookeeper image: "${CLICKHOUSE_IMAGE:-ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:23.3.19.33.altinitystable}" - hostname: clickhouse-04.dev.ci + hostname: clickhouse-04.local extra_hosts: - - "clickhouse-04.dev.ci:127.0.0.1" # Add entry to /etc/hosts file + - "clickhouse-04.local:127.0.0.1" # Add entry to /etc/hosts file profiles: ["multi_node"] volumes: - ./test_distributed_migrations/config/clickhouse/macros-04.xml:/etc/clickhouse-server/config.d/macros.xml diff --git a/snuba/settings/__init__.py b/snuba/settings/__init__.py index d26951574d..fd2df07301 100644 --- a/snuba/settings/__init__.py +++ b/snuba/settings/__init__.py @@ -82,7 +82,7 @@ CLUSTERS: Sequence[Mapping[str, Any]] = [ { - "host": os.environ.get("CLICKHOUSE_HOST", "clickhouse.dev.local"), + "host": os.environ.get("CLICKHOUSE_HOST", "localhost"), "port": int(os.environ.get("CLICKHOUSE_PORT", 9000)), "max_connections": int(os.environ.get("CLICKHOUSE_MAX_CONNECTIONS", 1)), "block_connections": bool( From ec5078be029ce58d3c8896bf10bc99fd148cb84e Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Tue, 10 Sep 2024 18:47:26 -0700 Subject: [PATCH 09/10] Fix --- snuba/admin/views.py | 12 +++++++++++- snuba/settings/__init__.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index b65ba90b63..11dcc334b5 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -1,6 +1,7 @@ from __future__ import annotations import io +import socket import sys import time from contextlib import redirect_stdout @@ -568,6 +569,15 @@ def clickhouse_trace_query() -> Response: ) +def hostname_resolves(hostname: str) -> bool: + try: + socket.gethostbyname(hostname) + except socket.error: + return False + else: + return True + + def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData]: summarized_trace_output = trace_output.summarized_trace_output node_name_to_query_id = { @@ -577,7 +587,7 @@ def parse_trace_for_query_ids(trace_output: TraceOutput) -> List[QueryTraceData] logger.info("node to query id mapping: {}".format(node_name_to_query_id)) return [ QueryTraceData( - host=node_name, + host=node_name if hostname_resolves(node_name) else "127.0.0.1", port=9000, query_id=query_id, node_name=node_name, diff --git a/snuba/settings/__init__.py b/snuba/settings/__init__.py index fd2df07301..0d23df207e 100644 --- a/snuba/settings/__init__.py +++ b/snuba/settings/__init__.py @@ -82,7 +82,7 @@ CLUSTERS: Sequence[Mapping[str, Any]] = [ { - "host": os.environ.get("CLICKHOUSE_HOST", "localhost"), + "host": os.environ.get("CLICKHOUSE_HOST", "127.0.0.1"), "port": int(os.environ.get("CLICKHOUSE_PORT", 9000)), "max_connections": int(os.environ.get("CLICKHOUSE_MAX_CONNECTIONS", 1)), "block_connections": bool( From 7260bf1b095a27e43f658e97e485a8afc49e2621 Mon Sep 17 00:00:00 2001 From: Onkar Deshpande Date: Thu, 12 Sep 2024 10:09:48 -0700 Subject: [PATCH 10/10] Review comments --- snuba/admin/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snuba/admin/views.py b/snuba/admin/views.py index 11dcc334b5..ac6a54d503 100644 --- a/snuba/admin/views.py +++ b/snuba/admin/views.py @@ -508,7 +508,7 @@ def clickhouse_trace_query() -> Response: ) ) system_query_result, counter = None, 0 - while counter < 60: + while counter < 30: # There is a race between the trace query and the 'SELECT ProfileEvents...' query. ClickHouse does not immediately # return the rows for 'SELECT ProfileEvents...' query. To make it return rows, sleep between the query executions. system_query_result = run_system_query_on_host_with_sql(