From ae38f0f80915dbef767e9c5546b9804045a370e2 Mon Sep 17 00:00:00 2001 From: gqp <446105468@qq.com> Date: Tue, 7 Jan 2025 11:11:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20CC=E6=8E=A5=E5=8F=A3=E9=99=90=E9=A2=91?= =?UTF-8?q?=E4=BC=98=E5=8C=96=20(closed=20#2531)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/backend/subscription/commons.py | 3 + apps/backend/subscription/tools.py | 186 ++++++++++++++++++++--- apps/backend/tests/subscription/utils.py | 22 +++ apps/node_man/constants.py | 11 +- common/api/modules/cc.py | 9 ++ 5 files changed, 205 insertions(+), 26 deletions(-) diff --git a/apps/backend/subscription/commons.py b/apps/backend/subscription/commons.py index 851f853e3..1b40b2e95 100644 --- a/apps/backend/subscription/commons.py +++ b/apps/backend/subscription/commons.py @@ -77,6 +77,9 @@ def get_host_by_inst(bk_biz_id, inst_list): bk_obj_id_list = [topo_data["bk_obj_id"] for topo_data in topo_data_list] for inst in inst_list: + if "bk_obj_id" not in inst: + continue + # 处理各种类型的节点 if inst["bk_obj_id"] == "biz": bk_biz_ids.append(bk_biz_id) diff --git a/apps/backend/subscription/tools.py b/apps/backend/subscription/tools.py index f40e68782..c29804cde 100644 --- a/apps/backend/subscription/tools.py +++ b/apps/backend/subscription/tools.py @@ -16,6 +16,7 @@ import math import os import pprint +import random import typing from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed @@ -41,6 +42,7 @@ from apps.component.esbclient import client_v2 from apps.core.concurrent import controller from apps.core.concurrent.cache import FuncCacheDecorator +from apps.core.concurrent.retry import RetryHandler from apps.core.ipchooser.tools.base import HostQuerySqlHelper from apps.node_man import constants, models from apps.node_man import tools as node_man_tools @@ -251,6 +253,7 @@ def create_host_key(data: Dict) -> str: @SetupObserve(counter=metrics.app_common_method_requests_total, get_labels_func=get_call_resource_labels_func) +@RetryHandler(interval=3, retry_times=2) def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]: """ 查询主机所属拓扑关系 @@ -275,7 +278,13 @@ def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]: {"bk_host_id": bk_host_ids[count * constants.QUERY_CMDB_LIMIT : (count + 1) * constants.QUERY_CMDB_LIMIT]} for count in range(math.ceil(len(bk_host_ids) / constants.QUERY_CMDB_LIMIT)) ] - host_biz_relations = request_multi_thread(client_v2.cc.find_host_biz_relations, param_list, get_data=lambda x: x) + host_biz_relations = batch_call( + func=client_v2.cc.find_host_biz_relations, + params_list=param_list, + interval=constants.FIND_HOST_BIZ_RELATIONS_INTERVAL, + extend_result=True, + ) + return host_biz_relations @@ -382,17 +391,17 @@ def get_modules_by_inst_list(inst_list, module_to_topo): return module_ids, no_module_inst_list -def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo): - module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo) - if not module_ids: - return [] - - if len(module_ids) <= models.GlobalSettings.get_config( - models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value, constants.QUERY_MODULE_ID_THRESHOLD - ): - params = [ +def get_service_instance_ids(bk_biz_id: int, module_ids: List[int]) -> List[int]: + """ + 查询服务实例ID列表 + :param bk_biz_id: 业务ID + :param module_ids: 模块id列表 + """ + service_instances = batch_call( + batch_request, + params_list=[ { - "func": CCApi.list_service_instance_detail, + "func": CCApi.list_service_instance, "params": { "bk_biz_id": int(bk_biz_id), "with_name": True, @@ -404,11 +413,71 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo): "limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT, } for bk_module_id in module_ids - ] + ], + extend_result=True, + interval=constants.LIST_SERVICE_INSTANCE_INTERVAL, + ) - service_instances = batch_call( - batch_request, params, extend_result=True, interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL - ) + return [service_instance["id"] for service_instance in service_instances] + + +@RetryHandler(interval=3, retry_times=2) +def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo): + module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo) + if not module_ids: + return [] + + service_instance_module_id_threshold = models.GlobalSettings.get_config( + key=models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value, + default={}, + ) + + if len(module_ids) <= service_instance_module_id_threshold.get(str(bk_biz_id), constants.QUERY_MODULE_ID_THRESHOLD): + # 随机挑选一种方式获取服务实例详情，分摊list_service_instance_detail压力 + # 1. 通过模块id分片查询所有的[服务实例ID列表]，再通过[服务实例ID列表]一次性筛选服务实例详情，避免了分片查询 + # 结果 -> n次list_service_instance查询 + 1次list_service_instance_detail查询 + # 2. 通过模块id分片查询服务实例详情 + # 结果 -> n次list_service_instance_detail查询 + + # 如果module_ids只有一个，没必要使用第一种方式，一定会出现一次list_service_instance_detail查询 + if len(module_ids) > 1 and random.random() < 0.5: + service_instance_ids = get_service_instance_ids(bk_biz_id, list(module_ids)) + if not service_instance_ids: + return [] + + service_instances = batch_request( + func=CCApi.list_service_instance_detail, + params={ + "bk_biz_id": int(bk_biz_id), + "with_name": True, + "no_request": True, + "service_instance_ids": service_instance_ids, + }, + sort="id", + interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL, + limit=constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT, + ) + else: + service_instances = batch_call( + func=batch_request, + params_list=[ + { + "func": CCApi.list_service_instance_detail, + "params": { + "bk_biz_id": int(bk_biz_id), + "with_name": True, + "bk_module_id": bk_module_id, + # CC 接口统一使用后台访问 + "no_request": True, + }, + "sort": "id", + "limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT, + } + for bk_module_id in module_ids + ], + extend_result=True, + interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL, + ) else: params = {"bk_biz_id": int(bk_biz_id), "with_name": True, "no_request": True} service_instances = batch_request( @@ -426,7 +495,54 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo): return service_instances -@FuncCacheDecorator(cache_time=1 * constants.TimeUnit.MINUTE) +@RetryHandler(interval=3, retry_times=2) +def get_service_instance_by_set_templates(bk_biz_id: int, set_template_ids: List[int]): + """ + 通过集群模板获取服务实例详情 + :param bk_biz_id: 业务ID + :param set_template_ids: 集群模板id列表 + """ + params = [ + { + "func": CCApi.list_service_instance_by_set_template, + "params": { + "bk_biz_id": int(bk_biz_id), + "set_template_id": set_template_id, + # CC 接口统一使用后台访问 + "no_request": True, + }, + "sort": "id", + "limit": constants.QUERY_CMDB_LIMIT, + } + for set_template_id in set_template_ids + ] + + service_instance_ids: List[int] = [ + service_instance["id"] + for service_instance in batch_call( + func=batch_request, + params_list=params, + extend_result=True, + interval=constants.LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL, + ) + ] + + service_instance_details = batch_request( + CCApi.list_service_instance_detail, + params={ + "bk_biz_id": int(bk_biz_id), + "service_instance_ids": service_instance_ids, + "no_request": True, + }, + sort="id", + limit=constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT, + interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL, + ) + + return service_instance_details + + +@FuncCacheDecorator(cache_time=15 * constants.TimeUnit.MINUTE) def fetch_biz_info_map(fields: typing.Optional[typing.List[str]] = None) -> typing.Dict[str, typing.Dict]: """ 查询所有业务 @@ -621,10 +737,13 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None): bk_host_ids.append(host["bk_host_id"]) bk_cloud_ids.append(host["bk_cloud_id"]) - host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail") - host_biz_map = {} - for host in host_relations: - host_biz_map[host["bk_host_id"]] = host["bk_biz_id"] + if bk_biz_id: + host_biz_map = {_host["bk_host_id"]: bk_biz_id for _host in hosts} + else: + host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail") + host_biz_map = {} + for host in host_relations: + host_biz_map[host["bk_host_id"]] = host["bk_biz_id"] cloud_id_name_map = models.Cloud.cloud_id_name_map(get_cache=True) @@ -635,7 +754,10 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None): host_key_dict = {} host_id_dict = {} for _host in hosts: - _host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]] + if bk_biz_id: + _host["bk_biz_id"] = bk_biz_id + else: + _host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]] _host["bk_biz_name"] = ( all_biz_info.get(_host["bk_biz_id"], {}).get("bk_biz_name", "") if _host["bk_biz_id"] != settings.BK_CMDB_RESOURCE_POOL_BIZ_ID @@ -883,7 +1005,7 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str, instances = [] bk_biz_id = scope["bk_biz_id"] - if bk_biz_id: + if bk_biz_id and scope["object_type"] == models.Subscription.ObjectType.SERVICE: module_to_topo = get_module_to_topo_dict(bk_biz_id) else: module_to_topo = {} @@ -957,12 +1079,26 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str, instances = add_host_module_info(host_biz_relations, instances) else: + template_ids = [node["bk_inst_id"] for node in scope["nodes"]] + # 补充服务实例中的信息 # 转化模板为节点，**注意不可在get_service_instance_by_inst之后才转换** nodes = set_template_scope_nodes(scope) - instances.extend( - [{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)] - ) + + if scope["node_type"] == models.Subscription.NodeType.SET_TEMPLATE: + instances.extend( + [ + {"service": inst} + for inst in get_service_instance_by_set_templates( + bk_biz_id=bk_biz_id, + set_template_ids=template_ids, + ) + ] + ) + else: + instances.extend( + [{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)] + ) if not need_register: # 补充必要的主机或实例相关信息 diff --git a/apps/backend/tests/subscription/utils.py b/apps/backend/tests/subscription/utils.py index 70f19767f..fc4da0acb 100644 --- a/apps/backend/tests/subscription/utils.py +++ b/apps/backend/tests/subscription/utils.py @@ -3327,6 +3327,28 @@ def list_service_instance_detail(cls, *args, **kwargs): SERVICE_DETAIL["info"] = [x for x in SERVICE_DETAIL["info"] if x["id"] == 10] return SERVICE_DETAIL + @classmethod + def list_service_instance(cls, *args, **kwargs): + return { + "count": 49, + "info": [ + { + "bk_biz_id": 1, + "id": 10, + "name": "127.0.0.1_gse_agent", + "labels": None, + "service_template_id": 14, + "bk_host_id": 1, + "bk_module_id": 12, + "creator": "cc_system", + "modifier": "cc_system", + "create_time": "2019-07-09T13:06:54.384+08:00", + "last_time": "2019-07-09T13:06:54.384+08:00", + "bk_supplier_account": "0", + } + ], + } + @classmethod def find_host_by_topo(cls, *args, **kwargs): return {"count": 4, "info": LIST_BIZ_HOSTS_WITHOUT_INFO} diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index cde490d4a..45d34ad5f 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -599,7 +599,16 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: # list_service_instance_detail接口调用参数配置 LIST_SERVICE_INSTANCE_DETAIL_LIMIT = 1000 -LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.2 +LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.3 + +# list_service_instance_by_set_template接口调用间隔 +LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL = 0.3 + +# find_host_biz_relations接口调用间隔 +FIND_HOST_BIZ_RELATIONS_INTERVAL = 0.3 + +# list_service_instance接口调用间隔 +LIST_SERVICE_INSTANCE_INTERVAL = 0.3 # redis键名模板 REDIS_NEED_DELETE_HOST_IDS_KEY_TPL = f"{settings.APP_CODE}:node_man:need_delete_host_ids:list" diff --git a/common/api/modules/cc.py b/common/api/modules/cc.py index 4837be105..dd9fe809c 100644 --- a/common/api/modules/cc.py +++ b/common/api/modules/cc.py @@ -256,3 +256,12 @@ def __init__(self): before_request=add_esb_info_before_request, api_name="list_service_instance_detail", ) + self.list_service_instance_by_set_template = DataAPI( + method="POST", + url=CC_APIGATEWAY_ROOT_V2 + "list_service_instance_by_set_template/", + module=self.MODULE, + simple_module=self.SIMPLE_MODULE, + description="通过集群模版查询关联的服务实例列表", + before_request=add_esb_info_before_request, + api_name="list_service_instance_by_set_template", + )