From f1d2d6ca1f539009ea5197a77708e3d57649430f Mon Sep 17 00:00:00 2001 From: dylan Date: Wed, 5 Aug 2020 11:26:39 -0700 Subject: [PATCH 1/3] [TOOLS-1530] Add --pmap option to include partition analysis in collect info files. --- asadm.py | 8 +++++--- lib/basiccontroller.py | 9 +++++---- lib/controllerlib.py | 3 ++- lib/utils/conf.py | 8 ++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/asadm.py b/asadm.py index 623f4a8b..24c92671 100755 --- a/asadm.py +++ b/asadm.py @@ -125,7 +125,7 @@ def critical(self, msg, *args, **kwargs): class AerospikeShell(cmd.Cmd): def __init__(self, admin_version, seeds, user=None, password=None, auth_mode=AuthMode.INTERNAL, use_services_alumni=False, use_services_alt=False, log_path="", mode=AdminMode.LIVE_CLUSTER, - ssl_context=None, only_connect_seed=False, execute_only_mode=False, timeout=5): + ssl_context=None, only_connect_seed=False, execute_only_mode=False, timeout=5, get_pmap=False): # indicates shell created successfully and connected to cluster/collectinfo/logfile self.connected = True @@ -181,7 +181,7 @@ def __init__(self, admin_version, seeds, user=None, password=None, auth_mode=Aut self.ctrl = BasicRootController(seed_nodes=seeds, user=user, password=password, auth_mode=auth_mode, use_services_alumni=use_services_alumni, use_services_alt=use_services_alt, ssl_context=ssl_context, asadm_version=admin_version, - only_connect_seed=only_connect_seed, timeout=timeout) + only_connect_seed=only_connect_seed, timeout=timeout, get_pmap=get_pmap) if not self.ctrl.cluster.get_live_nodes(): self.do_exit('') @@ -624,7 +624,9 @@ def main(): mode=mode, ssl_context=ssl_context, only_connect_seed=cli_args.single_node, - execute_only_mode=execute_only_mode, timeout=cli_args.timeout) + execute_only_mode=execute_only_mode, + timeout=cli_args.timeout, + get_pmap=cli_args.pmap) use_yappi = False if cli_args.profile: diff --git a/lib/basiccontroller.py b/lib/basiccontroller.py index 39350dd5..a953d8ae 100644 --- a/lib/basiccontroller.py +++ b/lib/basiccontroller.py @@ -53,9 +53,9 @@ class BasicRootController(BaseController): def __init__(self, seed_nodes=[('127.0.0.1', 3000, None)], user=None, password=None, auth_mode=constants.AuthMode.INTERNAL, use_services_alumni=False, use_services_alt=False, ssl_context=None, - asadm_version='', only_connect_seed=False, timeout=5): + asadm_version='', only_connect_seed=False, timeout=5, get_pmap=False): - super(BasicRootController, self).__init__(asadm_version) + super(BasicRootController, self).__init__(asadm_version, get_pmap) # Create static instance of cluster BasicRootController.cluster = Cluster(seed_nodes, user, password, auth_mode, @@ -1288,7 +1288,8 @@ def _get_collectinfo_data_json(self, default_user, default_pwd, default_ssh_port latency_map = self._get_as_latency() - pmap_map = self._get_as_pmap() + if self.get_pmap: + pmap_map = self._get_as_pmap() sys_map = self.cluster.info_system_statistics(default_user=default_user, default_pwd=default_pwd, default_ssh_key=default_ssh_key, default_ssh_port=default_ssh_port, credential_file=credential_file, nodes=self.nodes, @@ -1313,7 +1314,7 @@ def _get_collectinfo_data_json(self, default_user, default_pwd, default_ssh_port if node in latency_map: dump_map[node]['as_stat']['latency'] = latency_map[node] - if node in pmap_map: + if self.get_pmap and node in pmap_map: dump_map[node]['as_stat']['pmap'] = pmap_map[node] # Get the cluster name and add one more level in map diff --git a/lib/controllerlib.py b/lib/controllerlib.py index 2d3e9cc5..186ee886 100644 --- a/lib/controllerlib.py +++ b/lib/controllerlib.py @@ -81,12 +81,13 @@ class BaseController(object): asadm_version = '' logger = None - def __init__(self, asadm_version=''): + def __init__(self, asadm_version='', get_pmap=False): # Create static instances of view / health_checker / asadm_version / # logger BaseController.view = view.CliView() BaseController.health_checker = HealthChecker() BaseController.asadm_version = asadm_version + BaseController.get_pmap = get_pmap BaseController.logger = logging.getLogger("asadm") # instance vars self.modifiers = set() diff --git a/lib/utils/conf.py b/lib/utils/conf.py index 8c37185e..dd57bc83 100644 --- a/lib/utils/conf.py +++ b/lib/utils/conf.py @@ -80,6 +80,8 @@ def __init__(self, adict): "execute": False, "log-analyser": False, "log-path": "", + + "pmap": False }, } @@ -417,6 +419,7 @@ def print_config_help(): print (" --single-node Enable asadm mode to connect only seed node. \n" " By default asadm connects to all nodes in cluster.") print (" --collectinfo Start asadm to run against offline collectinfo files.") + print (" --pmap Include partition map analysis in collect info file.") print (" --log-analyser Start asadm in log-analyser mode and analyse data from log files.") print (" -f --log-path=path Path of cluster collectinfo file or directory \n" " containing collectinfo and system info files.") @@ -615,6 +618,11 @@ def get_cli_args(): add_fn("--tls_crl_check", action="store_true") add_fn("--tls_crl_check_all", action="store_true") + ### collectinfo options ### + # include pmap analysis in collect info file. + # Usage, `asadm collectinfo --pmap` + add_fn("--pmap", action="store_true") + if have_argparse: return parser.parse_args() From ffab8decf1c30d01177fa226deae8455c50c10df Mon Sep 17 00:00:00 2001 From: dylan Date: Wed, 5 Aug 2020 15:37:28 -0700 Subject: [PATCH 2/3] get --pmap using class variable --- asadm.py | 11 +++++++---- lib/basiccontroller.py | 10 ++++++---- lib/controllerlib.py | 3 +-- lib/getcontroller.py | 6 ++++-- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/asadm.py b/asadm.py index 24c92671..621a833e 100755 --- a/asadm.py +++ b/asadm.py @@ -125,7 +125,7 @@ def critical(self, msg, *args, **kwargs): class AerospikeShell(cmd.Cmd): def __init__(self, admin_version, seeds, user=None, password=None, auth_mode=AuthMode.INTERNAL, use_services_alumni=False, use_services_alt=False, log_path="", mode=AdminMode.LIVE_CLUSTER, - ssl_context=None, only_connect_seed=False, execute_only_mode=False, timeout=5, get_pmap=False): + ssl_context=None, only_connect_seed=False, execute_only_mode=False, timeout=5): # indicates shell created successfully and connected to cluster/collectinfo/logfile self.connected = True @@ -181,7 +181,7 @@ def __init__(self, admin_version, seeds, user=None, password=None, auth_mode=Aut self.ctrl = BasicRootController(seed_nodes=seeds, user=user, password=password, auth_mode=auth_mode, use_services_alumni=use_services_alumni, use_services_alt=use_services_alt, ssl_context=ssl_context, asadm_version=admin_version, - only_connect_seed=only_connect_seed, timeout=timeout, get_pmap=get_pmap) + only_connect_seed=only_connect_seed, timeout=timeout) if not self.ctrl.cluster.get_live_nodes(): self.do_exit('') @@ -567,6 +567,10 @@ def main(): if cli_args.no_color: disable_coloring() + + if cli_args.pmap: + from lib.basiccontroller import CollectinfoController + CollectinfoController.get_pmap = True mode = AdminMode.LIVE_CLUSTER if cli_args.collectinfo: @@ -625,8 +629,7 @@ def main(): ssl_context=ssl_context, only_connect_seed=cli_args.single_node, execute_only_mode=execute_only_mode, - timeout=cli_args.timeout, - get_pmap=cli_args.pmap) + timeout=cli_args.timeout) use_yappi = False if cli_args.profile: diff --git a/lib/basiccontroller.py b/lib/basiccontroller.py index a953d8ae..2ff9c597 100644 --- a/lib/basiccontroller.py +++ b/lib/basiccontroller.py @@ -53,9 +53,9 @@ class BasicRootController(BaseController): def __init__(self, seed_nodes=[('127.0.0.1', 3000, None)], user=None, password=None, auth_mode=constants.AuthMode.INTERNAL, use_services_alumni=False, use_services_alt=False, ssl_context=None, - asadm_version='', only_connect_seed=False, timeout=5, get_pmap=False): + asadm_version='', only_connect_seed=False, timeout=5): - super(BasicRootController, self).__init__(asadm_version, get_pmap) + super(BasicRootController, self).__init__(asadm_version) # Create static instance of cluster BasicRootController.cluster = Cluster(seed_nodes, user, password, auth_mode, @@ -974,6 +974,8 @@ def _do_default(self, line): @CommandHelp('"collectinfo" is used to collect cluster info, aerospike conf file and system stats.') class CollectinfoController(BasicCommandController): + get_pmap = False + def __init__(self): self.modifiers = set(['with']) self.aslogfile = "" @@ -1288,7 +1290,7 @@ def _get_collectinfo_data_json(self, default_user, default_pwd, default_ssh_port latency_map = self._get_as_latency() - if self.get_pmap: + if CollectinfoController.get_pmap: pmap_map = self._get_as_pmap() sys_map = self.cluster.info_system_statistics(default_user=default_user, default_pwd=default_pwd, default_ssh_key=default_ssh_key, @@ -1314,7 +1316,7 @@ def _get_collectinfo_data_json(self, default_user, default_pwd, default_ssh_port if node in latency_map: dump_map[node]['as_stat']['latency'] = latency_map[node] - if self.get_pmap and node in pmap_map: + if CollectinfoController.get_pmap and node in pmap_map: dump_map[node]['as_stat']['pmap'] = pmap_map[node] # Get the cluster name and add one more level in map diff --git a/lib/controllerlib.py b/lib/controllerlib.py index 186ee886..2d3e9cc5 100644 --- a/lib/controllerlib.py +++ b/lib/controllerlib.py @@ -81,13 +81,12 @@ class BaseController(object): asadm_version = '' logger = None - def __init__(self, asadm_version='', get_pmap=False): + def __init__(self, asadm_version=''): # Create static instances of view / health_checker / asadm_version / # logger BaseController.view = view.CliView() BaseController.health_checker = HealthChecker() BaseController.asadm_version = asadm_version - BaseController.get_pmap = get_pmap BaseController.logger = logging.getLogger("asadm") # instance vars self.modifiers = set() diff --git a/lib/getcontroller.py b/lib/getcontroller.py index c5e0e1e0..851cb696 100644 --- a/lib/getcontroller.py +++ b/lib/getcontroller.py @@ -637,11 +637,13 @@ def get_pmap(self, nodes='all'): getter = GetStatisticsController(self.cluster) node_ids = util.Future(self.cluster.info, 'node', nodes=nodes).start() pmap_info = util.Future(self.cluster.info, 'partition-info', nodes=nodes).start() - service_stats = getter.get_service(nodes=nodes) - namespace_stats = getter.get_namespace(nodes=nodes) + service_stats = util.Future(getter.get_service, nodes=nodes).start() + namespace_stats = util.Future(getter.get_namespace, nodes=nodes).start() node_ids = node_ids.result() pmap_info = pmap_info.result() + service_stats = service_stats.result() + namespace_stats = namespace_stats.result() cluster_keys = {} for node in list(service_stats.keys()): From 1e00919e03cf4cbcad24acab32fda99ff3ebc790 Mon Sep 17 00:00:00 2001 From: dylan Date: Tue, 11 Aug 2020 11:11:07 -0700 Subject: [PATCH 3/3] change --pmap help message --- lib/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils/conf.py b/lib/utils/conf.py index dd57bc83..70bc5a3f 100644 --- a/lib/utils/conf.py +++ b/lib/utils/conf.py @@ -419,7 +419,7 @@ def print_config_help(): print (" --single-node Enable asadm mode to connect only seed node. \n" " By default asadm connects to all nodes in cluster.") print (" --collectinfo Start asadm to run against offline collectinfo files.") - print (" --pmap Include partition map analysis in collect info file.") + print (" --pmap Include partition map analysis in collectinfo files.") print (" --log-analyser Start asadm in log-analyser mode and analyse data from log files.") print (" -f --log-path=path Path of cluster collectinfo file or directory \n" " containing collectinfo and system info files.")