From 3fe5390e17e56ba0d5bf0e1d85fa91956235d313 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Mon, 29 Aug 2022 15:12:39 -0700 Subject: [PATCH 01/12] Device: Handle uptime field even when poller info is missing Signed-off-by: Dinesh Dutt --- suzieq/engines/pandas/device.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/suzieq/engines/pandas/device.py b/suzieq/engines/pandas/device.py index b17002c183..f1cd5f5488 100644 --- a/suzieq/engines/pandas/device.py +++ b/suzieq/engines/pandas/device.py @@ -84,6 +84,7 @@ def get(self, **kwargs): (df['status_y'] != 0) & (df['status_y'] != 200) & (df['status'] == "N/A"), 'neverpoll', df['status']) + if 'version' in df.columns: df['version'] = np.where(df.status_y == 418, 'unsupported', df.version) @@ -100,13 +101,13 @@ def get(self, **kwargs): df.address = np.where(df['address'] == 'N/A', df['hostname'], df['address']) - if 'uptime' in columns or columns == ['*']: - uptime_cols = (df['timestamp'] - - humanize_timestamp(df['bootupTimestamp']*1000, - self.cfg.get('analyzer', {}).get('timezone', - None))) - uptime_cols = pd.to_timedelta(uptime_cols, unit='s') - df.insert(len(df.columns)-1, 'uptime', uptime_cols) + if 'uptime' in columns or columns == ['*']: + uptime_cols = (df['timestamp'] - + humanize_timestamp(df['bootupTimestamp']*1000, + self.cfg.get('analyzer', {}).get('timezone', + None))) + uptime_cols = pd.to_timedelta(uptime_cols, unit='s') + df.insert(len(df.columns)-1, 'uptime', uptime_cols) if df.empty: return df[fields] From f53e1db0f69f31b17a05f55784172ab12464f3fe Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Tue, 30 Aug 2022 20:33:44 -0700 Subject: [PATCH 02/12] Topology: avoid crashing when namespace is unknown Signed-off-by: Dinesh Dutt --- suzieq/engines/pandas/topology.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/suzieq/engines/pandas/topology.py b/suzieq/engines/pandas/topology.py index a7c5b3b901..e0c5f5aa49 100644 --- a/suzieq/engines/pandas/topology.py +++ b/suzieq/engines/pandas/topology.py @@ -23,6 +23,13 @@ class TopologyObj(SqPandasEngine): '''Backend class to operate on virtual table, topology, with pandas''' + def __init__(self, baseobj): + super().__init__(baseobj) + self.lsdb = pd.DataFrame() + self._a_df = pd.DataFrame() + self._ip_table = pd.DataFrame() + self.nses = [] + @staticmethod def table_name(): '''Table name''' @@ -86,9 +93,6 @@ class Services: self._init_dfs(self._namespaces) if self._if_df.empty: return pd.DataFrame({'error': ['No interfaces data found']}) - self.lsdb = pd.DataFrame() - self._a_df = pd.DataFrame() - self._ip_table = pd.DataFrame() fields = self.schema.get_display_fields(columns) From ba2da088c7eac2b5aa88b93bdecbb96d7860e9c8 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Tue, 30 Aug 2022 20:34:02 -0700 Subject: [PATCH 03/12] path: fix logic for check of ipaddr is an interface ipaddr Signed-off-by: Dinesh Dutt --- suzieq/engines/pandas/path.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/suzieq/engines/pandas/path.py b/suzieq/engines/pandas/path.py index cd1819b565..71f3641f86 100644 --- a/suzieq/engines/pandas/path.py +++ b/suzieq/engines/pandas/path.py @@ -119,10 +119,10 @@ def _init_dfs(self, ns, source, dest): if ':' in source: self._src_df = self._if_df[self._if_df.ip6AddressList.astype(str) - .str.contains(source + "/")] + .str.startswith(source + "/")] else: self._src_df = self._if_df[self._if_df.ipAddressList.astype(str) - .str.contains(source + "/")] + .str.startswith(source + "/")] if self._src_df.empty: # TODO: No host with this src addr. Is addr a local ARP entry? @@ -147,10 +147,10 @@ def _init_dfs(self, ns, source, dest): if ':' in dest: self._dest_df = self._if_df[self._if_df.ip6AddressList.astype(str) - .str.contains(dest + "/")] + .str.startswith(dest + "/")] else: self._dest_df = self._if_df[self._if_df.ipAddressList.astype(str) - .str.contains(dest + "/")] + .str.startswith(dest + "/")] if self._dest_df.empty: # Check if addr is in any switch's local ARP @@ -929,10 +929,10 @@ def get(self, **kwargs) -> pd.DataFrame: if not nhdf.empty: if srcvers == 4: nhdf = nhdf.query( - f'ipAddressList.str.contains("{ndst}")') + f'ipAddressList.str.startswith("{ndst}/")') else: nhdf = nhdf.query( - f'ip6AddressList.str.contains("{ndst}")') + f'ip6AddressList.str.startswith("{ndst}/")') if not nhdf.empty: ifmac = nhdf.macaddr.unique().tolist() if (not macaddr) or (macaddr in ifmac): From 19c659fa815630561de3ea7c63cc3bf9ebced459 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Wed, 31 Aug 2022 06:26:38 -0700 Subject: [PATCH 04/12] Add common tests to catch common errors in commands Signed-off-by: Dinesh Dutt --- tests/integration/sqcmds/common-samples/common-errors.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/integration/sqcmds/common-samples/common-errors.yml diff --git a/tests/integration/sqcmds/common-samples/common-errors.yml b/tests/integration/sqcmds/common-samples/common-errors.yml new file mode 100644 index 0000000000..71a201979c --- /dev/null +++ b/tests/integration/sqcmds/common-samples/common-errors.yml @@ -0,0 +1,6 @@ +description: Testing some common errors independent of NOS +tests: +- command: topology summarize --namespace=whatever --format=json + data-directory: tests/data/parquet/ + marks: topology summarize + output: '{}' From 5224ff6e97496593f3804f0c188f5cea291bab7f Mon Sep 17 00:00:00 2001 From: Claudio Lorina Date: Thu, 1 Sep 2022 15:36:31 +0200 Subject: [PATCH 05/12] sq-poller: store previous result if there aren't changes Signed-off-by: Claudio Lorina --- suzieq/poller/worker/services/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/suzieq/poller/worker/services/service.py b/suzieq/poller/worker/services/service.py index ca2ef9b728..21d308012f 100644 --- a/suzieq/poller/worker/services/service.py +++ b/suzieq/poller/worker/services/service.py @@ -628,6 +628,7 @@ async def commit_data(self, result: Dict, namespace: str, hostname: str): hostname=[hostname], namespace=[namespace]).query('active') prev_res = df.to_dict('records') + self.previous_results[key] = prev_res if result or prev_res: adds, dels = self.get_diff(prev_res, result) From 35021a75d194bc02bd1eab81a83baf72880e293d Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Thu, 1 Sep 2022 22:19:13 -0700 Subject: [PATCH 06/12] Device: incorrectly expecting != instead of ! in version string, fix Signed-off-by: Dinesh Dutt --- suzieq/engines/pandas/device.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/suzieq/engines/pandas/device.py b/suzieq/engines/pandas/device.py index f1cd5f5488..f56b9bd9f1 100644 --- a/suzieq/engines/pandas/device.py +++ b/suzieq/engines/pandas/device.py @@ -117,9 +117,11 @@ def get(self, **kwargs): df = df.loc[df.status.isin(status)] if os_version: opdict = {'>': operator.gt, '<': operator.lt, '>=': operator.ge, - '<=': operator.le, '=': operator.eq, '!=': operator.ne} + '<=': operator.le, '=': operator.eq, '!': operator.ne} op = operator.eq for osv in os_version: + # Introduced in 0.19.1, we do this for backwards compatibility + osv = osv.replace('!=', '!') for elem, val in opdict.items(): if osv.startswith(elem): osv = osv.replace(elem, '') From e41f9027bd4b680d698835b0ea6319e34691a9c0 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Thu, 1 Sep 2022 23:22:34 -0700 Subject: [PATCH 07/12] Add additional test for device version Signed-off-by: Dinesh Dutt --- .../integration/sqcmds/common-samples/not.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/integration/sqcmds/common-samples/not.yml b/tests/integration/sqcmds/common-samples/not.yml index 16b417d30a..58fbc4fed1 100644 --- a/tests/integration/sqcmds/common-samples/not.yml +++ b/tests/integration/sqcmds/common-samples/not.yml @@ -102769,3 +102769,24 @@ tests: ["ae1", "ae2", "ge-0/0/0"], "vlan": 10, "timestamp": 1631009089744}, {"namespace": "vmx", "hostname": "CRP-DIS-SW01", "vlanName": "vl20", "state": "active", "interfaces": ["ae1", "ae2", "ge-0/0/0"], "vlan": 20, "timestamp": 1631009089744}]' +- command: device show --version="!9.3(4)" --namespace=nxos --format=json + data-directory: tests/data/parquet/ + marks: device show negate + output: '[{"namespace": "nxos", "hostname": "server101", "model": "vm", "version": + "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive", + "address": "10.255.2.204", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256203}, + {"namespace": "nxos", "hostname": "server102", "model": "vm", "version": "18.04.3 + LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive", "address": + "10.255.2.39", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256290}, {"namespace": + "nxos", "hostname": "server301", "model": "vm", "version": "18.04.3 LTS", "vendor": + "Ubuntu", "architecture": "x86-64", "status": "alive", "address": "10.255.2.140", + "bootupTimestamp": 1619182381.0, "timestamp": 1619275256319}, {"namespace": "nxos", + "hostname": "server302", "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu", + "architecture": "x86-64", "status": "alive", "address": "10.255.2.114", "bootupTimestamp": + 1619182381.0, "timestamp": 1619275256394}, {"namespace": "nxos", "hostname": "firewall01", + "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64", + "status": "alive", "address": "10.255.2.249", "bootupTimestamp": 1619013132.0, + "timestamp": 1619275256497}, {"namespace": "nxos", "hostname": "dcedge01", "model": + "vqfx-10000", "version": "19.4R1.10", "vendor": "Juniper", "architecture": "", + "status": "alive", "address": "10.255.2.250", "bootupTimestamp": 1619013129.329, + "timestamp": 1619275258329}]' From 5c79e639b9516edf196b733aee1ac21a5ead7bb6 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Sun, 4 Sep 2022 06:08:07 -0700 Subject: [PATCH 08/12] Ensure consistent hostname setting During the initial node discovery, we retrieve the hostname for a device and use that. However, on initializing the SSH for the first time, we also reinit the hostname, and sometimes from a different command than whats present in the initial node discovery phase. The hostname needs to be whatever is advertised in LLDP because that is how we find peers of an interface and is essential for many tasks that we do. This diff in hostname computed during node discovery from the one in polling can cause us to recreate incorrect hostnames. This patch ensures that we only ever use the hostname computed during ssh setup after node discovery. Signed-off-by: Dinesh Dutt --- suzieq/poller/worker/nodes/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/suzieq/poller/worker/nodes/node.py b/suzieq/poller/worker/nodes/node.py index 7922f7f5f9..572432c496 100644 --- a/suzieq/poller/worker/nodes/node.py +++ b/suzieq/poller/worker/nodes/node.py @@ -409,7 +409,11 @@ async def _parse_device_type_hostname(self, output, _) -> None: f'Detected {devtype} for {self.address}:{self.port},' f' {hostname}') self._set_devtype(devtype, version_str) - self._set_hostname(hostname) + # We don't set the hostname here because the real hostname + # is retrieved via a different command on some platforms + # and can contain the FQDN. The hostname stored with a + # record needs to be one that is also used in LLDP so that we + # can find interface peers self.current_exception = None async def _detect_node_type(self): From d2f057298d8fb6be6e01f4449aeb93a4a6063907 Mon Sep 17 00:00:00 2001 From: LucaNicosia Date: Sun, 4 Sep 2022 15:24:59 +0200 Subject: [PATCH 09/12] add query_str to path Signed-off-by: LucaNicosia --- suzieq/cli/sqcmds/PathCmd.py | 2 ++ suzieq/engines/pandas/path.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/suzieq/cli/sqcmds/PathCmd.py b/suzieq/cli/sqcmds/PathCmd.py index a21e1d8d82..2c6aaf2aed 100644 --- a/suzieq/cli/sqcmds/PathCmd.py +++ b/suzieq/cli/sqcmds/PathCmd.py @@ -25,6 +25,7 @@ def __init__( src: str = "", dest: str = "", vrf: str = "", + query_str: str = ' ', ) -> None: super().__init__( engine=engine, @@ -34,6 +35,7 @@ def __init__( view=view, namespace=namespace, columns=columns, + query_str=query_str, format=format, sqobj=PathObj ) diff --git a/suzieq/engines/pandas/path.py b/suzieq/engines/pandas/path.py index 71f3641f86..3c1fc1fe87 100644 --- a/suzieq/engines/pandas/path.py +++ b/suzieq/engines/pandas/path.py @@ -752,6 +752,7 @@ def get(self, **kwargs) -> pd.DataFrame: src = kwargs.get("src", None) dest = kwargs.get("dest", None) dvrf = kwargs.get("vrf", "") + query_str = kwargs.pop('query_str', "") if not src or not dest: raise AttributeError("Must specify trace source and dest") @@ -1120,7 +1121,9 @@ def get(self, **kwargs) -> pd.DataFrame: # This occurs when a path traversal terminates due to an error such # as loop detected final_paths = paths - return self._path_cons_result(final_paths) + return self._handle_user_query_str( + self._path_cons_result(final_paths), query_str)\ + .reset_index(drop=True) def _path_cons_result(self, paths): df_plist = [] From 71976f43336fc85703d86410d5878baccae82203 Mon Sep 17 00:00:00 2001 From: LucaNicosia Date: Sun, 4 Sep 2022 15:25:16 +0200 Subject: [PATCH 10/12] test fixes Signed-off-by: LucaNicosia --- tests/integration/sqcmds/cumulus-samples/badquery.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/sqcmds/cumulus-samples/badquery.yml b/tests/integration/sqcmds/cumulus-samples/badquery.yml index 768b6a5851..05c4f191b6 100644 --- a/tests/integration/sqcmds/cumulus-samples/badquery.yml +++ b/tests/integration/sqcmds/cumulus-samples/badquery.yml @@ -75,8 +75,9 @@ tests: - command: path show --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json data-directory: tests/data/parquet/ + error: + error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]' marks: path show badquery cumulus - output: '[]' - command: topology show --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' data-directory: tests/data/parquet/ @@ -158,8 +159,9 @@ tests: - command: path summarize --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json data-directory: tests/data/parquet/ + error: + error: '{"error": {"0": "ERROR: UserQueryError: name ''vendor'' is not defined"}}' marks: path summarize badquery cumulus - output: '[]' - command: topology summarize --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname' data-directory: tests/data/parquet/ @@ -241,8 +243,9 @@ tests: - command: path unique --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json --columns='hostname' data-directory: tests/data/parquet/ + error: + error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]' marks: path unique badquery cumulus - output: '[]' - command: topology unique --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname' data-directory: tests/data/parquet/ From 3e2cd03423b85ddd0de351e7a301880208f577c5 Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Tue, 6 Sep 2022 04:33:56 -0700 Subject: [PATCH 11/12] Bump version to 0.19.1 Signed-off-by: Dinesh Dutt --- pyproject.toml | 2 +- suzieq/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7889149e44..c08d558822 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "suzieq" -version = "0.19.0" +version = "0.19.1" description = "A framework and application for network observability" readme = 'README.md' repository = 'https://github.com/netenglabs/suzieq' diff --git a/suzieq/version.py b/suzieq/version.py index 88295496ab..7afe6ef469 100755 --- a/suzieq/version.py +++ b/suzieq/version.py @@ -2,7 +2,7 @@ """Store the Suzieq version string.""" -SUZIEQ_VERSION = "0.19.0" +SUZIEQ_VERSION = "0.19.1" if __name__ == '__main__': print(SUZIEQ_VERSION) From d4559d00ee2cc5f27e9e3b4b295e59635b04a4dc Mon Sep 17 00:00:00 2001 From: Dinesh Dutt Date: Tue, 6 Sep 2022 04:34:45 -0700 Subject: [PATCH 12/12] Updated release notes for 0.19.1 Signed-off-by: Dinesh Dutt --- docs/release-notes.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/release-notes.md b/docs/release-notes.md index d394454cfe..70ab25f0a7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,15 @@ # Release Notes +## 0.19.1 (Sep 6, 2022) + +This patch release for the 0.19.0 release includes the following important fixes: + +* Fixes bug in path whereby an address such as 10.1.1.110 was incorrectly matched with 10.1.1.1 causing path problems +* Fixes inconsistent use of "!" in filtering by device version string +* Fixes intermittent crashes in device show (or the status page in the GUI) when using columns=* +* Fixes incorrect hostname setting on NXOS devices in some conditions when the hostname is a hostname that includes the domain name +* Add missing keyword query-str to path commands + ## 0.19.0 (Aug 22, 2022) The 19th release of SuzieQ contains bug fixes and improvements to key functionalities such as the REST API and endpoint tracker. Here's a detailed list of key features of this release: