diff --git a/docs/release-notes.md b/docs/release-notes.md index d394454cfe..70ab25f0a7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -1,5 +1,15 @@ # Release Notes +## 0.19.1 (Sep 6, 2022) + +This patch release for the 0.19.0 release includes the following important fixes: + +* Fixes bug in path whereby an address such as 10.1.1.110 was incorrectly matched with 10.1.1.1 causing path problems +* Fixes inconsistent use of "!" in filtering by device version string +* Fixes intermittent crashes in device show (or the status page in the GUI) when using columns=* +* Fixes incorrect hostname setting on NXOS devices in some conditions when the hostname is a hostname that includes the domain name +* Add missing keyword query-str to path commands + ## 0.19.0 (Aug 22, 2022) The 19th release of SuzieQ contains bug fixes and improvements to key functionalities such as the REST API and endpoint tracker. Here's a detailed list of key features of this release: diff --git a/pyproject.toml b/pyproject.toml index 7889149e44..c08d558822 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "suzieq" -version = "0.19.0" +version = "0.19.1" description = "A framework and application for network observability" readme = 'README.md' repository = 'https://github.com/netenglabs/suzieq' diff --git a/suzieq/cli/sqcmds/PathCmd.py b/suzieq/cli/sqcmds/PathCmd.py index a21e1d8d82..2c6aaf2aed 100644 --- a/suzieq/cli/sqcmds/PathCmd.py +++ b/suzieq/cli/sqcmds/PathCmd.py @@ -25,6 +25,7 @@ def __init__( src: str = "", dest: str = "", vrf: str = "", + query_str: str = ' ', ) -> None: super().__init__( engine=engine, @@ -34,6 +35,7 @@ def __init__( view=view, namespace=namespace, columns=columns, + query_str=query_str, format=format, sqobj=PathObj ) diff --git a/suzieq/engines/pandas/device.py b/suzieq/engines/pandas/device.py index b17002c183..f56b9bd9f1 100644 --- a/suzieq/engines/pandas/device.py +++ b/suzieq/engines/pandas/device.py @@ -84,6 +84,7 @@ def get(self, **kwargs): (df['status_y'] != 0) & (df['status_y'] != 200) & (df['status'] == "N/A"), 'neverpoll', df['status']) + if 'version' in df.columns: df['version'] = np.where(df.status_y == 418, 'unsupported', df.version) @@ -100,13 +101,13 @@ def get(self, **kwargs): df.address = np.where(df['address'] == 'N/A', df['hostname'], df['address']) - if 'uptime' in columns or columns == ['*']: - uptime_cols = (df['timestamp'] - - humanize_timestamp(df['bootupTimestamp']*1000, - self.cfg.get('analyzer', {}).get('timezone', - None))) - uptime_cols = pd.to_timedelta(uptime_cols, unit='s') - df.insert(len(df.columns)-1, 'uptime', uptime_cols) + if 'uptime' in columns or columns == ['*']: + uptime_cols = (df['timestamp'] - + humanize_timestamp(df['bootupTimestamp']*1000, + self.cfg.get('analyzer', {}).get('timezone', + None))) + uptime_cols = pd.to_timedelta(uptime_cols, unit='s') + df.insert(len(df.columns)-1, 'uptime', uptime_cols) if df.empty: return df[fields] @@ -116,9 +117,11 @@ def get(self, **kwargs): df = df.loc[df.status.isin(status)] if os_version: opdict = {'>': operator.gt, '<': operator.lt, '>=': operator.ge, - '<=': operator.le, '=': operator.eq, '!=': operator.ne} + '<=': operator.le, '=': operator.eq, '!': operator.ne} op = operator.eq for osv in os_version: + # Introduced in 0.19.1, we do this for backwards compatibility + osv = osv.replace('!=', '!') for elem, val in opdict.items(): if osv.startswith(elem): osv = osv.replace(elem, '') diff --git a/suzieq/engines/pandas/path.py b/suzieq/engines/pandas/path.py index cd1819b565..3c1fc1fe87 100644 --- a/suzieq/engines/pandas/path.py +++ b/suzieq/engines/pandas/path.py @@ -119,10 +119,10 @@ def _init_dfs(self, ns, source, dest): if ':' in source: self._src_df = self._if_df[self._if_df.ip6AddressList.astype(str) - .str.contains(source + "/")] + .str.startswith(source + "/")] else: self._src_df = self._if_df[self._if_df.ipAddressList.astype(str) - .str.contains(source + "/")] + .str.startswith(source + "/")] if self._src_df.empty: # TODO: No host with this src addr. Is addr a local ARP entry? @@ -147,10 +147,10 @@ def _init_dfs(self, ns, source, dest): if ':' in dest: self._dest_df = self._if_df[self._if_df.ip6AddressList.astype(str) - .str.contains(dest + "/")] + .str.startswith(dest + "/")] else: self._dest_df = self._if_df[self._if_df.ipAddressList.astype(str) - .str.contains(dest + "/")] + .str.startswith(dest + "/")] if self._dest_df.empty: # Check if addr is in any switch's local ARP @@ -752,6 +752,7 @@ def get(self, **kwargs) -> pd.DataFrame: src = kwargs.get("src", None) dest = kwargs.get("dest", None) dvrf = kwargs.get("vrf", "") + query_str = kwargs.pop('query_str', "") if not src or not dest: raise AttributeError("Must specify trace source and dest") @@ -929,10 +930,10 @@ def get(self, **kwargs) -> pd.DataFrame: if not nhdf.empty: if srcvers == 4: nhdf = nhdf.query( - f'ipAddressList.str.contains("{ndst}")') + f'ipAddressList.str.startswith("{ndst}/")') else: nhdf = nhdf.query( - f'ip6AddressList.str.contains("{ndst}")') + f'ip6AddressList.str.startswith("{ndst}/")') if not nhdf.empty: ifmac = nhdf.macaddr.unique().tolist() if (not macaddr) or (macaddr in ifmac): @@ -1120,7 +1121,9 @@ def get(self, **kwargs) -> pd.DataFrame: # This occurs when a path traversal terminates due to an error such # as loop detected final_paths = paths - return self._path_cons_result(final_paths) + return self._handle_user_query_str( + self._path_cons_result(final_paths), query_str)\ + .reset_index(drop=True) def _path_cons_result(self, paths): df_plist = [] diff --git a/suzieq/engines/pandas/topology.py b/suzieq/engines/pandas/topology.py index a7c5b3b901..e0c5f5aa49 100644 --- a/suzieq/engines/pandas/topology.py +++ b/suzieq/engines/pandas/topology.py @@ -23,6 +23,13 @@ class TopologyObj(SqPandasEngine): '''Backend class to operate on virtual table, topology, with pandas''' + def __init__(self, baseobj): + super().__init__(baseobj) + self.lsdb = pd.DataFrame() + self._a_df = pd.DataFrame() + self._ip_table = pd.DataFrame() + self.nses = [] + @staticmethod def table_name(): '''Table name''' @@ -86,9 +93,6 @@ class Services: self._init_dfs(self._namespaces) if self._if_df.empty: return pd.DataFrame({'error': ['No interfaces data found']}) - self.lsdb = pd.DataFrame() - self._a_df = pd.DataFrame() - self._ip_table = pd.DataFrame() fields = self.schema.get_display_fields(columns) diff --git a/suzieq/poller/worker/nodes/node.py b/suzieq/poller/worker/nodes/node.py index 7922f7f5f9..572432c496 100644 --- a/suzieq/poller/worker/nodes/node.py +++ b/suzieq/poller/worker/nodes/node.py @@ -409,7 +409,11 @@ async def _parse_device_type_hostname(self, output, _) -> None: f'Detected {devtype} for {self.address}:{self.port},' f' {hostname}') self._set_devtype(devtype, version_str) - self._set_hostname(hostname) + # We don't set the hostname here because the real hostname + # is retrieved via a different command on some platforms + # and can contain the FQDN. The hostname stored with a + # record needs to be one that is also used in LLDP so that we + # can find interface peers self.current_exception = None async def _detect_node_type(self): diff --git a/suzieq/poller/worker/services/service.py b/suzieq/poller/worker/services/service.py index ca2ef9b728..21d308012f 100644 --- a/suzieq/poller/worker/services/service.py +++ b/suzieq/poller/worker/services/service.py @@ -628,6 +628,7 @@ async def commit_data(self, result: Dict, namespace: str, hostname: str): hostname=[hostname], namespace=[namespace]).query('active') prev_res = df.to_dict('records') + self.previous_results[key] = prev_res if result or prev_res: adds, dels = self.get_diff(prev_res, result) diff --git a/suzieq/version.py b/suzieq/version.py index 88295496ab..7afe6ef469 100755 --- a/suzieq/version.py +++ b/suzieq/version.py @@ -2,7 +2,7 @@ """Store the Suzieq version string.""" -SUZIEQ_VERSION = "0.19.0" +SUZIEQ_VERSION = "0.19.1" if __name__ == '__main__': print(SUZIEQ_VERSION) diff --git a/tests/integration/sqcmds/common-samples/common-errors.yml b/tests/integration/sqcmds/common-samples/common-errors.yml new file mode 100644 index 0000000000..71a201979c --- /dev/null +++ b/tests/integration/sqcmds/common-samples/common-errors.yml @@ -0,0 +1,6 @@ +description: Testing some common errors independent of NOS +tests: +- command: topology summarize --namespace=whatever --format=json + data-directory: tests/data/parquet/ + marks: topology summarize + output: '{}' diff --git a/tests/integration/sqcmds/common-samples/not.yml b/tests/integration/sqcmds/common-samples/not.yml index 16b417d30a..58fbc4fed1 100644 --- a/tests/integration/sqcmds/common-samples/not.yml +++ b/tests/integration/sqcmds/common-samples/not.yml @@ -102769,3 +102769,24 @@ tests: ["ae1", "ae2", "ge-0/0/0"], "vlan": 10, "timestamp": 1631009089744}, {"namespace": "vmx", "hostname": "CRP-DIS-SW01", "vlanName": "vl20", "state": "active", "interfaces": ["ae1", "ae2", "ge-0/0/0"], "vlan": 20, "timestamp": 1631009089744}]' +- command: device show --version="!9.3(4)" --namespace=nxos --format=json + data-directory: tests/data/parquet/ + marks: device show negate + output: '[{"namespace": "nxos", "hostname": "server101", "model": "vm", "version": + "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive", + "address": "10.255.2.204", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256203}, + {"namespace": "nxos", "hostname": "server102", "model": "vm", "version": "18.04.3 + LTS", "vendor": "Ubuntu", "architecture": "x86-64", "status": "alive", "address": + "10.255.2.39", "bootupTimestamp": 1619182381.0, "timestamp": 1619275256290}, {"namespace": + "nxos", "hostname": "server301", "model": "vm", "version": "18.04.3 LTS", "vendor": + "Ubuntu", "architecture": "x86-64", "status": "alive", "address": "10.255.2.140", + "bootupTimestamp": 1619182381.0, "timestamp": 1619275256319}, {"namespace": "nxos", + "hostname": "server302", "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu", + "architecture": "x86-64", "status": "alive", "address": "10.255.2.114", "bootupTimestamp": + 1619182381.0, "timestamp": 1619275256394}, {"namespace": "nxos", "hostname": "firewall01", + "model": "vm", "version": "18.04.3 LTS", "vendor": "Ubuntu", "architecture": "x86-64", + "status": "alive", "address": "10.255.2.249", "bootupTimestamp": 1619013132.0, + "timestamp": 1619275256497}, {"namespace": "nxos", "hostname": "dcedge01", "model": + "vqfx-10000", "version": "19.4R1.10", "vendor": "Juniper", "architecture": "", + "status": "alive", "address": "10.255.2.250", "bootupTimestamp": 1619013129.329, + "timestamp": 1619275258329}]' diff --git a/tests/integration/sqcmds/cumulus-samples/badquery.yml b/tests/integration/sqcmds/cumulus-samples/badquery.yml index 768b6a5851..05c4f191b6 100644 --- a/tests/integration/sqcmds/cumulus-samples/badquery.yml +++ b/tests/integration/sqcmds/cumulus-samples/badquery.yml @@ -75,8 +75,9 @@ tests: - command: path show --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json data-directory: tests/data/parquet/ + error: + error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]' marks: path show badquery cumulus - output: '[]' - command: topology show --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' data-directory: tests/data/parquet/ @@ -158,8 +159,9 @@ tests: - command: path summarize --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json data-directory: tests/data/parquet/ + error: + error: '{"error": {"0": "ERROR: UserQueryError: name ''vendor'' is not defined"}}' marks: path summarize badquery cumulus - output: '[]' - command: topology summarize --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname' data-directory: tests/data/parquet/ @@ -241,8 +243,9 @@ tests: - command: path unique --src='172.16.1.101' --dest='172.16.3.202' --namespace='ospf-ibgp' --query-str='vendor == "Cisco" and mtu==1500' --format=json --columns='hostname' data-directory: tests/data/parquet/ + error: + error: '[{"error": "ERROR: UserQueryError: name ''vendor'' is not defined"}]' marks: path unique badquery cumulus - output: '[]' - command: topology unique --query-str='vendor == "Cisco" and mtu==1500' --format=json --namespace='ospf-single dual-evpn ospf-ibgp' --columns='hostname' data-directory: tests/data/parquet/