From 0b6e9fdde5aa8a68c9f7e8922e09adb9b91efe62 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 23 May 2024 11:03:48 +0200 Subject: [PATCH 01/33] objects: add domain and host sets New objects can be used inside the firewall configuration. --- setup.py | 2 +- src/nethsec/objects/__init__.py | 586 ++++++++++++++++++++++++++++++++ tests/test_firewall.py | 6 +- tests/test_objects.py | 245 +++++++++++++ 4 files changed, 837 insertions(+), 2 deletions(-) create mode 100644 src/nethsec/objects/__init__.py create mode 100644 tests/test_objects.py diff --git a/setup.py b/setup.py index 2f1f3f2a..9a3ee25c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ url = "https://github.com/NethServer/python3-nethsec", license = "GPLv3", package_dir = {'': 'src'}, - packages = ['nethsec', 'nethsec.utils', 'nethsec.firewall', 'nethsec.mwan', 'nethsec.dpi', 'nethsec.ipsec', 'nethsec.ovpn', 'nethsec.users', 'nethsec.reverse_proxy', 'nethsec.inventory', 'nethsec.conntrack', 'nethsec.ldif'], + packages = ['nethsec', 'nethsec.utils', 'nethsec.firewall', 'nethsec.mwan', 'nethsec.dpi', 'nethsec.ipsec', 'nethsec.ovpn', 'nethsec.users', 'nethsec.reverse_proxy', 'nethsec.inventory', 'nethsec.conntrack', 'nethsec.ldif', 'nethsec.objects'], requires = [ "pyuci" ], classifiers = [ "Programming Language :: Python :: 3", diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py new file mode 100644 index 00000000..2dcd5af7 --- /dev/null +++ b/src/nethsec/objects/__init__.py @@ -0,0 +1,586 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +''' +Objects utilities +''' +import ipaddress +import json +import os +import subprocess + +from euci import EUci + +from nethsec import utils, firewall + +# Generic + +def is_object_id(id): + """ + Check if an id is an object id. + + Args: + id: id to check + + Returns: + True if id is an object id, False otherwise + """ + return id.startswith('objects/') or id.startswith('dhcp/') or id.startswith('users/') + +def _validate_object(uci, id): + """ + Check if the object exists. + + Args: + id: id to check + + Returns: + True if object exists, False otherwise + """ + database, id = id.split('/') + try: + uci.get(database, id) + except: + raise utils.ValidationError('id', 'object_does_not_exists', id) + +def get_object(uci, database_id): + """ + Get object from objects config. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + object from config or None if not found + """ + try: + database, id = database_id.split('/') + return uci.get_all(database, id) + except: + return None + +def is_used_object(uci, database_id): + """ + Check if an object is used in firewall config. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + matches = [] + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_src', default=None) == database_id or uci.get('firewall', section, 'ns_dst', default=None) == database_id: + matches.append(f'firewall/{section}') + return len(matches) > 0, matches + +def get_object_ips(uci, database_id): + """ + Get all IP addresses from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + a list of unique IP addresses from the object + """ + ips = [] + + obj = get_object(uci, database_id) + database, id = database_id.split('/') + + if not obj: + return ips + + if database == 'dhcp': + ip = obj.get('ip') + if ip: + ips.append(ip) + elif database == 'users': + openvpn_ipaddr = obj.get('openvpn_ipaddr') + if openvpn_ipaddr: + ips.append(openvpn_ipaddr) + elif database == 'objects': + ipaddr = obj.get('ipaddr') + if ipaddr: + for ip in ipaddr: + if is_object_id(ip): + ips.extend(get_object_ips(uci, ip)) + else: + ips.append(ip) + + return list(set(ips)) # Convert the list to a set to remove duplicates, then convert it back to a list + +def get_object_first_ip(uci, database_id): + """ + Get the first IP address from an object. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + the first IP address from the object + """ + ips = get_object_ips(uci, database_id) + if ips: + return ips[0] + return None + +# Domain set + +def is_used_domain_set(uci, id): + """ + Check if domain set is used in firewall config. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + A tuple with: + - True if domain set is used in firewall config, False otherwise + - a list of firewall sections where domain set is used + """ + return is_used_object(uci, f'objects/{id}') + +def get_domain_set_ipsets(uci, id): + """ + Get ipsets linked to domain set. + + Args: + uci: EUci pointer + id: id of domain set + + Returns: + a dictionary with + - `firewall`: the ipset id linked to domain set from firewall config + - `dhcp`: the ipset id linked to domain set from dhcp config + """ + ipsets = {"firewall": None, "dhcp": None} + for section in utils.get_all_by_type(uci, "firewall", "ipset"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["firewall"] = section + break + for section in utils.get_all_by_type(uci, "dhcp", "ipset"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + ipsets["dhcp"] = section + break + return ipsets + +def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int = 600) -> str: + """ + Add domain set to objects config. + + Args: + uci: EUci pointer + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `600` seconds + + Returns: + id of domain set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + id = utils.get_random_id() + uci.set('objects', id, 'domain') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', domains) + uci.save('objects') + + # create ipset inside dhcp config + ipset = utils.get_random_id() + uci.set('dhcp', ipset, 'ipset') + uci.set('dhcp', ipset, 'name', [name]) + uci.set('dhcp', ipset, 'domain', domains) + uci.set('dhcp', ipset, 'table_family', 'inet') + uci.set('dhcp', ipset, 'ns_link', f'objects/{id}') + uci.save('dhcp') + + # create ipset inside firewall config + ipset = utils.get_random_id() + uci.set('firewall', ipset, 'ipset') + uci.set('firewall', ipset, 'name', name) + uci.set('firewall', ipset, 'family', family) + uci.set('firewall', ipset, 'timeout', timeout) + uci.set('firewall', ipset, 'counters', '1') + uci.set('firewall', ipset, 'match', 'ip') + uci.set('firewall', ipset, 'ns_link', f'objects/{id}') + uci.save('firewall') + return id + +def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], timeout: int = 600) -> str: + """ + Edit domain set in objects config. + + Args: + uci: EUci pointer + id: id of domain set to edit + name: name of domain set + family: can be `ipv4` or `ipv6` + domains: a list of valid DNS names + timeout: the timeout in seconds for the DNS resolution, default is `600` seconds + + Returns: + id of domain set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + if family not in ['ipv4', 'ipv6']: + raise utils.ValidationError('family', 'invalid_family', family) + if timeout < 0: + raise utils.ValidationError('timeout', 'invalid_timeout', timeout) + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'timeout', timeout) + uci.set('objects', id, 'domain', domains) + uci.save('objects') + + # update ipset inside dhcp config + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('dhcp', section, 'name', [name]) + uci.set('dhcp', section, 'domain', domains) + uci.set('dhcp', section, 'ns_tag', ['automated']) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.set('firewall', section, 'name', name) + uci.set('firewall', section, 'family', family) + uci.set('firewall', section, 'timeout', timeout) + uci.set('firewall', section, 'ns_tag', ['automated']) + uci.save('firewall') + break + return id + +def delete_domain_set(uci, id: str) -> str: + """ + Delete domain set from objects config. + + Args: + uci: EUci pointer + id: id of domain set to delete + + Returns: + name of domain set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "domain_set_does_not_exists", id) + uci.delete('objects', id) + uci.save('objects') + for section in uci.get_all("dhcp"): + if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('dhcp', section) + uci.save('dhcp') + break + for section in uci.get_all("firewall"): + if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': + uci.delete('firewall', section) + uci.save('firewall') + break + return id + +def list_domain_sets(uci) -> list: + """ + Get all domain sets from objects config + + Args: + uci: EUci pointer + + Returns: + a list of all domain sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'domain': + rule = uci.get_all('objects', section) + rule['id'] = section + used, matches = is_used_domain_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +# Host set + +def _validate_host_set_ipaddr(uci, ipaddr: str, family: str): + if is_object_id(ipaddr): + return _validate_object(uci, ipaddr) + if family == 'ipv4': + return _validate_host_set_ipaddr_v4(ipaddr) + elif family == 'ipv6': + return _validate_host_set_ipaddr_v6(ipaddr) + +def _validate_host_set_ipaddr_v4(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv4Network(ipaddr) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv4Address(start) + ipaddress.IPv4Address(end) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv4 + try: + ipaddress.IPv4Address(ipaddr) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def _validate_host_set_ipaddr_v6(ipaddr: str): + if '/' in ipaddr: + # validate CIDR + try: + ipaddress.IPv6Network(ipaddr) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + elif '-' in ipaddr: + start, end = ipaddr.split('-') + try: + ipaddress.IPv6Address(start) + ipaddress.IPv6Address(end) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + else: + # validate IPv6 + try: + ipaddress.IPv6Address(ipaddr) + except ipaddress.AddressValueError: + raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) + return True + +def add_host_set(uci, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Add host set to objects config. + + Args: + uci: EUci pointer + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was added + """ + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + # check name contains only number and letters + if not name.isalnum(): + raise utils.ValidationError('name', 'invalid_name', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, ipaddr, family) + id = utils.get_random_id() + uci.set('objects', id, 'host') + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def edit_host_set(uci, id: str, name: str, family: str, ipaddrs: list[str]) -> str: + """ + Edit host set in objects config. + + Args: + uci: EUci pointer + id: id of host set to edit + name: name of host set + family: can be `ipv4` or `ipv6` + ipaddrs: a list of IP addresses + + Returns: + id of host set config that was edited + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + if len(name) > 16: + raise utils.ValidationError('name', 'name_too_long', name) + for ipaddr in ipaddrs: + _validate_host_set_ipaddr(uci, ipaddr, family) + uci.set('objects', id, 'name', name) + uci.set('objects', id, 'family', family) + uci.set('objects', id, 'ipaddr', ipaddrs) + uci.save('objects') + return id + +def delete_host_set(uci, id: str) -> str: + """ + Delete host set from objects config. + + Args: + uci: EUci pointer + id: id of host set to delete + + Returns: + name of host set config that was deleted + """ + if not uci.get('objects', id, default=None): + raise utils.ValidationError("id", "host_set_does_not_exists", id) + uci.delete('objects', id) + uci.save('objects') + return id + +def is_used_host_set(uci, id): + """ + Check if host set is used in firewall config. + + Args: + uci: EUci pointer + id: id of host set + + Returns: + A tuple with: + - True if host set is used in firewall config, False otherwise + - a list of firewall sections where host set is used + """ + return is_used_object(uci, f'objects/{id}') + +def list_host_sets(uci) -> list: + """ + Get all host sets from objects config + + Args: + uci: EUci pointer + + Returns: + a list of all host sets + """ + sets = [] + for section in uci.get_all("objects"): + if uci.get('objects', section) == 'host': + rule = uci.get_all('objects', section) + rule['id'] = section + used, matches = is_used_host_set(uci, section) + rule['used'] = used + rule['matches'] = matches + sets.append(rule) + return sets + +# Rules + +def update_redirect_rules(uci): + """ + Update redirect rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + changed_sections: list of changed objects, each object is in the form of `/` + """ + for section in utils.get_all_by_type(uci, 'firewall', 'redirect'): + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_dst: + ipaddr = get_object_first_ip(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database == "objects" and obj_type == "domain": + ipsets = get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src_net") + try: + uci.delete('firewall', f"{section}_ipset") + except: + pass + else: + uci.set('firewall', section, 'ipset', f"{id}_ipset") + uci.set('firewall', f"{section}_ipset", "ipset") + uci.set('firewall', f"{section}_ipset", "name", f"{id}_ipset") + uci.set('firewall', f"{section}_ipset", "match", "src_net") + uci.set('firewall', f"{section}_ipset", "enabled", "1") + uci.set('firewall', f"{section}_ipset", "entry", get_object_ips(uci, ns_src)) + + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + ns_src = uci.get('firewall', section, 'ns_src', default=None) + if ns_dst: + ipaddr = get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + ipaddr = get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + uci.save('firewall') + +def update_firewall_rules(uci): + """ + Update firewall rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + """ + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + keep_ipset = False + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_src: + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database =="objects" and obj_type == "domain": + keep_ipset = True + ipsets = get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") + try: + uci.delete('firewall', section, 'src_ip') + except: + pass + else: + ipaddr = get_object_ips(uci, ns_src) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + try: + uci.delete('firewall', section, 'ipset') + except: + pass + if ns_dst: + database, id = ns_dst.split('/') + obj_type = uci.get(database, id) + if database =="objects" and obj_type == "domain": + ipsets = get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") + try: + uci.delete('firewall', section, 'dest_ip') + except: + pass + else: + ipaddr = get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if not keep_ipset: # do not delete ipset from src if is a domain + try: + uci.delete('firewall', section, 'ipset') + except: + pass + uci.save('firewall') \ No newline at end of file diff --git a/tests/test_firewall.py b/tests/test_firewall.py index 78be74bc..41bea6d6 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -368,6 +368,9 @@ 1704874398 ac:57:26:00:24:8c 192.168.1.219 test2 01:dc:57:26:00:25:8c """ +objects_db = """ +""" + # Setup fake ip command output ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' mock_ip_stdout = MagicMock() @@ -386,6 +389,8 @@ def _setup_db(tmp_path): fp.write(dhcp_db) with tmp_path.joinpath('netmap').open('w') as fp: fp.write(netmap_db) + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) return EUci(confdir=tmp_path.as_posix()) def test_add_interface_to_zone(tmp_path): @@ -1029,4 +1034,3 @@ def test_list_netmap_devices(mock_run, tmp_path): assert 'br-lan' in device_names assert 'eth1.4' in device_names assert {'device': 'br-lan', 'interface': 'lan'} in devices - \ No newline at end of file diff --git a/tests/test_objects.py b/tests/test_objects.py new file mode 100644 index 00000000..48d4e939 --- /dev/null +++ b/tests/test_objects.py @@ -0,0 +1,245 @@ +from nethsec import utils +from nethsec.utils import ValidationError +import pytest +from euci import EUci, UciExceptionNotFound + +from nethsec import firewall, objects + +objects_db = """ +""" + +firewall_db = """ +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' + +config redirect 'redirect1' + option ns_src '' + option ipset 'redirect1_ipset' + +config ipset 'redirect1_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect2' + option ns_src '' +""" + +dhcp_db = """ +config domain 'ns_8bec5896' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config host 'ns_8dcab636' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' +""" + +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" +""" + +def _setup_db(tmp_path): + # setup fake dbs + with tmp_path.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with tmp_path.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) + with tmp_path.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) + return EUci(confdir=tmp_path.as_posix()) + +def test_add_doman_set(tmp_path): + u = _setup_db(tmp_path) + id1 = objects.add_domain_set(u, "mydomainset", "ipv4", ["test1.com", "test2.com"]) + assert u.get("objects", id1, "name") == "mydomainset" + assert u.get("objects", id1, "family") == "ipv4" + assert u.get_all("objects", id1, "domain") == ("test1.com", "test2.com") + assert u.get("objects", id1, "timeout") == "600" + + linked = firewall.get_all_linked(u, f"objects/{id1}") + assert linked['firewall'] is not None + assert u.get('firewall', linked['firewall'][0], 'ns_link') == f"objects/{id1}" + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset" + assert u.get('firewall', linked['firewall'][0], 'family') == 'ipv4' + assert u.get('firewall', linked['firewall'][0], 'timeout') == '600' + + assert linked['dhcp'] is not None + assert u.get('dhcp', linked['dhcp'][0], 'ns_link') == f"objects/{id1}" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset",) + assert u.get_all('dhcp', linked['dhcp'][0], 'domain') == ("test1.com", "test2.com") + + id2 = objects.add_domain_set(u, "mydomainset2", "ipv6", ["test3.com", "test4.com"], 600) + assert u.get("objects", id2, "name") == "mydomainset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id2, "timeout") == "600" + linked = firewall.get_all_linked(u, f"objects/{id2}") + assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset2" + assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset2",) + +def test_edit_domain_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_domain_set(u, "mydomainset3", "ipv4", ["test1.com", "test2.com"]) + objects.edit_domain_set(u, id, "mydomainset3b", "ipv6", ["test3.com", "test4.com"], 600) + assert u.get("objects", id, "name") == "mydomainset3b" + assert u.get("objects", id, "family") == "ipv6" + assert u.get_all("objects", id, "domain") == ("test3.com", "test4.com") + assert u.get("objects", id, "timeout") == "600" + +def test_delete_domain_set(tmp_path): + u = _setup_db(tmp_path) + with pytest.raises(ValidationError): + objects.delete_domain_set(u, "notpresent") + id = objects.add_domain_set(u, "mydomainset4", "ipv4", ["test1.com", "test2.com"]) + assert objects.delete_domain_set(u, id) == id + linked = firewall.get_all_linked(u, f"objects/{id}") + assert linked['firewall'] == [] + assert linked['dhcp'] == [] + +def test_is_used_domain_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r1', 'ns_dst', f"objects/{id}") + used, matches = objects.is_used_domain_set(u, id) + assert used + assert matches == ["firewall/r1"] + +def test_list_domain_sets(tmp_path): + u = _setup_db(tmp_path) + sets = objects.list_domain_sets(u) + assert len(sets) == 4 + +def test_add_host_set(tmp_path): + u = _setup_db(tmp_path) + with pytest.raises(ValidationError): + objects.add_host_set(u, "myhostset", "ipv4", ["a.b.c.d", "e.f.g.h"]) + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10"]) + assert u.get("objects", id1, "name") == "myhostset" + assert u.get_all("objects", id1, "ipaddr") == ("1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10") + assert u.get("objects", id1, "family") == "ipv4" + id2 = objects.add_host_set(u, "myhostset2", "ipv6", ["2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890"]) + assert u.get("objects", id2, "name") == "myhostset2" + assert u.get("objects", id2, "family") == "ipv6" + assert u.get_all("objects", id2, "ipaddr") == ("2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890") + +def test_edit_host_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_host_set(u, "myhostset3", "ipv4", ["6.7.8.9"]) + objects.edit_host_set(u, id, "myhostset3b", "ipv4", ["1.1.1.1", "2.2.2.2"]) + assert u.get("objects", id, "name") == "myhostset3b" + assert u.get_all("objects", id, "ipaddr") == ("1.1.1.1", "2.2.2.2") + +def test_delete_host_set(tmp_path): + u = _setup_db(tmp_path) + with pytest.raises(ValidationError): + objects.delete_host_set(u, "notpresent") + id = objects.add_host_set(u, "myhostset4", "ipv4", ["6.7.8.9"]) + assert objects.delete_host_set(u, id) == id + +def test_is_used_host_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) + u.set('firewall', 'r3', 'ns_src', f"objects/{id}") + used, matches = objects.is_used_host_set(u, id) + assert used + assert matches == ["firewall/r3"] + +def test_list_host_sets(tmp_path): + u = _setup_db(tmp_path) + sets = objects.list_host_sets(u) + assert len(sets) == 4 + +def test_is_used_object(tmp_path): + u = _setup_db(tmp_path) + used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") + assert used + assert matches == ["firewall/r1"] + assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False + +def test_get_object(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + obj = objects.get_object(u, id) + +def test_get_object_ips(tmp_path): + u = _setup_db(tmp_path) + id0 = objects.add_host_set(u, "myhostset0", "ipv4", ["4.5.6.7"]) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) + ips = objects.get_object_ips(u, f"objects/{id}") + assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order + +def test_update_redirect_rules(tmp_path): + u = _setup_db(tmp_path) + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "redirect1", "ns_src", f"objects/{domain1}") # domain can be used only as source + u.set('firewall', 'redirect1', 'ns_dst', f"dhcp/ns_8bec5896") + objects.update_redirect_rules(u) + assert u.get("firewall", "redirect1", "dest_ip") == "7.8.9.1" + assert u.get("firewall", "redirect1", "ipset") == f"{ipsets['firewall']} src_net" + u.set("firewall", "redirect2", "ns_src", f"objects/{host1}") + u.set('firewall', 'redirect2', 'ns_dst', f"users/ns_user1") + objects.update_redirect_rules(u) + assert u.get("firewall", "redirect2", "dest_ip") == "10.10.10.22" + assert u.get("firewall", "redirect2", "ipset") == f"{host1}_ipset" + assert u.get("firewall", "redirect2_ipset") + +def test_update_firewall_rules(tmp_path): + u = _setup_db(tmp_path) + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "r1", "ns_dst", f"objects/{domain1}") + u.set("firewall", "r1", "ns_src", f"objects/{host1}") + objects.update_firewall_rules(u) + assert set(u.get_all("firewall", "r1", "src_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + assert u.get("firewall", "r1", "ipset") == f"{ipsets['firewall']} dst" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r1", "dest_ip") + + u.set("firewall", "r2", "ns_dst", f"objects/{host1}") + u.set("firewall", "r2", "ns_src", f"objects/{domain1}") + objects.update_firewall_rules(u) + + assert u.get("firewall", "r2", "ipset") == f"{ipsets['firewall']} src" + assert set(u.get_all("firewall", "r2", "dest_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r2", "src_ip") + + u.set("firewall", "r3", "ns_src", "dhcp/ns_8bec5896") + u.set("firewall", "r3", "ns_dst", "users/ns_user1") + objects.update_firewall_rules(u) + assert u.get_all("firewall", "r3", "src_ip") == ("7.8.9.1",) + assert u.get_all("firewall", "r3", "dest_ip") == ("10.10.10.22",) + + u.set("firewall", "r4", "ns_src", f"objects/{domain1}") + u.set("firewall", "r4", "dest_ip", "1.2.3.4") + objects.update_firewall_rules(u) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r4", "ns_dst") \ No newline at end of file From ffa7213246c032f54e5fbb62b71f1c36b3c350fb Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 28 May 2024 15:17:39 +0200 Subject: [PATCH 02/33] firewall: move updates rules functions Functions moved from objects package --- src/nethsec/firewall/__init__.py | 103 ++++++++++++++++++++++++++++++- src/nethsec/objects/__init__.py | 99 ----------------------------- tests/test_firewall.py | 101 +++++++++++++++++++++++++++++- tests/test_objects.py | 82 ++++-------------------- 4 files changed, 211 insertions(+), 174 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 17f19b25..c36dd749 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -15,7 +15,7 @@ from euci import EUci -from nethsec import utils +from nethsec import utils, objects PROTOCOLS = ['tcp', 'udp', 'udplite', 'icmp', 'esp', 'ah', 'sctp'] TARGETS = ['ACCEPT', 'DROP', 'REJECT'] @@ -1866,4 +1866,103 @@ def list_netmap_devices(uci) -> list: devices.append({"device": device['ifname'], "interface": utils.get_interface_from_device(uci, device['ifname'])}) except: pass - return devices \ No newline at end of file + return devices + +# Objects + +def update_redirect_rules(uci): + """ + Update redirect rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + changed_sections: list of changed objects, each object is in the form of `/` + """ + for section in utils.get_all_by_type(uci, 'firewall', 'redirect'): + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_dst: + ipaddr = objects.get_object_first_ip(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database == "objects" and obj_type == "domain": + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src_net") + try: + uci.delete('firewall', f"{section}_ipset") + except: + pass + else: + uci.set('firewall', section, 'ipset', f"{id}_ipset") + uci.set('firewall', f"{section}_ipset", "ipset") + uci.set('firewall', f"{section}_ipset", "name", f"{id}_ipset") + uci.set('firewall', f"{section}_ipset", "match", "src_net") + uci.set('firewall', f"{section}_ipset", "enabled", "1") + uci.set('firewall', f"{section}_ipset", "entry", objects.get_object_ips(uci, ns_src)) + + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + ns_src = uci.get('firewall', section, 'ns_src', default=None) + if ns_dst: + ipaddr = objects.get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if ns_src: + ipaddr = objects.get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + uci.save('firewall') + +def update_firewall_rules(uci): + """ + Update firewall rules with ipset field set and ns_src set. + + Args: + uci: EUci pointer + """ + for section in utils.get_all_by_type(uci, 'firewall', 'rule'): + keep_ipset = False + ns_src = uci.get('firewall', section, 'ns_src', default=None) + ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + if ns_src: + database, id = ns_src.split('/') + obj_type = uci.get(database, id) + if database =="objects" and obj_type == "domain": + keep_ipset = True + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") + try: + uci.delete('firewall', section, 'src_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_src) + if ipaddr: + uci.set('firewall', section, 'src_ip', ipaddr) + try: + uci.delete('firewall', section, 'ipset') + except: + pass + if ns_dst: + database, id = ns_dst.split('/') + obj_type = uci.get(database, id) + if database =="objects" and obj_type == "domain": + ipsets = objects.get_domain_set_ipsets(uci, id) + uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") + try: + uci.delete('firewall', section, 'dest_ip') + except: + pass + else: + ipaddr = objects.get_object_ips(uci, ns_dst) + if ipaddr: + uci.set('firewall', section, 'dest_ip', ipaddr) + if not keep_ipset: # do not delete ipset from src if is a domain + try: + uci.delete('firewall', section, 'ipset') + except: + pass + uci.save('firewall') \ No newline at end of file diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 2dcd5af7..bbef641a 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -485,102 +485,3 @@ def list_host_sets(uci) -> list: rule['matches'] = matches sets.append(rule) return sets - -# Rules - -def update_redirect_rules(uci): - """ - Update redirect rules with ipset field set and ns_src set. - - Args: - uci: EUci pointer - changed_sections: list of changed objects, each object is in the form of `/` - """ - for section in utils.get_all_by_type(uci, 'firewall', 'redirect'): - ns_src = uci.get('firewall', section, 'ns_src', default=None) - ns_dst = uci.get('firewall', section, 'ns_dst', default=None) - if ns_dst: - ipaddr = get_object_first_ip(uci, ns_dst) - if ipaddr: - uci.set('firewall', section, 'dest_ip', ipaddr) - if ns_src: - database, id = ns_src.split('/') - obj_type = uci.get(database, id) - if database == "objects" and obj_type == "domain": - ipsets = get_domain_set_ipsets(uci, id) - uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src_net") - try: - uci.delete('firewall', f"{section}_ipset") - except: - pass - else: - uci.set('firewall', section, 'ipset', f"{id}_ipset") - uci.set('firewall', f"{section}_ipset", "ipset") - uci.set('firewall', f"{section}_ipset", "name", f"{id}_ipset") - uci.set('firewall', f"{section}_ipset", "match", "src_net") - uci.set('firewall', f"{section}_ipset", "enabled", "1") - uci.set('firewall', f"{section}_ipset", "entry", get_object_ips(uci, ns_src)) - - for section in utils.get_all_by_type(uci, 'firewall', 'rule'): - ns_dst = uci.get('firewall', section, 'ns_dst', default=None) - ns_src = uci.get('firewall', section, 'ns_src', default=None) - if ns_dst: - ipaddr = get_object_ips(uci, ns_dst) - if ipaddr: - uci.set('firewall', section, 'dest_ip', ipaddr) - if ns_src: - ipaddr = get_object_ips(uci, ns_dst) - if ipaddr: - uci.set('firewall', section, 'src_ip', ipaddr) - uci.save('firewall') - -def update_firewall_rules(uci): - """ - Update firewall rules with ipset field set and ns_src set. - - Args: - uci: EUci pointer - """ - for section in utils.get_all_by_type(uci, 'firewall', 'rule'): - keep_ipset = False - ns_src = uci.get('firewall', section, 'ns_src', default=None) - ns_dst = uci.get('firewall', section, 'ns_dst', default=None) - if ns_src: - database, id = ns_src.split('/') - obj_type = uci.get(database, id) - if database =="objects" and obj_type == "domain": - keep_ipset = True - ipsets = get_domain_set_ipsets(uci, id) - uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") - try: - uci.delete('firewall', section, 'src_ip') - except: - pass - else: - ipaddr = get_object_ips(uci, ns_src) - if ipaddr: - uci.set('firewall', section, 'src_ip', ipaddr) - try: - uci.delete('firewall', section, 'ipset') - except: - pass - if ns_dst: - database, id = ns_dst.split('/') - obj_type = uci.get(database, id) - if database =="objects" and obj_type == "domain": - ipsets = get_domain_set_ipsets(uci, id) - uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") - try: - uci.delete('firewall', section, 'dest_ip') - except: - pass - else: - ipaddr = get_object_ips(uci, ns_dst) - if ipaddr: - uci.set('firewall', section, 'dest_ip', ipaddr) - if not keep_ipset: # do not delete ipset from src if is a domain - try: - uci.delete('firewall', section, 'ipset') - except: - pass - uci.save('firewall') \ No newline at end of file diff --git a/tests/test_firewall.py b/tests/test_firewall.py index 41bea6d6..eeacb169 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -3,7 +3,7 @@ import pytest from euci import EUci, UciExceptionNotFound -from nethsec import firewall +from nethsec import firewall, objects from pytest_mock import MockFixture from unittest.mock import MagicMock, patch @@ -120,6 +120,35 @@ option src_ip '192.168.1.44' option target 'SNAT' option snat_ip '10.20.30.5' + +config redirect 'redirect3' + option ns_src '' + option ipset 'redirect3_ipset' + +config ipset 'redirect3_ipset' + option name 'redirect1_ipset' + option match 'src_net' + option enabled '1' + list entry '6.7.8.9' + +config redirect 'redirect4' + option ns_src '' + +config rule 'r1' + option name 'r1' + option ns_dst 'dhcp/ns_8dcab636' + +config rule 'r2' + option name 'r2' + option ns_dst '' + +config rule 'r3' + option name 'r3' + option ns_src '' + +config rule 'r4' + option ns_dst '' + option ns_src '' """ network_db = """ @@ -333,6 +362,11 @@ config domain option name 'test3.test.org' option ip 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543' + +config domain 'ns_dhcp1' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' """ netmap_db = """ @@ -371,6 +405,14 @@ objects_db = """ """ +user_db = """ +config user 'ns_user1' + option name "john" + option database "main" + option label "John Doe" + option openvpn_ipaddr "10.10.10.22" +""" + # Setup fake ip command output ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' mock_ip_stdout = MagicMock() @@ -391,6 +433,8 @@ def _setup_db(tmp_path): fp.write(netmap_db) with tmp_path.joinpath('objects').open('w') as fp: fp.write(objects_db) + with tmp_path.joinpath('users').open('w') as fp: + fp.write(user_db) return EUci(confdir=tmp_path.as_posix()) def test_add_interface_to_zone(tmp_path): @@ -831,11 +875,12 @@ def test_list_host_suggestions(mocker, tmp_path): mock_isfile = mocker.patch('os.path.isfile') mock_isfile.return_value = True suggestions = firewall.list_host_suggestions(u) - assert len(suggestions) == 9 + assert len(suggestions) == 10 assert suggestions == [ {'value': '192.168.100.1', 'label': 'test.name.org', 'type': 'domain'}, {'value': '192.168.100.2', 'label': 'test2.giacomo.org', 'type': 'host'}, {'value': 'ac0d:b0e6:ee9e:172e:7f64:ea08:ed22:1543', 'label': 'test3.test.org', 'type': 'domain'}, + {'value': '7.8.9.1', 'label': 'host1', 'type': 'domain'}, {'value': '192.168.100.238', 'label': 'lan', 'type': 'network'}, {'value': '2001:db80::2/64', 'label': 'wan6', 'type': 'network'}, {'value': '10.0.0.22', 'label': 'bond1', 'type': 'network'}, @@ -1034,3 +1079,55 @@ def test_list_netmap_devices(mock_run, tmp_path): assert 'br-lan' in device_names assert 'eth1.4' in device_names assert {'device': 'br-lan', 'interface': 'lan'} in devices + +def test_update_redirect_rules(tmp_path): + u = _setup_db(tmp_path) + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "redirect3", "ns_src", f"objects/{domain1}") # domain can be used only as source + u.set('firewall', 'redirect3', 'ns_dst', f"dhcp/ns_dhcp1") + firewall.update_redirect_rules(u) + print(u.get_all("firewall", "redirect3")) + assert u.get("firewall", "redirect3", "dest_ip") == "7.8.9.1" + assert u.get("firewall", "redirect3", "ipset") == f"{ipsets['firewall']} src_net" + u.set("firewall", "redirect4", "ns_src", f"objects/{host1}") + u.set('firewall', 'redirect4', 'ns_dst', f"users/ns_user1") + firewall.update_redirect_rules(u) + assert u.get("firewall", "redirect4", "dest_ip") == "10.10.10.22" + assert u.get("firewall", "redirect4", "ipset") == f"{host1}_ipset" + assert u.get("firewall", "redirect4_ipset") + +def test_update_firewall_rules(tmp_path): + u = _setup_db(tmp_path) + domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) + ipsets = objects.get_domain_set_ipsets(u, domain1) + host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) + u.set("firewall", "r1", "ns_dst", f"objects/{domain1}") + u.set("firewall", "r1", "ns_src", f"objects/{host1}") + firewall.update_firewall_rules(u) + assert set(u.get_all("firewall", "r1", "src_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + assert u.get("firewall", "r1", "ipset") == f"{ipsets['firewall']} dst" + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r1", "dest_ip") + + u.set("firewall", "r2", "ns_dst", f"objects/{host1}") + u.set("firewall", "r2", "ns_src", f"objects/{domain1}") + firewall.update_firewall_rules(u) + + assert u.get("firewall", "r2", "ipset") == f"{ipsets['firewall']} src" + assert set(u.get_all("firewall", "r2", "dest_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r2", "src_ip") + + u.set("firewall", "r3", "ns_src", "dhcp/ns_dhcp1") + u.set("firewall", "r3", "ns_dst", "users/ns_user1") + firewall.update_firewall_rules(u) + assert u.get_all("firewall", "r3", "src_ip") == ("7.8.9.1",) + assert u.get_all("firewall", "r3", "dest_ip") == ("10.10.10.22",) + + u.set("firewall", "r4", "ns_src", f"objects/{domain1}") + u.set("firewall", "r4", "dest_ip", "1.2.3.4") + firewall.update_firewall_rules(u) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", "r4", "ns_dst") \ No newline at end of file diff --git a/tests/test_objects.py b/tests/test_objects.py index 48d4e939..61e511b2 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -9,21 +9,12 @@ """ firewall_db = """ -config rule 'r1' - option name 'r1' +config rule 'r5' + option name 'r5' option ns_dst 'dhcp/ns_8dcab636' -config rule 'r2' - option name 'r2' - option ns_dst '' - -config rule 'r3' - option name 'r3' - option ns_src '' - -config rule 'r4' - option ns_dst '' - option ns_src '' +config rule 'r6' + option name 'r6' config redirect 'redirect1' option ns_src '' @@ -124,15 +115,15 @@ def test_delete_domain_set(tmp_path): def test_is_used_domain_set(tmp_path): u = _setup_db(tmp_path) id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) - u.set('firewall', 'r1', 'ns_dst', f"objects/{id}") + u.set('firewall', 'r5', 'ns_dst', f"objects/{id}") used, matches = objects.is_used_domain_set(u, id) assert used - assert matches == ["firewall/r1"] + assert matches == ["firewall/r5"] def test_list_domain_sets(tmp_path): u = _setup_db(tmp_path) sets = objects.list_domain_sets(u) - assert len(sets) == 4 + assert len(sets) == 6 def test_add_host_set(tmp_path): u = _setup_db(tmp_path) @@ -164,21 +155,21 @@ def test_delete_host_set(tmp_path): def test_is_used_host_set(tmp_path): u = _setup_db(tmp_path) id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) - u.set('firewall', 'r3', 'ns_src', f"objects/{id}") + u.set('firewall', 'r6', 'ns_src', f"objects/{id}") used, matches = objects.is_used_host_set(u, id) assert used - assert matches == ["firewall/r3"] + assert matches == ["firewall/r6"] def test_list_host_sets(tmp_path): u = _setup_db(tmp_path) sets = objects.list_host_sets(u) - assert len(sets) == 4 + assert len(sets) == 6 def test_is_used_object(tmp_path): u = _setup_db(tmp_path) used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") assert used - assert matches == ["firewall/r1"] + assert matches == ["firewall/r5"] assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False def test_get_object(tmp_path): @@ -192,54 +183,3 @@ def test_get_object_ips(tmp_path): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) ips = objects.get_object_ips(u, f"objects/{id}") assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order - -def test_update_redirect_rules(tmp_path): - u = _setup_db(tmp_path) - domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) - ipsets = objects.get_domain_set_ipsets(u, domain1) - host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) - u.set("firewall", "redirect1", "ns_src", f"objects/{domain1}") # domain can be used only as source - u.set('firewall', 'redirect1', 'ns_dst', f"dhcp/ns_8bec5896") - objects.update_redirect_rules(u) - assert u.get("firewall", "redirect1", "dest_ip") == "7.8.9.1" - assert u.get("firewall", "redirect1", "ipset") == f"{ipsets['firewall']} src_net" - u.set("firewall", "redirect2", "ns_src", f"objects/{host1}") - u.set('firewall', 'redirect2', 'ns_dst', f"users/ns_user1") - objects.update_redirect_rules(u) - assert u.get("firewall", "redirect2", "dest_ip") == "10.10.10.22" - assert u.get("firewall", "redirect2", "ipset") == f"{host1}_ipset" - assert u.get("firewall", "redirect2_ipset") - -def test_update_firewall_rules(tmp_path): - u = _setup_db(tmp_path) - domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) - ipsets = objects.get_domain_set_ipsets(u, domain1) - host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) - u.set("firewall", "r1", "ns_dst", f"objects/{domain1}") - u.set("firewall", "r1", "ns_src", f"objects/{host1}") - objects.update_firewall_rules(u) - assert set(u.get_all("firewall", "r1", "src_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) - assert u.get("firewall", "r1", "ipset") == f"{ipsets['firewall']} dst" - with pytest.raises(UciExceptionNotFound): - u.get("firewall", "r1", "dest_ip") - - u.set("firewall", "r2", "ns_dst", f"objects/{host1}") - u.set("firewall", "r2", "ns_src", f"objects/{domain1}") - objects.update_firewall_rules(u) - - assert u.get("firewall", "r2", "ipset") == f"{ipsets['firewall']} src" - assert set(u.get_all("firewall", "r2", "dest_ip")) == set(objects.get_object_ips(u, f"objects/{host1}")) - with pytest.raises(UciExceptionNotFound): - u.get("firewall", "r2", "src_ip") - - u.set("firewall", "r3", "ns_src", "dhcp/ns_8bec5896") - u.set("firewall", "r3", "ns_dst", "users/ns_user1") - objects.update_firewall_rules(u) - assert u.get_all("firewall", "r3", "src_ip") == ("7.8.9.1",) - assert u.get_all("firewall", "r3", "dest_ip") == ("10.10.10.22",) - - u.set("firewall", "r4", "ns_src", f"objects/{domain1}") - u.set("firewall", "r4", "dest_ip", "1.2.3.4") - objects.update_firewall_rules(u) - with pytest.raises(UciExceptionNotFound): - u.get("firewall", "r4", "ns_dst") \ No newline at end of file From 80fc83859a862f782245a5dab91873253fa36a12 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 28 May 2024 15:51:02 +0200 Subject: [PATCH 03/33] firewall & objects: improve domain set --- src/nethsec/firewall/__init__.py | 10 ++++------ src/nethsec/objects/__init__.py | 15 +++++++++++++++ tests/test_objects.py | 7 +++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index c36dd749..70236f90 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1928,10 +1928,9 @@ def update_firewall_rules(uci): ns_src = uci.get('firewall', section, 'ns_src', default=None) ns_dst = uci.get('firewall', section, 'ns_dst', default=None) if ns_src: - database, id = ns_src.split('/') - obj_type = uci.get(database, id) - if database =="objects" and obj_type == "domain": + if objects.is_domain_set(uci, ns_src): keep_ipset = True + id = ns_src.split('/')[1] ipsets = objects.get_domain_set_ipsets(uci, id) uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} src") try: @@ -1947,9 +1946,8 @@ def update_firewall_rules(uci): except: pass if ns_dst: - database, id = ns_dst.split('/') - obj_type = uci.get(database, id) - if database =="objects" and obj_type == "domain": + if objects.is_domain_set(uci, ns_dst): + id = ns_dst.split('/')[1] ipsets = objects.get_domain_set_ipsets(uci, id) uci.set('firewall', section, 'ipset', f"{ipsets['firewall']} dst") try: diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index bbef641a..35771fc3 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -139,6 +139,21 @@ def get_object_first_ip(uci, database_id): # Domain set +def is_domain_set(uci, database_id): + """ + Check if an object is a domain set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a domain set, False otherwise + """ + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database =="objects" and obj_type == "domain" + def is_used_domain_set(uci, id): """ Check if domain set is used in firewall config. diff --git a/tests/test_objects.py b/tests/test_objects.py index 61e511b2..764812bc 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -183,3 +183,10 @@ def test_get_object_ips(tmp_path): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) ips = objects.get_object_ips(u, f"objects/{id}") assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order + +def test_is_domain_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain_set(u, f"objects/{id}") == True + assert objects.is_domain_set(u, "dhcp/ns_8dcab636") == False + assert objects.is_domain_set(u, "users/ns_user1") == False \ No newline at end of file From af18b83cb2cc087d33cd028eee9506c6b0f1199d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 28 May 2024 16:51:56 +0200 Subject: [PATCH 04/33] firewall & objects: add validation functions --- src/nethsec/firewall/__init__.py | 2 +- src/nethsec/objects/__init__.py | 89 ++++++++++++++++++++++++++++++-- tests/test_objects.py | 27 +++++++++- 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 70236f90..52c8c4b3 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1882,7 +1882,7 @@ def update_redirect_rules(uci): ns_src = uci.get('firewall', section, 'ns_src', default=None) ns_dst = uci.get('firewall', section, 'ns_dst', default=None) if ns_dst: - ipaddr = objects.get_object_first_ip(uci, ns_dst) + ipaddr = objects.get_object_ip(uci, ns_dst) if ipaddr: uci.set('firewall', section, 'dest_ip', ipaddr) if ns_src: diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 35771fc3..4c6bb2b6 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -121,7 +121,7 @@ def get_object_ips(uci, database_id): return list(set(ips)) # Convert the list to a set to remove duplicates, then convert it back to a list -def get_object_first_ip(uci, database_id): +def get_object_ip(uci, database_id): """ Get the first IP address from an object. @@ -150,9 +150,12 @@ def is_domain_set(uci, database_id): Returns: True if object is a domain set, False otherwise """ - database, id = database_id.split('/') - obj_type = uci.get(database, id) - return database =="objects" and obj_type == "domain" + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database =="objects" and obj_type == "domain" + except: + return False def is_used_domain_set(uci, id): """ @@ -500,3 +503,81 @@ def list_host_sets(uci) -> list: rule['matches'] = matches sets.append(rule) return sets + +def is_host_set(uci, database_id): + """ + Check if an object is a host set. + + Args: + uci: EUci pointer + id: id of the object in the form of `/` + + Returns: + True if object is a host set, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "objects" and obj_type == "host" + except: + return False + +# Host + +def is_host(uci, database_id): + """ + Check if an object is a host. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a host, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "host" + except: + return False + +# Domain + +def is_domain(uci, database_id): + """ + Check if an object is a domain. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a domain, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "dhcp" and obj_type == "domain" + except: + return False + +# VPN user + +def is_vpn_user(uci, database_id): + """ + Check if an object is a VPN user. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + True if object is a VPN user, False otherwise + """ + try: + database, id = database_id.split('/') + obj_type = uci.get(database, id) + return database == "users" and obj_type == "user" and uci.get(database, id, 'openvpn_ipaddr', default=None) != None + except: + return False \ No newline at end of file diff --git a/tests/test_objects.py b/tests/test_objects.py index 764812bc..115513d0 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -50,6 +50,9 @@ option database "main" option label "John Doe" option openvpn_ipaddr "10.10.10.22" + +config user 'ns_user2' + option name "user2" """ def _setup_db(tmp_path): @@ -189,4 +192,26 @@ def test_is_domain_set(tmp_path): id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) assert objects.is_domain_set(u, f"objects/{id}") == True assert objects.is_domain_set(u, "dhcp/ns_8dcab636") == False - assert objects.is_domain_set(u, "users/ns_user1") == False \ No newline at end of file + assert objects.is_domain_set(u, "users/ns_user1") == False + +def test_is_domain(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + assert objects.is_domain(u, f"objects/{id}") == False + assert objects.is_domain(u, "dhcp/ns_8bec5896") + +def test_is_host(tmp_path): + u = _setup_db(tmp_path) + assert objects.is_host(u, "dhcp/ns_8bec5896") == False + assert objects.is_host(u, "dhcp/ns_8dcab636") + +def test_is_vpn_user(tmp_path): + u = _setup_db(tmp_path) + assert objects.is_vpn_user(u, "users/ns_user1") + assert objects.is_vpn_user(u, "users/ns_user2") == False + +def test_is_host_set(tmp_path): + u = _setup_db(tmp_path) + id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + assert objects.is_host_set(u, f"objects/{id}") + assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False \ No newline at end of file From f7dded06bdbbf9916bf0459bec7c26b8b7e9fd53 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 28 May 2024 17:46:24 +0200 Subject: [PATCH 05/33] mwan: add objects support --- src/nethsec/mwan/__init__.py | 89 ++++++++++++++++++++++++++++++++++-- tests/test_mwan.py | 68 ++++++++++++++++++++++++++- 2 files changed, 151 insertions(+), 6 deletions(-) diff --git a/src/nethsec/mwan/__init__.py b/src/nethsec/mwan/__init__.py index 847d5b63..bcd3d6bc 100644 --- a/src/nethsec/mwan/__init__.py +++ b/src/nethsec/mwan/__init__.py @@ -16,9 +16,40 @@ import uci from euci import EUci -from nethsec import utils +from nethsec import utils, objects from nethsec.utils import ValidationError +def _is_valid_src(e_uci: EUci, database_id: str): + """ + Validate the given object for source. + Source objects can be only: + - dhcp reservation + - dns domain + - vpn user + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) + +def _is_valid_dst(e_uci: EUci, database_id: str): + """ + Validate the given object for destination. + Destination objects can be only: + - domain set + + Args: + e_uci: EUci instance + database_id: id of object + + Returns: + True if object is valid, False otherwise + """ + return objects.is_domain_set(e_uci, database_id) def __generate_metric(e_uci: EUci) -> int: """ @@ -123,7 +154,8 @@ def __store_member(e_uci: EUci, interface_name: str, metric: int, weight: int) - def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, source_address: str = None, source_port: str = None, - destination_address: str = None, destination_port: str = None, sticky: bool = False) -> str: + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None) -> str: """ Stores a rule for mwan3 @@ -137,6 +169,8 @@ def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, destination_address: destination addresses to match destination_port: destination ports to match or range sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address Returns: name of the rule created @@ -154,6 +188,11 @@ def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, raise ValidationError('name', 'unique', name) if e_uci.get('mwan3', policy, default=None) is None: raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) + e_uci.set('mwan3', rule_config_name, 'rule') e_uci.set('mwan3', rule_config_name, 'label', name) e_uci.set('mwan3', rule_config_name, 'use_policy', policy) @@ -171,6 +210,10 @@ def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, e_uci.set('mwan3', rule_config_name, 'dest_ip', destination_address) if destination_port is not None: e_uci.set('mwan3', rule_config_name, 'dest_port', destination_port.replace('-', ':')) + if ns_src is not None: + e_uci.set('mwan3', rule_config_name, 'ns_src', ns_src) + if ns_dst is not None: + e_uci.set('mwan3', rule_config_name, 'ns_dst', ns_dst) e_uci.save('mwan3') order_rules(e_uci, [rule_config_name] + list(rules)) @@ -428,6 +471,10 @@ def index_rules(e_uci: EUci) -> list[dict]: rule_data['destination_port'] = rule_value['dest_port'].replace(':', '-') if 'sticky' in rule_value: rule_data['sticky'] = rule_value['sticky'] == '1' + if 'ns_src' in rule_value: + rule_data['ns_src'] = rule_value['ns_src'] + if 'ns_dst' in rule_value: + rule_data['ns_dst'] = rule_value['ns_dst'] data.append(rule_data) return data @@ -495,7 +542,8 @@ def delete_rule(e_uci: EUci, name: str): def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = None, source_address: str = None, source_port: str = None, - destination_address: str = None, destination_port: str = None, sticky: bool = False): + destination_address: str = None, destination_port: str = None, sticky: bool = False, + ns_src: str = None, ns_dst: str = None): """ Edits a mwan3 rule. @@ -510,6 +558,8 @@ def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = N destination_address: CIDR notation of destination address destination_port: port or port range sticky: whether to use sticky connections + ns_src: source object, it overrides source_address + ns_dst: destination object, it overrides destination_address Raises: ValidationError: if name is not valid or policy is not valid @@ -519,6 +569,10 @@ def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = N if e_uci.get('mwan3', policy, default=None) is None: raise ValidationError('policy', 'invalid', policy) + if ns_src and not _is_valid_src(e_uci, ns_src): + raise ValidationError('ns_src', 'invalid_object', ns_src) + if ns_dst and not _is_valid_dst(e_uci, ns_dst): + raise ValidationError('ns_dst', 'invalid_object', ns_dst) e_uci.set('mwan3', name, 'use_policy', policy) e_uci.set('mwan3', name, 'label', label) # test if sticky is True of False, if not raise an error @@ -539,7 +593,10 @@ def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = N e_uci.set('mwan3', name, 'src_ip', source_address) if destination_address is not None: e_uci.set('mwan3', name, 'dest_ip', destination_address) - + if ns_src is not None: + e_uci.set('mwan3', name, 'ns_src', ns_src) + if ns_dst is not None: + e_uci.set('mwan3', name, 'ns_dst', ns_dst) e_uci.save('mwan3') return f'mwan3.{name}' @@ -597,3 +654,27 @@ def get_default_config(e_uci: EUci) -> dict: dict with default configuration """ return e_uci.get_all('ns-api', 'defaults_mwan') + + +def update_rules(e_uci: EUci): + """ + Updates mwan3 rules with objects addresses + + Args: + e_uci: euci instance + """ + for rule in utils.get_all_by_type(e_uci, 'mwan3', 'rule'): + ns_src = e_uci.get('mwan3', rule, 'ns_src', default=None) + ns_dst = e_uci.get('mwan3', rule, 'ns_dst', default=None) + if ns_src: + e_uci.set('mwan3', rule, 'src_ip', objects.get_object_ip(e_uci, ns_src)) + if ns_dst: # this can be only a domain set + id = ns_dst.split('/')[1] + ipsets = objects.get_domain_set_ipsets(e_uci, id) + e_uci.set('mwan3', rule, 'ipset', f"{ipsets['firewall']} dst") + try: + e_uci.delete('mwan3', rule, 'dest_ip') + except: + pass + + e_uci.save('mwan3') diff --git a/tests/test_mwan.py b/tests/test_mwan.py index 2e4fd513..3a4c7796 100644 --- a/tests/test_mwan.py +++ b/tests/test_mwan.py @@ -3,7 +3,7 @@ import pytest from euci import EUci -from nethsec import mwan +from nethsec import mwan, objects from nethsec.utils import ValidationError network_db = """ @@ -77,6 +77,28 @@ option quality_recovery_packet_loss '10' """ +objects_db = """ +""" + +dhcp_db = """ +config domain 'ns_domain_mwan' + option ip '7.8.9.1' + option name 'host1' + option ns_description 'Host 1' + +config host 'ns_host_mwan' + option ip '192.168.100.5' + option mac 'fe:54:00:6a:50:bf' + option dns '1' + option name 'host2' + option ns_description 'host2' +""" + +users_db = """ +""" + +firewall_db = """ +""" @pytest.fixture def e_uci(tmp_path: pathlib.Path) -> EUci: @@ -90,6 +112,14 @@ def e_uci(tmp_path: pathlib.Path) -> EUci: fp.write(ns_api_db) with conf_dir.joinpath('mwan3').open('w') as fp: fp.write('') + with conf_dir.joinpath('objects').open('w') as fp: + fp.write(objects_db) + with conf_dir.joinpath('dhcp').open('w') as fp: + fp.write(dhcp_db) + with conf_dir.joinpath('users').open('w') as fp: + fp.write(users_db) + with conf_dir.joinpath('firewall').open('w') as fp: + fp.write(firewall_db) return EUci(confdir=conf_dir.as_posix(), savedir=save_dir.as_posix()) @@ -306,6 +336,16 @@ def test_store_rule(e_uci, mocker): assert e_uci.get('mwan3', 'ns_rule_1', 'dest_port') == '22,443' assert e_uci.get('mwan3', 'ns_rule_1', 'sticky') == '1' + domain_id = objects.add_domain_set(e_uci, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) + id = mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{domain_id}") + id = id.split('.')[1] + assert e_uci.get('mwan3', id, 'ns_src') == "dhcp/ns_host_mwan" + assert e_uci.get('mwan3', id, 'ns_dst') == f"objects/{domain_id}" + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst="objects/invalid") + with pytest.raises(ValueError): + mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src=f"objects/{domain_id}") + def test_unique_rule(e_uci, mocker): mocker.patch('subprocess.run') mwan.store_policy(e_uci, 'default', [ @@ -499,6 +539,9 @@ def test_edit_rule(e_uci, mocker): assert e_uci.get('mwan3', 'ns_default_rule', 'dest_ip') == '0.0.0.0/0' assert e_uci.get('mwan3', 'ns_default_rule', 'dest_port') == '4040:8080' + assert mwan.edit_rule(e_uci, 'ns_default_rule', 'ns_cool_policy', 'new label2!', ns_src="dhcp/ns_host_mwan") + assert e_uci.get('mwan3', 'ns_default_rule', 'ns_src') == "dhcp/ns_host_mwan" + def test_cant_edit_invalid_rule(e_uci, mocker): mocker.patch('subprocess.run') @@ -525,7 +568,6 @@ def test_cant_edit_invalid_rule(e_uci, mocker): assert e.value.args[1] == 'invalid' assert e.value.args[2] == 'ns_cool_policy' - def test_policy_type(e_uci, mocker): mocker.patch('subprocess.run') mwan.store_policy(e_uci, 'custom', [ @@ -536,3 +578,25 @@ def test_policy_type(e_uci, mocker): } ]) assert mwan.index_policies(e_uci)[0]['type'] == 'custom' + +def test_update_rules(e_uci, mocker): + mocker.patch('subprocess.run') + mwan.store_policy(e_uci, 'cool policy', [ + { + 'name': 'RED_3', + 'metric': '10', + 'weight': '100', + }, + { + 'name': 'RED_1', + 'metric': '10', + 'weight': '100', + } + ]) + domain_id = objects.add_domain_set(e_uci, "mydomainset7", "ipv4", ["test1.com", "test2.com"]) + id = mwan.store_rule(e_uci, 'rule_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"objects/{domain_id}") + id = id.split('.')[1] + ipsets = objects.get_domain_set_ipsets(e_uci, domain_id) + mwan.update_rules(e_uci) + assert e_uci.get('mwan3', id, 'src_ip') == '7.8.9.1' + assert e_uci.get('mwan3', id, 'ipset') == f"{ipsets['firewall']} dst" From c2a1113677cb04e51c27f1e79091db54659b0cce Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 29 May 2024 14:27:23 +0200 Subject: [PATCH 06/33] mwan: expand object targets Make sure to expand objects on every rule change. --- src/nethsec/mwan/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nethsec/mwan/__init__.py b/src/nethsec/mwan/__init__.py index bcd3d6bc..843d79b3 100644 --- a/src/nethsec/mwan/__init__.py +++ b/src/nethsec/mwan/__init__.py @@ -215,8 +215,8 @@ def store_rule(e_uci: EUci, name: str, policy: str, protocol: str = None, if ns_dst is not None: e_uci.set('mwan3', rule_config_name, 'ns_dst', ns_dst) - e_uci.save('mwan3') order_rules(e_uci, [rule_config_name] + list(rules)) + update_rules(e_uci) # update rules with objects and save mwan3 config return f'mwan3.{rule_config_name}' @@ -597,7 +597,7 @@ def edit_rule(e_uci: EUci, name: str, policy: str, label: str, protocol: str = N e_uci.set('mwan3', name, 'ns_src', ns_src) if ns_dst is not None: e_uci.set('mwan3', name, 'ns_dst', ns_dst) - e_uci.save('mwan3') + update_rules(e_uci) # update rules with objects and save mwan3 config return f'mwan3.{name}' From edb04318c0cb6d3b4612eed3725f729b97bad48f Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 29 May 2024 16:15:32 +0200 Subject: [PATCH 07/33] objects: expose exists_object function --- src/nethsec/objects/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 4c6bb2b6..3d4fe683 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -31,7 +31,7 @@ def is_object_id(id): """ return id.startswith('objects/') or id.startswith('dhcp/') or id.startswith('users/') -def _validate_object(uci, id): +def object_exists(uci, id): """ Check if the object exists. @@ -347,7 +347,7 @@ def list_domain_sets(uci) -> list: def _validate_host_set_ipaddr(uci, ipaddr: str, family: str): if is_object_id(ipaddr): - return _validate_object(uci, ipaddr) + return object_exists(uci, ipaddr) if family == 'ipv4': return _validate_host_set_ipaddr_v4(ipaddr) elif family == 'ipv6': From effc33dbbcac575ca02d9cd5be3f556d640595e7 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 29 May 2024 16:16:31 +0200 Subject: [PATCH 08/33] firewall: add objects to API functions --- src/nethsec/firewall/__init__.py | 53 ++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 52c8c4b3..84ea434a 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1386,7 +1386,7 @@ def validate_port_format(port: str) -> bool: return False return True -def validate_rule(src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str): +def validate_rule(src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, ns_src: str, ns_dst: str): """ Validate rule. @@ -1399,17 +1399,29 @@ def validate_rule(src: str, src_ip: list[str], dest: str, dest_ip: list[str], pr dest_port: a list of destination ports, each element cna be be a port number, a comma-separated list of port numbers or a range with `-` (eg. 80-90) target: target, must be one of 'ACCEPT', 'REJECT', 'DROP' service: service name + ns_src: an object in the form `/` + ns_dst: an object in the form `/` Raises: ValidationError: if rule is invalid """ - for s in src_ip: - if not validate_address_format(s): - raise utils.ValidationError('src_ip', 'invalid_format', s) - for d in dest_ip: - if not validate_address_format(d): - raise utils.ValidationError('dest_ip', 'invalid_format', d) - if src == dest: + if ns_src: + if not objects.object_exists(ns_src): + raise utils.ValidationError('ns_src', 'object_not_found', ns_src) + else: # check source only if not using objects + for s in src_ip: + if not validate_address_format(s): + raise utils.ValidationError('src_ip', 'invalid_format', s) + if ns_dst: + if not objects.object_exists(ns_dst): + raise utils.ValidationError('ns_dst', 'object_not_found', ns_dst) + else: # check destiation only if not using objects + for d in dest_ip: + if not validate_address_format(d): + raise utils.ValidationError('dest_ip', 'invalid_format', d) + if ns_src and ns_dst and objects.is_domain_set(ns_src) and objects.is_domain_set(ns_dst): + raise utils.ValidationError('dest', 'domain_set_conflict', dest) + if (not ns_src and not ns_dst) and src == dest: # check only if not using objects raise utils.ValidationError('dest', 'same_zone', dest) if target not in TARGETS: raise utils.ValidationError('target', 'invalid_target', target) @@ -1442,7 +1454,7 @@ def get_service_by_name(name: str) -> dict: return None def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, - enabled: bool = True, log: bool = False, tag = []) -> None: + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None) -> None: """ Set up a rule in the firewall config. @@ -1461,6 +1473,8 @@ def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, enabled: if True, rule is enabled; if False, rule is disabled log: if True, log traffic tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` """ uci.set('firewall', id, 'name', name) uci.set('firewall', id, 'src', src) @@ -1490,6 +1504,8 @@ def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, uci.set('firewall', id, 'enabled', '1' if enabled else '0') uci.set('firewall', id, 'log', '1' if log else '0') uci.set('firewall', id, 'ns_tag', tag) + uci.set('firewall', id, 'ns_src', ns_src) + uci.set('firewall', id, 'ns_dst', ns_dst) uci.save('firewall') def split_firewall_config(uci): @@ -1554,7 +1570,7 @@ def reorder_firewall_config(uci): uci.save('firewall') def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, - enabled: bool = True, log: bool = False, tag = [], add_to_top: bool = False) -> str: + enabled: bool = True, log: bool = False, tag = [], add_to_top: bool = False, ns_src: str = None, ns_dst: str = None) -> str: """ Add rule to firewall config. @@ -1574,16 +1590,18 @@ def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: li log: if True, log traffic tag: list of optional tags add_to_top: if True, add rule to the top of the list, otherwise add to the bottom + ns_src: an object in the form `/` + ns_dst: an object in the form `/` Returns: name of rule config that was added """ - validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service) + validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) rule = utils.get_random_id() uci.set('firewall', rule, 'rule') - setup_rule(uci, rule, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag) - uci.save('firewall') + setup_rule(uci, rule, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst) reorder_firewall_config(uci) + update_firewall_rules(uci) # expand objects and save if add_to_top: rule_type = uci.get_all('firewall', rule) @@ -1599,7 +1617,7 @@ def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: li return rule def edit_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, - enabled: bool = True, log: bool = False, tag = []) -> str: + enabled: bool = True, log: bool = False, tag = [], ns_src: str = None, ns_dst: str = None) -> str: """ Edit rule in firewall config. @@ -1618,14 +1636,17 @@ def edit_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, d enabled: if True, rule is enabled, if False, rule is disabled log: if True, log traffic tag: list of optional tags + ns_src: an object in the form `/` + ns_dst: an object in the form `/` Returns: name of rule config that was edited """ if not uci.get('firewall', id, default=None): raise utils.ValidationError("id", "rule_does_not_exists", id) - validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service) - setup_rule(uci, id, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag) + validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + setup_rule(uci, id, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst) + update_firewall_rules(uci) # expand objects and save return id def list_nat_rules(uci) -> list: From b7dd4e5b07b33eed3edc66bbfc39163a78174955 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 09:34:49 +0200 Subject: [PATCH 09/33] objects: tests, use fixture --- tests/test_objects.py | 77 ++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/tests/test_objects.py b/tests/test_objects.py index 115513d0..e2934788 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -1,8 +1,7 @@ -from nethsec import utils +import pathlib from nethsec.utils import ValidationError import pytest from euci import EUci, UciExceptionNotFound - from nethsec import firewall, objects objects_db = """ @@ -55,8 +54,18 @@ option name "user2" """ -def _setup_db(tmp_path): - # setup fake dbs +mwan3_db = """ +""" + +dpi_db = """ +""" + +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() with tmp_path.joinpath('objects').open('w') as fp: fp.write(objects_db) with tmp_path.joinpath('firewall').open('w') as fp: @@ -65,10 +74,13 @@ def _setup_db(tmp_path): fp.write(dhcp_db) with tmp_path.joinpath('users').open('w') as fp: fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) return EUci(confdir=tmp_path.as_posix()) -def test_add_doman_set(tmp_path): - u = _setup_db(tmp_path) +def test_add_doman_set(u): id1 = objects.add_domain_set(u, "mydomainset", "ipv4", ["test1.com", "test2.com"]) assert u.get("objects", id1, "name") == "mydomainset" assert u.get("objects", id1, "family") == "ipv4" @@ -96,8 +108,7 @@ def test_add_doman_set(tmp_path): assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset2" assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset2",) -def test_edit_domain_set(tmp_path): - u = _setup_db(tmp_path) +def test_edit_domain_set(u): id = objects.add_domain_set(u, "mydomainset3", "ipv4", ["test1.com", "test2.com"]) objects.edit_domain_set(u, id, "mydomainset3b", "ipv6", ["test3.com", "test4.com"], 600) assert u.get("objects", id, "name") == "mydomainset3b" @@ -105,8 +116,7 @@ def test_edit_domain_set(tmp_path): assert u.get_all("objects", id, "domain") == ("test3.com", "test4.com") assert u.get("objects", id, "timeout") == "600" -def test_delete_domain_set(tmp_path): - u = _setup_db(tmp_path) +def test_delete_domain_set(u): with pytest.raises(ValidationError): objects.delete_domain_set(u, "notpresent") id = objects.add_domain_set(u, "mydomainset4", "ipv4", ["test1.com", "test2.com"]) @@ -115,21 +125,18 @@ def test_delete_domain_set(tmp_path): assert linked['firewall'] == [] assert linked['dhcp'] == [] -def test_is_used_domain_set(tmp_path): - u = _setup_db(tmp_path) +def test_is_used_domain_set(u): id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) u.set('firewall', 'r5', 'ns_dst', f"objects/{id}") used, matches = objects.is_used_domain_set(u, id) assert used assert matches == ["firewall/r5"] -def test_list_domain_sets(tmp_path): - u = _setup_db(tmp_path) +def test_list_domain_sets(u): sets = objects.list_domain_sets(u) assert len(sets) == 6 -def test_add_host_set(tmp_path): - u = _setup_db(tmp_path) +def test_add_host_set(u): with pytest.raises(ValidationError): objects.add_host_set(u, "myhostset", "ipv4", ["a.b.c.d", "e.f.g.h"]) id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "4.5.6.0/24", "192.168.1.3-192.168.1.10"]) @@ -141,77 +148,65 @@ def test_add_host_set(tmp_path): assert u.get("objects", id2, "family") == "ipv6" assert u.get_all("objects", id2, "ipaddr") == ("2001:db8:3333:4444:5555:6666:7777:8888", "2001:db8::/95", "2001:db8:3333:4444:5555:6666:7777:8888-2001:db8:3333:4444:5555:6666:7777:8890") -def test_edit_host_set(tmp_path): - u = _setup_db(tmp_path) +def test_edit_host_set(u): id = objects.add_host_set(u, "myhostset3", "ipv4", ["6.7.8.9"]) objects.edit_host_set(u, id, "myhostset3b", "ipv4", ["1.1.1.1", "2.2.2.2"]) assert u.get("objects", id, "name") == "myhostset3b" assert u.get_all("objects", id, "ipaddr") == ("1.1.1.1", "2.2.2.2") -def test_delete_host_set(tmp_path): - u = _setup_db(tmp_path) +def test_delete_host_set(u): with pytest.raises(ValidationError): objects.delete_host_set(u, "notpresent") id = objects.add_host_set(u, "myhostset4", "ipv4", ["6.7.8.9"]) assert objects.delete_host_set(u, id) == id -def test_is_used_host_set(tmp_path): - u = _setup_db(tmp_path) +def test_is_used_host_set(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) u.set('firewall', 'r6', 'ns_src', f"objects/{id}") used, matches = objects.is_used_host_set(u, id) assert used assert matches == ["firewall/r6"] -def test_list_host_sets(tmp_path): - u = _setup_db(tmp_path) +def test_list_host_sets(u): sets = objects.list_host_sets(u) assert len(sets) == 6 -def test_is_used_object(tmp_path): - u = _setup_db(tmp_path) +def test_is_used_object(u): used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") assert used assert matches == ["firewall/r5"] assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False -def test_get_object(tmp_path): - u = _setup_db(tmp_path) +def test_get_object(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) obj = objects.get_object(u, id) -def test_get_object_ips(tmp_path): - u = _setup_db(tmp_path) +def test_get_object_ips(u): id0 = objects.add_host_set(u, "myhostset0", "ipv4", ["4.5.6.7"]) id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "dhcp/ns_8bec5896", "users/ns_user1", f"objects/{id0}"]) ips = objects.get_object_ips(u, f"objects/{id}") assert set(ips) == set(["1.2.3.4", "7.8.9.1", "10.10.10.22", "4.5.6.7"]) # check with set to ignore order -def test_is_domain_set(tmp_path): - u = _setup_db(tmp_path) +def test_is_domain_set(u): id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) assert objects.is_domain_set(u, f"objects/{id}") == True assert objects.is_domain_set(u, "dhcp/ns_8dcab636") == False assert objects.is_domain_set(u, "users/ns_user1") == False -def test_is_domain(tmp_path): - u = _setup_db(tmp_path) +def test_is_domain(u): id = objects.add_domain_set(u, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) assert objects.is_domain(u, f"objects/{id}") == False assert objects.is_domain(u, "dhcp/ns_8bec5896") -def test_is_host(tmp_path): - u = _setup_db(tmp_path) +def test_is_host(u): assert objects.is_host(u, "dhcp/ns_8bec5896") == False assert objects.is_host(u, "dhcp/ns_8dcab636") -def test_is_vpn_user(tmp_path): - u = _setup_db(tmp_path) +def test_is_vpn_user(u): assert objects.is_vpn_user(u, "users/ns_user1") assert objects.is_vpn_user(u, "users/ns_user2") == False -def test_is_host_set(tmp_path): - u = _setup_db(tmp_path) +def test_is_host_set(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) assert objects.is_host_set(u, f"objects/{id}") - assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False \ No newline at end of file + assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False From 3114596631613df3adb76ed82622609d4d979b71 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 09:48:36 +0200 Subject: [PATCH 10/33] firewall: tests, use fixture --- src/nethsec/firewall/__init__.py | 6 +- tests/test_firewall.py | 186 +++++++++++++------------------ 2 files changed, 80 insertions(+), 112 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 84ea434a..81cd17c0 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1504,8 +1504,10 @@ def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, uci.set('firewall', id, 'enabled', '1' if enabled else '0') uci.set('firewall', id, 'log', '1' if log else '0') uci.set('firewall', id, 'ns_tag', tag) - uci.set('firewall', id, 'ns_src', ns_src) - uci.set('firewall', id, 'ns_dst', ns_dst) + if ns_src: + uci.set('firewall', id, 'ns_src', ns_src) + if ns_dst: + uci.set('firewall', id, 'ns_dst', ns_dst) uci.save('firewall') def split_firewall_config(uci): diff --git a/tests/test_firewall.py b/tests/test_firewall.py index eeacb169..0ac766a8 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -6,7 +6,7 @@ from nethsec import firewall, objects from pytest_mock import MockFixture from unittest.mock import MagicMock, patch - +import pathlib firewall_db = """ config zone lan1 option name 'lan' @@ -418,8 +418,13 @@ mock_ip_stdout = MagicMock() mock_ip_stdout.configure_mock(**{"stdout": ip_json}) -def _setup_db(tmp_path): - # setup fake dbs +@pytest.fixture +def u(tmp_path: pathlib.Path) -> EUci: + # setup fake dbs + conf_dir = tmp_path.joinpath('conf') + conf_dir.mkdir() + save_dir = tmp_path.joinpath('save') + save_dir.mkdir() with tmp_path.joinpath('firewall').open('w') as fp: fp.write(firewall_db) fp.write(zone_testing_db) @@ -437,8 +442,7 @@ def _setup_db(tmp_path): fp.write(user_db) return EUci(confdir=tmp_path.as_posix()) -def test_add_interface_to_zone(tmp_path): - u = _setup_db(tmp_path) +def test_add_interface_to_zone(u): z1 = firewall.add_interface_to_zone(u, "interface1", "lan") assert z1 == 'lan1' assert 'interface1' in u.get_all('firewall', 'lan1', 'network') @@ -446,35 +450,29 @@ def test_add_interface_to_zone(tmp_path): z1 = firewall.add_interface_to_zone(u, "interface2", "lan") assert 'interface2' in u.get_all('firewall', 'lan1', 'network') -def test_remove_interface_from_zone(tmp_path): - u = _setup_db(tmp_path) +def test_remove_interface_from_zone(u): z1 = firewall.remove_interface_from_zone(u, 'interface1', "lan") assert(not 'interface1' in u.get_all('firewall', 'lan1', 'network')) -def test_add_device_to_zone(tmp_path): - u = _setup_db(tmp_path) +def test_add_device_to_zone(u): z1 = firewall.add_device_to_zone(u, "vnet1", "lan") assert z1 == 'lan1' assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') assert firewall.add_device_to_zone(u, "vnet1", "blue") == None -def test_add_device_to_lan(tmp_path): - u = _setup_db(tmp_path) +def test_add_device_to_lan(u): assert firewall.add_device_to_lan(u, "vnet1") == 'lan1' assert 'vnet1' in u.get_all('firewall', 'lan1', 'device') -def test_add_device_to_wan(tmp_path): - u = _setup_db(tmp_path) +def test_add_device_to_wan(u): assert firewall.add_device_to_wan(u, "vnet2") == 'wan1f' assert 'vnet2' in u.get_all('firewall', 'wan1f', 'device') -def test_remove_device_from_zone(tmp_path): - u = _setup_db(tmp_path) +def test_remove_device_from_zone(u): firewall.remove_device_from_zone(u, 'vnet1', "lan") assert(not 'vnet2' in u.get_all('firewall', 'lan1', 'device')) -def test_add_service(tmp_path): - u = _setup_db(tmp_path) +def test_add_service(u): rule = firewall.add_service(u, "my_service", "443", "tcp", "nginx/_lan") assert rule is not None assert rule == "ns_allow_my_service" @@ -488,30 +486,26 @@ def test_add_service(tmp_path): assert firewall.add_service(u, "my-service2", "456", ["tcp", "udp"]) == "ns_allow_my_service2" assert u.get_all('firewall', 'ns_allow_my_service2', 'proto') == ("tcp", "udp") -def test_block_service(tmp_path): - u = _setup_db(tmp_path) +def test_block_service(u): firewall.add_service(u, "my-service", "443", "tcp") assert firewall.remove_service(u, "my-service") == "ns_allow_my_service" with pytest.raises(UciExceptionNotFound): u.get('firewall', 'ns_allow_my_service') -def test_disable_service(tmp_path): - u = _setup_db(tmp_path) +def test_disable_service(u): firewall.add_service(u, "my-service", "443", "tcp") assert firewall.disable_service(u, "my-service") == "ns_allow_my_service" assert u.get("firewall", "ns_allow_my_service", "enabled") == "0" assert firewall.disable_service(u, "non-existing") == None -def test_enable_service(tmp_path): - u = _setup_db(tmp_path) +def test_enable_service(u): firewall.add_service(u, "my-service", "1234", "tcp") firewall.disable_service(u, "my-service") assert firewall.enable_service(u, "my-service") == "ns_allow_my_service" assert u.get("firewall", "ns_allow_my_service", "enabled") assert firewall.enable_service(u, "non-existing") == None -def test_add_vpn_interface(tmp_path): - u = _setup_db(tmp_path) +def test_add_vpn_interface(u): assert firewall.add_vpn_interface(u, 'test!vpn', 'tuntest') == 'test_vpn' assert u.get('network', 'test_vpn') == 'interface' assert u.get('network', 'test_vpn', 'proto') == 'none' @@ -520,8 +514,7 @@ def test_add_vpn_interface(tmp_path): i = firewall.add_vpn_interface(u, 'p2p', 'ppp10', 'torrent/server1') assert u.get('network', 'p2p', 'ns_link') == 'torrent/server1' -def test_add_trusted_zone(tmp_path): - u = _setup_db(tmp_path) +def test_add_trusted_zone(u): assert firewall.add_trusted_zone(u, 'toolongnameforzone') == (None, None) (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') @@ -552,8 +545,7 @@ def test_add_trusted_zone(tmp_path): assert u.get("firewall", forwardings[1], 'ns_link') == link assert u.get("firewall", forwardings[2], 'ns_link') == link -def test_duplicated_add_trusted_zone(tmp_path): - u = _setup_db(tmp_path) +def test_duplicated_add_trusted_zone(u): (zone, forwardings) = firewall.add_trusted_zone(u, 'mytrusted') assert zone is None assert forwardings is None @@ -565,8 +557,7 @@ def test_duplicated_add_trusted_zone(tmp_path): trusted = trusted + 1 assert trusted == 1 -def test_add_trusted_zone_with_networks(tmp_path): - u = _setup_db(tmp_path) +def test_add_trusted_zone_with_networks(u): interface = firewall.add_vpn_interface(u, 'testvpn2', 'tuntest2') zone, forwardings = firewall.add_trusted_zone(u, 'mytrusted2', list(interface)) assert u.get_all("firewall", zone, 'network') == tuple(interface) @@ -575,8 +566,7 @@ def test_apply(): # Already tested in pyuci assert 1 -def test_add_template_rule(tmp_path): - u = _setup_db(tmp_path) +def test_add_template_rule(u): rule = firewall.add_template_rule(u, 'ns_test_rule', 'tcp', '443', 'test1/key1') assert u.get("firewall", rule) == "rule" assert u.get("firewall", rule, "proto") == "tcp" @@ -599,8 +589,7 @@ def test_add_template_rule(tmp_path): assert u.get("firewall", rule, "target") == "ACCEPT" assert u.get("firewall", rule, "ns_tag") == "automated" -def test_add_template_zone(tmp_path): - u = _setup_db(tmp_path) +def test_add_template_zone(u): (zone, forwardings) = firewall.add_template_zone(u, 'ns_blue', ["lan", "lan2"], link="mydb/mykey" ) assert zone is not None assert u.get("firewall", zone) == "zone" @@ -623,8 +612,7 @@ def test_add_template_zone(tmp_path): assert zone is None assert forwardings is None -def test_add_template_service_group(tmp_path): - u = _setup_db(tmp_path) +def test_add_template_service_group(u): sections = firewall.add_template_service_group(u, "ns_web_secure") assert len(sections) == 2 assert u.get("firewall", sections[0]) == "rule" @@ -651,8 +639,7 @@ def test_add_template_service_group(tmp_path): assert u.get("firewall", sections[0], "ns_link") == "db/mykey" assert u.get("firewall", sections[1], "ns_link") == "db/mykey" -def test_get_all_linked(tmp_path): - u = _setup_db(tmp_path) +def test_get_all_linked(u): link = "mytestdb/mykey" sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) @@ -668,8 +655,7 @@ def test_get_all_linked(tmp_path): assert interface in linked['network'] -def test_disable_linked_rules(tmp_path): - u = _setup_db(tmp_path) +def test_disable_linked_rules(u): link = "mytestdb/mykey" sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) @@ -686,8 +672,7 @@ def test_disable_linked_rules(tmp_path): for f in forwardings: assert u.get("network", f, "enabled", default="XX") == "XX" # option must not be set -def test_delete_linked_sections(tmp_path): - u = _setup_db(tmp_path) +def test_delete_linked_sections(u): link = "mytestdb/mykey" sections = firewall.add_template_service_group(u, "ns_web_secure", "blue", "yellow", link=link) rule = firewall.add_service(u, "my_service", "443", "tcp", link=link) @@ -706,8 +691,7 @@ def test_delete_linked_sections(tmp_path): for f in forwardings: u.get("firewall", f) -def test_is_ipv6_enabled(tmp_path): - u = _setup_db(tmp_path) +def test_is_ipv6_enabled(u): assert firewall.is_ipv6_enabled(u) == True u.delete("network", 'lan6') assert firewall.is_ipv6_enabled(u) == True @@ -720,15 +704,13 @@ def test_is_ipv6_enabled(tmp_path): u.delete("network", 'wan6c') assert firewall.is_ipv6_enabled(u) == False -def test_disable_ipv6_firewall(tmp_path): - u = _setup_db(tmp_path) +def test_disable_ipv6_firewall(u): assert u.get("firewall", "v6rule", "enabled", default="1") == "1" firewall.disable_ipv6_firewall(u) assert u.get("firewall", "v6rule", "enabled", default="1") == "0" -def test_list_zones(tmp_path): - u = _setup_db(tmp_path) +def test_list_zones(u): assert firewall.list_zones(u)["ns_lan"]["name"] == "lan" assert firewall.list_zones(u)["ns_lan"]["input"] == "ACCEPT" assert firewall.list_zones(u)["ns_lan"]["output"] == "ACCEPT" @@ -741,21 +723,18 @@ def test_list_zones(tmp_path): assert firewall.list_zones(u)["ns_wan"]["network"] == ("wan6", "RED_2", "RED_3", "RED_1") -def list_zones_no_aliases(tmp_path): - u = _setup_db(tmp_path) +def list_zones_no_aliases(u): assert firewall.list_zones_no_aliases(u)["ns_lan"]["network"] == ("GREEN_1",) -def test_list_forwardings(tmp_path): - u = _setup_db(tmp_path) +def test_list_forwardings(u): assert firewall.list_forwardings(u)["ns_lan2guests"]["src"] == "lan" assert firewall.list_forwardings(u)["ns_lan2guests"]["dest"] == "guests" assert firewall.list_forwardings(u)["ns_guests2wan"]["src"] == "guests" assert firewall.list_forwardings(u)["ns_guests2wan"]["dest"] == "wan" -def test_add_zone(tmp_path): - u = _setup_db(tmp_path) +def test_add_zone(u): assert firewall.add_zone(u, "new_zone", "REJECT", "DROP", True, ["lan"], ["lan", "guest"]) == ( "ns_new_zone", {"ns_new_zone2wan", "ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) assert u.get("firewall", "ns_new_zone", "name") == "new_zone" @@ -771,8 +750,7 @@ def test_add_zone(tmp_path): assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" -def test_edit_zone(tmp_path): - u = _setup_db(tmp_path) +def test_edit_zone(u): assert firewall.edit_zone(u, "new_zone", "DROP", "ACCEPT", False, ["lan"], ["lan", "guest"]) == ( "ns_new_zone", {"ns_new_zone2lan", "ns_lan2new_zone", "ns_guest2new_zone"}) assert u.get("firewall", "ns_new_zone", "name") == "new_zone" @@ -786,15 +764,13 @@ def test_edit_zone(tmp_path): assert u.get("firewall", "ns_guest2new_zone", "src") == "guest" assert u.get("firewall", "ns_guest2new_zone", "dest") == "new_zone" -def test_delete_zone(tmp_path): - u = _setup_db(tmp_path) +def test_delete_zone(u): assert firewall.delete_zone(u, "ns_new_zone") == ( "ns_new_zone", {"ns_new_zone2lan", "ns_guest2new_zone", "ns_lan2new_zone"}) with pytest.raises(Exception) as e: firewall.delete_zone(u, "not_a_zone") -def test_get_rule_by_name(tmp_path): - u = _setup_db(tmp_path) +def test_get_rule_by_name(u): assert firewall.get_rule_by_name(u, "Allow-DHCPv6") == ( "v6rule", {"name": "Allow-DHCPv6", "src": "wan", "proto": "udp", "dest_port": "546", "family": "ipv6", "target": "ACCEPT", "enabled": "0"} @@ -806,14 +782,12 @@ def test_get_rule_by_name(tmp_path): assert firewall.get_rule_by_name(u, "not_a_rule") == (None, None) assert firewall.get_rule_by_name(u, "Not-Automated", "automated") == (None, None) -def test_add_default_ipv6_rules(tmp_path): - u = _setup_db(tmp_path) +def test_add_default_ipv6_rules(u): # one rule should be skipped because it already exists assert len(firewall.add_default_ipv6_rules(u)) == 3 assert firewall.add_default_ipv6_rules(u) == [] -def test_resolve_address(tmp_path): - u = _setup_db(tmp_path) +def test_resolve_address(u): assert firewall.resolve_address(u, "192.168.100.1") == {"value": "192.168.100.1", "type": "domain", "label": "test.name.org"} assert firewall.resolve_address(u, "192.168.100.2") == {"value": "192.168.100.2", "type": "host", "label": "test2.giacomo.org"} assert firewall.resolve_address(u, "192.168.100.238") == {"value": "192.168.100.238", "type": "interface", "label": "lan"} @@ -821,8 +795,7 @@ def test_resolve_address(tmp_path): assert firewall.resolve_address(u, "10.0.0.22") == {"value": "10.0.0.22", "type": "interface", "label": "bond1"} assert firewall.resolve_address(u, "2001:db80::2/64") == {"value": "2001:db80::2/64", "type": "interface", "label": "wan6"} -def test_list_forward_rules(tmp_path): - u = _setup_db(tmp_path) +def test_list_forward_rules(u): rules = firewall.list_forward_rules(u) assert len(rules) > 0 for r in rules: @@ -834,8 +807,7 @@ def test_list_forward_rules(tmp_path): assert 'ns_tag' in r assert 'proto' in r -def test_list_output_rules(tmp_path): - u = _setup_db(tmp_path) +def test_list_output_rules(u): rules = firewall.list_output_rules(u) assert len(rules) > 0 for r in rules: @@ -847,9 +819,7 @@ def test_list_output_rules(tmp_path): assert 'ns_tag' in r assert 'proto' in r - -def test_list_input_rules(tmp_path): - u = _setup_db(tmp_path) +def test_list_input_rules(u): rules = firewall.list_input_rules(u) assert len(rules) > 0 for r in rules: @@ -861,7 +831,7 @@ def test_list_input_rules(tmp_path): assert 'ns_tag' in r assert 'proto' in r -def test_list_service_suggestions(mocker): +def test_list_service_suggestions(u, mocker): mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) mock_isfile = mocker.patch('os.path.isfile') mock_isfile.return_value = True @@ -869,8 +839,7 @@ def test_list_service_suggestions(mocker): assert len(services) == 5 assert services == [{'id': 'ftp', 'proto': ['tcp'], 'port': 21}, {'id': 'ssh', 'proto': ['tcp', 'udp'], 'port': 22}, {'id': 'time', 'proto': ['udp'], 'port': 37}, {'id': 'www', 'proto': ['tcp'], 'port': 80}, {'id': 'kerberos', 'proto': ['tcp', 'udp'], 'port': 88}] -def test_list_host_suggestions(mocker, tmp_path): - u = _setup_db(tmp_path) +def test_list_host_suggestions(u, mocker): mocker.patch('builtins.open', mocker.mock_open(read_data=lease_file)) mock_isfile = mocker.patch('os.path.isfile') mock_isfile.return_value = True @@ -889,8 +858,7 @@ def test_list_host_suggestions(mocker, tmp_path): {'value': '192.168.1.219', 'label': 'test2', 'type': 'lease'}, ] -def test_add_rule(tmp_path, mocker): - u = _setup_db(tmp_path) +def test_add_rule(u, mocker): mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) mock_isfile = mocker.patch('os.path.isfile') mock_isfile.return_value = True @@ -909,8 +877,7 @@ def test_add_rule(tmp_path, mocker): assert u.get_all("firewall", rid, "ns_tag") == ("tag1",) assert u.get("firewall", rid, "ns_link", default="notpresent") == "notpresent" -def test_edit_rule(tmp_path, mocker): - u = _setup_db(tmp_path) +def test_edit_rule(u, mocker): mocker.patch('builtins.open', mocker.mock_open(read_data=services_file)) mock_isfile = mocker.patch('os.path.isfile') mock_isfile.return_value = True @@ -939,22 +906,19 @@ def test_edit_rule(tmp_path, mocker): assert u.get_all("firewall", rid, "proto") == ('tcp',) assert u.get("firewall", rid, "dest_port") == "80" -def test_delete_rule(tmp_path): - u = _setup_db(tmp_path) +def test_delete_rule(u): ids = firewall.list_rule_ids(u) id_to_delete = ids.pop() firewall.delete_rule(u, id_to_delete) assert id_to_delete not in firewall.list_rule_ids(u) -def test_disable_rule(tmp_path): - u = _setup_db(tmp_path) +def test_disable_rule(u): ids = firewall.list_rule_ids(u) id_to_disable = ids.pop() firewall.disable_rule(u, id_to_disable) assert u.get("firewall", id_to_disable, "enabled") == "0" -def test_enable_rule(tmp_path): - u = _setup_db(tmp_path) +def test_enable_rule(u): ids = firewall.list_rule_ids(u) id_to_enable = ids.pop() firewall.disable_rule(u, id_to_enable) @@ -962,13 +926,12 @@ def test_enable_rule(tmp_path): firewall.enable_rule(u, id_to_enable) assert u.get("firewall", id_to_enable, "enabled") == "1" -def test_order_rules(tmp_path, mocker): +def test_order_rules(u, mocker): # The firewall.order_rules function uses the uci binary to reorder the rules # The test is usefull because uci behaves differently on a real machine assert True -def test_list_nat_rules(tmp_path): - u = _setup_db(tmp_path) +def test_list_nat_rules(u): names = [] for r in firewall.list_nat_rules(u): names.append(r.get("name")) @@ -978,8 +941,7 @@ def test_list_nat_rules(tmp_path): assert "SNAT_NSEC7_style" in names assert "Allow-DHCPv6" not in names -def test_add_nat_rule(tmp_path): - u = _setup_db(tmp_path) +def test_add_nat_rule(u): id1 = firewall.add_nat_rule(u, "myrule", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") assert u.get("firewall", id1, "name") == "myrule" assert u.get("firewall", id1, "target") == "SNAT" @@ -997,8 +959,7 @@ def test_add_nat_rule(tmp_path): with pytest.raises(UciExceptionNotFound): u.get("firewall", id2, "src_ip") -def test_edit_nat_rule(tmp_path): - u = _setup_db(tmp_path) +def test_edit_nat_rule(u): id = firewall.add_nat_rule(u, "myrule4", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") firewall.edit_nat_rule(u, id, "myrule4b", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "3.3.3.3") assert u.get("firewall", id, "name") == "myrule4b" @@ -1009,23 +970,20 @@ def test_edit_nat_rule(tmp_path): assert u.get_all("firewall", id, "proto") == ("all",) assert u.get("firewall", id, "src") == "lan" -def test_delete_nat_rule(tmp_path): - u = _setup_db(tmp_path) +def test_delete_nat_rule(u): with pytest.raises(ValidationError): firewall.delete_nat_rule(u, "notpresent") id = firewall.add_nat_rule(u, "myrule5", "SNAT", "lan", "1.2.3.4", "6.7.8.9", "1.1.1.1") assert firewall.delete_nat_rule(u, id) == id -def test_list_netmap_rules(tmp_path): - u = _setup_db(tmp_path) +def test_list_netmap_rules(u): names = [] for r in firewall.list_netmap_rules(u): names.append(r.get("name")) assert "source_nat1" in names assert "dest_nat1" in names -def test_add_netmap_rule(tmp_path): - u = _setup_db(tmp_path) +def test_add_netmap_rule(u): with pytest.raises(ValidationError): id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") id1 = firewall.add_netmap_rule(u, "myrule", "10.50.51.0/24", "", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") @@ -1046,8 +1004,7 @@ def test_add_netmap_rule(tmp_path): assert u.get("netmap", id2, "map_from") == "10.10.10.0/24" assert u.get("netmap", id2, "map_to") == "192.168.1.0/24" -def test_edit_netmap_rule(tmp_path): - u = _setup_db(tmp_path) +def test_edit_netmap_rule(u): id = firewall.add_netmap_rule(u, "myrule3", "", "10.50.50.0/24", ["eth0"], ["eth1"], "10.10.10.0/24", "192.168.1.0/24") firewall.edit_netmap_rule(u, id, "myrule3b", "", "10.50.51.0/24", [], [], "10.10.11.0/24", "192.168.2.0/24") assert u.get("netmap", id, "name") == "myrule3b" @@ -1061,17 +1018,28 @@ def test_edit_netmap_rule(tmp_path): assert u.get("netmap", id, "map_from") == "10.10.11.0/24" assert u.get("netmap", id, "map_to") == "192.168.2.0/24" -def test_delete_netmap_rule(tmp_path): - u = _setup_db(tmp_path) +def test_delete_netmap_rule(u): + with pytest.raises(ValidationError): + firewall.delete_netmap_rule(u, "notpresent") + id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") + assert firewall.delete_netmap_rule(u, id) == id + +@patch("nethsec.utils.subprocess.run") +def test_list_netmap_devices(mock_run, u): + u.get_all("netmap", id, "device_out") + with pytest.raises(UciExceptionNotFound): + u.get_all("netmap", id, "device_in") + assert u.get("netmap", id, "map_from") == "10.10.11.0/24" + assert u.get("netmap", id, "map_to") == "192.168.2.0/24" + +def test_delete_netmap_rule(u): with pytest.raises(ValidationError): firewall.delete_netmap_rule(u, "notpresent") id = firewall.add_netmap_rule(u, "myrule3b", "", "10.50.51.0/24", None, None, "10.10.11.0/24", "192.168.2.0/24") assert firewall.delete_netmap_rule(u, id) == id @patch("nethsec.utils.subprocess.run") -def test_list_netmap_devices(mock_run, tmp_path): - # setup mock - u = _setup_db(tmp_path) +def test_list_netmap_devices(mock_run, u): mock_run.return_value = mock_ip_stdout devices = firewall.list_netmap_devices(u) device_names = [d.get("device") for d in devices] @@ -1080,8 +1048,7 @@ def test_list_netmap_devices(mock_run, tmp_path): assert 'eth1.4' in device_names assert {'device': 'br-lan', 'interface': 'lan'} in devices -def test_update_redirect_rules(tmp_path): - u = _setup_db(tmp_path) +def test_update_redirect_rules(u): domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) ipsets = objects.get_domain_set_ipsets(u, domain1) host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) @@ -1098,8 +1065,7 @@ def test_update_redirect_rules(tmp_path): assert u.get("firewall", "redirect4", "ipset") == f"{host1}_ipset" assert u.get("firewall", "redirect4_ipset") -def test_update_firewall_rules(tmp_path): - u = _setup_db(tmp_path) +def test_update_firewall_rules(u): domain1 = objects.add_domain_set(u, "d1", "ipv4", ["test1.com", "test2.com"]) ipsets = objects.get_domain_set_ipsets(u, domain1) host1 = objects.add_host_set(u, "h1", "ipv4", ["192.168.168.1", "users/ns_user1"]) @@ -1130,4 +1096,4 @@ def test_update_firewall_rules(tmp_path): u.set("firewall", "r4", "dest_ip", "1.2.3.4") firewall.update_firewall_rules(u) with pytest.raises(UciExceptionNotFound): - u.get("firewall", "r4", "ns_dst") \ No newline at end of file + u.get("firewall", "r4", "ns_dst") From 06219a20249b1804a63d084b6c7d6da5dfca9884 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 10:33:16 +0200 Subject: [PATCH 11/33] objects: improve checks --- src/nethsec/objects/__init__.py | 25 ++++++++++--- tests/test_objects.py | 65 ++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 3d4fe683..43f43d91 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -31,21 +31,22 @@ def is_object_id(id): """ return id.startswith('objects/') or id.startswith('dhcp/') or id.startswith('users/') -def object_exists(uci, id): +def object_exists(uci, database_id): """ Check if the object exists. Args: - id: id to check + database_id: id to check in the form of `/` Returns: True if object exists, False otherwise """ - database, id = id.split('/') try: + database, id = database_id.split('/') uci.get(database, id) + return True except: - raise utils.ValidationError('id', 'object_does_not_exists', id) + return False def get_object(uci, database_id): """ @@ -66,7 +67,10 @@ def get_object(uci, database_id): def is_used_object(uci, database_id): """ - Check if an object is used in firewall config. + Check if an object is used in: + - firewall config + - mwan3 config + - dpi config Args: uci: EUci pointer @@ -81,6 +85,12 @@ def is_used_object(uci, database_id): for section in uci.get_all("firewall"): if uci.get('firewall', section, 'ns_src', default=None) == database_id or uci.get('firewall', section, 'ns_dst', default=None) == database_id: matches.append(f'firewall/{section}') + for section in uci.get_all("mwan3"): + if uci.get('mwan3', section, 'ns_src', default=None) == database_id or uci.get('mwan3', section, 'ns_dst', default=None) == database_id: + matches.append(f'mwan3/{section}') + for section in uci.get_all("dpi"): + if uci.get('dpi', section, 'source', default=None) == database_id: + matches.append(f'dpi/{section}') return len(matches) > 0, matches def get_object_ips(uci, database_id): @@ -347,7 +357,10 @@ def list_domain_sets(uci) -> list: def _validate_host_set_ipaddr(uci, ipaddr: str, family: str): if is_object_id(ipaddr): - return object_exists(uci, ipaddr) + if not object_exists(uci, ipaddr): + raise utils.ValidationError('ipaddr', 'object_does_not_exists', ipaddr) + else: + return # validation is ok if family == 'ipv4': return _validate_host_set_ipaddr_v4(ipaddr) elif family == 'ipv6': diff --git a/tests/test_objects.py b/tests/test_objects.py index e2934788..c9fc020f 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -5,6 +5,18 @@ from nethsec import firewall, objects objects_db = """ +config domain 'myset' + option name 'MySet' + option description 'Mydomain set' + option family 'ipv4' + option timeout '600' + list domain 'www.nethsecurity.org' + list domain 'www.nethserver.org' + +config host 'ns_04fadb5c' + option name 'h1' + option family 'ipv4' + list ipaddr '1.2.3.4' """ firewall_db = """ @@ -15,6 +27,10 @@ config rule 'r6' option name 'r6' +config rule 'r7' + option name 'r7' + option ns_src 'objects/ns_04fadb5c' + config redirect 'redirect1' option ns_src '' option ipset 'redirect1_ipset' @@ -41,6 +57,17 @@ option dns '1' option name 'host2' option ns_description 'host2' + +config domain 'ns_9e7f705e' + option ip '1.2.3.4' + option name 'test1.domain' + +config host 'ns_271ca281' + option ip '192.168.100.22' + option mac 'fe:54:00:6a:4a:a1' + option dns '1' + option name 'reserve1' + option ns_description 'reserve1' """ user_db = """ @@ -55,9 +82,35 @@ """ mwan3_db = """ +config rule 'ns_r1' + option label 'r1' + option use_policy 'ns_default' + option sticky '0' + option proto 'tcp' + option src_ip '1.2.3.4' + option dest_ip '1.1.1.1' + option ns_src 'users/ns_user2' """ dpi_db = """ +config rule + option action 'block' + list application 'netify.twitter' + list source 'dhcp/ns_9e7f705e' + option description 'Block Twitter for user Goofy' + option enabled '1' + +config exemption + option criteria 'dhcp/ns_271ca281' + option description 'Important host' + option enabled '1' + +config rule 'ns_e775b8a7' + option enabled '1' + option device 'br-lan' + option action 'block' + list application 'netify.amazon-prime' + """ @pytest.fixture @@ -134,7 +187,7 @@ def test_is_used_domain_set(u): def test_list_domain_sets(u): sets = objects.list_domain_sets(u) - assert len(sets) == 6 + assert len(sets) == 7 def test_add_host_set(u): with pytest.raises(ValidationError): @@ -169,13 +222,21 @@ def test_is_used_host_set(u): def test_list_host_sets(u): sets = objects.list_host_sets(u) - assert len(sets) == 6 + assert len(sets) == 7 def test_is_used_object(u): used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") assert used assert matches == ["firewall/r5"] assert objects.is_used_object(u, "dhcp/ns_8bec5896")[0] == False + assert objects.is_used_object(u, "users/ns_user2")[0] == True + used, matches = objects.is_used_object(u, "objects/ns_04fadb5c") + assert used + assert matches == ["firewall/r7"] + +def test_object_exists(u): + assert objects.object_exists(u, "users/ns_user1") + assert objects.object_exists(u, "uknown") == False def test_get_object(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) From 7cb06688e537e0c11bdabe6f71c61d9dbefa6978 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 14:21:15 +0200 Subject: [PATCH 12/33] mwan: expand objects support --- src/nethsec/mwan/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/nethsec/mwan/__init__.py b/src/nethsec/mwan/__init__.py index 843d79b3..6ab2462a 100644 --- a/src/nethsec/mwan/__init__.py +++ b/src/nethsec/mwan/__init__.py @@ -26,6 +26,7 @@ def _is_valid_src(e_uci: EUci, database_id: str): - dhcp reservation - dns domain - vpn user + - a singleton host set Args: e_uci: EUci instance @@ -34,6 +35,9 @@ def _is_valid_src(e_uci: EUci, database_id: str): Returns: True if object is valid, False otherwise """ + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id) + return objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) def _is_valid_dst(e_uci: EUci, database_id: str): @@ -41,6 +45,10 @@ def _is_valid_dst(e_uci: EUci, database_id: str): Validate the given object for destination. Destination objects can be only: - domain set + - dhcp reservation + - dns domain + - vpn user + - a singleton host set Args: e_uci: EUci instance @@ -49,7 +57,10 @@ def _is_valid_dst(e_uci: EUci, database_id: str): Returns: True if object is valid, False otherwise """ - return objects.is_domain_set(e_uci, database_id) + if objects.is_host_set(e_uci, database_id): + return objects.is_singleton_host_set(e_uci, database_id) + + return objects.is_domain_set(e_uci, database_id) or objects.is_host(e_uci, database_id) or objects.is_domain(e_uci, database_id) or objects.is_vpn_user(e_uci, database_id) def __generate_metric(e_uci: EUci) -> int: """ From 3b23a1a198c6d8d43fec3ac28bc14b48092ff9fe Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 14:22:08 +0200 Subject: [PATCH 13/33] objects: add singleton host sets --- src/nethsec/objects/__init__.py | 27 +++++++++++++++++++++++++++ tests/test_objects.py | 11 +++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 43f43d91..523afc7c 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -535,6 +535,33 @@ def is_host_set(uci, database_id): except: return False +# in mathematical terms, a singleton set is a set with exactly one element +def is_singleton_host_set(uci, database_id, allow_cidr=False): + """ + Check if an object is a host set with a single IP address. + The IP must not be an IP range. + If `allow_cidr` is True, the IP can be in CIDR notation. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + allow_cidr: allow CIDR notation + + Returns: + True if object is a singleton host set, False otherwise + """ + if is_host_set(uci, database_id): + obj = get_object(uci, database_id) + if obj and len(obj.get('ipaddr')) == 1: + ip = obj.get('ipaddr')[0] + if '-' in ip: + return False + if allow_cidr: + return True + else: + return '/' not in ip + return False + # Host def is_host(uci, database_id): diff --git a/tests/test_objects.py b/tests/test_objects.py index c9fc020f..854b662b 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -224,6 +224,17 @@ def test_list_host_sets(u): sets = objects.list_host_sets(u) assert len(sets) == 7 +def test_is_singleton_host_set(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "5.6.7.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id1}") == False + id2 = objects.add_host_set(u, "myhostset2", "ipv4", ["8.8.8.8"]) + assert objects.is_singleton_host_set(u, f"objects/{id2}") == True + id3 = objects.add_host_set(u, "myhostset3", "ipv4", ["192.168.7.2-192.168.7.3"]) + assert objects.is_singleton_host_set(u, f"objects/{id3}") == False + id4 = objects.add_host_set(u, "myhostset4", "ipv4", ["192.168.0.0/24"]) + assert objects.is_singleton_host_set(u, f"objects/{id4}") == False + assert objects.is_singleton_host_set(u, f"objects/{id4}", allow_cidr=True) + def test_is_used_object(u): used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") assert used From 55b7447bb101016ed1b984443a87a9dd86459d76 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 14:53:39 +0200 Subject: [PATCH 14/33] objects: detect loop on host sets --- src/nethsec/objects/__init__.py | 27 +++++++++++++++++++++++---- tests/test_objects.py | 6 ++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 523afc7c..f0bc6145 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -355,12 +355,31 @@ def list_domain_sets(uci) -> list: # Host set -def _validate_host_set_ipaddr(uci, ipaddr: str, family: str): +def _has_loop(uci, id, ipaddr, depth=0): + if depth > 2: + return True + if is_object_id(ipaddr): + if ipaddr == id: + return True + obj = get_object(uci, ipaddr) + if obj: + for ip in obj.get('ipaddr'): + if _has_loop(uci, id, ip, depth + 1): + return True + return False + +def _validate_host_set_ipaddr(uci, id, ipaddr: str, family: str): if is_object_id(ipaddr): if not object_exists(uci, ipaddr): raise utils.ValidationError('ipaddr', 'object_does_not_exists', ipaddr) else: - return # validation is ok + if id and is_host_set(uci, id): + # check loop + if _has_loop(uci, id, ipaddr): + raise utils.ValidationError('ipaddr', 'loop_detected', ipaddr) + else: + return # validation is ok + if family == 'ipv4': return _validate_host_set_ipaddr_v4(ipaddr) elif family == 'ipv6': @@ -429,7 +448,7 @@ def add_host_set(uci, name: str, family: str, ipaddrs: list[str]) -> str: if not name.isalnum(): raise utils.ValidationError('name', 'invalid_name', name) for ipaddr in ipaddrs: - _validate_host_set_ipaddr(uci, ipaddr, family) + _validate_host_set_ipaddr(uci, '', ipaddr, family) id = utils.get_random_id() uci.set('objects', id, 'host') uci.set('objects', id, 'name', name) @@ -457,7 +476,7 @@ def edit_host_set(uci, id: str, name: str, family: str, ipaddrs: list[str]) -> s if len(name) > 16: raise utils.ValidationError('name', 'name_too_long', name) for ipaddr in ipaddrs: - _validate_host_set_ipaddr(uci, ipaddr, family) + _validate_host_set_ipaddr(uci, f'objects/{id}', ipaddr, family) uci.set('objects', id, 'name', name) uci.set('objects', id, 'family', family) uci.set('objects', id, 'ipaddr', ipaddrs) diff --git a/tests/test_objects.py b/tests/test_objects.py index 854b662b..3ba491c1 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -235,6 +235,12 @@ def test_is_singleton_host_set(u): assert objects.is_singleton_host_set(u, f"objects/{id4}") == False assert objects.is_singleton_host_set(u, f"objects/{id4}", allow_cidr=True) +def test_add_host_set_loopback(u): + id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) + id2 = objects.add_host_set(u, "myhostset2", "ipv4", [f"objects/{id1}"]) + with pytest.raises(ValidationError): + objects.edit_host_set(u, id1, "myhostset", "ipv4", [f"objects/{id2}"]) + def test_is_used_object(u): used, matches = objects.is_used_object(u, "dhcp/ns_8dcab636") assert used From b4f938e77992c908fa5b29e465a429e51b1a2d78 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 15:07:49 +0200 Subject: [PATCH 15/33] objects: improve is_used_object Check if the object is used inside another host set object --- src/nethsec/objects/__init__.py | 8 ++++++++ tests/test_objects.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index f0bc6145..ed9a9764 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -71,6 +71,7 @@ def is_used_object(uci, database_id): - firewall config - mwan3 config - dpi config + - existing host set Args: uci: EUci pointer @@ -91,6 +92,13 @@ def is_used_object(uci, database_id): for section in uci.get_all("dpi"): if uci.get('dpi', section, 'source', default=None) == database_id: matches.append(f'dpi/{section}') + for section in uci.get_all("objects"): + try: + ips = uci.get_all('objects', section, 'ipaddr') + if database_id in ips: + matches.append(f'objects/{section}') + except: + continue return len(matches) > 0, matches def get_object_ips(uci, database_id): diff --git a/tests/test_objects.py b/tests/test_objects.py index 3ba491c1..d9898758 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -17,6 +17,11 @@ option name 'h1' option family 'ipv4' list ipaddr '1.2.3.4' + +config host 'ns_12345' + option name 'h1' + option family 'ipv4' + list ipaddr 'dhcp/ns_112233' """ firewall_db = """ @@ -51,6 +56,11 @@ option name 'host1' option ns_description 'Host 1' +config domain 'ns_112233' + option ip '7.8.9.1' + option name 'used_host' + option ns_description 'Used Host' + config host 'ns_8dcab636' option ip '192.168.100.5' option mac 'fe:54:00:6a:50:bf' @@ -222,7 +232,7 @@ def test_is_used_host_set(u): def test_list_host_sets(u): sets = objects.list_host_sets(u) - assert len(sets) == 7 + assert len(sets) == 8 def test_is_singleton_host_set(u): id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "5.6.7.8"]) @@ -250,6 +260,9 @@ def test_is_used_object(u): used, matches = objects.is_used_object(u, "objects/ns_04fadb5c") assert used assert matches == ["firewall/r7"] + used, matches = objects.is_used_object(u, "dhcp/ns_112233") + assert used + assert matches == ["objects/ns_12345"] def test_object_exists(u): assert objects.object_exists(u, "users/ns_user1") From 5457237f984f2f52248e085485f58732ebd476c6 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 16:27:54 +0200 Subject: [PATCH 16/33] objects: prevent deletion of used host and domain sets --- src/nethsec/objects/__init__.py | 4 ++++ tests/test_objects.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index ed9a9764..073af444 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -326,6 +326,8 @@ def delete_domain_set(uci, id: str) -> str: """ if not uci.get('objects', id, default=None): raise utils.ValidationError("id", "domain_set_does_not_exists", id) + if is_used_domain_set(uci, id)[0]: + raise utils.ValidationError("id", "domain_set_is_used", id) uci.delete('objects', id) uci.save('objects') for section in uci.get_all("dhcp"): @@ -504,6 +506,8 @@ def delete_host_set(uci, id: str) -> str: """ if not uci.get('objects', id, default=None): raise utils.ValidationError("id", "host_set_does_not_exists", id) + if is_used_host_set(uci, id)[0]: + raise utils.ValidationError("id", "host_set_is_used", id) uci.delete('objects', id) uci.save('objects') return id diff --git a/tests/test_objects.py b/tests/test_objects.py index d9898758..50bd2a1c 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -36,6 +36,9 @@ option name 'r7' option ns_src 'objects/ns_04fadb5c' +config rule 'r8' + option name 'r8' + config redirect 'redirect1' option ns_src '' option ipset 'redirect1_ipset' @@ -187,6 +190,10 @@ def test_delete_domain_set(u): linked = firewall.get_all_linked(u, f"objects/{id}") assert linked['firewall'] == [] assert linked['dhcp'] == [] + id = objects.add_domain_set(u, "mydomainsettd", "ipv4", ["test1.com", "test2.com"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_domain_set(u, id) def test_is_used_domain_set(u): id = objects.add_domain_set(u, "used1", "ipv4", ["test1.com", "test2.com"]) @@ -197,7 +204,7 @@ def test_is_used_domain_set(u): def test_list_domain_sets(u): sets = objects.list_domain_sets(u) - assert len(sets) == 7 + assert len(sets) == 8 def test_add_host_set(u): with pytest.raises(ValidationError): @@ -222,6 +229,10 @@ def test_delete_host_set(u): objects.delete_host_set(u, "notpresent") id = objects.add_host_set(u, "myhostset4", "ipv4", ["6.7.8.9"]) assert objects.delete_host_set(u, id) == id + id = objects.add_host_set(u, "myhostsettd", "ipv4", ["2.2.2.2"]) + u.set('firewall', 'r8', 'ns_dst', f"objects/{id}") + with pytest.raises(ValidationError): + objects.delete_host_set(u, id) def test_is_used_host_set(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.1.1.1"]) @@ -232,7 +243,7 @@ def test_is_used_host_set(u): def test_list_host_sets(u): sets = objects.list_host_sets(u) - assert len(sets) == 8 + assert len(sets) == 9 def test_is_singleton_host_set(u): id1 = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4", "5.6.7.8"]) From 3420a10051e9973e482a808c52207f8021fbb3ec Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 30 May 2024 18:15:13 +0200 Subject: [PATCH 17/33] objects: delete, improve returned value on error On error, return the records where the object is used. Such records could be shown inside the UI. Example: {"validation": {"errors": [{"parameter": "id", "message": "host_set_is_used", "value": ["firewall/ns_allow_OpenVPNRW1"]}]}} --- src/nethsec/objects/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 073af444..f22bb0ea 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -326,8 +326,9 @@ def delete_domain_set(uci, id: str) -> str: """ if not uci.get('objects', id, default=None): raise utils.ValidationError("id", "domain_set_does_not_exists", id) - if is_used_domain_set(uci, id)[0]: - raise utils.ValidationError("id", "domain_set_is_used", id) + used, matches = is_used_domain_set(uci, id) + if used: + raise utils.ValidationError("id", "domain_set_is_used", matches) uci.delete('objects', id) uci.save('objects') for section in uci.get_all("dhcp"): @@ -506,8 +507,9 @@ def delete_host_set(uci, id: str) -> str: """ if not uci.get('objects', id, default=None): raise utils.ValidationError("id", "host_set_does_not_exists", id) - if is_used_host_set(uci, id)[0]: - raise utils.ValidationError("id", "host_set_is_used", id) + used, matches = is_used_host_set(uci, id) + if used: + raise utils.ValidationError("id", "host_set_is_used", matches) uci.delete('objects', id) uci.save('objects') return id From d6cb43b1b5a61dd33c6bec8f43b335b28d31b501 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 31 May 2024 14:02:20 +0200 Subject: [PATCH 18/33] objects: add list --- src/nethsec/firewall/__init__.py | 13 +++ src/nethsec/objects/__init__.py | 135 ++++++++++++++++++++++++++++--- tests/test_firewall.py | 3 + tests/test_objects.py | 37 +++++++++ 4 files changed, 179 insertions(+), 9 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 81cd17c0..4be2164c 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1315,6 +1315,19 @@ def list_host_suggestions(uci): ret = ret + list_active_leases() return ret +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) + def validate_address_format(address: str) -> bool: """ Validate address format. diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index f22bb0ea..8755656f 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -343,12 +343,13 @@ def delete_domain_set(uci, id: str) -> str: break return id -def list_domain_sets(uci) -> list: +def list_domain_sets(uci, used_info = True) -> list: """ Get all domain sets from objects config Args: uci: EUci pointer + used_info: include used and matches info Returns: a list of all domain sets @@ -358,9 +359,10 @@ def list_domain_sets(uci) -> list: if uci.get('objects', section) == 'domain': rule = uci.get_all('objects', section) rule['id'] = section - used, matches = is_used_domain_set(uci, section) - rule['used'] = used - rule['matches'] = matches + if used_info: + used, matches = is_used_domain_set(uci, section) + rule['used'] = used + rule['matches'] = matches sets.append(rule) return sets @@ -529,12 +531,13 @@ def is_used_host_set(uci, id): """ return is_used_object(uci, f'objects/{id}') -def list_host_sets(uci) -> list: +def list_host_sets(uci, used_info = True) -> list: """ Get all host sets from objects config Args: uci: EUci pointer + used_info: include used and matches info Returns: a list of all host sets @@ -544,9 +547,10 @@ def list_host_sets(uci) -> list: if uci.get('objects', section) == 'host': rule = uci.get_all('objects', section) rule['id'] = section - used, matches = is_used_host_set(uci, section) - rule['used'] = used - rule['matches'] = matches + if used_info: + used, matches = is_used_host_set(uci, section) + rule['used'] = used + rule['matches'] = matches sets.append(rule) return sets @@ -653,4 +657,117 @@ def is_vpn_user(uci, database_id): obj_type = uci.get(database, id) return database == "users" and obj_type == "user" and uci.get(database, id, 'openvpn_ipaddr', default=None) != None except: - return False \ No newline at end of file + return False + +# API suggestions functions + +# Each element of the list should contain the following fields: +# - `id`: the id of the object +# - `name`: the name of the object +# - `type`: the type of the object +# - `family`: the family of the object (optional) +# If expand flag is set to True, the list should contain all IP addresses of the object + +def list_vpn_users(uci, expand=False): + """ + Get all VPN users from users config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + + Returns: + a list of all VPN users + """ + users = [] + for section in uci.get_all("users"): + user = {} + if uci.get('users', section) == 'user' and uci.get('users', section, 'openvpn_ipaddr', default=None) != None: + obj = uci.get_all('users', section) + user['id'] = f"users/{section}" + user['name'] = obj.get('name') + user['type'] = 'vpn_user' + user['family'] = 'ipv4' + if expand: + user['ipaddr'] = [obj.get('openvpn_ipaddr')] + users.append(user) + return users + +def list_dhcp_static_leases(uci, expand=False): + """ + Get all DHCP static leases from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + + Returns: + a list of all DHCP static leases + """ + leases = [] + for section in uci.get_all("dhcp"): + lease = {} + if uci.get('dhcp', section) == 'host': + obj = uci.get_all('dhcp', section) + lease['id'] = f"dhcp/{section}" + lease['name'] = obj.get('name') + lease['type'] = 'dhcp_static_lease' + lease['family'] = 'ipv4' + if expand: + lease['ipaddr'] = [obj.get('ip')] + leases.append(lease) + return leases + +def list_dns_records(uci, expand=False): + """ + Get all DNS records from dhcp config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + + Returns: + a list of all DNS records + """ + records = [] + for section in uci.get_all("dhcp"): + record = {} + if uci.get('dhcp', section) == 'domain': + obj = uci.get_all('dhcp', section) + record['id'] = f"dhcp/{section}" + record['name'] = obj.get('name') + record['type'] = 'dns_record' + record['family'] = 'ipv4' + if expand: + record['ipaddr'] = [obj.get('ip')] + records.append(record) + return records + +def list_all_objects(uci, expand=False): + """ + Get all objects from objects, dhcp, and users config + + Args: + uci: EUci pointer + expand: expand the list with all IP addresses of the object + + Returns: + a list of all objects + """ + hsets = [] + dsets = [] + for h in list_host_sets(uci, False): + h['type'] = 'host_set' + if not expand: + del[h['ipaddr']] + hsets.append(h) + + for d in list_domain_sets(uci, False): + d['type'] = 'domain_set' + if not expand: + del[d['domain']] + dsets.append(d) + vpn_users = list_vpn_users(uci, expand) + dhcp_static_leases = list_dhcp_static_leases(uci, expand) + dns_records = list_dns_records(uci, expand) + return hsets + dsets + vpn_users + dhcp_static_leases + dns_records \ No newline at end of file diff --git a/tests/test_firewall.py b/tests/test_firewall.py index 0ac766a8..d39d70bd 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -1097,3 +1097,6 @@ def test_update_firewall_rules(u): firewall.update_firewall_rules(u) with pytest.raises(UciExceptionNotFound): u.get("firewall", "r4", "ns_dst") + +def list_object_suggestions(u): + assert False diff --git a/tests/test_objects.py b/tests/test_objects.py index 50bd2a1c..f5082c0e 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -312,3 +312,40 @@ def test_is_host_set(u): id = objects.add_host_set(u, "myhostset", "ipv4", ["1.2.3.4"]) assert objects.is_host_set(u, f"objects/{id}") assert objects.is_host_set(u, "dhcp/ns_8dcab636") == False + +def test_list_vpn_users(u): + users = objects.list_vpn_users(u) + assert len(users) == 1 + users = objects.list_vpn_users(u, True) + assert 'ipaddr' in users[0] + +def test_list_dns_records(u): + records = objects.list_dns_records(u) + assert len(records) == 3 + records = objects.list_dns_records(u, True) + for r in records: + assert 'ipaddr' in r + +def test_list_dhcp_static_leases(u): + leases = objects.list_dhcp_static_leases(u) + assert len(leases) == 2 + leases = objects.list_dhcp_static_leases(u, True) + for l in leases: + assert 'ipaddr' in l + +def test_list_all_objects(u): + objs = objects.list_all_objects(u) + assert len(objs) == len(objects.list_domain_sets(u)) + len(objects.list_host_sets(u)) + len(objects.list_vpn_users(u)) + len(objects.list_dhcp_static_leases(u)) + len(objects.list_dns_records(u)) + objs = objects.list_all_objects(u, True) + for o in objs: + if o['type'] == 'host_set': + assert 'ipaddr' in o + elif o['type'] == 'domain_set': + assert 'domain' in o + elif o['type'] == 'vpn_user': + assert 'ipaddr' in o + elif o['type'] == 'dhcp_static_lease': + assert 'ipaddr' in o + elif o['type'] == 'dns_record': + assert 'ipaddr' in o + \ No newline at end of file From 51d660a0074fc7051353ec6780491b0a6c87347c Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 31 May 2024 18:05:04 +0200 Subject: [PATCH 19/33] firewall: add list_objects_suggestions --- src/nethsec/firewall/__init__.py | 15 ++++++++++++++- src/nethsec/objects/__init__.py | 2 ++ tests/test_firewall.py | 5 +++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 4be2164c..c80b599b 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1999,4 +1999,17 @@ def update_firewall_rules(uci): uci.delete('firewall', section, 'ipset') except: pass - uci.save('firewall') \ No newline at end of file + uci.save('firewall') + +def list_object_suggestions(uci, expand = False): + """ + Get all objects from objects config + + Args: + uci: EUci pointer + expand: if True, expand object details + + Returns: + a list of all objects, each object is a dict with keys value, label, type + """ + return objects.list_all_objects(uci, expand) \ No newline at end of file diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 8755656f..2fe8689a 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -547,6 +547,7 @@ def list_host_sets(uci, used_info = True) -> list: if uci.get('objects', section) == 'host': rule = uci.get_all('objects', section) rule['id'] = section + rule['singleton'] = is_singleton_host_set(uci, f'objects/{section}') if used_info: used, matches = is_used_host_set(uci, section) rule['used'] = used @@ -766,6 +767,7 @@ def list_all_objects(uci, expand=False): d['type'] = 'domain_set' if not expand: del[d['domain']] + del[d['timeout']] dsets.append(d) vpn_users = list_vpn_users(uci, expand) dhcp_static_leases = list_dhcp_static_leases(uci, expand) diff --git a/tests/test_firewall.py b/tests/test_firewall.py index d39d70bd..f28632f2 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -1098,5 +1098,6 @@ def test_update_firewall_rules(u): with pytest.raises(UciExceptionNotFound): u.get("firewall", "r4", "ns_dst") -def list_object_suggestions(u): - assert False +def test_list_object_suggestions(u): + obj = firewall.list_object_suggestions(u) + assert len(obj) == 9 From febdde690a8d722f6e6db6e9d908b2d31f971624 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 4 Jun 2024 10:39:37 +0200 Subject: [PATCH 20/33] objects & firewall: only one function to list objects Reuse the same code from multiple API calls --- src/nethsec/objects/__init__.py | 19 +++++++++++-------- tests/test_firewall.py | 2 +- tests/test_objects.py | 14 +++++++++++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 2fe8689a..be712761 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -744,9 +744,9 @@ def list_dns_records(uci, expand=False): records.append(record) return records -def list_all_objects(uci, expand=False): +def list_objects(uci, include_domain_sets=True, singleton_only=False, expand=False): """ - Get all objects from objects, dhcp, and users config + Get objects from objects, dhcp, and users config Args: uci: EUci pointer @@ -758,17 +758,20 @@ def list_all_objects(uci, expand=False): hsets = [] dsets = [] for h in list_host_sets(uci, False): + if singleton_only and not h['singleton']: + continue h['type'] = 'host_set' if not expand: del[h['ipaddr']] hsets.append(h) - for d in list_domain_sets(uci, False): - d['type'] = 'domain_set' - if not expand: - del[d['domain']] - del[d['timeout']] - dsets.append(d) + if include_domain_sets: + for d in list_domain_sets(uci, False): + d['type'] = 'domain_set' + if not expand: + del[d['domain']] + del[d['timeout']] + dsets.append(d) vpn_users = list_vpn_users(uci, expand) dhcp_static_leases = list_dhcp_static_leases(uci, expand) dns_records = list_dns_records(uci, expand) diff --git a/tests/test_firewall.py b/tests/test_firewall.py index f28632f2..1af1c5d1 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -1099,5 +1099,5 @@ def test_update_firewall_rules(u): u.get("firewall", "r4", "ns_dst") def test_list_object_suggestions(u): - obj = firewall.list_object_suggestions(u) + obj = objects.list_objects(u) assert len(obj) == 9 diff --git a/tests/test_objects.py b/tests/test_objects.py index f5082c0e..dbc1625d 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -333,10 +333,10 @@ def test_list_dhcp_static_leases(u): for l in leases: assert 'ipaddr' in l -def test_list_all_objects(u): - objs = objects.list_all_objects(u) +def test_list_objects(u): + objs = objects.list_objects(u) assert len(objs) == len(objects.list_domain_sets(u)) + len(objects.list_host_sets(u)) + len(objects.list_vpn_users(u)) + len(objects.list_dhcp_static_leases(u)) + len(objects.list_dns_records(u)) - objs = objects.list_all_objects(u, True) + objs = objects.list_objects(u, expand=True) for o in objs: if o['type'] == 'host_set': assert 'ipaddr' in o @@ -348,4 +348,12 @@ def test_list_all_objects(u): assert 'ipaddr' in o elif o['type'] == 'dns_record': assert 'ipaddr' in o + objs = objects.list_objects(u, include_domain_sets=False) + for o in objs: + assert o['type'] != 'domain_set' + objs = objects.list_objects(u, singleton_only=True) + for o in objs: + if o['type'] == 'host_set': + assert o['singleton'] + \ No newline at end of file From 9a1fde9a2d6cbdedfc79e88a2af5a12765f884eb Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 6 Jun 2024 15:41:28 +0200 Subject: [PATCH 21/33] objects: add subtype --- src/nethsec/objects/__init__.py | 9 +++++++++ tests/test_objects.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index be712761..ddcf42b8 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -548,6 +548,15 @@ def list_host_sets(uci, used_info = True) -> list: rule = uci.get_all('objects', section) rule['id'] = section rule['singleton'] = is_singleton_host_set(uci, f'objects/{section}') + if rule['singleton']: + # set subtype to CIDR, range or host + ip = get_object_ip(uci, f'objects/{section}') + if '/' in ip: + rule['subtype'] = 'cidr' + elif '-' in ip: + rule['subtype'] = 'range' + else: + rule['subtype'] = 'host' if used_info: used, matches = is_used_host_set(uci, section) rule['used'] = used diff --git a/tests/test_objects.py b/tests/test_objects.py index dbc1625d..95214b6b 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -352,8 +352,9 @@ def test_list_objects(u): for o in objs: assert o['type'] != 'domain_set' objs = objects.list_objects(u, singleton_only=True) + print(objs) for o in objs: if o['type'] == 'host_set': assert o['singleton'] - + assert o['subtype'] \ No newline at end of file From 092b0bccfa7befb6ea7553c31d67c5336c0ffaeb Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 6 Jun 2024 16:19:35 +0200 Subject: [PATCH 22/33] objects: add get_reference_info --- src/nethsec/objects/__init__.py | 30 +++++++++++++++++++++++++++++- tests/test_objects.py | 8 +++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index ddcf42b8..497d083e 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -784,4 +784,32 @@ def list_objects(uci, include_domain_sets=True, singleton_only=False, expand=Fal vpn_users = list_vpn_users(uci, expand) dhcp_static_leases = list_dhcp_static_leases(uci, expand) dns_records = list_dns_records(uci, expand) - return hsets + dsets + vpn_users + dhcp_static_leases + dns_records \ No newline at end of file + return hsets + dsets + vpn_users + dhcp_static_leases + dns_records + +def get_info(uci, database_id): + """ + Get the info of the object. + + Args: + uci: EUci pointer + database_id: id of the object in the form of `/` + + Returns: + a dictionary with the following fields: + - `database`: the database of the object + - `id`: the id of the object + - `name`: the name of the object + - `type`: the type of the object + """ + try: + database, id = database_id.split('/') + type = uci.get(database, id) + name = uci.get(database, id, 'name', default=None) + if not name: + name = uci.get(database, id, 'label', default=None) + if not name: + name = id + return {'database': database, 'id': id, 'name': name, 'type': type} + except: + return None + diff --git a/tests/test_objects.py b/tests/test_objects.py index 95214b6b..b9d44422 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -352,9 +352,15 @@ def test_list_objects(u): for o in objs: assert o['type'] != 'domain_set' objs = objects.list_objects(u, singleton_only=True) - print(objs) for o in objs: if o['type'] == 'host_set': assert o['singleton'] assert o['subtype'] + +def test_get_reference_info(u): + ref = objects.get_info(u, "dhcp/ns_8dcab636") + assert ref['type'] == 'host' + assert ref['name'] == 'host2' + assert ref['id'] == 'ns_8dcab636' + assert objects.get_info(u, "unknown") == None \ No newline at end of file From bd2e3c8f8fa3132da38f0a6196c4b8b23715b991 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 6 Jun 2024 16:34:51 +0200 Subject: [PATCH 23/33] mwan: fix tests --- tests/test_mwan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mwan.py b/tests/test_mwan.py index 3a4c7796..e18684d9 100644 --- a/tests/test_mwan.py +++ b/tests/test_mwan.py @@ -337,7 +337,7 @@ def test_store_rule(e_uci, mocker): assert e_uci.get('mwan3', 'ns_rule_1', 'sticky') == '1' domain_id = objects.add_domain_set(e_uci, "mydomainset6", "ipv4", ["test1.com", "test2.com"]) - id = mwan.store_rule(e_uci, 'rule_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{domain_id}") + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_default', 'udp', ns_src="dhcp/ns_host_mwan", ns_dst=f"objects/{domain_id}") id = id.split('.')[1] assert e_uci.get('mwan3', id, 'ns_src') == "dhcp/ns_host_mwan" assert e_uci.get('mwan3', id, 'ns_dst') == f"objects/{domain_id}" @@ -594,7 +594,7 @@ def test_update_rules(e_uci, mocker): } ]) domain_id = objects.add_domain_set(e_uci, "mydomainset7", "ipv4", ["test1.com", "test2.com"]) - id = mwan.store_rule(e_uci, 'rule_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"objects/{domain_id}") + id = mwan.store_rule(e_uci, 'r_with_obj', 'ns_cool_policy', 'udp', ns_src="dhcp/ns_domain_mwan", ns_dst=f"objects/{domain_id}") id = id.split('.')[1] ipsets = objects.get_domain_set_ipsets(e_uci, domain_id) mwan.update_rules(e_uci) From 25c88965f1b7b860285aa3c80e92f511847d2acb Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 18 Jun 2024 09:27:28 +0200 Subject: [PATCH 24/33] objects: improve IP address validation Make sure to always return a validation to the UI when an IP address or a CIDR is not valid. --- src/nethsec/objects/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 497d083e..a2a3140c 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -403,20 +403,20 @@ def _validate_host_set_ipaddr_v4(ipaddr: str): # validate CIDR try: ipaddress.IPv4Network(ipaddr) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) elif '-' in ipaddr: start, end = ipaddr.split('-') try: ipaddress.IPv4Address(start) ipaddress.IPv4Address(end) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) else: # validate IPv4 try: ipaddress.IPv4Address(ipaddr) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) return True @@ -425,20 +425,20 @@ def _validate_host_set_ipaddr_v6(ipaddr: str): # validate CIDR try: ipaddress.IPv6Network(ipaddr) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) elif '-' in ipaddr: start, end = ipaddr.split('-') try: ipaddress.IPv6Address(start) ipaddress.IPv6Address(end) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) else: # validate IPv6 try: ipaddress.IPv6Address(ipaddr) - except ipaddress.AddressValueError: + except: raise utils.ValidationError('ipaddr', 'invalid_ipaddr', ipaddr) return True From e79c28f2a87ed91e9daef4bca4ee9593858fc533 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 18 Jun 2024 14:14:24 +0200 Subject: [PATCH 25/33] firewall: refactor list_objects output Changes for the UI: - always add subtype - use consistent id format - add used feild to simple objects like dns records --- src/nethsec/objects/__init__.py | 39 ++++++++++++++++++++++++++------- tests/test_firewall.py | 10 +++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index a2a3140c..d9c433c0 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -557,6 +557,8 @@ def list_host_sets(uci, used_info = True) -> list: rule['subtype'] = 'range' else: rule['subtype'] = 'host' + else: + rule['subtype'] = 'host_set' if used_info: used, matches = is_used_host_set(uci, section) rule['used'] = used @@ -678,13 +680,14 @@ def is_vpn_user(uci, database_id): # - `family`: the family of the object (optional) # If expand flag is set to True, the list should contain all IP addresses of the object -def list_vpn_users(uci, expand=False): +def list_vpn_users(uci, expand=False, used_info=False): """ Get all VPN users from users config Args: uci: EUci pointer expand: expand the list with all IP addresses of the object + used_info: include used and matches info Returns: a list of all VPN users @@ -697,19 +700,25 @@ def list_vpn_users(uci, expand=False): user['id'] = f"users/{section}" user['name'] = obj.get('name') user['type'] = 'vpn_user' + user['subtype'] = 'vpn_user' user['family'] = 'ipv4' if expand: user['ipaddr'] = [obj.get('openvpn_ipaddr')] + if used_info: + used, matches = is_used_object(uci, f'users/{section}') + user['used'] = used + user['matches'] = matches users.append(user) return users -def list_dhcp_static_leases(uci, expand=False): +def list_dhcp_static_leases(uci, expand=False, used_info=False): """ Get all DHCP static leases from dhcp config Args: uci: EUci pointer expand: expand the list with all IP addresses of the object + used_info: include used and matches info Returns: a list of all DHCP static leases @@ -722,19 +731,25 @@ def list_dhcp_static_leases(uci, expand=False): lease['id'] = f"dhcp/{section}" lease['name'] = obj.get('name') lease['type'] = 'dhcp_static_lease' + lease['subtype'] = 'dhcp_static_lease' lease['family'] = 'ipv4' if expand: lease['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + lease['used'] = used + lease['matches'] = matches leases.append(lease) return leases -def list_dns_records(uci, expand=False): +def list_dns_records(uci, expand=False, used_info=False): """ Get all DNS records from dhcp config Args: uci: EUci pointer expand: expand the list with all IP addresses of the object + used_info: include used and matches info Returns: a list of all DNS records @@ -747,9 +762,14 @@ def list_dns_records(uci, expand=False): record['id'] = f"dhcp/{section}" record['name'] = obj.get('name') record['type'] = 'dns_record' + record['subtype'] = 'dns_record' record['family'] = 'ipv4' if expand: record['ipaddr'] = [obj.get('ip')] + if used_info: + used, matches = is_used_object(uci, f'dhcp/{section}') + record['used'] = used + record['matches'] = matches records.append(record) return records @@ -766,24 +786,27 @@ def list_objects(uci, include_domain_sets=True, singleton_only=False, expand=Fal """ hsets = [] dsets = [] - for h in list_host_sets(uci, False): + for h in list_host_sets(uci, True): if singleton_only and not h['singleton']: continue + h['id'] = f"objects/{h['id']}" h['type'] = 'host_set' if not expand: del[h['ipaddr']] hsets.append(h) if include_domain_sets: - for d in list_domain_sets(uci, False): + for d in list_domain_sets(uci, True): + d['id'] = f"objects/{d['id']}" d['type'] = 'domain_set' + d['subtype'] = 'domain_set' if not expand: del[d['domain']] del[d['timeout']] dsets.append(d) - vpn_users = list_vpn_users(uci, expand) - dhcp_static_leases = list_dhcp_static_leases(uci, expand) - dns_records = list_dns_records(uci, expand) + vpn_users = list_vpn_users(uci, expand, True) + dhcp_static_leases = list_dhcp_static_leases(uci, expand, True) + dns_records = list_dns_records(uci, expand, True) return hsets + dsets + vpn_users + dhcp_static_leases + dns_records def get_info(uci, database_id): diff --git a/tests/test_firewall.py b/tests/test_firewall.py index 1af1c5d1..d83545df 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -413,6 +413,12 @@ option openvpn_ipaddr "10.10.10.22" """ +mwan3_db = """ +""" + +dpi_db = """ +""" + # Setup fake ip command output ip_json='[{"ifindex":9,"ifname":"vnet3","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"virbr2","operstate":"UNKNOWN","group":"default","txqlen":1000,"link_type":"ether","address":"fe:62:31:19:0b:29","broadcast":"ff:ff:ff:ff:ff:ff","addr_info":[{"family":"inet6","local":"fe80::fc62:31ff:fe19:b29","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]},{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":3,"ifname":"eth1","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":4,"ifname":"eth2","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:75:1c:c1","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":5,"ifname":"eth3","flags":["BROADCAST","MULTICAST"],"mtu":1500,"qdisc":"noop","operstate":"DOWN","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":6,"ifname":"ifb-dns","flags":["BROADCAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":32,"link_type":"ether","address":"72:79:65:12:07:07","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":7,"ifname":"br-lan","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:6a:50:bf","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":9,"ifname":"bond-bond1","flags":["BROADCAST","MULTICAST","MASTER","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:ad:6f:63","broadcast":"ff:ff:ff:ff:ff:ff"},{"ifindex":8,"ifname":"tuntunsubnet","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1500,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":500,"link_type":"none"}, {"ifindex":69,"ifname":"pppoe-w1","flags":["POINTOPOINT","MULTICAST","NOARP","UP","LOWER_UP"],"mtu":1492,"qdisc":"fq_codel","operstate":"UNKNOWN","linkmode":"DEFAULT","group":"default","txqlen":3,"link_type":"ppp"},{"ifindex":20,"link":"eth1","ifname":"eth1.4","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1500,"qdisc":"noqueue","master":"br-lan","operstate":"UP","linkmode":"DEFAULT","group":"default","txqlen":1000,"link_type":"ether","address":"52:54:00:20:82:a6","broadcast":"ff:ff:ff:ff:ff:ff"}]' mock_ip_stdout = MagicMock() @@ -440,6 +446,10 @@ def u(tmp_path: pathlib.Path) -> EUci: fp.write(objects_db) with tmp_path.joinpath('users').open('w') as fp: fp.write(user_db) + with tmp_path.joinpath('mwan3').open('w') as fp: + fp.write(mwan3_db) + with tmp_path.joinpath('dpi').open('w') as fp: + fp.write(dpi_db) return EUci(confdir=tmp_path.as_posix()) def test_add_interface_to_zone(u): From 1c5ea39cf8d780ff92c6831fab065dcd5ba2628d Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 19 Jun 2024 12:23:34 +0200 Subject: [PATCH 26/33] objects: fix host set validation Changes: - do not fail if an object does not contain ipaddr field - do not validate IP format for objects --- src/nethsec/objects/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index d9c433c0..48c07c42 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -376,7 +376,7 @@ def _has_loop(uci, id, ipaddr, depth=0): return True obj = get_object(uci, ipaddr) if obj: - for ip in obj.get('ipaddr'): + for ip in obj.get('ipaddr', []): if _has_loop(uci, id, ip, depth + 1): return True return False @@ -390,8 +390,7 @@ def _validate_host_set_ipaddr(uci, id, ipaddr: str, family: str): # check loop if _has_loop(uci, id, ipaddr): raise utils.ValidationError('ipaddr', 'loop_detected', ipaddr) - else: - return # validation is ok + return # validation is ok if family == 'ipv4': return _validate_host_set_ipaddr_v4(ipaddr) From b795838d3ebad4b966e3486b91ff4e1dc1eb835c Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 24 Jun 2024 16:13:06 +0200 Subject: [PATCH 27/33] firewall: fix add_rule and edit_rule --- src/nethsec/firewall/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index c80b599b..4a2dad30 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1399,7 +1399,7 @@ def validate_port_format(port: str) -> bool: return False return True -def validate_rule(src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, ns_src: str, ns_dst: str): +def validate_rule(uci, src: str, src_ip: list[str], dest: str, dest_ip: list[str], proto: list, dest_port: list[str], target: str, service: str, ns_src: str, ns_dst: str): """ Validate rule. @@ -1419,20 +1419,20 @@ def validate_rule(src: str, src_ip: list[str], dest: str, dest_ip: list[str], pr ValidationError: if rule is invalid """ if ns_src: - if not objects.object_exists(ns_src): + if not objects.object_exists(uci, ns_src): raise utils.ValidationError('ns_src', 'object_not_found', ns_src) else: # check source only if not using objects for s in src_ip: if not validate_address_format(s): raise utils.ValidationError('src_ip', 'invalid_format', s) if ns_dst: - if not objects.object_exists(ns_dst): + if not objects.object_exists(uci, ns_dst): raise utils.ValidationError('ns_dst', 'object_not_found', ns_dst) else: # check destiation only if not using objects for d in dest_ip: if not validate_address_format(d): raise utils.ValidationError('dest_ip', 'invalid_format', d) - if ns_src and ns_dst and objects.is_domain_set(ns_src) and objects.is_domain_set(ns_dst): + if ns_src and ns_dst and objects.is_domain_set(uci, ns_src) and objects.is_domain_set(uci, ns_dst): raise utils.ValidationError('dest', 'domain_set_conflict', dest) if (not ns_src and not ns_dst) and src == dest: # check only if not using objects raise utils.ValidationError('dest', 'same_zone', dest) @@ -1611,7 +1611,7 @@ def add_rule(uci, name: str, src: str, src_ip: list[str], dest: str, dest_ip: li Returns: name of rule config that was added """ - validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) rule = utils.get_random_id() uci.set('firewall', rule, 'rule') setup_rule(uci, rule, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst) @@ -1659,7 +1659,7 @@ def edit_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, d """ if not uci.get('firewall', id, default=None): raise utils.ValidationError("id", "rule_does_not_exists", id) - validate_rule(src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) + validate_rule(uci, src, src_ip, dest, dest_ip, proto, dest_port, target, service, ns_src, ns_dst) setup_rule(uci, id, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled, log, tag, ns_src, ns_dst) update_firewall_rules(uci) # expand objects and save return id From b66e92125c0b60b4959a7d96cfee9d3da616dfe9 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Wed, 26 Jun 2024 16:56:12 +0200 Subject: [PATCH 28/33] objects: domain set, fix ipset usage Firewall rules must refer the ipset by name and not by id. --- src/nethsec/objects/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 48c07c42..c654dac7 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -206,7 +206,7 @@ def get_domain_set_ipsets(uci, id): ipsets = {"firewall": None, "dhcp": None} for section in utils.get_all_by_type(uci, "firewall", "ipset"): if uci.get('firewall', section, 'ns_link', default=None) == f'objects/{id}': - ipsets["firewall"] = section + ipsets["firewall"] = uci.get("firewall", section, "name", default='') break for section in utils.get_all_by_type(uci, "dhcp", "ipset"): if uci.get('dhcp', section, 'ns_link', default=None) == f'objects/{id}': From 6dc5a15e8c6eb325ed9bb4335462f66347a007a5 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 27 Jun 2024 09:09:57 +0200 Subject: [PATCH 29/33] objects: fix ip address retrieval Make sure that if the object is invalide, the function returns an empty IP list. This fixes an issue with creation of port forward rules not using objects. --- src/nethsec/objects/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index c654dac7..2ad68d7f 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -113,6 +113,8 @@ def get_object_ips(uci, database_id): a list of unique IP addresses from the object """ ips = [] + if not database_id: + return ips obj = get_object(uci, database_id) database, id = database_id.split('/') From 3c72f1176fb8f008a8b6807cfd7538eb9a29355c Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Thu, 27 Jun 2024 11:33:03 +0200 Subject: [PATCH 30/33] objects: domain set, increase timeout Increase the timeout from 10 to 11 minutes to prevent potential issues with the cron job. The cron job runs every 10 minutes, and if it is delayed, the IP set might be empty for a brief period. Extending the timeout ensures the IP set remains populated even if the cron job runs late. --- src/nethsec/objects/__init__.py | 8 ++++---- tests/test_objects.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/nethsec/objects/__init__.py b/src/nethsec/objects/__init__.py index 2ad68d7f..265ea198 100644 --- a/src/nethsec/objects/__init__.py +++ b/src/nethsec/objects/__init__.py @@ -216,7 +216,7 @@ def get_domain_set_ipsets(uci, id): break return ipsets -def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int = 600) -> str: +def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int = 660) -> str: """ Add domain set to objects config. @@ -225,7 +225,7 @@ def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int name: name of domain set family: can be `ipv4` or `ipv6` domains: a list of valid DNS names - timeout: the timeout in seconds for the DNS resolution, default is `600` seconds + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds Returns: id of domain set config that was added @@ -268,7 +268,7 @@ def add_domain_set(uci, name: str, family: str, domains: list[str], timeout: int uci.save('firewall') return id -def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], timeout: int = 600) -> str: +def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], timeout: int = 660) -> str: """ Edit domain set in objects config. @@ -278,7 +278,7 @@ def edit_domain_set(uci, id: str, name: str, family: str, domains: list[str], ti name: name of domain set family: can be `ipv4` or `ipv6` domains: a list of valid DNS names - timeout: the timeout in seconds for the DNS resolution, default is `600` seconds + timeout: the timeout in seconds for the DNS resolution, default is `660` seconds Returns: id of domain set config that was edited diff --git a/tests/test_objects.py b/tests/test_objects.py index b9d44422..8ebd5ef8 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -151,25 +151,25 @@ def test_add_doman_set(u): assert u.get("objects", id1, "name") == "mydomainset" assert u.get("objects", id1, "family") == "ipv4" assert u.get_all("objects", id1, "domain") == ("test1.com", "test2.com") - assert u.get("objects", id1, "timeout") == "600" + assert u.get("objects", id1, "timeout") == "660" linked = firewall.get_all_linked(u, f"objects/{id1}") assert linked['firewall'] is not None assert u.get('firewall', linked['firewall'][0], 'ns_link') == f"objects/{id1}" assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset" assert u.get('firewall', linked['firewall'][0], 'family') == 'ipv4' - assert u.get('firewall', linked['firewall'][0], 'timeout') == '600' + assert u.get('firewall', linked['firewall'][0], 'timeout') == '660' assert linked['dhcp'] is not None assert u.get('dhcp', linked['dhcp'][0], 'ns_link') == f"objects/{id1}" assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset",) assert u.get_all('dhcp', linked['dhcp'][0], 'domain') == ("test1.com", "test2.com") - id2 = objects.add_domain_set(u, "mydomainset2", "ipv6", ["test3.com", "test4.com"], 600) + id2 = objects.add_domain_set(u, "mydomainset2", "ipv6", ["test3.com", "test4.com"], 300) assert u.get("objects", id2, "name") == "mydomainset2" assert u.get("objects", id2, "family") == "ipv6" assert u.get_all("objects", id2, "domain") == ("test3.com", "test4.com") - assert u.get("objects", id2, "timeout") == "600" + assert u.get("objects", id2, "timeout") == "300" linked = firewall.get_all_linked(u, f"objects/{id2}") assert u.get('firewall', linked['firewall'][0], 'name') == "mydomainset2" assert u.get_all('dhcp', linked['dhcp'][0], 'name') == ("mydomainset2",) @@ -363,4 +363,4 @@ def test_get_reference_info(u): assert ref['name'] == 'host2' assert ref['id'] == 'ns_8dcab636' assert objects.get_info(u, "unknown") == None - \ No newline at end of file + From fe5f84e55e3238578941cdb08d58bb1fb270dbaa Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 28 Jun 2024 14:59:54 +0200 Subject: [PATCH 31/33] firewall: fix object removal from rule --- src/nethsec/firewall/__init__.py | 11 +++++++++++ tests/test_firewall.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 4a2dad30..7fb25316 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1519,8 +1519,18 @@ def setup_rule(uci, id: str, name: str, src: str, src_ip: list[str], dest: str, uci.set('firewall', id, 'ns_tag', tag) if ns_src: uci.set('firewall', id, 'ns_src', ns_src) + else: + try: + uci.delete('firewall', id, 'ns_src') + except: + pass if ns_dst: uci.set('firewall', id, 'ns_dst', ns_dst) + else: + try: + uci.delete('firewall', id, 'ns_dst') + except: + pass uci.save('firewall') def split_firewall_config(uci): @@ -1963,6 +1973,7 @@ def update_firewall_rules(uci): keep_ipset = False ns_src = uci.get('firewall', section, 'ns_src', default=None) ns_dst = uci.get('firewall', section, 'ns_dst', default=None) + name = uci.get('firewall', section, 'name', default=None) if ns_src: if objects.is_domain_set(uci, ns_src): keep_ipset = True diff --git a/tests/test_firewall.py b/tests/test_firewall.py index d83545df..9f3c5696 100644 --- a/tests/test_firewall.py +++ b/tests/test_firewall.py @@ -1111,3 +1111,15 @@ def test_update_firewall_rules(u): def test_list_object_suggestions(u): obj = objects.list_objects(u) assert len(obj) == 9 + +def test_edit_rule_remove_object(u): + host1 = objects.add_host_set(u, "h1", "ipv4", ["1.2.3.4"]) + # def add_rule(uci, name, src, src_ip, dest, dest_ip, proto, dest_port, target, service, enabled=True, log=False, tag=[], add_to_top=False, ns_src=None, ns_dst=None): + idf1 = firewall.add_rule(u, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, [], False, f"objects/{host1}", "") + # remove object from rule + firewall.edit_rule(u, idf1, "forward1", "*", [], "wan", [], [], [], "REJECT", "*", True, False, []) + with pytest.raises(UciExceptionNotFound): + u.get("firewall", idf1, "src_ip") + assert u.get('firewall', idf1, 'ns_src', default='NONE') == 'NONE' + firewall.delete_rule(u, idf1) + objects.delete_host_set(u, host1) \ No newline at end of file From e867dbb6210c3b993ed1fe6251e2341559797250 Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Fri, 28 Jun 2024 15:59:36 +0200 Subject: [PATCH 32/33] firewall: better handle domain conflict Make sure the UI highlight the ns_dst field. --- src/nethsec/firewall/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nethsec/firewall/__init__.py b/src/nethsec/firewall/__init__.py index 7fb25316..e5bec3b3 100644 --- a/src/nethsec/firewall/__init__.py +++ b/src/nethsec/firewall/__init__.py @@ -1433,7 +1433,7 @@ def validate_rule(uci, src: str, src_ip: list[str], dest: str, dest_ip: list[str if not validate_address_format(d): raise utils.ValidationError('dest_ip', 'invalid_format', d) if ns_src and ns_dst and objects.is_domain_set(uci, ns_src) and objects.is_domain_set(uci, ns_dst): - raise utils.ValidationError('dest', 'domain_set_conflict', dest) + raise utils.ValidationError('ns_dst', 'domain_set_conflict', ns_dst) if (not ns_src and not ns_dst) and src == dest: # check only if not using objects raise utils.ValidationError('dest', 'same_zone', dest) if target not in TARGETS: @@ -2023,4 +2023,4 @@ def list_object_suggestions(uci, expand = False): Returns: a list of all objects, each object is a dict with keys value, label, type """ - return objects.list_all_objects(uci, expand) \ No newline at end of file + return objects.list_all_objects(uci, expand) From 4f12bd950f77c7dc281a297684f4f99de4c7086e Mon Sep 17 00:00:00 2001 From: Giacomo Sanchietti Date: Tue, 18 Jun 2024 14:15:31 +0200 Subject: [PATCH 33/33] python3-nethsec: bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9a3ee25c..b9a1d441 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name = 'nethsec', - version = '0.0.66', + version = '0.0.67', author = 'Giacomo Sanchietti', author_email = 'giacomo.sanchietti@nethesis.it', description = 'Utilities for NethSecurity development',