From 46f0fdbee6ca1410207a316b562cddbbc70e8080 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 29 Jul 2024 15:53:02 -0600 Subject: [PATCH 1/2] ignore private-space IP addresses in Zeek intel generation --- scripts/malcolm_utils.py | 17 +++++++++++++++++ shared/bin/zeek_threat_feed_utils.py | 24 +++++++++++++++++------- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/scripts/malcolm_utils.py b/scripts/malcolm_utils.py index df1846b7f..208ef2d2e 100644 --- a/scripts/malcolm_utils.py +++ b/scripts/malcolm_utils.py @@ -352,6 +352,23 @@ def isipaddress(value): return result +################################################################################################### +# check a string or list to see if something is a private IP address or subnet +def isprivateip(value): + result = True + try: + if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set): + for v in value: + result = result and ipaddress.ip_network(value).is_private + if not result: + break + else: + result = ipaddress.ip_network(value).is_private + except Exception: + result = False + return result + + ################################################################################################### # return the primary IP (the one with a default route) on the local box def get_primary_ip(): diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 837a71f5a..1341a243b 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -37,7 +37,7 @@ import requests import urllib3 -from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson +from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson, isprivateip # keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields ZEEK_INTEL_INDICATOR = 'indicator' @@ -316,9 +316,14 @@ def map_stix_indicator_to_zeek( parsed = urlparse(ioc_value) scheme = f"{parsed.scheme}://" ioc_value = parsed.geturl().replace(scheme, "", 1) - elif zeek_type == "ADDR" and re.match(".+/.+", ioc_value): - # elevate to subnet if possible - zeek_type = "SUBNET" + elif zeek_type == "ADDR": + if not isprivateip(ioc_value): + if re.match(".+/.+", ioc_value): + # elevate to subnet if possible + zeek_type = "SUBNET" + else: + # ignore private IP-space ADDR avlues + continue # ... "fields containing only a hyphen are considered to be null values" zeekItem = defaultdict(lambda: '-') @@ -395,9 +400,14 @@ def map_misp_attribute_to_zeek( parsed = urlparse(attribute_value) scheme = f"{parsed.scheme}://" attribute_value = parsed.geturl().replace(scheme, "", 1) - elif zeek_type == "ADDR" and re.match(".+/.+", attribute_value): - # elevate to subnet if possible - zeek_type = "SUBNET" + elif zeek_type == "ADDR": + if not isprivateip(attribute_value): + if re.match(".+/.+", attribute_value): + # elevate to subnet if possible + zeek_type = "SUBNET" + else: + # ignore private IP-space ADDR avlues + continue # ... "fields containing only a hyphen are considered to be null values" zeekItem = defaultdict(lambda: '-') From dd5084c1c93fc633183c013857568c4a65c71560 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 30 Jul 2024 08:38:55 -0600 Subject: [PATCH 2/2] directory checks before pruning files --- filebeat/scripts/clean-processed-folder.py | 49 +++++++++++++--------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/filebeat/scripts/clean-processed-folder.py b/filebeat/scripts/clean-processed-folder.py index 19be4ade8..b21e9841b 100755 --- a/filebeat/scripts/clean-processed-folder.py +++ b/filebeat/scripts/clean-processed-folder.py @@ -104,16 +104,22 @@ def pruneFiles(): return # look for regular Zeek files in the processed/ directory - zeekFoundFiles = [ - (os.path.join(root, filename)) - for root, dirnames, filenames in os.walk(zeekProcessedDir) - for filename in filenames - ] + zeekFoundFiles = ( + [ + (os.path.join(root, filename)) + for root, dirnames, filenames in os.walk(zeekProcessedDir) + for filename in filenames + ] + if os.path.isdir(zeekProcessedDir) + else [] + ) # look for rotated files from live zeek instance - zeekRotatedFiles = [ - (os.path.join(root, filename)) for root, dirnames, filenames in os.walk(zeekLiveDir) for filename in filenames - ] + zeekRotatedFiles = ( + [(os.path.join(root, filename)) for root, dirnames, filenames in os.walk(zeekLiveDir) for filename in filenames] + if os.path.isdir(zeekLiveDir) + else [] + ) # look up the filebeat registry file and try to read it fbReg = None @@ -128,18 +134,20 @@ def pruneFiles(): checkFile(file, filebeatReg=None, checkLogs=False, checkArchives=True) # clean up any broken symlinks in the Zeek current/ directory - for current in os.listdir(zeekCurrentDir): - currentFileSpec = os.path.join(zeekCurrentDir, current) - if os.path.islink(currentFileSpec) and not os.path.exists(currentFileSpec): - print(f'removing dead symlink "{currentFileSpec}"') - silentRemove(currentFileSpec) + if os.path.isdir(zeekCurrentDir): + for current in os.listdir(zeekCurrentDir): + currentFileSpec = os.path.join(zeekCurrentDir, current) + if os.path.islink(currentFileSpec) and not os.path.exists(currentFileSpec): + print(f'removing dead symlink "{currentFileSpec}"') + silentRemove(currentFileSpec) # clean up any old and empty directories in Zeek processed/ directory cleanDirSeconds = min(i for i in (cleanLogSeconds, cleanZipSeconds) if i > 0) candidateDirs = [] - for root, dirs, files in os.walk(zeekProcessedDir, topdown=False): - if root and dirs: - candidateDirs += [os.path.join(root, tmpDir) for tmpDir in dirs] + if os.path.isdir(zeekProcessedDir): + for root, dirs, files in os.walk(zeekProcessedDir, topdown=False): + if root and dirs: + candidateDirs += [os.path.join(root, tmpDir) for tmpDir in dirs] candidateDirs = list(set(candidateDirs)) candidateDirs.sort(reverse=True) candidateDirs.sort(key=len, reverse=True) @@ -155,10 +163,11 @@ def pruneFiles(): # check the suricata logs (live and otherwise) as well for surDir in [suricataDir, suricataLiveDir]: - for eve in os.listdir(surDir): - eveFile = os.path.join(surDir, eve) - if os.path.isfile(eveFile): - checkFile(eveFile, filebeatReg=fbReg, checkLogs=True, checkArchives=False) + if os.path.isdir(surDir): + for eve in os.listdir(surDir): + eveFile = os.path.join(surDir, eve) + if os.path.isfile(eveFile): + checkFile(eveFile, filebeatReg=fbReg, checkLogs=True, checkArchives=False) def main():