Merge branch 'dev' into bump_nuclei_version

bbot/core/core.py

@@ -34,8 +34,6 @@ def __init__(self):
         self._logger = None
         self._files_config = None
 
-        self.bbot_sudo_pass = None
-
         self._config = None
         self._custom_config = None
 

bbot/core/event/base.py

@@ -171,6 +171,7 @@ def __init__(
         self._module_priority = None
         self._resolved_hosts = set()
         self.dns_children = dict()
+        self.raw_dns_records = dict()
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
@@ -379,10 +380,20 @@ def discovery_path(self):
         """
         This event's full discovery context, including those of all its parents
         """
-        parent_path = []
+        discovery_path = []
         if self.parent is not None and self.parent is not self:
-            parent_path = self.parent.discovery_path
-        return parent_path + [[self.id, self.discovery_context]]
+            discovery_path = self.parent.discovery_path
+        return discovery_path + [self.discovery_context]
+
+    @property
+    def parent_chain(self):
+        """
+        This event's full discovery context, including those of all its parents
+        """
+        parent_chain = []
+        if self.parent is not None and self.parent is not self:
+            parent_chain = self.parent.parent_chain
+        return parent_chain + [self.id]
 
     @property
     def words(self):
@@ -770,6 +781,7 @@ def json(self, mode="json", siem_friendly=False):
         # discovery context
         j["discovery_context"] = self.discovery_context
         j["discovery_path"] = self.discovery_path
+        j["parent_chain"] = self.parent_chain
 
         # normalize non-primitive python objects
         for k, v in list(j.items()):
@@ -909,6 +921,10 @@ def _data_human(self):
     def discovery_path(self):
         return []
 
+    @property
+    def parent_chain(self):
+        return []
+
 
 class FINISHED(BaseEvent):
     """
@@ -1053,6 +1069,17 @@ def __init__(self, *args, **kwargs):
             if parent_module_type == "DNS":
                 self.dns_resolve_distance += 1
         # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
+        # tag subdomain / domain
+        if is_subdomain(self.host):
+            self.add_tag("subdomain")
+        elif is_domain(self.host):
+            self.add_tag("domain")
+        # tag private IP
+        try:
+            if self.host.is_private:
+                self.add_tag("private-ip")
+        except AttributeError:
+            pass
 
 
 class IP_RANGE(DnsEvent):
@@ -1069,13 +1096,6 @@ def _host(self):
 
 
 class DNS_NAME(DnsEvent):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if is_subdomain(self.data):
-            self.add_tag("subdomain")
-        elif is_domain(self.data):
-            self.add_tag("domain")
-
     def sanitize_data(self, data):
         return validators.validate_host(data)
 
@@ -1498,7 +1518,7 @@ class FILESYSTEM(DictPathEvent):
     pass
 
 
-class RAW_DNS_RECORD(DictHostEvent):
+class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     # don't emit raw DNS records for affiliates
     _always_emit_tags = ["target"]
 

bbot/core/helpers/command.py

@@ -295,7 +295,7 @@ def _prepare_command_kwargs(self, command, kwargs):
     if sudo and os.geteuid() != 0:
         self.depsinstaller.ensure_root()
         env["SUDO_ASKPASS"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve())
-        env["BBOT_SUDO_PASS"] = self.depsinstaller._sudo_password
+        env["BBOT_SUDO_PASS"] = self.depsinstaller.encrypted_sudo_pw
         kwargs["env"] = env
 
         PATH = os.environ.get("PATH", "")

bbot/core/helpers/depsinstaller/installer.py

@@ -10,10 +10,11 @@
 from threading import Lock
 from itertools import chain
 from contextlib import suppress
+from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
 
-from ..misc import can_sudo_without_password, os_platform
+from ..misc import can_sudo_without_password, os_platform, rm_at_exit
 
 log = logging.getLogger("bbot.core.helpers.depsinstaller")
 
@@ -29,14 +30,13 @@ def __init__(self, parent_helper):
         http_timeout = self.web_config.get("http_timeout", 30)
         os.environ["ANSIBLE_TIMEOUT"] = str(http_timeout)
 
+        # cache encrypted sudo pass
         self.askpass_filename = "sudo_askpass.py"
+        self._sudo_password = None
+        self._sudo_cache_setup = False
+        self._setup_sudo_cache()
         self._installed_sudo_askpass = False
-        self._sudo_password = os.environ.get("BBOT_SUDO_PASS", None)
-        if self._sudo_password is None:
-            if self.core.bbot_sudo_pass is not None:
-                self._sudo_password = self.core.bbot_sudo_pass
-            elif can_sudo_without_password():
-                self._sudo_password = ""
+
         self.data_dir = self.parent_helper.cache_dir / "depsinstaller"
         self.parent_helper.mkdir(self.data_dir)
         self.setup_status_cache = self.data_dir / "setup_status.json"
@@ -314,20 +314,27 @@ def write_setup_status(self):
 
     def ensure_root(self, message=""):
         self._install_sudo_askpass()
+        # skip if we've already done this
+        if self._sudo_password is not None:
+            return
         with self.ensure_root_lock:
-            if os.geteuid() != 0 and self._sudo_password is None:
-                if message:
-                    log.warning(message)
-                while not self._sudo_password:
-                    # sleep for a split second to flush previous log messages
-                    sleep(0.1)
-                    password = getpass.getpass(prompt="[USER] Please enter sudo password: ")
-                    if self.parent_helper.verify_sudo_password(password):
-                        log.success("Authentication successful")
-                        self._sudo_password = password
-                        self.core.bbot_sudo_pass = password
-                    else:
-                        log.warning("Incorrect password")
+            # first check if the environment variable is set
+            _sudo_password = os.environ.get("BBOT_SUDO_PASS", None)
+            if _sudo_password is not None or os.geteuid() == 0 or can_sudo_without_password():
+                # if we're already root or we can sudo without a password, there's no need to prompt
+                return
+
+            if message:
+                log.warning(message)
+            while not self._sudo_password:
+                # sleep for a split second to flush previous log messages
+                sleep(0.1)
+                _sudo_password = getpass.getpass(prompt="[USER] Please enter sudo password: ")
+                if self.parent_helper.verify_sudo_password(_sudo_password):
+                    log.success("Authentication successful")
+                    self._sudo_password = _sudo_password
+                else:
+                    log.warning("Incorrect password")
 
     def install_core_deps(self):
         to_install = set()
@@ -343,6 +350,38 @@ def install_core_deps(self):
             self.ensure_root()
             self.apt_install(list(to_install))
 
+    def _setup_sudo_cache(self):
+        if not self._sudo_cache_setup:
+            self._sudo_cache_setup = True
+            # write temporary encryption key, to be deleted upon scan completion
+            self._sudo_temp_keyfile = self.parent_helper.temp_filename()
+            # remove it at exit
+            rm_at_exit(self._sudo_temp_keyfile)
+            # generate random 32-byte key
+            random_key = token_bytes(32)
+            # write key to file and set secure permissions
+            self._sudo_temp_keyfile.write_bytes(random_key)
+            self._sudo_temp_keyfile.chmod(0o600)
+            # export path to environment variable, for use in askpass script
+            os.environ["BBOT_SUDO_KEYFILE"] = str(self._sudo_temp_keyfile.resolve())
+
+    @property
+    def encrypted_sudo_pw(self):
+        if self._sudo_password is None:
+            return ""
+        return self._encrypt_sudo_pw(self._sudo_password)
+
+    def _encrypt_sudo_pw(self, pw):
+        from Crypto.Cipher import AES
+        from Crypto.Util.Padding import pad
+
+        key = self._sudo_temp_keyfile.read_bytes()
+        cipher = AES.new(key, AES.MODE_CBC)
+        ct_bytes = cipher.encrypt(pad(pw.encode(), AES.block_size))
+        iv = cipher.iv.hex()
+        ct = ct_bytes.hex()
+        return f"{iv}:{ct}"
+
     def _install_sudo_askpass(self):
         if not self._installed_sudo_askpass:
             self._installed_sudo_askpass = True

bbot/core/helpers/depsinstaller/sudo_askpass.py

@@ -1,5 +1,41 @@
 #!/usr/bin/env python3
-
 import os
+import sys
+from pathlib import Path
+from Crypto.Cipher import AES
+from Crypto.Util.Padding import unpad
+
+ENV_VAR_NAME = "BBOT_SUDO_PASS"
+KEY_ENV_VAR_PATH = "BBOT_SUDO_KEYFILE"
+
+
+def decrypt_password(encrypted_data, key):
+    iv, ciphertext = encrypted_data.split(":")
+    iv = bytes.fromhex(iv)
+    ct = bytes.fromhex(ciphertext)
+    cipher = AES.new(key, AES.MODE_CBC, iv)
+    pt = unpad(cipher.decrypt(ct), AES.block_size)
+    return pt.decode("utf-8")
+
+
+def main():
+    encrypted_password = os.environ.get(ENV_VAR_NAME, "")
+    # remove variable from environment once we've got it
+    os.environ.pop(ENV_VAR_NAME, None)
+    encryption_keypath = Path(os.environ.get(KEY_ENV_VAR_PATH, ""))
+
+    if not encrypted_password or not encryption_keypath.is_file():
+        print("Error: Encrypted password or encryption key not found in environment variables.", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        key = encryption_keypath.read_bytes()
+        decrypted_password = decrypt_password(encrypted_password, key)
+        print(decrypted_password, end="")
+    except Exception as e:
+        print(f'Error decrypting password "{encrypted_password}": {str(e)}', file=sys.stderr)
+        sys.exit(1)
+
 
-print(os.environ.get("BBOT_SUDO_PASS", ""), end="")
+if __name__ == "__main__":
+    main()

bbot/core/helpers/dns/brute.py

@@ -15,15 +15,17 @@ class DNSBrute:
     >>> results = await self.helpers.dns.brute(self, domain, subdomains)
     """
 
-    nameservers_url = (
+    _nameservers_url = (
         "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
     )
 
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.dns_config = self.parent_helper.config.get("dns", {})
         self.num_canaries = 100
-        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
+        self.max_resolvers = self.dns_config.get("brute_threads", 1000)
+        self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url)
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
@@ -39,18 +41,12 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
             type = "A"
         type = str(type).strip().upper()
 
-        domain_wildcard_rdtypes = set()
-        for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
-            for rdtype, results in rdtypes.items():
-                if results:
-                    domain_wildcard_rdtypes.add(rdtype)
-        if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
-            self.log.info(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
+        wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
+        if wildcard_rdtypes:
+            self.log.hugewarning(
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
             )
             return []
-        else:
-            self.log.debug(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
 
         canaries = self.gen_random_subdomains(self.num_canaries)
         canaries_list = list(canaries)
@@ -148,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain):
 
     async def resolver_file(self):
         if self._resolver_file is None:
-            self._resolver_file = await self.parent_helper.wordlist(
+            self._resolver_file_original = await self.parent_helper.wordlist(
                 self.nameservers_url,
                 cache_hrs=24 * 7,
             )
+            nameservers = set(self.parent_helper.read_file(self._resolver_file_original))
+            nameservers.difference_update(self.parent_helper.dns.system_resolvers)
+            # exclude system nameservers from brute-force
+            # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail
+            self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)
         return self._resolver_file
 
     def gen_random_subdomains(self, n=50):

bbot/core/helpers/dns/dns.py

@@ -66,7 +66,7 @@ def __init__(self, parent_helper):
         self.resolver.timeout = self.timeout
         self.resolver.lifetime = self.timeout
 
-        self.runaway_limit = self.config.get("runaway_limit", 5)
+        self.runaway_limit = self.dns_config.get("runaway_limit", 5)
 
         # wildcard handling
         self.wildcard_disable = self.dns_config.get("wildcard_disable", False)
@@ -117,8 +117,11 @@ def brute(self):
             self._brute = DNSBrute(self.parent_helper)
         return self._brute
 
-    @async_cachedmethod(lambda self: self._is_wildcard_cache)
-    async def is_wildcard(self, query, ips=None, rdtype=None):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_cache,
+        key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)),
+    )
+    async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
         Use this method to check whether a *host* is a wildcard entry
 
@@ -150,9 +153,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         Note:
             `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
         """
-        if [ips, rdtype].count(None) == 1:
-            raise ValueError("Both ips and rdtype must be specified")
-
         query = self._wildcard_prevalidation(query)
         if not query:
             return {}
@@ -161,15 +161,17 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         if is_domain(query):
             return {}
 
-        return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype)
+        return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records)
 
-    @async_cachedmethod(lambda self: self._is_wildcard_domain_cache)
-    async def is_wildcard_domain(self, domain, log_info=False):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes)))
+    )
+    async def is_wildcard_domain(self, domain, rdtypes):
         domain = self._wildcard_prevalidation(domain)
         if not domain:
             return {}
 
-        return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False)
+        return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes)
 
     def _wildcard_prevalidation(self, host):
         if self.wildcard_disable:
@@ -192,8 +194,8 @@ def _wildcard_prevalidation(self, host):
 
         return host
 
-    async def _mock_dns(self, mock_data):
+    async def _mock_dns(self, mock_data, custom_lookup_fn=None):
         from .mock import MockResolver
 
-        self.resolver = MockResolver(mock_data)
-        await self.run_and_return("_mock_dns", mock_data=mock_data)
+        self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn)
+        await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn)