diff --git a/Dockerfile.new b/Dockerfile.new new file mode 100644 index 0000000000..8964869e53 --- /dev/null +++ b/Dockerfile.new @@ -0,0 +1,88 @@ +# +# Spiderfoot Dockerfile +# +# http://www.spiderfoot.net +# +# Written by: Michael Pellon +# Updated by: Chandrapal +# Updated by: Steve Micallef +# Updated by: Steve Bate +# -> Inspired by https://github.com/combro2k/dockerfiles/tree/master/alpine-spiderfoot +# +# Usage: +# +# sudo docker build -t spiderfoot . +# sudo docker run -p 5001:5001 --security-opt no-new-privileges spiderfoot +# +# Using Docker volume for spiderfoot data +# +# sudo docker run -p 5001:5001 -v /mydir/spiderfoot:/var/lib/spiderfoot spiderfoot +# +# Using SpiderFoot remote command line with web server +# +# docker run --rm -it spiderfoot sfcli.py -s http://my.spiderfoot.host:5001/ +# +# Running spiderfoot commands without web server (can optionally specify volume) +# +# sudo docker run --rm spiderfoot sf.py -h +# +# Running a shell in the container for maintenance +# sudo docker run -it --entrypoint /bin/sh spiderfoot +# +# Running spiderfoot unit tests in container +# +# sudo docker build -t spiderfoot-test --build-arg REQUIREMENTS=test/requirements.txt . +# sudo docker run --rm spiderfoot-test -m pytest --flake8 . + +FROM kalilinux/kali-rolling:latest AS build +ARG REQUIREMENTS=requirements.txt +#RUN apt update && apt install -y gcc git curl python3 python3-dev swig libtinyxml-dev \ +# python3-dev musl-dev openssl-dev libffi-dev libxslt-dev libxml2-dev jpeg-dev \ +# openjpeg-dev zlib-dev cargo rust +RUN python3 -m venv /opt/venv +RUN apt update && apt install -y gcc git curl python3 python3-dev swig libtinyxml-dev \ + python3-dev musl-dev libssl-dev libffi-dev libxslt-dev libxml2-dev libjpeg-dev \ + libopenjp2-7-dev zlib1g-dev cargo rust-all +RUN python3 -m venv /opt/venv +ENV PATH="/opt/venv/bin":$PATH +COPY $REQUIREMENTS requirements.txt ./ +RUN ls +RUN echo "$REQUIREMENTS" +RUN pip3 install -U pip +RUN pip3 install -r "$REQUIREMENTS" + + + +FROM kalilinux/kali-rolling:latest +WORKDIR /home/spiderfoot + +# Place database and logs outside installation directory +ENV SPIDERFOOT_DATA /var/lib/spiderfoot +ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log +ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache + +# Run everything as one command so that only one layer is created +RUN apt update && apt -y install python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \ + && adduser -u 99 -G users -h /home/spiderfoot -s /sbin/nologin \ + -g "SpiderFoot User" -D spiderfoot \ + && rm -rf /var/cache/apt/* \ + && rm -rf /lib/apt/db \ + && rm -rf /root/.cache \ + && mkdir -p $SPIDERFOOT_DATA || true \ + && mkdir -p $SPIDERFOOT_LOGS || true \ + && mkdir -p $SPIDERFOOT_CACHE || true \ + && chown spiderfoot:users $SPIDERFOOT_DATA \ + && chown spiderfoot:users $SPIDERFOOT_LOGS \ + && chown spiderfoot:users $SPIDERFOOT_CACHE + +COPY . . +COPY --from=build /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +USER spiderfoot + +EXPOSE 5001 + +# Run the application. +ENTRYPOINT ["/opt/venv/bin/python"] +CMD ["sf.py", "-l", "0.0.0.0:5001"] diff --git a/modules/sfp_deepinfo.py b/modules/sfp_deepinfo.py new file mode 100755 index 0000000000..32e7177b94 --- /dev/null +++ b/modules/sfp_deepinfo.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------- +# Name: sfp_deepinfo +# Purpose: Query Deepinfo using their API +# +# Author: Ceylan Bozogullarindan +# +# Created: 16/04:/2022 +# Copyright: (c) Deepinfo 2023 +# Licence: GPL +# ------------------------------------------------------------------------------- + +import json +import time +from spiderfoot import SpiderFootEvent, SpiderFootPlugin + +class sfp_deepinfo(SpiderFootPlugin): + meta = { + 'name': "Deepinfo", + 'summary': "Obtain Passive DNS and other information from Deepinfo", + 'flags': ["apikey"], + 'useCases': ["Investigate", "Passive"], + 'categories': ["Search Engines"], + 'dataSource': { + 'website': "https://deepinfo.com/", + 'model': "COMMERCIAL_ONLY", + 'references': [ + "https://docs.deepinfo.com/docs/getting-started" + ], + 'apiKeyInstructions': [ + "Visit https://deepinfo.info/request-demo", + "Request a demo account", + "Navigate to https://platform.deepinfo.com/app/settings/organization/api-keys", + "The API key is listed under 'API Keys'" + ], + 'favIcon': "https://ik.imagekit.io/deepinfo/test/favicon/favicon-96x96.png", + 'logo': "https://ik.imagekit.io/deepinfo/test/favicon/favicon-96x96.png", + 'description': "Deepinfo provides relevant data and insights that empower " + "cybersecurity professionals by providing access to an extensive database and reliable indicators." + "Deepinfo has the data you need to understand what's going on on the Internet, we are dealing with " + "terabytes of data, hundreds of data sources, billions of lines of raw data.", + } + } + + # Default options + opts = { + "api_key": "", + } + + # Option descriptions + optdescs = { + "api_key": "Deepinfo API key.", + } + + # Be sure to completely clear any class variables in setup() + # or you run the risk of data persisting between scan runs. + + results = None + errorState = False + + def setup(self, sfc, userOpts=dict()): + self.sf = sfc + self.results = self.tempStorage() + + # Clear / reset any other class member variables here + # or you risk them persisting between threads. + + for opt in list(userOpts.keys()): + self.opts[opt] = userOpts[opt] + + # What events is this module interested in for input + def watchedEvents(self): + return ["DOMAIN_NAME"] + + # What events this module produces + def producedEvents(self): + return ["DOMAIN_NAME", + "INTERNET_NAME" + ] + + # Search Deepinfo + def query(self, qry, page=1, accum=None): + url = f"https://api.deepinfo.com/v1/discovery/subdomain-finder?domain={qry}&page={page}" + request = None + headers = {"apikey": self.opts['api_key']} + res = self.sf.fetchUrl(url, + useragent="SpiderFoot", headers=headers, + postData=request) + + if res['code'] not in ["200"]: + self.error("Deepinfo API key seems to have been rejected or you have exceeded usage limits for the month.") + self.errorState = True + return None + + if res['content'] is None: + self.info("No Deepinfo info found for " + qry) + return None + + try: + info = json.loads(res['content']) + self.info("result_count {0}, page {1}".format(info.get("result_count"), page)) + if info.get("result_count", 0) > 100: + domains = [item.get("punycode", "") for item in info.get("results", [])] + if len(domains) >= 100: + # Avoid throttling + time.sleep(1) + if accum: + accum.extend(domains) + else: + accum = domains + return self.query(qry, page + 1, accum) + else: + # We are at the last page + accum.extend(domains) + return accum + else: + return info.get('results', []) + except Exception as e: + self.error("Error processing JSON response from Deepinfo: " + str(e)) + return None + + # Handle events sent to this module + def handleEvent(self, event): + eventName = event.eventType + srcModuleName = event.module + eventData = event.data + + if self.errorState: + return + + self.debug(f"Received event, {eventName}, from {srcModuleName}") + + if self.opts['api_key'] == "": + self.error("You enabled sfp_deepinfo but did not set an API uid/secret!") + self.errorState = True + return + + # Don't look up stuff twice + if eventData in self.results: + self.debug(f"Skipping {eventData}, already checked.") + return + + self.results[eventData] = True + + if eventName in ["DOMAIN_NAME"]: + domain = eventData + rec = self.query(domain) + myres = list() + if rec is not None: + for h in rec: + if h == "": + continue + if h.lower() not in myres: + myres.append(h.lower()) + else: + continue + e = SpiderFootEvent("INTERNET_NAME", h, + self.__name__, event) + self.notifyListeners(e)