Merge pull request #55 from osint-dev-team/develop

Add workers. RabbitMQ queue.

.dockerignore

@@ -4,3 +4,5 @@
 !server.py
 !requirements.txt
 !docker/server/wait-for-it.sh
+!docker/consumer/wait-for-it.sh
+!docker/consumer/consumer.py

.gitignore

@@ -2,6 +2,7 @@
 .idea/
 .DS_Store
 results/*.json
+src/drivers/chromedriver_*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Makefile

@@ -1,13 +1,20 @@
 .PHONY: up up_build test clean
 
 up:
-	docker-compose up -d
+	docker-compose up -d --scale consumer=5
 
-up_build:
-	docker-compose up -d --build --force-recreate
+up_log:
+	docker-compose up --scale consumer=5
+
+build:
+	docker-compose up -d --build --force-recreate --scale consumer=5
+
+build_log:
+	docker-compose up --build --force-recreate --scale consumer=5
 
 tests:
 	python3 -W ignore:ResourceWarning -m unittest discover -v -b
 
 clean:
 	docker-compose down
+	docker volume rm osint-framework_postgres -f

README.md

@@ -48,11 +48,11 @@ python3 cli.py -h
 
 To run the framework as a web service via docker and docker-compose:  
 ```bash
-make up
+make up_log
 ```
 or
 ```bash
-docker-compose up
+docker-compose up --scale consumer=5
 ```
 ## As a separated module
 Basic:

docker-compose.yml

@@ -4,6 +4,7 @@ services:
   db:
     container_name: osint-framework-db
     image: postgres:alpine
+    restart: unless-stopped
     environment:
       POSTGRES_HOST_AUTH_METHOD: trust
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
@@ -14,21 +15,68 @@ services:
       interval: 30s
       timeout: 5s
       retries: 5
+    ports:
+      - "127.0.0.1:5432:5432"
     volumes:
       - postgres:/data/postgres
     networks:
       - postgres
+  rabbitmq:
+    container_name: osint-framework-rabbitmq
+    image: rabbitmq:alpine
+    restart: unless-stopped
+    healthcheck:
+      test: nc -z localhost 5672
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "127.0.0.1:5672:5672"
+    networks:
+      - rabbitmq
+  consumer:
+    image: osint-framework-consumer:1.0
+    restart: on-failure
+    depends_on:
+      - db
+      - rabbitmq
+    healthcheck:
+      test: nc -z osint-framework-rabbitmq 5672
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    environment:
+      POSTGRES_DATABASE: ${POSTGRES_DATABASE:-osint}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
+      POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
+      POSTGRES_HOST: ${POSTGRES_HOST:-osint-framework-db}
+      POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+      RABBITMQ_HOST: ${RABBITMQ_HOST:-osint-framework-rabbitmq}
+      RABBITMQ_PORT: ${RABBITMQ_PORT:-5672}
+      LOG_HANDLER: ${LOG_HANDLER:-stream}
+    build:
+      context: .
+      target: osint-framework-consumer
+      dockerfile: docker/consumer/Dockerfile
+    networks:
+      - postgres
+      - rabbitmq
   server:
     container_name: osint-framework-server
     image: osint-framework-server:1.0
+    restart: on-failure
     depends_on:
       - db
+      - rabbitmq
     environment:
       POSTGRES_DATABASE: ${POSTGRES_DATABASE:-osint}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
       POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
       POSTGRES_HOST: ${POSTGRES_HOST:-osint-framework-db}
       POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+      RABBITMQ_HOST: ${RABBITMQ_HOST:-osint-framework-rabbitmq}
+      RABBITMQ_PORT: ${RABBITMQ_PORT:-5672}
+      LOG_HANDLER: ${LOG_HANDLER:-stream}
     build:
       context: .
       target: osint-framework-server
@@ -42,8 +90,11 @@ services:
       - "8888:8888"
     networks:
       - postgres
+      - rabbitmq
 networks:
   postgres:
     driver: bridge
+  rabbitmq:
+    driver: bridge
 volumes:
     postgres:

docker/consumer/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.8-alpine as osint-framework-consumer
+
+LABEL org.label-schema.name="OSINT Framework Consumer" \
+      org.label-schema.description="OSINT Framework Consumer" \
+      org.label-schema.license="GPL-2.0"
+
+COPY . /app/
+COPY docker/server/wait-for-it.sh /app/wait-for-it.sh
+COPY docker/consumer/consumer.py /app/consumer.py
+WORKDIR /app
+RUN apk add --no-cache --virtual .build_deps build-base libffi-dev gcc musl-dev && \
+    apk add --no-cache postgresql-dev bash curl && \
+    # If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+    apk add --no-cache unzip libexif udev chromium chromium-chromedriver xvfb && \
+    pip install --no-cache-dir -r requirements.txt && \
+    # If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+    pip install --no-cache-dir pyvirtualdisplay && \
+    apk del .build_deps && \
+    chmod +x wait-for-it.sh
+
+# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+ENV DISPLAY=:99
+
+ENTRYPOINT ["./wait-for-it.sh", "-t", "5", "osint-framework-rabbitmq:5672", "--", "python3", "consumer.py"]

docker/consumer/consumer.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+
+from logging import basicConfig, INFO
+
+from src.queue.consumer import Consumer
+
+basicConfig(level=INFO)
+
+if __name__ == "__main__":
+    consumer = Consumer()
+    consumer.start_consuming()

docker/consumer/wait-for-it.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    else
+        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+    fi
+    WAITFORIT_start_ts=$(date +%s)
+    while :
+    do
+        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+            WAITFORIT_result=$?
+        else
+            (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+            WAITFORIT_result=$?
+        fi
+        if [[ $WAITFORIT_result -eq 0 ]]; then
+            WAITFORIT_end_ts=$(date +%s)
+            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    else
+        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+    fi
+    WAITFORIT_PID=$!
+    trap "kill -INT -$WAITFORIT_PID" INT
+    wait $WAITFORIT_PID
+    WAITFORIT_RESULT=$?
+    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+    fi
+    return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        WAITFORIT_hostport=(${1//:/ })
+        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+        shift 1
+        ;;
+        --child)
+        WAITFORIT_CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        WAITFORIT_QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        WAITFORIT_STRICT=1
+        shift 1
+        ;;
+        -h)
+        WAITFORIT_HOST="$2"
+        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        WAITFORIT_HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        WAITFORIT_PORT="$2"
+        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        WAITFORIT_PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        WAITFORIT_TIMEOUT="$2"
+        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        WAITFORIT_TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        WAITFORIT_CLI=("$@")
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# Check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+
+WAITFORIT_BUSYTIMEFLAG=""
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+    WAITFORIT_ISBUSY=1
+    # Check if busybox timeout uses -t flag
+    # (recent Alpine versions don't support -t anymore)
+    if timeout &>/dev/stdout | grep -q -e '-t '; then
+        WAITFORIT_BUSYTIMEFLAG="-t"
+    fi
+else
+    WAITFORIT_ISBUSY=0
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+    wait_for
+    WAITFORIT_RESULT=$?
+    exit $WAITFORIT_RESULT
+else
+    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        WAITFORIT_RESULT=$?
+    else
+        wait_for
+        WAITFORIT_RESULT=$?
+    fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+        exit $WAITFORIT_RESULT
+    fi
+    exec "${WAITFORIT_CLI[@]}"
+else
+    exit $WAITFORIT_RESULT
+fi

docker/server/Dockerfile

@@ -9,9 +9,16 @@ COPY docker/server/wait-for-it.sh /app/wait-for-it.sh
 WORKDIR /app
 RUN apk add --no-cache --virtual .build_deps build-base libffi-dev gcc musl-dev && \
     apk add --no-cache postgresql-dev bash curl && \
+    # If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+    apk add --no-cache unzip libexif udev chromium chromium-chromedriver xvfb && \
     pip install --no-cache-dir -r requirements.txt && \
+    # If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+    pip install --no-cache-dir pyvirtualdisplay && \
     apk del .build_deps && \
     chmod +x wait-for-it.sh
 
+# If you don't want to use selenium-based/chromedriver-based modules - comment the following line:
+ENV DISPLAY=:99
+
 EXPOSE 8888
 ENTRYPOINT ["./wait-for-it.sh", "-t", "5", "osint-framework-db:5432", "--", "python3", "server.py"]