From 4e763d6ba99a62486bfae907eb08571bdb47f4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 17:42:51 +0100 Subject: [PATCH 1/8] chore(x509): Separate CA/certificate generation and saving them on disk --- .../.zfs-source/home/vlt-os/scripts/pki.py | 17 +++++++++++++++-- vulture_os/toolkit/system/x509.py | 18 ++---------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/home/jails.apache/.zfs-source/home/vlt-os/scripts/pki.py b/home/jails.apache/.zfs-source/home/vlt-os/scripts/pki.py index 4d26fdb8a..c21cce9c1 100644 --- a/home/jails.apache/.zfs-source/home/vlt-os/scripts/pki.py +++ b/home/jails.apache/.zfs-source/home/vlt-os/scripts/pki.py @@ -66,10 +66,14 @@ pki["organizational_unit"] ) + with open("/var/db/pki/ca.pem", "wb") as cert_file: + cert_file.write(cacert_pem) + with open("/var/db/pki/ca.key", "wb") as key_file: + key_file.write(cakey_pem) + """ Build node certificate (overwrite if it exist) """ hostname = subprocess.check_output(['hostname']).strip().decode('utf-8') - # TODO give ca_cert and ca_key - _, _ = mk_signed_cert_files( + cert_pem, key_pem = mk_signed_cert_files( hostname, pki["country"], pki["state"], @@ -81,6 +85,15 @@ cakey_pem ) + node_pem = cert_pem + key_pem + + with open("/var/db/pki/node.pem", "wb") as node_file: + node_file.write(node_pem) + with open("/var/db/pki/node.cert", "wb") as cert_file: + cert_file.write(cert_pem) + with open("/var/db/pki/node.key", "wb") as key_file: + key_file.write(key_pem) + """ Generate Diffie hellman configuration """ os.system("openssl dhparam -out /var/db/pki/dh2048.pem 2048") diff --git a/vulture_os/toolkit/system/x509.py b/vulture_os/toolkit/system/x509.py index 182fb1303..ff962f438 100644 --- a/vulture_os/toolkit/system/x509.py +++ b/vulture_os/toolkit/system/x509.py @@ -175,7 +175,7 @@ def mk_cert_builder(serial: int) -> x509.CertificateBuilder: return builder -def mk_ca_cert_files(CN: str, C: str, ST: str, L: str, O: str, OU: str) -> tuple([x509.Certificate, rsa.RSAPrivateKey]): +def mk_ca_cert_files(CN: str, C: str, ST: str, L: str, O: str, OU: str) -> tuple[bytes, bytes]: """Write CA cacert files (cert + key). :param CN: Common Name field @@ -189,11 +189,6 @@ def mk_ca_cert_files(CN: str, C: str, ST: str, L: str, O: str, OU: str) -> tuple ca_cert_pem = get_cert_PEM(ca_cert) private_key_pem = get_key_PEM(private_key) - with open("/var/db/pki/ca.pem", "wb") as cert_file: - cert_file.write(ca_cert_pem) - with open("/var/db/pki/ca.key", "wb") as key_file: - key_file.write(private_key_pem) - return ca_cert_pem, private_key_pem @@ -247,14 +242,5 @@ def mk_signed_cert_files(CN: str, C: str, ST: str, L: str, O: str, OU: str, seri cert_pem = get_cert_PEM(cert) key_pem = get_key_PEM(private_key) - node_pem = cert_pem + key_pem - - with open("/var/db/pki/node.pem", "wb") as node_file: - node_file.write(node_pem) - with open("/var/db/pki/node.cert", "wb") as cert_file: - cert_file.write(cert_pem) - with open("/var/db/pki/node.key", "wb") as key_file: - key_file.write(key_pem) - - return cert, private_key + return cert_pem, key_pem From f26f270e1c79d181316aafe59e1afc1245bae991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 18:04:51 +0100 Subject: [PATCH 2/8] chore(settings): Use and set HOSTNAME value from settings file --- CHANGELOG | 1 + vulture_os/gui/context_processors.py | 3 +-- vulture_os/gui/crontab/feed.py | 6 +++--- vulture_os/gui/views/install.py | 10 +++++----- vulture_os/portal/settings.py | 4 +++- vulture_os/services/filebeat/models.py | 4 ++-- vulture_os/services/pf/models.py | 4 ++-- vulture_os/system/cluster/form.py | 3 +-- vulture_os/system/cluster/models.py | 4 ++-- vulture_os/toolkit/log/log_utils.py | 4 ++-- vulture_os/toolkit/mongodb/mongo_base.py | 7 +++---- vulture_os/toolkit/redis/redis_base.py | 6 ++++-- vulture_os/vulture_os/settings.py | 4 +++- vulture_os/workflow/models.py | 3 +-- 14 files changed, 33 insertions(+), 30 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a5fa1cd6b..9da67dd88 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [API_PARSER] [CORTEX_XDR] Remove fields used for backward compatibility ### Changed - [API_PARSER] [NETSKOPE] Add the possiblity to retrieve 3 new kinds of logs +- [SYSTEM] [SETTINGS] Use Hostname value from Django settings ### Fixed - [API_PARSER] [NETSKOPE] Remove duplicated logs diff --git a/vulture_os/gui/context_processors.py b/vulture_os/gui/context_processors.py index 6e76bf750..67a9342a7 100644 --- a/vulture_os/gui/context_processors.py +++ b/vulture_os/gui/context_processors.py @@ -33,7 +33,6 @@ from services.service import Service from system.cluster.models import Cluster from system.system import System -from toolkit.network.network import get_hostname from workflow.workflow import Workflows import logging @@ -65,7 +64,7 @@ def admin_media(request): 'VERSION': settings.VERSION, 'CURRENT_NODE': node_name, 'DEV_MODE': settings.DEV_MODE, - 'TITLE': get_hostname(), + 'TITLE': settings.HOSTNAME, 'COLLAPSE': request.session.get('collapse') } diff --git a/vulture_os/gui/crontab/feed.py b/vulture_os/gui/crontab/feed.py index 0c1d4ad05..f610fda78 100644 --- a/vulture_os/gui/crontab/feed.py +++ b/vulture_os/gui/crontab/feed.py @@ -36,7 +36,7 @@ from django.utils.timezone import now as timezone_now from gui.models.rss import RSS -from toolkit.network.network import get_hostname, get_proxy +from toolkit.network.network import get_proxy from applications.reputation_ctx.models import ReputationContext from services.rsyslogd.rsyslog import restart_service as restart_rsyslog_service from system.exceptions import VultureSystemError @@ -109,13 +109,13 @@ def security_update(node_logger=None): logger.info("Crontab::security_update: No vulnerability found.") elif "is vulnerable" in res: logger.info("Crontab::security_update: Security problem found : {}".format(res)) - security_alert("Security problem found on node {}".format(get_hostname()), "danger", res) + security_alert("Security problem found on node {}".format(settings.HOSTNAME), "danger", res) except subprocess.CalledProcessError as e: if e.stdout.decode("utf-8").startswith("0 problem"): logger.info("Crontab::security_update: No vulnerability found.") elif "is vulnerable" in e.stdout.decode("utf-8"): logger.info("Crontab::security_update: Security problem found : {}".format(e.stdout.decode('utf-8'))) - security_alert("Security problem found on node {}".format(get_hostname()), "danger", + security_alert("Security problem found on node {}".format(settings.HOSTNAME), "danger", e.stdout.decode("utf-8")) else: logger.error("Crontab::security_update: Failed to retrieve vulnerabilities : " diff --git a/vulture_os/gui/views/install.py b/vulture_os/gui/views/install.py index f9b21ef51..e77aa5b6e 100755 --- a/vulture_os/gui/views/install.py +++ b/vulture_os/gui/views/install.py @@ -31,7 +31,7 @@ from system.cluster.models import Cluster, Node from system.pki.models import X509Certificate, TLSProfile from system.users.models import User -from toolkit.network.network import get_hostname, get_management_ip +from toolkit.network.network import get_management_ip from toolkit.mongodb.mongo_base import MongoBase from toolkit.redis.redis_base import RedisBase from toolkit.system.secret_key import set_key @@ -69,7 +69,7 @@ def cluster_create(admin_user=None, admin_password=None): """ Create the local node """ RC_NETWORK_CONF = "network" node, new_node = Node.objects.get_or_create( - name=get_hostname(), + name=settings.HOSTNAME, management_ip=get_management_ip() ) @@ -186,7 +186,7 @@ def cluster_create(admin_user=None, admin_password=None): """ Update uri of internal Log Forwarder """ logfwd = LogOMMongoDB.objects.get() - logfwd.uristr = "mongodb://{}:9091/?replicaset=Vulture&ssl=true".format(get_hostname()) + logfwd.uristr = "mongodb://{}:9091/?replicaset=Vulture&ssl=true".format(settings.HOSTNAME) logfwd.x509_certificate = node_cert logfwd.save() @@ -307,7 +307,7 @@ def cluster_join(master_hostname, master_ip, secret_key, ca_cert=None, cert=None infos = requests.post( "https://{}:8000/api/system/pki/get_cert/".format(master_ip), headers={'Cluster-api-key': secret_key}, - data={'node_name': get_hostname()}, + data={'node_name': settings.HOSTNAME}, verify=False ).json() @@ -361,7 +361,7 @@ def cluster_join(master_hostname, master_ip, secret_key, ca_cert=None, cert=None infos = requests.post( "https://{}:8000/api/system/cluster/add/".format(master_ip), headers={'Cluster-api-key': secret_key}, - data={'ip': get_management_ip(), 'name': get_hostname()}, + data={'ip': get_management_ip(), 'name': settings.HOSTNAME}, verify=False ) diff --git a/vulture_os/portal/settings.py b/vulture_os/portal/settings.py index d89d47a4f..526c6ba3c 100644 --- a/vulture_os/portal/settings.py +++ b/vulture_os/portal/settings.py @@ -8,6 +8,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(__file__)) +HOSTNAME = get_hostname() + # Retrieving Django SECRET_KEY try: from vulture_os.secret_key import SECRET_KEY @@ -102,7 +104,7 @@ 'ENGINE': 'djongo', 'NAME': 'vulture', "CLIENT": { - 'host': get_hostname(), + 'host': HOSTNAME, 'port': 9091, 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', diff --git a/vulture_os/services/filebeat/models.py b/vulture_os/services/filebeat/models.py index 3a590d994..f7e5e741c 100644 --- a/vulture_os/services/filebeat/models.py +++ b/vulture_os/services/filebeat/models.py @@ -30,7 +30,7 @@ from services.frontend.models import Frontend from system.config.models import Config from system.cluster.models import Node -from toolkit.network.network import get_hostname, JAIL_ADDRESSES +from toolkit.network.network import JAIL_ADDRESSES # Required exceptions imports @@ -52,7 +52,7 @@ def to_template(self): return { 'frontends': Frontend.objects.filter(enabled=True, mode="filebeat"), 'tenants_name': Config.objects.get().internal_tenants.name, - 'nodes': Node.objects.exclude(name=get_hostname()), + 'nodes': Node.objects.exclude(name=settings.HOSTNAME), 'jail_addresses': JAIL_ADDRESSES, } diff --git a/vulture_os/services/pf/models.py b/vulture_os/services/pf/models.py index f35daf3e8..239d69b0d 100644 --- a/vulture_os/services/pf/models.py +++ b/vulture_os/services/pf/models.py @@ -30,7 +30,7 @@ # Django project imports from applications.reputation_ctx.models import DATABASES_PATH from system.cluster.models import Cluster, Node -from toolkit.network.network import get_hostname, JAIL_ADDRESSES +from toolkit.network.network import JAIL_ADDRESSES from toolkit.network.network import get_sanitized_proxy # Required exceptions imports @@ -54,7 +54,7 @@ def to_template(self): :return Dictionnary of configuration parameters """ return { - 'nodes': Node.objects.exclude(name=get_hostname()), + 'nodes': Node.objects.exclude(name=settings.HOSTNAME), 'global_config': Cluster.get_global_config(), 'jail_addresses': JAIL_ADDRESSES, 'databases_path': DATABASES_PATH, diff --git a/vulture_os/system/cluster/form.py b/vulture_os/system/cluster/form.py index c3f2d202a..fde34f2a4 100644 --- a/vulture_os/system/cluster/form.py +++ b/vulture_os/system/cluster/form.py @@ -29,7 +29,6 @@ # Django project imports from system.cluster.models import Node, NetworkAddress -from toolkit.network.network import get_hostname # Logger configuration imports import logging @@ -39,7 +38,7 @@ class NodeForm(ModelForm): scanner_ip = ModelChoiceField( - queryset=NetworkAddress.objects.filter(nic__node__name=get_hostname(), + queryset=NetworkAddress.objects.filter(nic__node__name=settings.HOSTNAME, carp_vhid=0).only(*NetworkAddress.str_attrs()), widget=Select(attrs={'class': 'form-control select2'}), required=False, diff --git a/vulture_os/system/cluster/models.py b/vulture_os/system/cluster/models.py index b8ea3b0c7..7cd858a31 100644 --- a/vulture_os/system/cluster/models.py +++ b/vulture_os/system/cluster/models.py @@ -25,7 +25,7 @@ from system.config.models import Config -from toolkit.network.network import get_hostname, is_valid_ip4, is_valid_ip6, is_valid_hostname, is_loopback +from toolkit.network.network import is_valid_ip4, is_valid_ip6, is_valid_hostname, is_loopback from toolkit.network.route import get_route_interface from toolkit.mongodb.mongo_base import MongoBase from toolkit.redis.redis_base import RedisBase @@ -651,7 +651,7 @@ class Meta: @staticmethod def get_current_node(): try: - hostname = get_hostname() + hostname = settings.HOSTNAME return Node.objects.get(name=hostname) except Node.DoesNotExist: logger.error("Cluster:get_current_node: Current node not found. Are we a pending replica ?") diff --git a/vulture_os/toolkit/log/log_utils.py b/vulture_os/toolkit/log/log_utils.py index 97ff883da..9cf6c3685 100644 --- a/vulture_os/toolkit/log/log_utils.py +++ b/vulture_os/toolkit/log/log_utils.py @@ -24,7 +24,7 @@ __doc__ = 'Database handler for logging' from toolkit.mongodb.mongo_base import MongoBase -from toolkit.network.network import get_hostname +from django.conf import settings from django.utils import timezone import logging @@ -65,7 +65,7 @@ def emit(self, record): 'filename': record.filename, 'message': record.msg, 'source': record.name, - 'node': get_hostname() + 'node': settings.HOSTNAME, }) except Exception: diff --git a/vulture_os/toolkit/mongodb/mongo_base.py b/vulture_os/toolkit/mongodb/mongo_base.py index 8ce8fc3e2..6491c763e 100644 --- a/vulture_os/toolkit/mongodb/mongo_base.py +++ b/vulture_os/toolkit/mongodb/mongo_base.py @@ -25,7 +25,6 @@ from pymongo import MongoClient, ReadPreference from pymongo.errors import OperationFailure, AutoReconnect -from toolkit.network.network import get_hostname from django.conf import settings from re import search as re_search import subprocess @@ -54,11 +53,11 @@ def __init__(self): @staticmethod def get_local_node(): - return '{}:9091'.format(get_hostname()) + return '{}:9091'.format(settings.HOSTNAME) @staticmethod def get_local_uri(): - return 'mongodb://{}:9091'.format(get_hostname()) + return 'mongodb://{}:9091'.format(settings.HOSTNAME) @staticmethod def get_replicaset_uri(): @@ -248,7 +247,7 @@ def repl_initiate(self): """ config = { '_id': 'Vulture', 'members': [ - {'_id': 0, 'host': "{}:9091".format(get_hostname())} + {'_id': 0, 'host': "{}:9091".format(settings.HOSTNAME)} ] } try: diff --git a/vulture_os/toolkit/redis/redis_base.py b/vulture_os/toolkit/redis/redis_base.py index 04b2e8f21..ad4eb2f27 100644 --- a/vulture_os/toolkit/redis/redis_base.py +++ b/vulture_os/toolkit/redis/redis_base.py @@ -25,7 +25,9 @@ from ast import literal_eval from redis import Redis, RedisError -from toolkit.network.network import get_hostname, get_management_ip +from toolkit.network.network import get_management_ip + +from django.conf import settings import logging logger = logging.getLogger('debug') @@ -66,7 +68,7 @@ def get_master(self, node=None): if node: return node else: - return get_hostname() + return settings.HOSTNAME elif redis_info.get('role') == 'slave': return redis_info.get('master_host') except Exception as e: diff --git a/vulture_os/vulture_os/settings.py b/vulture_os/vulture_os/settings.py index 43d0b78ba..e4f1ea045 100644 --- a/vulture_os/vulture_os/settings.py +++ b/vulture_os/vulture_os/settings.py @@ -23,6 +23,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SETTINGS_DIR = os.path.abspath(os.path.dirname(__file__)) +HOSTNAME = get_hostname() + # Retrieving Django SECRET_KEY try: from vulture_os.secret_key import SECRET_KEY @@ -137,7 +139,7 @@ 'ENGINE': 'djongo', 'NAME': 'vulture', "CLIENT": { - 'host': get_hostname(), + 'host': HOSTNAME, 'port': 9091, 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', diff --git a/vulture_os/workflow/models.py b/vulture_os/workflow/models.py index be1ebd5df..2dd745102 100644 --- a/vulture_os/workflow/models.py +++ b/vulture_os/workflow/models.py @@ -39,7 +39,6 @@ from services.frontend.models import Frontend from services.haproxy.haproxy import HAPROXY_OWNER, HAPROXY_PERMS, HAPROXY_PATH from system.cluster.models import Cluster, Node -from toolkit.network.network import get_hostname # Extern modules imports from jinja2 import Environment, FileSystemLoader @@ -326,7 +325,7 @@ def generate_conf(self): jinja2_env = Environment(loader=FileSystemLoader(JINJA_PATH)) template = jinja2_env.get_template(JINJA_TEMPLATE) return template.render({'conf': self.to_template(), - 'nodes': Node.objects.exclude(name=get_hostname()), + 'nodes': Node.objects.exclude(name=settings.HOSTNAME), 'global_config': Cluster.get_global_config().to_dict(fields=['public_token', 'portal_cookie_name'])}) # In ALL exceptions, associate an error message # The exception instantiation MUST be IN except statement, to retrieve traceback in __init__ From 785cdadfd20bdab477eb797013104b9c1541f3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 18:05:51 +0100 Subject: [PATCH 3/8] chore(mongo_base): Connect using settings' configurations --- CHANGELOG | 1 + vulture_os/toolkit/mongodb/mongo_base.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 9da67dd88..564401dd3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [API_PARSER] [NETSKOPE] Add the possiblity to retrieve 3 new kinds of logs - [SYSTEM] [SETTINGS] Use Hostname value from Django settings ### Fixed +- [SYSTEM] [MONGODB] Always refer to settings to connect to mongodb - [API_PARSER] [NETSKOPE] Remove duplicated logs diff --git a/vulture_os/toolkit/mongodb/mongo_base.py b/vulture_os/toolkit/mongodb/mongo_base.py index 6491c763e..5c2dd4bbf 100644 --- a/vulture_os/toolkit/mongodb/mongo_base.py +++ b/vulture_os/toolkit/mongodb/mongo_base.py @@ -198,19 +198,19 @@ def connect_with_retries(self, retries, node=None, primary=True, timeout=2): def connect(self, node=None, primary=True, timeout_ms=5000): try: + args = settings.DATABASES['default']['CLIENT'] + args['serverSelectionTimeoutMS'] = timeout_ms + if node: - host = 'mongodb://{}'.format(node) + args['host'] = f'mongodb://{node}' else: - host = self.get_replicaset_uri() - - args = {'host': host, - 'ssl': True, - 'tlsCertificateKeyFile': "/var/db/pki/node.pem", - 'tlsCAFile': "/var/db/pki/ca.pem", - 'read_preference': ReadPreference.PRIMARY_PREFERRED, - "serverSelectionTimeoutMS": timeout_ms} + args['host'] = self.get_replicaset_uri() + if primary: - args['replicaset'] = "Vulture" + args['read_preference'] = ReadPreference.PRIMARY + else: + args['read_preference'] = ReadPreference.PRIMARY_PREFERRED + self.db = MongoClient(**args) # Execute a request to test connection (pymongo doesn't try connecting until a command is executed on client) self.db.admin.command('ping') From e95dcda2a5b872ccad65e5808f9bab15751f8bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 19:49:22 +0100 Subject: [PATCH 4/8] chore(settings): variabilize paths in project --- CHANGELOG | 1 + vulture_os/applications/backend/models.py | 5 ++- vulture_os/applications/logfwd/models.py | 4 +- .../applications/reputation_ctx/models.py | 3 +- vulture_os/authentication/openid/models.py | 3 +- .../authentication/portal_template/models.py | 3 +- .../authentication/user_portal/models.py | 3 +- vulture_os/darwin/access_control/models.py | 3 +- vulture_os/darwin/policy/models.py | 20 ++++----- vulture_os/gui/crontab/pki.py | 9 ++-- vulture_os/gui/views/install.py | 19 +++++---- vulture_os/portal/settings.py | 36 ++++++++++------ vulture_os/portal/system/sso_clients.py | 5 ++- vulture_os/services/darwin/darwin.py | 5 ++- vulture_os/services/filebeat/filebeat.py | 7 ++-- vulture_os/services/frontend/models.py | 9 ++-- vulture_os/services/haproxy/haproxy.py | 10 ++--- vulture_os/services/logrotate/logrotate.py | 5 ++- vulture_os/services/openvpn/models.py | 7 ++-- vulture_os/services/pf/pf.py | 2 +- vulture_os/services/rsyslogd/rsyslog.py | 7 ++-- vulture_os/services/strongswan/models.py | 11 ++--- vulture_os/system/config/models.py | 7 ++-- vulture_os/system/pki/models.py | 14 ++++--- vulture_os/toolkit/auth/ldap_client.py | 7 +++- vulture_os/toolkit/auth/radius_client.py | 3 +- vulture_os/toolkit/http/utils.py | 6 ++- vulture_os/toolkit/log/lognormalizer.py | 2 +- vulture_os/toolkit/redis/redis_base.py | 3 +- vulture_os/toolkit/yara/yara.py | 9 ++-- vulture_os/vulture_os/settings.py | 41 +++++++++++-------- vulture_os/workflow/models.py | 3 +- 32 files changed, 162 insertions(+), 110 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 564401dd3..c56ade9fc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [SYSTEM] [MONGODB] Always refer to settings to connect to mongodb - [API_PARSER] [NETSKOPE] Remove duplicated logs +- [SYSTEM] [PATHS] Variabilize all system paths and allow to configure them from settings ## [2.19.0] - 2025-01-16 diff --git a/vulture_os/applications/backend/models.py b/vulture_os/applications/backend/models.py index c9cfc4c6a..9e8a2433a 100644 --- a/vulture_os/applications/backend/models.py +++ b/vulture_os/applications/backend/models.py @@ -39,6 +39,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join # Required exceptions imports from jinja2.exceptions import (TemplateAssertionError, TemplateNotFound, TemplatesNotFound, TemplateRuntimeError, @@ -118,13 +119,13 @@ ) # Jinja template for backends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/applications/backend/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "applications/backend/config/") JINJA_TEMPLATE = "haproxy_backend.conf" BACKEND_OWNER = HAPROXY_OWNER BACKEND_PERMS = HAPROXY_PERMS -UNIX_SOCKET_PATH = "/var/sockets/rsyslog" +UNIX_SOCKET_PATH = path_join(settings.SOCKETS_PATH, "rsyslog") class Backend(models.Model): diff --git a/vulture_os/applications/logfwd/models.py b/vulture_os/applications/logfwd/models.py index 68cf81b0e..b454eb33f 100644 --- a/vulture_os/applications/logfwd/models.py +++ b/vulture_os/applications/logfwd/models.py @@ -35,6 +35,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join import pymongo import hashlib @@ -81,8 +82,7 @@ ('stream', "Stream mode, using xadd"), ) -CONF_PATH = "/usr/local/etc/rsyslog.d/10-applications.conf" -JINJA_PATH = "/home/vlt-os/vulture_os/applications/logfwd/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "applications/logfwd/config/") class LogOM (models.Model): diff --git a/vulture_os/applications/reputation_ctx/models.py b/vulture_os/applications/reputation_ctx/models.py index b494669c4..c89605383 100644 --- a/vulture_os/applications/reputation_ctx/models.py +++ b/vulture_os/applications/reputation_ctx/models.py @@ -34,6 +34,7 @@ # Extern modules imports from gzip import decompress as gzip_decompress +from os.path import join as path_join import requests from requests.auth import HTTPBasicAuth, HTTPDigestAuth from re import compile as re_compile @@ -75,7 +76,7 @@ 'digest': HTTPDigestAuth } -DATABASES_PATH = "/var/db/darwin" +DATABASES_PATH = path_join(settings.DBS_PATH, "darwin") DATABASES_OWNER = "vlt-os:vlt-conf" DATABASES_PERMS = "644" diff --git a/vulture_os/authentication/openid/models.py b/vulture_os/authentication/openid/models.py index 9d6c0f8b0..10f476008 100644 --- a/vulture_os/authentication/openid/models.py +++ b/vulture_os/authentication/openid/models.py @@ -32,6 +32,7 @@ # Django project imports from authentication.base_repository import BaseRepository from authentication.user_scope.models import UserScope +from os.path import join as path_join from system.exceptions import VultureSystemConfigError from system.pki.models import CERT_OWNER, CERT_PERMS from toolkit.auth.authy_client import AuthyClient @@ -253,7 +254,7 @@ def to_html_template(self): } def get_jwt_key_filename(self): - return f"/var/db/pki/openid-{self.pk}.pub" + return path_join(settings.DBS_PATH, "pki/openid-{self.pk}.pub") @staticmethod def jwt_validate_with_certificate(jwt_signature_type): diff --git a/vulture_os/authentication/portal_template/models.py b/vulture_os/authentication/portal_template/models.py index bb27aef2d..cc54caba3 100644 --- a/vulture_os/authentication/portal_template/models.py +++ b/vulture_os/authentication/portal_template/models.py @@ -34,6 +34,7 @@ from system.config.models import write_conf # Extern modules imports +from os.path import join as path_join # Required exceptions imports @@ -308,7 +309,7 @@ def write_on_disk(self): continue try: - with open("/usr/local/etc/haproxy.d/templates/portal_%s_%s.conf" % (str(self.id), tpl), + with open(path_join(settings.LOCALETC_PATH, "haproxy.d/templates/portal_%s_%s.conf" % (str(self.id), tpl)), 'w') as f: html = getattr(self, tpl) if tpl not in ["email_subject", "email_body", "email_register_subject", "email_register_body"]: diff --git a/vulture_os/authentication/user_portal/models.py b/vulture_os/authentication/user_portal/models.py index 448c4eba5..9d38a315b 100644 --- a/vulture_os/authentication/user_portal/models.py +++ b/vulture_os/authentication/user_portal/models.py @@ -49,6 +49,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join # Required exceptions imports from jinja2.exceptions import (TemplateAssertionError, TemplateNotFound, TemplatesNotFound, TemplateRuntimeError, @@ -61,7 +62,7 @@ logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('gui') -JINJA_PATH = "/home/vlt-os/vulture_os/authentication/user_portal/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "authentication/user_portal/config/") JINJA_TEMPLATE = "haproxy_portal.conf" diff --git a/vulture_os/darwin/access_control/models.py b/vulture_os/darwin/access_control/models.py index 3f7baee99..209e16d81 100644 --- a/vulture_os/darwin/access_control/models.py +++ b/vulture_os/darwin/access_control/models.py @@ -34,6 +34,7 @@ from bson import ObjectId from hashlib import sha1 import logging +from os.path import join as path_join import json logging.config.dictConfig(settings.LOG_SETTINGS) @@ -114,7 +115,7 @@ ] # Jinja template for backends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/darwin/access_control/config" +JINJA_PATH = path_join(settings.BASE_DIR, "darwin/access_control/config") JINJA_TEST_TEMPLATE = "haproxy_test.conf" diff --git a/vulture_os/darwin/policy/models.py b/vulture_os/darwin/policy/models.py index 264913ac7..ad8556695 100644 --- a/vulture_os/darwin/policy/models.py +++ b/vulture_os/darwin/policy/models.py @@ -44,24 +44,24 @@ logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('gui') -JINJA_PATH = "/home/vlt-os/vulture_os/darwin/log_viewer/config/" +JINJA_PATH = os_path.join(settings.BASE_DIR, "darwin/log_viewer/config/") -SOCKETS_PATH = "/var/sockets/darwin" -FILTERS_PATH = "/home/darwin/filters" -CONF_PATH = "/home/darwin/conf" -DATA_PATH = "/var/db/darwin" +SOCKETS_PATH = os_path.join(settings.SOCKETS_PATH, "darwin") +FILTERS_PATH = os_path.join(settings.HOMES_PATH, "darwin/filters") +CONF_PATH = os_path.join(settings.HOMES_PATH, "darwin/conf") +DATA_PATH = os_path.join(settings.DBS_PATH, "darwin") TEMPLATE_OWNER = "darwin:vlt-web" TEMPLATE_PERMS = "644" -REDIS_SOCKET_PATH = "/var/sockets/redis/redis.sock" +REDIS_SOCKET_PATH = os_path.join(settings.SOCKETS_PATH, "redis/redis.sock") ALERTS_REDIS_LIST_NAME = "darwin_alerts" ALERTS_REDIS_CHANNEL_NAME = "darwin.alerts" -ALERTS_LOG_FILEPATH = "/var/log/darwin/alerts.log" +ALERTS_LOG_FILEPATH = os_path.join(settings.LOGS_PATH,"darwin/alerts.log") -DGA_MODELS_PATH = CONF_PATH + '/fdgad/' -VAST_MODELS_PATH = CONF_PATH + '/fvast/' -VAML_MODELS_PATH = CONF_PATH + '/fvaml/' +DGA_MODELS_PATH = os_path.join(CONF_PATH, 'fdgad/') +VAST_MODELS_PATH = os_path.join(CONF_PATH, 'fvast/') +VAML_MODELS_PATH = os_path.join(CONF_PATH, 'fvaml/') DARWIN_LOGLEVEL_CHOICES = ( ('CRITICAL', 'Critical'), diff --git a/vulture_os/gui/crontab/pki.py b/vulture_os/gui/crontab/pki.py index 52862e75c..412e4413e 100644 --- a/vulture_os/gui/crontab/pki.py +++ b/vulture_os/gui/crontab/pki.py @@ -28,6 +28,9 @@ import subprocess import os.path +from django.conf import settings + +CONF_PATH = os.path.join(settings.DBS_PATH, "acme") def update_crl(): """ @@ -44,7 +47,7 @@ def acme_update(): """ :return: Run acme.sh to automatically renew Let's encrypt certificates """ - subprocess.check_output(["/usr/local/sbin/acme.sh", "--cron", "--home", "/var/db/acme/.acme.sh"]) + subprocess.check_output(["/usr/local/sbin/acme.sh", "--cron", "--home", os.path.join(CONF_PATH, ".acme.sh")]) """ Now update certificate database""" need_restart = False @@ -53,8 +56,8 @@ def acme_update(): subject = crypto_cert.subject common_name_obj = subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0] cn = common_name_obj.value - if os.path.isfile("/var/db/acme/.acme.sh/{}/{}.cer".format(cn, cn)): - with open("/var/db/acme/.acme.sh/{}/{}.cer".format(cn, cn)) as file_cert: + if os.path.isfile(os.path.join(CONF_PATH, ".acme.sh/{}/{}.cer".format(cn, cn))): + with open(os.path.join(CONF_PATH, ".acme.sh/{}/{}.cer".format(cn, cn))) as file_cert: pem_cert = file_cert.read() cert.cert = pem_cert cert.save() diff --git a/vulture_os/gui/views/install.py b/vulture_os/gui/views/install.py index e77aa5b6e..6239ee0ca 100755 --- a/vulture_os/gui/views/install.py +++ b/vulture_os/gui/views/install.py @@ -28,6 +28,7 @@ from django.conf import settings from django.contrib.auth.models import Group from django.utils.crypto import get_random_string +from os.path import join as path_join from system.cluster.models import Cluster, Node from system.pki.models import X509Certificate, TLSProfile from system.users.models import User @@ -91,16 +92,16 @@ def cluster_create(admin_user=None, admin_password=None): node.api_request('toolkit.network.network.write_management_ips') """ Obtain Certificates and store it into mongoDB """ - with open("/var/db/pki/ca.pem") as f: + with open(path_join(settings.DBS_PATH, "pki/ca.pem")) as f: ca_cert = f.read() - with open("/var/db/pki/ca.key") as f: + with open(path_join(settings.DBS_PATH, "pki/ca.key")) as f: ca_key = f.read() - with open("/var/db/pki/node.cert") as f: + with open(path_join(settings.DBS_PATH, "pki/node.cert")) as f: cert = f.read() - with open("/var/db/pki/node.key") as f: + with open(path_join(settings.DBS_PATH, "pki/node.key")) as f: key = f.read() internal_ca = X509Certificate( @@ -325,19 +326,19 @@ def cluster_join(master_hostname, master_ip, secret_key, ca_cert=None, cert=None logger.error("Unable to retrieve Node certificate and key, check secret key") return False - with open("/var/tmp/ca.pem", "w") as f: + with open(path_join(settings.TMP_PATH, "ca.pem", "w")) as f: f.write(ca_cert) - with open("/var/tmp/ca.key", "w") as f: + with open(path_join(settings.TMP_PATH, "ca.key", "w")) as f: f.write(ca_key) - with open("/var/tmp/node.cert", "w") as f: + with open(path_join(settings.TMP_PATH, "node.cert", "w")) as f: f.write(cert) - with open("/var/tmp/node.key", "w") as f: + with open(path_join(settings.TMP_PATH, "node.key", "w")) as f: f.write(key) - with open("/var/tmp/node.pem", "w") as f: + with open(path_join(settings.TMP_PATH, "node.pem", "w")) as f: f.write(bundle) """ At this point we should have valid certificates: diff --git a/vulture_os/portal/settings.py b/vulture_os/portal/settings.py index 526c6ba3c..e8b8dfeb5 100644 --- a/vulture_os/portal/settings.py +++ b/vulture_os/portal/settings.py @@ -2,11 +2,21 @@ Django settings for vulture project. """ -import os +from os import path as os_path from toolkit.network.network import get_hostname from toolkit.system.secret_key import set_key -BASE_DIR = os.path.dirname(os.path.dirname(__file__)) +BASE_DIR = os_path.dirname(os_path.dirname(os_path.abspath(__file__))) +SETTINGS_DIR = os_path.abspath(os_path.dirname(__file__)) + +# Project folders +ROOT_PATH = "/" +DBS_PATH = os_path.join(ROOT_PATH, "var/db") +TMP_PATH = os_path.join(ROOT_PATH, "var/tmp") +LOGS_PATH = os_path.join(ROOT_PATH, "var/log") +SOCKETS_PATH = os_path.join(ROOT_PATH, "var/sockets") +HOMES_PATH = os_path.join(ROOT_PATH, "home") +LOCALETC_PATH = os_path.join(ROOT_PATH, "usr/local/etc") HOSTNAME = get_hostname() @@ -68,7 +78,7 @@ ) TEMPLATE_DIRS = ( - '/home/vlt-gui/vulture/portal/templates', + os_path.join(HOMES_PATH, 'vlt-gui/vulture/portal/templates'), ) TEMPLATE_CONTEXT_PROCESSORS = ( @@ -109,8 +119,8 @@ 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', 'SSL': True, - 'tlsCertificateKeyFile': '/var/db/pki/node.pem', - 'tlsCAFile': '/var/db/pki/ca.pem', + 'tlsCertificateKeyFile': os_path.join(DBS_PATH, 'pki/node.pem'), + 'tlsCAFile': os_path.join(DBS_PATH, 'pki/ca.pem'), 'READPREFERENCE': "primaryPreferred" }, } @@ -124,7 +134,7 @@ STATIC_URL = 'static/' -CACERT_DIR = '/var/db/mongodb/' +CACERT_DIR = os_path.join(DBS_PATH, 'mongodb/') MONGODBPORT = 9091 MONGODBARBPORT = 9092 REDISIP = '127.0.0.1' @@ -172,7 +182,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/portal_authentication.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/portal_authentication.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -181,7 +191,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/redis_events.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/redis_events.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -190,14 +200,14 @@ 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/authentication.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/authentication.log'), 'mode': 'a' }, 'debug': { 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/debug.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/debug.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -247,7 +257,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': 'INFO', 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/portal_authentication.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/portal_authentication.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -256,7 +266,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': 'DEBUG', 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/redis_events.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/redis_events.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -265,7 +275,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': 'INFO', 'formatter': 'verbose', - 'filename': '/var/log/vulture/portal/debug.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/portal/debug.log'), 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, diff --git a/vulture_os/portal/system/sso_clients.py b/vulture_os/portal/system/sso_clients.py index a5299a713..35076a0af 100644 --- a/vulture_os/portal/system/sso_clients.py +++ b/vulture_os/portal/system/sso_clients.py @@ -32,6 +32,7 @@ # Extern modules imports from json import dumps as json_dumps +from os.path import join as path_join from re import compile, IGNORECASE from requests import Session from requests.adapters import HTTPAdapter @@ -47,7 +48,7 @@ # Global variables vulture_custom_agent = 'Vulture/3 (FreeBSD; Vulture OS)' - +PKI_PATH = path_join(settings.DBS_PATH, "pki") class SSLAdapter(HTTPAdapter): @@ -90,7 +91,7 @@ def __init__( self, if ssl_context: # Only compatible with request-2.18.1 !!! self.session.mount("https://", SSLAdapter(ssl_context=ssl_context)) - self.verify_certificate = "/var/db/pki/" if ssl_context.verify_mode == CERT_REQUIRED else verify_certificate + self.verify_certificate = PKI_PATH if ssl_context.verify_mode == CERT_REQUIRED else verify_certificate self.client_side_cert = client_certificate logger.debug("SSOClient::_init_: SSL/TLS context successfully created") diff --git a/vulture_os/services/darwin/darwin.py b/vulture_os/services/darwin/darwin.py index 0f7b402a6..1c2e03128 100644 --- a/vulture_os/services/darwin/darwin.py +++ b/vulture_os/services/darwin/darwin.py @@ -44,6 +44,7 @@ # Extern modules imports from json import loads as json_loads +from os.path import join as path_join from subprocess import check_output, PIPE # Logger configuration imports @@ -52,10 +53,10 @@ logger = logging.getLogger('services') -DARWIN_PATH = "/home/darwin/conf" +DARWIN_PATH = path_join(settings.HOMES_PATH, "darwin/conf") DARWIN_PERMS = "640" DARWIN_OWNERS = "darwin:vlt-conf" -MANAGEMENT_SOCKET = "/var/sockets/darwin/darwin.sock" +MANAGEMENT_SOCKET = path_join(settings.SOCKETS_PATH, "darwin/darwin.sock") class DarwinService(Service): diff --git a/vulture_os/services/filebeat/filebeat.py b/vulture_os/services/filebeat/filebeat.py index aa5263401..71a7cc132 100644 --- a/vulture_os/services/filebeat/filebeat.py +++ b/vulture_os/services/filebeat/filebeat.py @@ -38,6 +38,7 @@ from system.exceptions import VultureSystemError # Extern modules imports +from os.path import join as path_join from subprocess import check_output, PIPE # Logger configuration imports @@ -45,9 +46,9 @@ logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('services') -JINJA_PATH = "/home/vlt-os/vulture_os/services/config/" -FILEBEAT_PATH = "/usr/local/etc/filebeat" -INPUTS_PATH = FILEBEAT_PATH + "/filebeat.yml" +JINJA_PATH = path_join(settings.BASE_DIR, "services/config/") +FILEBEAT_PATH = path_join(settings.LOCALETC_PATH, "filebeat") +INPUTS_PATH = path_join(FILEBEAT_PATH, "filebeat.yml") FILEBEAT_OWNER = "root:vlt-os" FILEBEAT_PERMS = "640" diff --git a/vulture_os/services/frontend/models.py b/vulture_os/services/frontend/models.py index 2e550baaf..686f378ac 100644 --- a/vulture_os/services/frontend/models.py +++ b/vulture_os/services/frontend/models.py @@ -48,6 +48,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join from re import search as re_search from requests import post import glob @@ -118,7 +119,7 @@ ) # Filebeat module list -FILEBEAT_MODULE_PATH = "/usr/local/etc/filebeat/modules.d" +FILEBEAT_MODULE_PATH = path_join(settings.LOCALETC_PATH, "filebeat/modules.d") FILEBEAT_MODULE_LIST = [('_custom', 'Custom Filebeat config')] FILEBEAT_MODULE_CONFIG = {'_custom': '# Be sure to select the appropriate "Filebeat listening mode"\n\ # Use the following variables when needed: \n\n\ @@ -151,14 +152,14 @@ def _getKey(item): # Jinja template for frontends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/services/frontend/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "services/frontend/config/") JINJA_TEMPLATE = "haproxy_frontend.conf" FRONTEND_OWNER = HAPROXY_OWNER FRONTEND_PERMS = HAPROXY_PERMS -UNIX_SOCKET_PATH = "/var/sockets/rsyslog" -LOG_API_PATH = "/var/log/api_parser" +UNIX_SOCKET_PATH = path_join(settings.SOCKETS_PATH, "rsyslog") +LOG_API_PATH = path_join(settings.LOGS_PATH, "api_parser") class Frontend(models.Model): """ Model used to generate fontends configuration of HAProxy """ diff --git a/vulture_os/services/haproxy/haproxy.py b/vulture_os/services/haproxy/haproxy.py index e2dfb0ae3..d12becf6e 100644 --- a/vulture_os/services/haproxy/haproxy.py +++ b/vulture_os/services/haproxy/haproxy.py @@ -38,10 +38,10 @@ # Required exceptions imports from services.exceptions import ServiceError, ServiceStatusError, ServiceTestConfigError - from subprocess import CalledProcessError # Extern modules imports +from os.path import join as path_join from subprocess import check_output, PIPE # Logger configuration imports @@ -50,14 +50,14 @@ logger = logging.getLogger('services') -HAPROXY_PATH = "/usr/local/etc/haproxy.d" -TEST_CONF_PATH = "/var/tmp/haproxy" +HAPROXY_PATH = path_join(settings.LOCALETC_PATH, "haproxy.d") +TEST_CONF_PATH = path_join(settings.TMP_PATH, "haproxy") HAPROXY_OWNER = "vlt-os:vlt-web" HAPROXY_PERMS = "644" -MANAGEMENT_SOCKET = "/var/sockets/haproxy/haproxy.sock" +MANAGEMENT_SOCKET = path_join(settings.SOCKETS_PATH, "haproxy/haproxy.sock") -JINJA_PATH = "/home/vlt-os/vulture_os/services/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "services/config/") JINJA_TEMPLATE = "haproxy_internals.cfg" diff --git a/vulture_os/services/logrotate/logrotate.py b/vulture_os/services/logrotate/logrotate.py index 0cfe6cb71..8b09c7b87 100644 --- a/vulture_os/services/logrotate/logrotate.py +++ b/vulture_os/services/logrotate/logrotate.py @@ -34,6 +34,7 @@ # Required exceptions imports # Extern modules imports +from os.path import join as path_join # Logger configuration imports import logging @@ -41,11 +42,11 @@ logger = logging.getLogger('services') -LOGROTATE_PATH = "/usr/local/etc/logrotate.d" +LOGROTATE_PATH = path_join(settings.LOCALETC_PATH, "logrotate.d") LOGROTATE_OWNER = "root:vlt-os" LOGROTATE_PERMS = "640" -JINJA_PATH = "/home/vlt-os/vulture_os/services/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "services/config/") JINJA_TEMPLATE = "logrotate.conf" diff --git a/vulture_os/services/openvpn/models.py b/vulture_os/services/openvpn/models.py index b47093796..f3931b7a3 100644 --- a/vulture_os/services/openvpn/models.py +++ b/vulture_os/services/openvpn/models.py @@ -34,6 +34,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join from services.exceptions import (ServiceJinjaError) from system.exceptions import VultureSystemConfigError @@ -56,10 +57,10 @@ # Jinja template for frontends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/services/openvpn/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "services/openvpn/config/") JINJA_TEMPLATE_OPENVPN = "client.conf" -CONF_PATH = "/usr/local/etc/openvpn" +CONF_PATH = path_join(settings.LOCALETC_PATH, "openvpn") OPENVPN_OWNER = "root:vlt-conf" @@ -153,7 +154,7 @@ def save_conf(self): """ Write configuration on disk """ conf = self.generate_conf() - params = ['/usr/local/etc/openvpn/openvpn_client.conf', conf['template_client'], OPENVPN_OWNER, "644"] + params = [path_join(CONF_PATH, 'openvpn_client.conf'), conf['template_client'], OPENVPN_OWNER, "644"] try: self.node.api_request('system.config.models.write_conf', config=params) except Exception: diff --git a/vulture_os/services/pf/pf.py b/vulture_os/services/pf/pf.py index 172711880..daa655439 100644 --- a/vulture_os/services/pf/pf.py +++ b/vulture_os/services/pf/pf.py @@ -48,7 +48,7 @@ logger = logging.getLogger('services') -PF_PATH = "/usr/local/etc" +PF_PATH = settings.LOCALETC_PATH PF_PERMS = "640" PF_OWNERS = "root:vlt-os" diff --git a/vulture_os/services/rsyslogd/rsyslog.py b/vulture_os/services/rsyslogd/rsyslog.py index 86c496d8f..a293e2935 100644 --- a/vulture_os/services/rsyslogd/rsyslog.py +++ b/vulture_os/services/rsyslogd/rsyslog.py @@ -40,6 +40,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join from re import search as re_search from subprocess import check_output, PIPE @@ -49,9 +50,9 @@ logger = logging.getLogger('services') -JINJA_PATH = "/home/vlt-os/vulture_os/services/rsyslogd/config/" -RSYSLOG_PATH = "/usr/local/etc/rsyslog.d" -INPUTS_PATH = RSYSLOG_PATH + "/00-system.conf" +JINJA_PATH = path_join(settings.BASE_DIR, "services/rsyslogd/config/") +RSYSLOG_PATH = path_join(settings.LOCALETC_PATH, "rsyslog.d") +INPUTS_PATH = path_join(RSYSLOG_PATH, "00-system.conf") RSYSLOG_PERMS = "640" RSYSLOG_OWNER = "vlt-os:wheel" diff --git a/vulture_os/services/strongswan/models.py b/vulture_os/services/strongswan/models.py index 918ac8b92..3fa834642 100644 --- a/vulture_os/services/strongswan/models.py +++ b/vulture_os/services/strongswan/models.py @@ -32,6 +32,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join from services.exceptions import ServiceJinjaError from system.exceptions import VultureSystemConfigError @@ -61,12 +62,12 @@ ) # Jinja template for frontends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/services/strongswan/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "services/strongswan/config/") JINJA_TEMPLATE_IPSEC = "ipsec.conf" JINJA_TEMPLATE_SECRETS = "ipsec.secrets" JINJA_TEMPLATE_STRONGSWAN = "strongswan.conf" -CONF_PATH = "/usr/local/etc/" +CONF_PATH = settings.LOCALETC_PATH STRONGSWAN_OWNER = "root:wheel" @@ -190,9 +191,9 @@ def save_conf(self): conf = self.generate_conf() - params_ipsec = ['/usr/local/etc/ipsec.conf', conf['template_ipsec'], STRONGSWAN_OWNER, "644"] - params_secrets = ['/usr/local/etc/ipsec.secrets', conf['template_secrets'], STRONGSWAN_OWNER, "600"] - params_strongswan = ['/usr/local/etc/strongswan.conf', conf['template_strongswan'], STRONGSWAN_OWNER, "644"] + params_ipsec = [path_join(CONF_PATH, 'ipsec.conf'), conf['template_ipsec'], STRONGSWAN_OWNER, "644"] + params_secrets = [path_join(CONF_PATH, 'ipsec.secrets'), conf['template_secrets'], STRONGSWAN_OWNER, "600"] + params_strongswan = [path_join(CONF_PATH, 'strongswan.conf'), conf['template_strongswan'], STRONGSWAN_OWNER, "644"] for params in (params_ipsec, params_secrets, params_strongswan): try: diff --git a/vulture_os/system/config/models.py b/vulture_os/system/config/models.py index 9191800eb..11ae59329 100644 --- a/vulture_os/system/config/models.py +++ b/vulture_os/system/config/models.py @@ -22,6 +22,7 @@ __email__ = "contact@vultureproject.org" __doc__ = 'Global Configuration main models' +from django.conf import settings from django.utils.translation import gettext_lazy as _ from django.forms.models import model_to_dict from djongo import models @@ -115,9 +116,9 @@ def write_conf(logger, args): else: file_path, file_content, owner, perm = args - # Write temporary file info /tmp dir, - # because everybody can write onto - temp_dir = "/var/tmp/" + # Write temporary file info tmp dir, + # because everybody can write onto + temp_dir = settings.TMP_PATH.rstrip('/') + '/' """ Create a temporary named file in {prefix} path """ tmpfile = mktemp(prefix=temp_dir) logger.debug("Config::write_conf: Writing into {}".format(tmpfile)) diff --git a/vulture_os/system/pki/models.py b/vulture_os/system/pki/models.py index 0321e4ff1..827b0537a 100644 --- a/vulture_os/system/pki/models.py +++ b/vulture_os/system/pki/models.py @@ -33,6 +33,7 @@ import datetime from cryptography import x509 +from os.path import join as path_join from system.exceptions import VultureSystemConfigError from toolkit.system.x509 import mk_signed_cert, get_cert_PEM, get_key_PEM @@ -140,7 +141,8 @@ ('required', 'Required') ) -CERT_PATH = "/var/db/pki" +CERT_PATH = path_join(settings.DBS_PATH, "pki") +ACME_PATH = path_join(settings.DBS_PATH, "acme") CERT_OWNER = "vlt-os:haproxy" CERT_PERMS = "640" @@ -212,7 +214,7 @@ def gen_letsencrypt(self, cn, name): try: proc = subprocess.Popen(['/usr/local/sbin/acme.sh', '--issue', '-d', cn, '--webroot', - '/var/db/acme/'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ACME_PATH], stdout=subprocess.PIPE, stderr=subprocess.PIPE) success, error = proc.communicate() if error: logger.error("X509Certificate::gen_letsencrypt(): {}".format(error.decode('utf-8'))) @@ -223,9 +225,9 @@ def gen_letsencrypt(self, cn, name): return False # Read certificate, private key and chain file - with open("/var/db/acme/.acme.sh/{}/{}.cer".format(cn, cn)) as pem_cert: - with open("/var/db/acme/.acme.sh/{}/{}.key".format(cn, cn)) as pem_key: - with open("/var/db/acme/.acme.sh/{}/fullchain.cer".format(cn)) as pem_chain: + with open(path_join(ACME_PATH, ".acme.sh/{}/{}.cer".format(cn, cn))) as pem_cert: + with open(path_join(ACME_PATH, ".acme.sh/{}/{}.key".format(cn, cn))) as pem_key: + with open(path_join(ACME_PATH, ".acme.sh/{}/fullchain.cer".format(cn))) as pem_chain: try: tmp_crt = x509.load_pem_x509_certificate(pem_cert.encode()) @@ -617,7 +619,7 @@ def to_dict(self, fields=None): @property def client_ca_cert_filename(self): - return "/var/db/pki/client_{}.crt".format(self.id) + return path_join(CERT_PATH, "client_{}.crt".format(self.id)) def save_conf(self): # self.x509_certificate.save_conf() diff --git a/vulture_os/toolkit/auth/ldap_client.py b/vulture_os/toolkit/auth/ldap_client.py index 5b0cb9b33..d036c61f2 100644 --- a/vulture_os/toolkit/auth/ldap_client.py +++ b/vulture_os/toolkit/auth/ldap_client.py @@ -26,6 +26,7 @@ # Django project imports from toolkit.auth.base_auth import BaseAuth +from django.conf import settings # Extern modules imports import copy @@ -33,6 +34,7 @@ import ldap.modlist as modlist from ldap.filter import escape_filter_chars from ldap.dn import escape_dn_chars +from os.path import join as path_join # Required exceptions imports @@ -42,6 +44,7 @@ import logging logger = logging.getLogger('authentication') +CERT_PATH = path_join(settings.DBS_PATH, "pki") class LDAPClient(BaseAuth): @@ -134,14 +137,14 @@ def __init__(self, settings): self.start_tls = False if settings.encryption_scheme == 'start-tls': self.start_tls = True - self._connection_settings[ldap.OPT_X_TLS_CACERTDIR] = '/var/db/pki' + self._connection_settings[ldap.OPT_X_TLS_CACERTDIR] = CERT_PATH self._connection_settings[ldap.OPT_X_TLS_REQUIRE_CERT] = ldap.OPT_X_TLS_NEVER self._connection_settings[ldap.OPT_X_TLS_NEWCTX] = 0 self._connection_settings[ldap.OPT_DEBUG_LEVEL] = 255 elif settings.encryption_scheme == 'ldaps': proto = 'ldaps' - self._connection_settings[ldap.OPT_X_TLS_CACERTDIR] = '/var/db/pki' + self._connection_settings[ldap.OPT_X_TLS_CACERTDIR] = CERT_PATH self._connection_settings[ldap.OPT_X_TLS_REQUIRE_CERT] = ldap.OPT_X_TLS_NEVER self._connection_settings[ldap.OPT_X_TLS_NEWCTX] = 0 self._connection_settings[ldap.OPT_DEBUG_LEVEL] = 255 diff --git a/vulture_os/toolkit/auth/radius_client.py b/vulture_os/toolkit/auth/radius_client.py index f155c0ee9..d3b738589 100644 --- a/vulture_os/toolkit/auth/radius_client.py +++ b/vulture_os/toolkit/auth/radius_client.py @@ -30,6 +30,7 @@ from toolkit.auth.base_auth import BaseAuth # Extern modules imports +from os.path import join as path_join from pyrad.client import Client from pyrad.client import Timeout from pyrad.dictionary import Dictionary @@ -44,7 +45,7 @@ logger = logging.getLogger('authentication') -DICTIONARY_PATH = "/usr/local/etc/radiusclient/dictionary" +DICTIONARY_PATH = path_join(settings.LOCALETC_PATH, "radiusclient/dictionary") class RadiusClient(BaseAuth): diff --git a/vulture_os/toolkit/http/utils.py b/vulture_os/toolkit/http/utils.py index 5200b22bc..1da9d6c71 100644 --- a/vulture_os/toolkit/http/utils.py +++ b/vulture_os/toolkit/http/utils.py @@ -25,9 +25,11 @@ # Django system imports # Django project imports +from django.conf import settings # Extern modules imports import json +from os.path import join as path_join import re import ssl from urllib.parse import urlencode @@ -48,6 +50,8 @@ vulture_custom_agent = 'Vulture/4 (FreeBSD; Vulture OS)' +CERT_PATH = path_join(settings.DBS_PATH, "pki") + class SSLAdapter(HTTPAdapter): """ "Transport adapter" that allows us to use TLSv1 """ @@ -192,7 +196,7 @@ def ssoPOST(logger, uri, req, data, cookie_data, app, cookie_from_fetch, ssl_con # requests version 2.18.1 needed for the following line session.mount("https://", SSLAdapter(ssl_context.protocol)) - verify_certificate = "/var/db/pki/" if ssl_context.verify_mode == ssl.CERT_REQUIRED else False + verify_certificate = CERT_PATH if ssl_context.verify_mode == ssl.CERT_REQUIRED else False client_side_cert = app.ssl_client_certificate if app.ssl_client_certificate else None if content_type == "json": diff --git a/vulture_os/toolkit/log/lognormalizer.py b/vulture_os/toolkit/log/lognormalizer.py index 659288117..6a1f54a22 100644 --- a/vulture_os/toolkit/log/lognormalizer.py +++ b/vulture_os/toolkit/log/lognormalizer.py @@ -43,7 +43,7 @@ logger = logging.getLogger('api') -TEST_CONF_PATH = "/tmp" +TEST_CONF_PATH = settings.TMP_PATH def test_lognormalizer(filename, rulebase_content, to_parse): diff --git a/vulture_os/toolkit/redis/redis_base.py b/vulture_os/toolkit/redis/redis_base.py index ad4eb2f27..182cad415 100644 --- a/vulture_os/toolkit/redis/redis_base.py +++ b/vulture_os/toolkit/redis/redis_base.py @@ -24,6 +24,7 @@ from ast import literal_eval +from os.path import join as path_join from redis import Redis, RedisError from toolkit.network.network import get_management_ip @@ -39,7 +40,7 @@ def __init__(self, node=None, port=None, password=None): self.node = node self.port = port self.password = password - self.db = '/var/sockets/redis/redis.sock' + self.db = path_join(settings.SOCKETS_PATH, 'redis/redis.sock') self._redis_client = None @property diff --git a/vulture_os/toolkit/yara/yara.py b/vulture_os/toolkit/yara/yara.py index 29aff100f..1d3cc15c8 100644 --- a/vulture_os/toolkit/yara/yara.py +++ b/vulture_os/toolkit/yara/yara.py @@ -23,6 +23,7 @@ __doc__ = 'System Utils Yara Toolkit' # Django project imports +from django.conf import settings from darwin.inspection.models import InspectionPolicy, InspectionRule, DEFAULT_YARA_CATEGORIES from django.core.exceptions import ObjectDoesNotExist from system.exceptions import VultureSystemConfigError @@ -54,17 +55,17 @@ def fetch_yara_rules(logger): logger.info("Yara::fetch_yara_rules:: extracting them...") try: - with open("/var/tmp/yara_rules.zip", "wb") as f: + with open(os.path.join(settings.TMP_PATH, "yara_rules.zip", "wb")) as f: f.write(doc.content) - with zipfile.ZipFile("/var/tmp/yara_rules.zip") as z: - z.extractall("/var/tmp/yara_rules/") + with zipfile.ZipFile(os.path.join(settings.TMP_PATH, "yara_rules.zip")) as z: + z.extractall(os.path.join(settings.TMP_PATH, "yara_rules/")) except Exception as e: logger.error("Yara::fetch_yara_rules:: {}".format(e), exc_info=1) raise rule_regex = re.compile(r'^\s*rule .*$') - for (baseRoot, baseDirs, baseFiles) in os.walk("/var/tmp/yara_rules/rules-master/"): + for (baseRoot, baseDirs, baseFiles) in os.walk(os.path.join(settings.TMP_PATH, "yara_rules/rules-master/")): for baseDir in baseDirs: for (root, dirs, files) in os.walk(os.path.join(baseRoot, baseDir)): for filename in files: diff --git a/vulture_os/vulture_os/settings.py b/vulture_os/vulture_os/settings.py index e4f1ea045..e7030abf3 100644 --- a/vulture_os/vulture_os/settings.py +++ b/vulture_os/vulture_os/settings.py @@ -15,13 +15,22 @@ along with Vulture 3. If not, see http://www.gnu.org/licenses/. """ -import os +from os import path as os_path from toolkit.network.network import get_hostname from toolkit.system.secret_key import set_key # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SETTINGS_DIR = os.path.abspath(os.path.dirname(__file__)) +BASE_DIR = os_path.dirname(os_path.dirname(os_path.abspath(__file__))) +SETTINGS_DIR = os_path.abspath(os_path.dirname(__file__))\ + +# Project folders +ROOT_PATH = "/" +DBS_PATH = os_path.join(ROOT_PATH, "var/db") +TMP_PATH = os_path.join(ROOT_PATH, "var/tmp") +LOGS_PATH = os_path.join(ROOT_PATH, "var/log") +SOCKETS_PATH = os_path.join(ROOT_PATH, "var/sockets") +HOMES_PATH = os_path.join(ROOT_PATH, "home") +LOCALETC_PATH = os_path.join(ROOT_PATH, "usr/local/etc") HOSTNAME = get_hostname() @@ -86,7 +95,7 @@ CRONTAB_COMMAND_PREFIX = "LANG=en_US.UTF-8" # Extend cronjobs with custom cronjobs -if os.path.exists(os.path.dirname(os.path.abspath(__file__)) + "/custom_cronjobs.py"): +if os_path.exists(os_path.dirname(os_path.abspath(__file__)) + "/custom_cronjobs.py"): try: from vulture_os.custom_cronjobs import CUSTOM_CRONJOBS CRONJOBS.extend(CUSTOM_CRONJOBS) @@ -144,8 +153,8 @@ 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', 'SSL': True, - 'tlsCertificateKeyFile': '/var/db/pki/node.pem', - 'tlsCAFile': '/var/db/pki/ca.pem', + 'tlsCertificateKeyFile': os_path.join(DBS_PATH, 'pki/node.pem'), + 'tlsCAFile': os_path.join(DBS_PATH, 'pki/ca.pem'), 'READPREFERENCE': "primaryPreferred" }, } @@ -195,7 +204,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.2/howto/static-files/ MEDIA_PATH = 'gui/static/img/' -STATIC_ROOT = os.path.join(BASE_DIR, "static") +STATIC_ROOT = os_path.join(BASE_DIR, "static") STATIC_URL = 'static/' LOG_SETTINGS = { @@ -236,7 +245,7 @@ 'debug': { 'level': LOG_LEVEL, 'class': 'logging.handlers.RotatingFileHandler', - 'filename': '/var/log/vulture/os/debug.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/debug.log'), 'formatter': 'verbose', 'mode': 'a', 'maxBytes': 10485760, @@ -250,21 +259,21 @@ 'api': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': '/var/log/vulture/os/api.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/api.log'), 'formatter': 'verbose', 'mode': 'a' }, 'gui': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': '/var/log/vulture/os/gui.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/gui.log'), 'formatter': 'verbose', 'mode': 'a' }, 'services': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': '/var/log/vulture/os/services.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/services.log'), 'formatter': 'verbose', 'mode': 'a' }, @@ -272,35 +281,35 @@ 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/os/cluster.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/cluster.log'), 'mode': 'a' }, 'crontab': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/os/crontab.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/crontab.log'), 'mode': 'a' }, 'api_parser': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'api_parser', - 'filename': '/var/log/vulture/os/api_parser.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/api_parser.log'), 'mode': 'a' }, 'authentication': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/os/authentication.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/authentication.log'), 'mode': 'a' }, 'system': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': '/var/log/vulture/os/system.log', + 'filename': os_path.join(LOGS_PATH, 'vulture/os/system.log'), 'mode': 'a' } }, diff --git a/vulture_os/workflow/models.py b/vulture_os/workflow/models.py index 2dd745102..0af4a6b6a 100644 --- a/vulture_os/workflow/models.py +++ b/vulture_os/workflow/models.py @@ -42,6 +42,7 @@ # Extern modules imports from jinja2 import Environment, FileSystemLoader +from os.path import join as path_join # Required exceptions imports from portal.system.exceptions import ACLError @@ -56,7 +57,7 @@ logger = logging.getLogger('gui') # Jinja template for backends rendering -JINJA_PATH = "/home/vlt-os/vulture_os/workflow/config/" +JINJA_PATH = path_join(settings.BASE_DIR, "workflow/config/") JINJA_TEMPLATE = "haproxy_portal.conf" WORKFLOW_OWNER = HAPROXY_OWNER From b333ad02968d151e0e3f2e9fbc16db61157dc129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 20:18:18 +0100 Subject: [PATCH 5/8] chore(logging): Correctly set handlers for logging in files --- vulture_os/system/cluster/api.py | 1 + vulture_os/system/config/models.py | 1 + vulture_os/system/config/views.py | 2 ++ vulture_os/system/pki/api.py | 3 ++- vulture_os/system/tenants/models.py | 2 ++ vulture_os/system/tenants/views.py | 2 ++ vulture_os/system/users/models.py | 2 ++ vulture_os/toolkit/api/responses.py | 2 ++ vulture_os/toolkit/auth/authy_client.py | 2 ++ vulture_os/toolkit/auth/base_auth.py | 2 ++ vulture_os/toolkit/auth/ldap_client.py | 1 + vulture_os/toolkit/auth/totp_client.py | 2 ++ vulture_os/toolkit/auth/vulturemail_client.py | 2 ++ vulture_os/toolkit/mongodb/mongo_base.py | 1 + vulture_os/toolkit/network/network.py | 1 - vulture_os/toolkit/network/smtp.py | 3 ++- vulture_os/toolkit/redis/redis_base.py | 1 + vulture_os/toolkit/yara/yara.py | 3 ++- 18 files changed, 29 insertions(+), 4 deletions(-) diff --git a/vulture_os/system/cluster/api.py b/vulture_os/system/cluster/api.py index 113e467ea..0df65dfef 100644 --- a/vulture_os/system/cluster/api.py +++ b/vulture_os/system/cluster/api.py @@ -51,6 +51,7 @@ import time import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('system') diff --git a/vulture_os/system/config/models.py b/vulture_os/system/config/models.py index 11ae59329..8d9e5abed 100644 --- a/vulture_os/system/config/models.py +++ b/vulture_os/system/config/models.py @@ -45,6 +45,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('gui') diff --git a/vulture_os/system/config/views.py b/vulture_os/system/config/views.py index 53ee8153e..801c01659 100644 --- a/vulture_os/system/config/views.py +++ b/vulture_os/system/config/views.py @@ -23,6 +23,7 @@ __doc__ = 'Config View' # Django system imports +from django.conf import settings from django.db.models import Q from django.http import HttpResponseRedirect, JsonResponse from django.shortcuts import render @@ -46,6 +47,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('services') diff --git a/vulture_os/system/pki/api.py b/vulture_os/system/pki/api.py index e7a29a8d5..d9bedf0fa 100644 --- a/vulture_os/system/pki/api.py +++ b/vulture_os/system/pki/api.py @@ -33,8 +33,9 @@ from system.pki.views import tls_profile_edit, pki_edit, pki_delete from django.views.generic.base import View from django.conf import settings -import logging +import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('system') diff --git a/vulture_os/system/tenants/models.py b/vulture_os/system/tenants/models.py index 0c853019a..f79ef564e 100644 --- a/vulture_os/system/tenants/models.py +++ b/vulture_os/system/tenants/models.py @@ -22,6 +22,7 @@ __email__ = "contact@vultureproject.org" __doc__ = 'Perimeter Configuration main models' +from django.conf import settings from django.forms.models import model_to_dict from djongo import models from applications.reputation_ctx.models import ReputationContext @@ -31,6 +32,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('gui') diff --git a/vulture_os/system/tenants/views.py b/vulture_os/system/tenants/views.py index f7db35cfb..f87f9a4b0 100644 --- a/vulture_os/system/tenants/views.py +++ b/vulture_os/system/tenants/views.py @@ -23,6 +23,7 @@ __doc__ = 'Tenants View' # Django system imports +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.http import HttpResponseRedirect, JsonResponse, HttpResponseForbidden from django.shortcuts import render @@ -42,6 +43,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('system') diff --git a/vulture_os/system/users/models.py b/vulture_os/system/users/models.py index 1858827a7..16c75d8af 100644 --- a/vulture_os/system/users/models.py +++ b/vulture_os/system/users/models.py @@ -24,6 +24,7 @@ # Django system imports +from django.conf import settings from django.contrib.auth import models as auth_models from djongo import models @@ -35,6 +36,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('gui') diff --git a/vulture_os/toolkit/api/responses.py b/vulture_os/toolkit/api/responses.py index 05ebd2dae..49aeed385 100644 --- a/vulture_os/toolkit/api/responses.py +++ b/vulture_os/toolkit/api/responses.py @@ -23,6 +23,7 @@ __doc__ = 'Response API toolkit functions' # Django system imports +from django.conf import settings from django.http import JsonResponse from django.urls import reverse @@ -34,6 +35,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('authentication') diff --git a/vulture_os/toolkit/auth/authy_client.py b/vulture_os/toolkit/auth/authy_client.py index 005674684..4f5b9bd2c 100644 --- a/vulture_os/toolkit/auth/authy_client.py +++ b/vulture_os/toolkit/auth/authy_client.py @@ -26,6 +26,7 @@ # Django system imports # Django project imports +from django.conf import settings from toolkit.auth.base_auth import BaseAuth # Required exceptions imports @@ -37,6 +38,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('authentication') diff --git a/vulture_os/toolkit/auth/base_auth.py b/vulture_os/toolkit/auth/base_auth.py index 5be377533..a23030975 100644 --- a/vulture_os/toolkit/auth/base_auth.py +++ b/vulture_os/toolkit/auth/base_auth.py @@ -23,6 +23,7 @@ __doc__ = 'Base authentication, mother of authentication wrappers' # Django system imports +from django.conf import settings from django.contrib.auth.hashers import make_password from django.utils.crypto import constant_time_compare, pbkdf2, get_random_string @@ -36,6 +37,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('authentication') diff --git a/vulture_os/toolkit/auth/ldap_client.py b/vulture_os/toolkit/auth/ldap_client.py index d036c61f2..ff5095702 100644 --- a/vulture_os/toolkit/auth/ldap_client.py +++ b/vulture_os/toolkit/auth/ldap_client.py @@ -42,6 +42,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('authentication') CERT_PATH = path_join(settings.DBS_PATH, "pki") diff --git a/vulture_os/toolkit/auth/totp_client.py b/vulture_os/toolkit/auth/totp_client.py index 181b48c51..a3ceeecef 100644 --- a/vulture_os/toolkit/auth/totp_client.py +++ b/vulture_os/toolkit/auth/totp_client.py @@ -24,6 +24,7 @@ # Django system imports +from django.conf import settings # Django project imports from authentication.totp_profiles.models import TOTPProfile @@ -37,6 +38,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('portal_authentication') diff --git a/vulture_os/toolkit/auth/vulturemail_client.py b/vulture_os/toolkit/auth/vulturemail_client.py index c4dcdcb9e..171a53085 100644 --- a/vulture_os/toolkit/auth/vulturemail_client.py +++ b/vulture_os/toolkit/auth/vulturemail_client.py @@ -24,6 +24,7 @@ # Django system imports +from django.conf import settings # Django project imports from system.cluster.models import Cluster @@ -38,6 +39,7 @@ # Logger configuration imports import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('authentication') diff --git a/vulture_os/toolkit/mongodb/mongo_base.py b/vulture_os/toolkit/mongodb/mongo_base.py index 5c2dd4bbf..05c3be459 100644 --- a/vulture_os/toolkit/mongodb/mongo_base.py +++ b/vulture_os/toolkit/mongodb/mongo_base.py @@ -30,6 +30,7 @@ import subprocess import logging +logging.config.dictConfig(settings.LOG_SETTINGS) # No database logging to prevent infinite loop logger = logging.getLogger('system') diff --git a/vulture_os/toolkit/network/network.py b/vulture_os/toolkit/network/network.py index 124fb6191..50ddfea1f 100644 --- a/vulture_os/toolkit/network/network.py +++ b/vulture_os/toolkit/network/network.py @@ -22,7 +22,6 @@ __email__ = "contact@vultureproject.org" __doc__ = 'System Utils Network Toolkit' - from toolkit.system.rc import get_rc_config, set_rc_config, remove_rc_config from ipaddress import IPv4Address, IPv6Address, ip_address, AddressValueError diff --git a/vulture_os/toolkit/network/smtp.py b/vulture_os/toolkit/network/smtp.py index dc5ffb6af..de3b530f2 100644 --- a/vulture_os/toolkit/network/smtp.py +++ b/vulture_os/toolkit/network/smtp.py @@ -23,10 +23,11 @@ __doc__ = 'SMTP Utils Network Toolkit' +from django.conf import settings from smtplib import SMTP import logging - +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('system') diff --git a/vulture_os/toolkit/redis/redis_base.py b/vulture_os/toolkit/redis/redis_base.py index 182cad415..cf8135c20 100644 --- a/vulture_os/toolkit/redis/redis_base.py +++ b/vulture_os/toolkit/redis/redis_base.py @@ -31,6 +31,7 @@ from django.conf import settings import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('debug') diff --git a/vulture_os/toolkit/yara/yara.py b/vulture_os/toolkit/yara/yara.py index 1d3cc15c8..5945ae67a 100644 --- a/vulture_os/toolkit/yara/yara.py +++ b/vulture_os/toolkit/yara/yara.py @@ -31,11 +31,12 @@ import subprocess from toolkit.network.network import get_proxy import requests -import logging import os import re import zipfile +import logging +logging.config.dictConfig(settings.LOG_SETTINGS) logger = logging.getLogger('system') From 582b1070458811b0dc875737f2e74b50f98d6b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 20 Dec 2024 20:32:10 +0100 Subject: [PATCH 6/8] fix(settings): Disable the deprecated 'database' logger --- vulture_os/portal/settings.py | 13 ++++--------- vulture_os/vulture_os/settings.py | 5 ----- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/vulture_os/portal/settings.py b/vulture_os/portal/settings.py index e8b8dfeb5..3145b2376 100644 --- a/vulture_os/portal/settings.py +++ b/vulture_os/portal/settings.py @@ -173,11 +173,6 @@ 'level': LOG_LEVEL, 'filters': ['require_debug_true'] }, - 'database': { - 'level': LOG_LEVEL, - 'class': 'toolkit.log.log_utils.DatabaseHandler', - 'type_logs': 'vulture', - }, 'file_portal_authentication': { 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, @@ -219,22 +214,22 @@ }, 'loggers': { 'portal_authentication': { - 'handlers': ['file_portal_authentication', 'database', 'console'], + 'handlers': ['file_portal_authentication', 'console'], 'propagate': True, 'level': LOG_LEVEL, }, 'redis_events': { - 'handlers': ['file_redis_events', 'database', 'console'], + 'handlers': ['file_redis_events', 'console'], 'propagate': True, 'level': LOG_LEVEL, }, 'authentication': { - 'handlers': ('authentication', 'database', 'console'), + 'handlers': ('authentication', 'console'), 'propagate': True, 'level': LOG_LEVEL, }, 'debug': { - 'handlers': ['debug', 'database', 'console'], + 'handlers': ['debug', 'console'], 'propagate': True, 'level': LOG_LEVEL, }, diff --git a/vulture_os/vulture_os/settings.py b/vulture_os/vulture_os/settings.py index e7030abf3..5c5a6e0b9 100644 --- a/vulture_os/vulture_os/settings.py +++ b/vulture_os/vulture_os/settings.py @@ -251,11 +251,6 @@ 'maxBytes': 10485760, 'backupCount': 5, }, - 'database': { - 'level': LOG_LEVEL, - 'class': 'toolkit.log.log_utils.DatabaseHandler', - 'type_logs': 'vulture', - }, 'api': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', From a8687b5031be880dce3f3977b65340274c75c482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 3 Jan 2025 14:57:26 +0100 Subject: [PATCH 7/8] feat(settings): variabilize settings with django-environ --- CHANGELOG | 2 + requirements.in | 1 + requirements.txt | 4 ++ vulture_os/portal/settings.py | 54 +++++++++++-------- vulture_os/toolkit/network/network.py | 8 ++- vulture_os/vulture_os/settings.py | 77 ++++++++++++++++----------- 6 files changed, 93 insertions(+), 53 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c56ade9fc..adadee1e5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- [SYSTEM] [SETTINGS] Allow to configure project with django-environ ### Removed - [API_PARSER] [CORTEX_XDR] Remove fields used for backward compatibility ### Changed diff --git a/requirements.in b/requirements.in index d56dcd38d..72570b67a 100644 --- a/requirements.in +++ b/requirements.in @@ -4,6 +4,7 @@ pymongo jinja2 iptools django-crontab +django-environ requests pyOpenSSL redis~=4.5 diff --git a/requirements.txt b/requirements.txt index 9ebbe702e..e86072b0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -330,6 +330,10 @@ django==4.2.11 \ django-crontab==0.7.1 \ --hash=sha256:1201810a212460aaaa48eb6a766738740daf42c1a4f6aafecfb1525036929236 # via -r requirements.in +django-environ==0.11.2 \ + --hash=sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05 \ + --hash=sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be + # via -r requirements.in django-jsoneditor==0.2.4 \ --hash=sha256:1d3dfca28f047feefa6ebc6f9541179eb815fb459b006faf3fb8d0fb2197d2df \ --hash=sha256:d7a639a7251e376126b5be64ea588c925c7a40d45e0e212f66ef475d2f0f90bb diff --git a/vulture_os/portal/settings.py b/vulture_os/portal/settings.py index 3145b2376..8b4becd1a 100644 --- a/vulture_os/portal/settings.py +++ b/vulture_os/portal/settings.py @@ -2,6 +2,7 @@ Django settings for vulture project. """ +import environ from os import path as os_path from toolkit.network.network import get_hostname from toolkit.system.secret_key import set_key @@ -9,16 +10,27 @@ BASE_DIR = os_path.dirname(os_path.dirname(os_path.abspath(__file__))) SETTINGS_DIR = os_path.abspath(os_path.dirname(__file__)) +env = environ.Env() +env.prefix = 'VULTURE_' +environ.Env.read_env(os_path.join(SETTINGS_DIR, '.env')) + # Project folders -ROOT_PATH = "/" -DBS_PATH = os_path.join(ROOT_PATH, "var/db") -TMP_PATH = os_path.join(ROOT_PATH, "var/tmp") -LOGS_PATH = os_path.join(ROOT_PATH, "var/log") -SOCKETS_PATH = os_path.join(ROOT_PATH, "var/sockets") -HOMES_PATH = os_path.join(ROOT_PATH, "home") -LOCALETC_PATH = os_path.join(ROOT_PATH, "usr/local/etc") +SYSTEM_ROOT_PATH = env.str("SYSTEM_ROOT_PATH", "/") +DBS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("DBS_PATH", "var/db")) +TMP_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("TMP_PATH", "var/tmp")) +LOGS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("LOGS_PATH", "var/log")) +SOCKETS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("SOCKETS_PATH", "var/sockets")) +HOMES_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("HOMES_PATH", "home")) +LOCALETC_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("LOCALETC_PATH", "usr/local/etc")) + +# Logging +ALL_LOGS_PATH = env.str("ALL_LOGS_PATH", None) +PORTAL_AUTHENTICATION_LOGS_PATH = env.str("PORTAL_AUTHENTICATION_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, 'vulture/portal/portal_authentication.log')) +REDIS_EVENTS_LOGS_PATH = env.str("REDIS_EVENTS_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, 'vulture/portal/redis_events.log')) +AUTHENTICATION_LOGS_PATH = env.str("AUTHENTICATION_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, 'vulture/portal/authentication.log')) +DEBUG_LOGS_PATH = env.str("DEBUG_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, 'vulture/portal/debug.log')) -HOSTNAME = get_hostname() +HOSTNAME = env.str("HOSTNAME", get_hostname()) # Retrieving Django SECRET_KEY try: @@ -34,9 +46,9 @@ except ImportError: pass -LOG_LEVEL = 'INFO' +LOG_LEVEL = env.str("LOG_LEVEL", "INFO") -DEBUG = False +DEBUG = env.bool("DEBUG", False) TEMPLATE_DEBUG = DEBUG ALLOWED_HOSTS = ["*"] @@ -114,13 +126,13 @@ 'ENGINE': 'djongo', 'NAME': 'vulture', "CLIENT": { - 'host': HOSTNAME, - 'port': 9091, + 'host': env.str('MONGODB_HOST', get_hostname()), + 'port': env.int('MONGODB_PORT', 9091), 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', - 'SSL': True, - 'tlsCertificateKeyFile': os_path.join(DBS_PATH, 'pki/node.pem'), - 'tlsCAFile': os_path.join(DBS_PATH, 'pki/ca.pem'), + 'SSL': env.bool('MONGODB_SSL', True), + 'tlsCertificateKeyFile': None if not env.bool('MONGODB_SSL', True) else env.str('MONGODB_CERT_FILE', os_path.join(DBS_PATH, 'pki/node.pem')), + 'tlsCAFile': None if not env.bool('MONGODB_SSL', True) else env.str('MONGODB_CA_FILE', os_path.join(DBS_PATH, 'pki/ca.pem')), 'READPREFERENCE': "primaryPreferred" }, } @@ -137,8 +149,8 @@ CACERT_DIR = os_path.join(DBS_PATH, 'mongodb/') MONGODBPORT = 9091 MONGODBARBPORT = 9092 -REDISIP = '127.0.0.1' -REDISPORT = '6379' +REDISIP = env.str('REDIS_HOST', '127.0.0.1') +REDISPORT = env.int('REDIS_PORT', 6379) OS = "FreeBSD" LOG_SETTINGS = { @@ -177,7 +189,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/portal/portal_authentication.log'), + 'filename': PORTAL_AUTHENTICATION_LOGS_PATH, 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -186,7 +198,7 @@ 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/portal/redis_events.log'), + 'filename': REDIS_EVENTS_LOGS_PATH, 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, @@ -195,14 +207,14 @@ 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/portal/authentication.log'), + 'filename': AUTHENTICATION_LOGS_PATH, 'mode': 'a' }, 'debug': { 'class': 'logging.handlers.RotatingFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/portal/debug.log'), + 'filename': DEBUG_LOGS_PATH, 'mode': 'a', 'maxBytes': 10485760, 'backupCount': 5, diff --git a/vulture_os/toolkit/network/network.py b/vulture_os/toolkit/network/network.py index 50ddfea1f..8d18fb409 100644 --- a/vulture_os/toolkit/network/network.py +++ b/vulture_os/toolkit/network/network.py @@ -30,6 +30,7 @@ import subprocess import logging import os +import platform import re from django.core.validators import URLValidator @@ -105,8 +106,11 @@ def get_hostname(): :return: A String, the hostname of the local node. """ - with open("/etc/host-hostname", "r") as f: - return f.read().strip() + if os.path.exists("/etc/host-hostname"): + with open("/etc/host-hostname", "r") as f: + return f.read().strip() + else: + return platform.node() def get_management_ip(): diff --git a/vulture_os/vulture_os/settings.py b/vulture_os/vulture_os/settings.py index 5c5a6e0b9..38811fb83 100644 --- a/vulture_os/vulture_os/settings.py +++ b/vulture_os/vulture_os/settings.py @@ -15,24 +15,42 @@ along with Vulture 3. If not, see http://www.gnu.org/licenses/. """ +import environ from os import path as os_path from toolkit.network.network import get_hostname from toolkit.system.secret_key import set_key # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os_path.dirname(os_path.dirname(os_path.abspath(__file__))) -SETTINGS_DIR = os_path.abspath(os_path.dirname(__file__))\ +SETTINGS_DIR = os_path.abspath(os_path.dirname(__file__)) -# Project folders -ROOT_PATH = "/" -DBS_PATH = os_path.join(ROOT_PATH, "var/db") -TMP_PATH = os_path.join(ROOT_PATH, "var/tmp") -LOGS_PATH = os_path.join(ROOT_PATH, "var/log") -SOCKETS_PATH = os_path.join(ROOT_PATH, "var/sockets") -HOMES_PATH = os_path.join(ROOT_PATH, "home") -LOCALETC_PATH = os_path.join(ROOT_PATH, "usr/local/etc") +env = environ.Env() +env.prefix = 'VULTURE_' +environ.Env.read_env(os_path.join(SETTINGS_DIR, '.env')) -HOSTNAME = get_hostname() +# Project folders +SYSTEM_ROOT_PATH = env.str("SYSTEM_ROOT_PATH", "/") +DBS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("DBS_PATH", "var/db")) +TMP_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("TMP_PATH", "var/tmp")) +LOGS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("LOGS_PATH", "var/log")) +APPLICATION_LOGS_PATH = os_path.join(LOGS_PATH, env.str("APPLICATION_LOGS_PATH", "vulture/os")) +SOCKETS_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("SOCKETS_PATH", "var/sockets")) +HOMES_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("HOMES_PATH", "home")) +LOCALETC_PATH = os_path.join(SYSTEM_ROOT_PATH, env.str("LOCALETC_PATH", "usr/local/etc")) + +# Logging +ALL_LOGS_PATH = env.str("ALL_LOGS_PATH", None) +DEBUG_LOGS_PATH = env.str("DEBUG_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/debug.log")) +API_LOGS_PATH = env.str("API_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/api.log")) +GUI_LOGS_PATH = env.str("GUI_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/gui.log")) +SERVICES_LOGS_PATH = env.str("SERVICES_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/services.log")) +DAEMON_LOGS_PATH = env.str("DAEMON_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/cluster.log")) +CRONTAB_LOGS_PATH = env.str("CRONTAB_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/crontab.log")) +API_PARSER_LOGS_PATH = env.str("API_PARSER_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/api_parser.log")) +AUTHENTICATION_LOGS_PATH = env.str("AUTHENTICATION_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/authentication.log")) +SYSTEM_LOGS_PATH = env.str("SYSTEM_LOGS_PATH", ALL_LOGS_PATH or os_path.join(LOGS_PATH, "vulture/os/system.log")) + +HOSTNAME = env.str("HOSTNAME", get_hostname()) # Retrieving Django SECRET_KEY try: @@ -48,11 +66,10 @@ except ImportError: pass -LOG_LEVEL = "INFO" +LOG_LEVEL = env.str("LOG_LEVEL", "INFO") -DEBUG = False -DEV_MODE = False -TEMPLATE_DEBUG = DEBUG +DEBUG = env.bool("DEBUG", False) +DEV_MODE = env.bool("DEV_MODE", False) ALLOWED_HOSTS = ["*"] @@ -148,13 +165,13 @@ 'ENGINE': 'djongo', 'NAME': 'vulture', "CLIENT": { - 'host': HOSTNAME, - 'port': 9091, + 'host': env.str('MONGODB_HOST', get_hostname()), + 'port': env.int('MONGODB_PORT', 9091), 'serverSelectionTimeoutMS': 5000, 'REPLICASET': 'Vulture', - 'SSL': True, - 'tlsCertificateKeyFile': os_path.join(DBS_PATH, 'pki/node.pem'), - 'tlsCAFile': os_path.join(DBS_PATH, 'pki/ca.pem'), + 'SSL': env.bool('MONGODB_SSL', True), + 'tlsCertificateKeyFile': None if not env.bool('MONGODB_SSL', True) else env.str('MONGODB_CERT_FILE', os_path.join(DBS_PATH, 'pki/node.pem')), + 'tlsCAFile': None if not env.bool('MONGODB_SSL', True) else env.str('MONGODB_CA_FILE', os_path.join(DBS_PATH, 'pki/ca.pem')), 'READPREFERENCE': "primaryPreferred" }, } @@ -162,8 +179,8 @@ DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -REDISIP = '127.0.0.1' -REDISPORT = '6379' +REDISIP = env.str('REDIS_HOST', '127.0.0.1') +REDISPORT = env.int('REDIS_PORT', 6379) LOGIN_URL = "/login/" @@ -245,7 +262,7 @@ 'debug': { 'level': LOG_LEVEL, 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/debug.log'), + 'filename': DEBUG_LOGS_PATH, 'formatter': 'verbose', 'mode': 'a', 'maxBytes': 10485760, @@ -254,21 +271,21 @@ 'api': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/api.log'), + 'filename': API_LOGS_PATH, 'formatter': 'verbose', 'mode': 'a' }, 'gui': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/gui.log'), + 'filename': GUI_LOGS_PATH, 'formatter': 'verbose', 'mode': 'a' }, 'services': { 'level': LOG_LEVEL, 'class': 'logging.handlers.WatchedFileHandler', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/services.log'), + 'filename': SERVICES_LOGS_PATH, 'formatter': 'verbose', 'mode': 'a' }, @@ -276,35 +293,35 @@ 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/cluster.log'), + 'filename': DAEMON_LOGS_PATH, 'mode': 'a' }, 'crontab': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/crontab.log'), + 'filename': CRONTAB_LOGS_PATH, 'mode': 'a' }, 'api_parser': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'api_parser', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/api_parser.log'), + 'filename': API_PARSER_LOGS_PATH, 'mode': 'a' }, 'authentication': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/authentication.log'), + 'filename': AUTHENTICATION_LOGS_PATH, 'mode': 'a' }, 'system': { 'class': 'logging.handlers.WatchedFileHandler', 'level': LOG_LEVEL, 'formatter': 'verbose', - 'filename': os_path.join(LOGS_PATH, 'vulture/os/system.log'), + 'filename': SYSTEM_LOGS_PATH, 'mode': 'a' } }, From ae98cefb2d2e33d555a048969ed25be8ceb08d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Bertin?= Date: Fri, 13 Dec 2024 16:11:23 +0100 Subject: [PATCH 8/8] tests: Deprecate old 'testing' folder and implement django tests or system certificates and workflows --- CHANGELOG | 3 + vulture_os/{ => system/tests}/__init__.py | 0 vulture_os/system/tests/test_models.py | 342 ++++++++++++++++++ vulture_os/testing/config.py | 29 -- vulture_os/testing/core/__init__.py | 0 vulture_os/testing/core/cleanup_module.py | 19 - vulture_os/testing/core/command_parse.py | 29 -- vulture_os/testing/core/modules_utils.py | 179 --------- vulture_os/testing/core/print_results.py | 59 --- vulture_os/testing/core/setup_module.py | 19 - vulture_os/testing/core/testing_module.py | 73 ---- vulture_os/testing/doc/getting_started | 42 --- .../testing/doc/setup&cleanup_modules.txt | 23 -- vulture_os/testing/doc/testing_modules.txt | 48 --- .../change_log_level_for_specific_test.txt | 9 - vulture_os/testing/modules/__init__.py | 0 .../modules/cleanup_modules/__init__.py | 0 .../testing/modules/setup_modules/__init__.py | 0 .../modules/testing_modules/__init__.py | 0 vulture_os/testing/modules/utils/__init__.py | 0 vulture_os/testing/testing_client.py | 62 ---- .../{testing => workflow/tests}/__init__.py | 0 vulture_os/workflow/tests/test_models.py | 263 ++++++++++++++ 23 files changed, 608 insertions(+), 591 deletions(-) rename vulture_os/{ => system/tests}/__init__.py (100%) create mode 100644 vulture_os/system/tests/test_models.py delete mode 100644 vulture_os/testing/config.py delete mode 100644 vulture_os/testing/core/__init__.py delete mode 100644 vulture_os/testing/core/cleanup_module.py delete mode 100644 vulture_os/testing/core/command_parse.py delete mode 100644 vulture_os/testing/core/modules_utils.py delete mode 100644 vulture_os/testing/core/print_results.py delete mode 100644 vulture_os/testing/core/setup_module.py delete mode 100644 vulture_os/testing/core/testing_module.py delete mode 100644 vulture_os/testing/doc/getting_started delete mode 100644 vulture_os/testing/doc/setup&cleanup_modules.txt delete mode 100644 vulture_os/testing/doc/testing_modules.txt delete mode 100644 vulture_os/testing/doc/tips_n_tricks/change_log_level_for_specific_test.txt delete mode 100644 vulture_os/testing/modules/__init__.py delete mode 100644 vulture_os/testing/modules/cleanup_modules/__init__.py delete mode 100644 vulture_os/testing/modules/setup_modules/__init__.py delete mode 100644 vulture_os/testing/modules/testing_modules/__init__.py delete mode 100644 vulture_os/testing/modules/utils/__init__.py delete mode 100644 vulture_os/testing/testing_client.py rename vulture_os/{testing => workflow/tests}/__init__.py (100%) create mode 100644 vulture_os/workflow/tests/test_models.py diff --git a/CHANGELOG b/CHANGELOG index adadee1e5..0eddf4bdb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -8,8 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - [SYSTEM] [SETTINGS] Allow to configure project with django-environ +- [TESTS] [SYSTEM] Implement Network test cases +- [TESTS] [WORKFLOW] Implement Workflow test cases ### Removed - [API_PARSER] [CORTEX_XDR] Remove fields used for backward compatibility +- [TESTS] Remove old 'testing' folder ### Changed - [API_PARSER] [NETSKOPE] Add the possiblity to retrieve 3 new kinds of logs - [SYSTEM] [SETTINGS] Use Hostname value from Django settings diff --git a/vulture_os/__init__.py b/vulture_os/system/tests/__init__.py similarity index 100% rename from vulture_os/__init__.py rename to vulture_os/system/tests/__init__.py diff --git a/vulture_os/system/tests/test_models.py b/vulture_os/system/tests/test_models.py new file mode 100644 index 000000000..78b56a65c --- /dev/null +++ b/vulture_os/system/tests/test_models.py @@ -0,0 +1,342 @@ +#!/home/vlt-os/env/bin/python +"""This file is part of Vulture OS. + +Vulture OS is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Vulture OS is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Vulture OS. If not, see http://www.gnu.org/licenses/. +""" + +__author__ = "Théo BERTIN" +__credits__ = [] +__license__ = "GPLv3" +__version__ = "4.0.0" +__maintainer__ = "Vulture OS" +__email__ = "contact@vultureproject.org" +__doc__ = 'Tests for System App' + +from django.test import TestCase +from unittest.mock import patch + +from system.cluster.models import NetworkInterfaceCard, NetworkAddress, NetworkAddressNIC + +class NetworkInterfaceCardTestCase(TestCase): + TEST_CASE_NAME=f"{__name__}" + def setUp(self): + from system.cluster.models import Node + self.node = Node.objects.create( + name=f"node_test_{self.TEST_CASE_NAME}", + ) + + def test_nic_creation(self): + self.assertIsNotNone( + NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + ) + + @patch('subprocess.check_output') + def test_get_list(self, mocked_check_output): + mocked_check_output.return_value = b"vtnet0 lo0 lo1 lo2 em1 lo3 lo4 lo5 lo6 tun0 pflog0" + nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + + interface_list = nic.get_list() + self.assertIsInstance(interface_list, list) + self.assertEquals(set(["vtnet0", "em1"]), set(interface_list)) + + def test_has_ipv4_simple(self): + nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + ) + nic_addr = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_ipv4 + ) + self.assertIsNotNone(nic) + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(nic_addr) + + self.assertTrue(nic.has_ipv4) + self.assertFalse(nic.has_ipv6) + + def test_has_ipv4_and_vlanv6(self): + nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + ) + net_addr_vlan = NetworkAddress.objects.create( + name = f"net_addr_vlan_test_{self.TEST_CASE_NAME}", + type="vlan", + ip="fe80::1", + prefix_or_netmask="64", + vlan=123 + ) + nic_addr = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_ipv4 + ) + nic_addr2 = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_vlan + ) + self.assertIsNotNone(nic) + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(net_addr_vlan) + self.assertIsNotNone(nic_addr) + self.assertIsNotNone(nic_addr2) + + self.assertEquals(nic.networkaddress_set.count(), 2) + + self.assertTrue(nic.has_ipv4) + self.assertFalse(nic.has_ipv6) + + def test_has_ipv6_simple(self): + nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="system", + ip="fe80::1", + prefix_or_netmask="64", + ) + nic_addr = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_ipv6 + ) + self.assertIsNotNone(nic) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(nic_addr) + + self.assertFalse(nic.has_ipv4) + self.assertTrue(nic.has_ipv6) + + + def test_has_ipv6_and_vlanv4(self): + nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="system", + ip="fe80::1", + prefix_or_netmask="64", + ) + net_addr_vlan = NetworkAddress.objects.create( + name = f"net_addr_vlan_test_{self.TEST_CASE_NAME}", + type="vlan", + ip="127.0.0.1", + prefix_or_netmask="24", + vlan=123 + ) + nic_addr = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_ipv6 + ) + nic_addr2 = NetworkAddressNIC.objects.create( + nic=nic, + network_address=net_addr_vlan + ) + self.assertIsNotNone(nic) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(net_addr_vlan) + self.assertIsNotNone(nic_addr) + self.assertIsNotNone(nic_addr2) + + self.assertEquals(nic.networkaddress_set.count(), 2) + + self.assertFalse(nic.has_ipv4) + self.assertTrue(nic.has_ipv6) + + +class NetworkAddressTestCase(TestCase): + TEST_CASE_NAME=f"{__name__}" + def setUp(self): + from system.cluster.models import Node + self.node = Node.objects.create( + name=f"node_test_{self.TEST_CASE_NAME}", + ) + self.nic0 = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + self.nic1 = NetworkInterfaceCard.objects.create( + dev = "vtnet1", + node=self.node, + ) + + def test_is_carp(self): + net_addr_with_carp = NetworkAddress.objects.create( + name = f"net_addr_with_carp_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + carp_vhid=42 + ) + net_addr_without_carp = NetworkAddress.objects.create( + name = f"net_addr_without_carp_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + carp_vhid=0 + ) + + self.assertIsNotNone(net_addr_with_carp) + self.assertIsNotNone(net_addr_without_carp) + self.assertTrue(net_addr_with_carp.is_carp) + self.assertFalse(net_addr_without_carp.is_carp) + + def test_version(self): + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="system", + ip="fe80::1", + prefix_or_netmask="64", + ) + net_addr_lagg = NetworkAddress.objects.create( + name = f"net_addr_lagg_test_{self.TEST_CASE_NAME}", + type="lagg", + ) + + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(net_addr_lagg) + self.assertEquals(net_addr_ipv4.version, 4) + self.assertEquals(net_addr_ipv6.version, 6) + # Oh that's ugly... + self.assertEquals(net_addr_lagg.version, '') + + def test_ip_cidr(self): + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="system", + ip="fe80::1", + prefix_or_netmask="64", + ) + net_addr_lagg = NetworkAddress.objects.create( + name = f"net_addr_lagg_test_{self.TEST_CASE_NAME}", + type="lagg", + ) + + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(net_addr_lagg) + self.assertEquals(net_addr_ipv4.ip_cidr, '127.0.0.1/24') + self.assertEquals(net_addr_ipv6.ip_cidr, 'fe80::1/64') + self.assertEquals(net_addr_lagg.ip_cidr, '') + + def test_family(self): + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="system", + ip="fe80::1", + prefix_or_netmask="64", + ) + net_addr_lagg = NetworkAddress.objects.create( + name = f"net_addr_lagg_test_{self.TEST_CASE_NAME}", + type="lagg", + ) + + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(net_addr_lagg) + self.assertEquals(net_addr_ipv4.family, 'inet') + self.assertEquals(net_addr_ipv6.family, 'inet6') + self.assertEquals(net_addr_lagg.family, '') + + def test_main_iface(self): + net_addr_ipv4 = NetworkAddress.objects.create( + name = f"net_addr_ipv4_test_{self.TEST_CASE_NAME}", + type="system", + ip="127.0.0.1", + prefix_or_netmask="24", + iface_id=0 + ) + NetworkAddressNIC.objects.create( + nic=self.nic0, + network_address=net_addr_ipv4 + ) + net_addr_ipv6 = NetworkAddress.objects.create( + name = f"net_addr_ipv6_test_{self.TEST_CASE_NAME}", + type="alias", + ip="fe80::1", + prefix_or_netmask="64", + iface_id=1 + ) + NetworkAddressNIC.objects.create( + nic=self.nic1, + network_address=net_addr_ipv6 + ) + net_addr_lagg = NetworkAddress.objects.create( + name = f"net_addr_lagg_test_{self.TEST_CASE_NAME}", + type="lagg", + iface_id=2, + ) + NetworkAddressNIC.objects.create( + nic=self.nic0, + network_address=net_addr_lagg + ) + NetworkAddressNIC.objects.create( + nic=self.nic1, + network_address=net_addr_lagg + ) + net_addr_vlan = NetworkAddress.objects.create( + name = f"net_addr_lagg_test_{self.TEST_CASE_NAME}", + type="vlan", + vlan=42, + iface_id=42, + ) + + self.assertIsNotNone(net_addr_ipv4) + self.assertIsNotNone(net_addr_ipv6) + self.assertIsNotNone(net_addr_lagg) + self.assertIsNotNone(net_addr_vlan) + self.assertEquals(net_addr_ipv4.main_iface, 'vtnet0') + self.assertEquals(net_addr_ipv6.main_iface, 'vtnet1_alias1') + self.assertEquals(net_addr_lagg.main_iface, 'lagg2') + self.assertEquals(net_addr_vlan.main_iface, 'vlan42') \ No newline at end of file diff --git a/vulture_os/testing/config.py b/vulture_os/testing/config.py deleted file mode 100644 index a0cfbf21d..000000000 --- a/vulture_os/testing/config.py +++ /dev/null @@ -1,29 +0,0 @@ -############################################ -# AUTOMATED TEST CONFIGURATION FILE # -############################################ - -# Setup modules used before any test is launched, executed one time befoer testing -SETUP_MODULES_DIRECTORY = 'modules.setup_modules' - -SETUP_MODULES = [ - -] - - -# Cleanup modules used after all tests are finished, executed one time after all the tests -CLEANUP_MODULES_DIRECTORY = 'modules.cleanup_modules' - -CLEANUP_MODULES = [ - -] - - -# Testing modules are the tests you want to execute, every test is executed one time -TESTING_MODULES_DIRECTORY = 'modules.testing_modules' - -TESTING_MODULES = [ - 'connection_tests', - 'cluster_connection', - 'node_system_status', - 'repository_status', -] diff --git a/vulture_os/testing/core/__init__.py b/vulture_os/testing/core/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/core/cleanup_module.py b/vulture_os/testing/core/cleanup_module.py deleted file mode 100644 index 2b7b14b23..000000000 --- a/vulture_os/testing/core/cleanup_module.py +++ /dev/null @@ -1,19 +0,0 @@ -class CleanupModule(object): - - def __init__(self): - pass - - def __str__(self): - return "CleanupModule" - - def set_up(self): - """ - This method is executed before every test of the module - """ - pass - - def tear_down(self): - """ - This method is executed after every test of the module - """ - pass diff --git a/vulture_os/testing/core/command_parse.py b/vulture_os/testing/core/command_parse.py deleted file mode 100644 index b81193c9d..000000000 --- a/vulture_os/testing/core/command_parse.py +++ /dev/null @@ -1,29 +0,0 @@ -import argparse - - -def command_parse(): - """ - Parse the command - :return: The parsed command - """ - - parser = argparse.ArgumentParser(description="The standalone testing framework for Vulture") - - parser.add_argument( - '--output', - dest='output_level', - nargs='?', - default='no', - choices=['overview', 'global', 'detailed', 'no'], - help="Output detail level (default: no)" - ) - - parser.add_argument( - '--no-log', - dest='save', - action='store_false', - default=True, - help="Disable the diagnostic logging in Vulture" - ) - - return parser.parse_args() diff --git a/vulture_os/testing/core/modules_utils.py b/vulture_os/testing/core/modules_utils.py deleted file mode 100644 index e8a03831b..000000000 --- a/vulture_os/testing/core/modules_utils.py +++ /dev/null @@ -1,179 +0,0 @@ -import importlib -import sys -import traceback -from contextlib import contextmanager -from time import time - -from testing_module import TestingModule - -sys.path.append("/home/vlt-gui/vulture") - -import logging -import logging.config - -logger = logging.getLogger('diagnostic') - - -@contextmanager -def get_modules_list(modules_files, modules_directory): - """ - Get all the modules and return them as a list - :return: The list of all testing modules - """ - - try: - modules_list = [] - for mod in modules_files: - modules_list.append(getattr(importlib.import_module(modules_directory + '.' + mod), "Module")()) - yield modules_list - finally: - pass - - -def get_methods(modules_list): - """ - Get all the test methods from the modules - :param modules_list: List containing the loaded modules - :return: Dict containing the loaded modules as keys and their test methods as values - """ - - test_modules = {} - for module in modules_list: - test_modules[module] = [method for method in dir(module) - if method not in dir(TestingModule) and callable(getattr(module, method))] - module.nb_test = len(test_modules[module]) - - return test_modules - - -def launch_setup(setup_modules, save, node_name, summary): - """ - Execute the setups of all the loaded setup modules - :param setup_modules: Dict containing the loaded modules as keys and their setup methods as values - :param save: Boolean determining if the logging is activated - :param node_name: The name of the node the tests are executed on - :param summary: A dict summarizing the tests results - :return failed_setup: Dict containing the failed setup method for every setup module - """ - - failed_setup = {} - for mod, methods in setup_modules.items(): - failed_methods = [] - for method in methods: - try: - getattr(mod, method)() - except: - failed_methods.append(method) - if save: - logger.critical("Test failure out of testing scope, please contact the Vulture project team.", - extra={ - 'test_module': str(mod), - 'test_name': method, - 'traceback': traceback.format_exc(), - 'node_name': node_name - }) - else: - print "[ERROR] <{}> {}:: Test failure out of testing scope, " \ - "please check the code twice.\n{}".format(str(mod), method, traceback.format_exc()) - if failed_methods: - failed_setup[str(mod)] = failed_methods - - if failed_setup: - summary['setup'] = False - summary['global_status'] = False - else: - summary['setup'] = True - return failed_setup - - -def launch_cleanup(cleanup_modules, failed_setup, save, node_name, summary): - """ - Execute the cleanups of all the loaded cleanup modules - :param cleanup_modules: Dict containing the loaded modules as keys and their cleanup methods as values - :param failed_setup: Dict containing the failed setup method for every setup module - :param save: Boolean determining if the logging is activated - :param node_name: The name of the node the tests are executed on - :param summary: A dict summarizing the tests results - """ - - summary['cleanup'] = True - for mod, methods in cleanup_modules.items(): - for method in methods: - try: - getattr(mod, method)() - except: - summary['cleanup'] = False - summary['global_status'] = False - if save: - logger.critical("Test failure out of testing scope, please contact the Vulture project team.", - extra={ - 'test_module': str(mod), - 'test_name': method, - 'traceback': traceback.format_exc(), - 'node_name': node_name - }) - else: - print "[ERROR] <{}> {}:: Test failure out of testing scope, " \ - "please check the code twice.\n{}".format(str(mod), method, traceback.format_exc()) - - -def launch_testing(test_modules, output_level, save, failed_setup, node_name, summary): - """ - Execute the tests of all the loaded modules - :param test_modules: Dict containing the loaded modules as keys and their test methods as values - :param output_level: The verbosity of the tests output - :param save: Boolean determining if the logging is activated - :param failed_setup: Dict containing the failed setup method for every setup module - :param node_name: The name of the node the tests are executed on - :param summary: A dict summarizing the tests results - """ - - tmp_modules = test_modules.copy() - for mod, methods in tmp_modules.items(): - - if set(mod.setup_modules).intersection(failed_setup): - test_modules.pop(mod) - continue - - summary[str(mod)] = {} - start = time() - - for method in methods: - try: - mod.set_up() - except: - pass - - try: - getattr(mod, method)() - summary[str(mod)][method] = True - - except AssertionError as e: - mod.test_failure_handler(method, e, traceback.format_exc(), output_level, save, node_name, summary) - - except: - summary[str(mod)][method] = False - summary['global_status'] = False - mod.nb_failed += 1 - if save: - logger.critical("Test failure out of testing scope, please contact the Vulture project team.", - extra={ - 'test_module': str(mod), - 'test_name': method, - 'traceback': traceback.format_exc(), - 'node_name': node_name - }) - else: - print "[ERROR] <{}> {}:: Test failure out of testing scope, " \ - "please check the code twice.\n{}".format(str(mod), method, traceback.format_exc()) - - try: - mod.tear_down() - except: - pass - - mod.time = time() - start - - if not mod.nb_failed and save: - logger.info("Test ended successfully", - extra={'test_module': str(mod), 'test_name': 'All', 'traceback': '-No Traceback-', 'node_name': node_name}) diff --git a/vulture_os/testing/core/print_results.py b/vulture_os/testing/core/print_results.py deleted file mode 100644 index 77b3fdfdc..000000000 --- a/vulture_os/testing/core/print_results.py +++ /dev/null @@ -1,59 +0,0 @@ - - -def print_detailed_results(test_modules): - """ - Print the detailed results of the tests - :param test_modules: Dict containing the loaded modules as keys and their test methods as values - """ - - for mod in test_modules: - - if mod.nb_failed: - print "--------------------------------------------------" - print " " * (25 - len(str(mod)) / 2) + str(mod) - print "--------------------------------------------------\n" - print mod.buffer - - -def print_summarized_results(test_modules): - """ - Print a array with the results overview by module - :param test_modules: Dict containing the loaded modules as keys and their test methods as values - """ - - # Set the minimal length - length = [ - len('Module'), - len('Time (seconds)'), - len('Failed'), - len('Total'), - ] - - # Search for the biggest length for each column - for mod in test_modules: - length[0] = length[0] if length[0] >= len(str(mod).strip()) else len(str(mod).strip()) - length[1] = length[1] if length[1] >= len(str(mod.time)) else len(str(mod.time)) - length[2] = length[2] if length[2] >= len(str(mod.nb_failed)) else len(str(mod.nb_failed)) - length[3] = length[3] if length[3] >= len(str(mod.nb_test)) else len(str(mod.nb_test)) - - # Create the separation line - sep_line = " *" + ("-" * (reduce(lambda a, b: a + b, length) + 11)) + "*" - print sep_line - - # Print the title row - line = " | Module" + (" " * (length[0] - 6)) - line += " | Time (seconds)" + (" " * (length[1] - 14)) - line += " | Failed" + (" " * (length[2] - 6)) - line += " | Total" + (" " * (length[3] - 5)) + " |" - print line - print sep_line - - # Print the overview results for each module - for mod in test_modules: - line = " | " + str(mod).strip() + (" " * (length[0] - len(str(mod).strip()))) - line += " | " + str(mod.time) + (" " * (length[1] - len(str(mod.time)))) - line += " | " + str(mod.nb_failed) + (" " * (length[2] - len(str(mod.nb_failed)))) - line += " | " + str(mod.nb_test) + (" " * (length[3] - len(str(mod.nb_test)))) + " |" - print line - - print sep_line diff --git a/vulture_os/testing/core/setup_module.py b/vulture_os/testing/core/setup_module.py deleted file mode 100644 index 6bc9e7f34..000000000 --- a/vulture_os/testing/core/setup_module.py +++ /dev/null @@ -1,19 +0,0 @@ -class SetupModule(object): - - def __init__(self): - pass - - def __str__(self): - return "SetupModule" - - def set_up(self): - """ - This method is executed before every test of the module - """ - pass - - def tear_down(self): - """ - This method is executed after every test of the module - """ - pass diff --git a/vulture_os/testing/core/testing_module.py b/vulture_os/testing/core/testing_module.py deleted file mode 100644 index 863467674..000000000 --- a/vulture_os/testing/core/testing_module.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys - -sys.path.append("/home/vlt-gui/vulture") - -import logging -import logging.config - -logger = logging.getLogger('diagnostic') - - -class TestingModule(object): - - def __init__(self): - self.nb_test = 0 - self.nb_failed = 0 - self.time = 0 - self.log_level = 'error' - self.buffer = "" - self.setup_modules = [] - - def __str__(self): - return "TestingModule" - - def test_failure_handler(self, test_name, exception, traceback_str, output_level, save, node_name, summary): - """ - Had the information about the test failed in the print buffer - :param test_name: Name of the failed test - :param exception: The AssertException raised by the test - :param traceback_str: The traceback text as a string - :param output_level: The verbosity of the tests output - :param save: Boolean determining if the logging is activated - :param node_name: The name of the node the tests are executed on - :param summary: A dict summarizing the tests results - """ - - self.nb_failed += 1 - summary[str(self)][test_name] = False - summary['global_status'] = False - if save: - level_logger = getattr(logger, self.log_level, logger.error) - level_logger(str(exception), - extra={ - 'test_module': str(self), - 'test_name': test_name, - 'traceback': traceback_str, - 'node_name': node_name - }) - - if output_level != 'overview' and output_level != 'no': - self.buffer += "{} : \"{}\"\n".format(test_name.strip(), exception) - - try: - self.buffer += "DESCRIPTION: '" + getattr(self, test_name).__doc__.strip() + "'\n" - except AttributeError: - self.buffer += "DESCRIPTION: '" + test_name + "'\n" - - if output_level == 'detailed' or output_level == 'full': - self.buffer += "{}".format(traceback_str) - self.buffer += "--------------------------------------------------\n\n" - - @staticmethod - def set_up(): - """ - This method is executed before every test of the module - """ - pass - - @staticmethod - def tear_down(): - """ - This method is executed after every test of the module - """ - pass diff --git a/vulture_os/testing/doc/getting_started b/vulture_os/testing/doc/getting_started deleted file mode 100644 index 9aaccbb5e..000000000 --- a/vulture_os/testing/doc/getting_started +++ /dev/null @@ -1,42 +0,0 @@ -### GETTING STARTED IN VULTURE'S AUTOMATED TESTS DEVELOPMENT ### - -I - Keep it simple - -The testing mechanism is simple: you create a module, containing unitary tests, that will test one Vulture function. -You can also create a setup and a cleanup module if you need a special test environment. -Finally, you just call the tests you want using a simple configuration file : - -_________________________________________________________________________________________ - -###################################### -# DEMO # -###################################### - -# Setup modules used before any test is launched, executed one time befoer testing -SETUP_MODULES_DIRECTORY = 'demo.setup_modules' - -SETUP_MODULES = [ - 'setup_module', - 'setup_fail', -] - - -# Cleanup modules used after all tests are finished, executed one time after all the tests -CLEANUP_MODULES_DIRECTORY = 'demo.cleanup_modules' - -CLEANUP_MODULES = [ - 'cleanup_module', - 'cleanup_fail', -] - - -# Testing modules are the tests you want to execute, every test is executed one time -TESTING_MODULES_DIRECTORY = 'demo.testing_modules' - -TESTING_MODULES = [ - 'test_module', - 'test_module2', - 'test_fails_modules', -] - -_________________________________________________________________________________________ diff --git a/vulture_os/testing/doc/setup&cleanup_modules.txt b/vulture_os/testing/doc/setup&cleanup_modules.txt deleted file mode 100644 index 04cf478b4..000000000 --- a/vulture_os/testing/doc/setup&cleanup_modules.txt +++ /dev/null @@ -1,23 +0,0 @@ -### BASIC UNDERSTANDING OF SETUP AND CLEANUP MODULES ### - -I - WTF are those ? - -Those modules are respectively executed before and after all the tests. -Remember, as every malloc() involve a free(), a setup involve a cleanup! -If a setup method fail, the associate testing modules and cleanup methods will not be executed. -There the set_up and tear_down methods are available as well. - - -II - Creating a cleanup / setup couple - -WARNING : The name of your modules and methods are important. There is a syntax to follow ! - -The MODULE'S NAME: - Defined in the __str__() method, the name of the module should be in the following format: - - For setup: "setup_ModuleName" - - For cleanup: "cleanup_ModuleName" - -The Methods: - The corresponding setup and cleanup methods should have the exact same name ! - For example, if I have a method named "yolo_file" which create a file named "yolo" in your setup module, you should have - a "yolo_file" method which remove the file named "yolo" in the corresponding cleanup module. \ No newline at end of file diff --git a/vulture_os/testing/doc/testing_modules.txt b/vulture_os/testing/doc/testing_modules.txt deleted file mode 100644 index ed8623f4b..000000000 --- a/vulture_os/testing/doc/testing_modules.txt +++ /dev/null @@ -1,48 +0,0 @@ -### BASIC UNDERSTANDING OF TESTING MODULES ### - -I - Creation of a basic testing module - -This part is really simple: - - First, import the TestingModule class ("from testing.core.testing_module import TestingModule") - - Then create a class named "Module" in your file which inherit from TestingModule ("class Module(TestingModule):") - - You can now name your test module using the __str__ method - - Finally, create a new method for each test (Don't forget the python doc to have a description of your methods ;) - - - -II - More advanced modules - -A - The corresponding setup / cleanup modules - - Simply define an __init__ method as follow and fill the self.setup list with the name of the setup modules: - -example: - -def __init__(self): - super(Module, self).__init__() - self.setup = [ - 'setup_Test', - ] - - -B - Defining the importance of a testing module - - The importance of a testing module is defined by the level of the logger used in error case. - The default logger for a failed test is error, for a failure out of testing scope the logger is critical and for a -successful test module execution (no error at all), the logger is info. - That's why we advice to use the 'warning', 'error'(default) or 'critical' loggers. - - To define the importance of the module, define an __init__ method as follow and fill the self.log_level: - -example: - -def __init__(self): - super(Module, self).__init__() - self.log_level = 'warning' - - -C - The set_up and tear_down methods - - The set_up() method is called before every testing method of the module. - The tear_down() method is called after every testing method of the module. - By default, these methods are empty. diff --git a/vulture_os/testing/doc/tips_n_tricks/change_log_level_for_specific_test.txt b/vulture_os/testing/doc/tips_n_tricks/change_log_level_for_specific_test.txt deleted file mode 100644 index 42adc7d61..000000000 --- a/vulture_os/testing/doc/tips_n_tricks/change_log_level_for_specific_test.txt +++ /dev/null @@ -1,9 +0,0 @@ -### HOW TO SET A DIFFERENT LOG LEVEL FOR A SPECIFIC TEST OF YOUR MODULE - - -This is a bit of a trick, but it shows that with some reflection you can do whatever you want. ;) - -1 - Set the 'self.log_level' at the beginning of your function. -2 - Use the tearDown function to reset the 'self.log_level' to the value you want for the rest of your testing module. - -That's it ! XD diff --git a/vulture_os/testing/modules/__init__.py b/vulture_os/testing/modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/modules/cleanup_modules/__init__.py b/vulture_os/testing/modules/cleanup_modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/modules/setup_modules/__init__.py b/vulture_os/testing/modules/setup_modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/modules/testing_modules/__init__.py b/vulture_os/testing/modules/testing_modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/modules/utils/__init__.py b/vulture_os/testing/modules/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vulture_os/testing/testing_client.py b/vulture_os/testing/testing_client.py deleted file mode 100644 index 371558739..000000000 --- a/vulture_os/testing/testing_client.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/home/vlt-gui/env/bin/python - -import json -import sys - -from config import (CLEANUP_MODULES, CLEANUP_MODULES_DIRECTORY, SETUP_MODULES, SETUP_MODULES_DIRECTORY, TESTING_MODULES, - TESTING_MODULES_DIRECTORY) -from core.command_parse import command_parse -from core.modules_utils import launch_testing, launch_setup, launch_cleanup, get_methods, get_modules_list -from core.print_results import print_detailed_results, print_summarized_results - -sys.path.append("/home/vlt-gui/vulture") - -from gui.models.system_settings import Cluster -from vulture_toolkit.log.settings import LOG_SETTINGS, LOG_SETTINGS_FALLBACK - -import logging - -try: - logging.config.dictConfig(LOG_SETTINGS) -except: - logging.config.dictConfig(LOG_SETTINGS_FALLBACK) - - -logger = logging.getLogger('diagnostic') - - -if __name__ == '__main__': - - parsing_result = command_parse() - node = Cluster.get_current_node() - node_name = node.name - - summary = { - 'global_status': True - } - - with get_modules_list(SETUP_MODULES, SETUP_MODULES_DIRECTORY) as modules: - setup_modules = get_methods(modules) - failed_setup = launch_setup(setup_modules, parsing_result.save, node_name, summary) - - with get_modules_list(TESTING_MODULES, TESTING_MODULES_DIRECTORY) as modules: - test_modules = get_methods(modules) - launch_testing(test_modules, parsing_result.output_level, parsing_result.save, failed_setup, node_name, summary) - - with get_modules_list(CLEANUP_MODULES, CLEANUP_MODULES_DIRECTORY) as modules: - cleanup_modules = get_methods(modules) - launch_cleanup(cleanup_modules, failed_setup, parsing_result.save, node_name, summary) - - if parsing_result.save: - try: - node.diagnostic = json.dumps(summary) - except: - node.diagnostic = "ERROR: Unable to store diagnostic summary properly" - - """ Don't dispatch signal to not affect performance """ - node.save(bootstrap=True) - - if parsing_result.output_level != 'no': - print_summarized_results(test_modules) - if parsing_result.output_level != 'overview': - print_detailed_results(test_modules) diff --git a/vulture_os/testing/__init__.py b/vulture_os/workflow/tests/__init__.py similarity index 100% rename from vulture_os/testing/__init__.py rename to vulture_os/workflow/tests/__init__.py diff --git a/vulture_os/workflow/tests/test_models.py b/vulture_os/workflow/tests/test_models.py new file mode 100644 index 000000000..e1b26dac4 --- /dev/null +++ b/vulture_os/workflow/tests/test_models.py @@ -0,0 +1,263 @@ +#!/home/vlt-os/env/bin/python +"""This file is part of Vulture OS. + +Vulture OS is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Vulture OS is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Vulture OS. If not, see http://www.gnu.org/licenses/. +""" + +__author__ = "Théo BERTIN" +__credits__ = [] +__license__ = "GPLv3" +__version__ = "4.0.0" +__maintainer__ = "Vulture OS" +__email__ = "contact@vultureproject.org" +__doc__ = 'Tests for Workflows' + +from django.test import TestCase +from django.utils.crypto import get_random_string +from workflow.models import Workflow + +from uuid import uuid4 + +class WorkflowTestCase(TestCase): + TEST_CASE_NAME=f"{__name__}" + def setUp(self): + from services.frontend.models import Frontend, Listener + from system.cluster.models import NetworkAddress, NetworkInterfaceCard, NetworkAddressNIC + from applications.backend.models import Backend, Server + from authentication.user_portal.form import UserAuthenticationForm + from authentication.openid.form import OpenIDRepositoryForm + from system.pki.models import X509Certificate, TLSProfile + from system.cluster.models import Node + from toolkit.system.x509 import mk_ca_cert_files, mk_signed_cert_files + + self.node = Node.objects.create( + name=f"node_test_{self.TEST_CASE_NAME}", + ) + self.nic = NetworkInterfaceCard.objects.create( + dev = "vtnet0", + node=self.node, + ) + self.netaddr = NetworkAddress.objects.create( + name=f"network_address_test_{self.TEST_CASE_NAME}", + type="alias", + ip="127.127.127.127", + prefix_or_netmask="24", + ) + NetworkAddressNIC.objects.create( + nic=self.nic, + network_address=self.netaddr, + ) + + cacert_pem, cakey_pem = mk_ca_cert_files( + "Vulture_PKI_" + get_random_string(16, 'abcdef0123456789'), + "FR", + "59", + "Lille", + "VultureProject.ORG", + "Internal PKI" + ) + cert_pem, key_pem = mk_signed_cert_files( + "vulture-test", + "FR", + "59", + "Lille", + "VultureProject.ORG", + "Internal PKI", + 2, + cacert_pem, + cakey_pem + ) + + self.node_cert = X509Certificate.objects.create( + is_vulture_ca=False, + is_ca=False, + is_external=False, + status='V', + cert=cert_pem.decode(), + key=key_pem.decode(), + chain=cacert_pem.decode(), + serial=1 + ) + self.tls_profile = TLSProfile.objects.create( + name=f"tlp_profile_test_{self.TEST_CASE_NAME}", + x509_certificate=self.node_cert, + + ) + self.frontend_tcp = Frontend.objects.create( + name=f"frontend_tcp_test_{self.TEST_CASE_NAME}", + mode="tcp", + ) + self.frontend_http = Frontend.objects.create( + name=f"frontend_http_test_{self.TEST_CASE_NAME}", + mode="http", + ) + self.listener_tcp = Listener.objects.create( + network_address=self.netaddr, + port=1234, + frontend=self.frontend_tcp + ) + self.listener_http = Listener.objects.create( + network_address=self.netaddr, + port=80, + frontend=self.frontend_http, + ) + self.listener_https = Listener.objects.create( + network_address=self.netaddr, + port=443, + frontend=self.frontend_http, + ) + self.backend_tcp = Backend.objects.create( + name=f"backend_tcp_test_{self.TEST_CASE_NAME}", + mode="tcp", + ) + self.backend_http = Backend.objects.create( + name=f"backend_http_test_{self.TEST_CASE_NAME}", + mode="http", + ) + self.serv_tcp = Server.objects.create( + mode="net", + target="10.10.10.10", + port=1234, + backend=self.backend_tcp + ) + self.serv_http = Server.objects.create( + mode="net", + target="10.10.10.10", + port=443, + backend=self.backend_http, + tls_profile=self.tls_profile, + ) + + # Don't + repo_form = OpenIDRepositoryForm({ + 'name': f"openid_test_{self.TEST_CASE_NAME}", + 'provider': 'openid', + 'provider_url': 'https://openid.test.fr', + 'client_id': uuid4(), + 'client_secret': uuid4(), + 'scopes': ['openid'] + }) + self.openid = repo_form.save() + + # Need to create through Form for djongo's f***ing ArrayReferenceField to work! + portal_form = UserAuthenticationForm({ + 'name': f"portal_with_openid_test_{self.TEST_CASE_NAME}", + 'auth_type': 'form', + 'lookup_ldap_attr': 'cn', + 'lookup_claim_attr': 'username', + 'auth_timeout': 900, + 'disconnect_url': 'test/disconnect', + 'external_fqdn': 'test', + 'oauth_client_id': uuid4(), + 'oauth_client_secret': uuid4(), + 'oauth_redirect_uris': 'https://www.test.fr', + 'oauth_timeout': 600, + 'repositories': [self.openid.pk] + }) + self.portal = portal_form.save() + + +####### +# TCP # +####### + def test_create_workflow_tcp(self): + Workflow.objects.create( + name=f"workflow_tcp_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_tcp, + backend=self.backend_tcp, + ) + + def test_generate_workflow_tcp_conf(self): + workflow = Workflow.objects.create( + name=f"workflow_tcp_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_tcp, + backend=self.backend_tcp, + ) + self.assertIsNotNone(workflow) + self.assertIsNotNone(workflow.generate_conf()) + +######## +# HTTP # +######## + def test_create_workflow_http(self): + Workflow.objects.create( + name=f"workflow_http_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + ) + + def test_generate_workflow_http_filename(self): + workflow = Workflow.objects.create( + name=f"workflow_http_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + ) + self.assertIsNotNone(workflow) + self.assertIsInstance(workflow.get_filename(), str) + self.assertIsNot(workflow.get_filename(), "") + + def test_generate_workflow_http_conf(self): + workflow = Workflow.objects.create( + name=f"workflow_http_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + ) + self.assertIsNotNone(workflow) + self.assertIsNotNone(workflow.generate_conf()) + + +######################### +# HTTP + AUTHENTICATION # +######################### + def test_create_workflow_http_auth(self): + workflow = Workflow.objects.create( + name=f"workflow_http_auth_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + authentication=self.portal, + ) + self.assertIsNotNone(workflow.authentication) + + def test_generate_workflow_http_auth_filename(self): + workflow = Workflow.objects.create( + name=f"workflow_http_auth_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + authentication=self.portal, + ) + self.assertIsNotNone(workflow) + self.assertIsInstance(workflow.get_filename(), str) + self.assertIsNot(workflow.get_filename(), "") + + def test_generate_workflow_http_auth_conf(self): + workflow = Workflow.objects.create( + name=f"workflow_http_auth_test_{self.TEST_CASE_NAME}", + frontend=self.frontend_http, + backend=self.backend_http, + fqdn="workflow.vulture.test", + public_dir="/", + authentication=self.portal, + ) + self.assertIsNotNone(workflow) + self.assertIsNotNone(workflow.to_template())