feat(testing): Variabilize settings and implement django tests

CHANGELOG

@@ -6,12 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 ## [Unreleased]
+### Added
+- [SYSTEM] [SETTINGS] Allow to configure project with django-environ
+- [TESTS] [SYSTEM] Implement Network test cases
+- [TESTS] [WORKFLOW] Implement Workflow test cases
 ### Removed
 - [API_PARSER] [CORTEX_XDR] Remove fields used for backward compatibility
+- [TESTS] Remove old 'testing' folder
 ### Changed
 - [API_PARSER] [NETSKOPE] Add the possiblity to retrieve 3 new kinds of logs
+- [SYSTEM] [SETTINGS] Use Hostname value from Django settings
 ### Fixed
+- [SYSTEM] [MONGODB] Always refer to settings to connect to mongodb
 - [API_PARSER] [NETSKOPE] Remove duplicated logs
+- [SYSTEM] [PATHS] Variabilize all system paths and allow to configure them from settings
 
 
 ## [2.19.0] - 2025-01-16

home/jails.apache/.zfs-source/home/vlt-os/scripts/pki.py

@@ -66,10 +66,14 @@
             pki["organizational_unit"]
         )
 
+    with open("/var/db/pki/ca.pem", "wb") as cert_file:
+        cert_file.write(cacert_pem)
+    with open("/var/db/pki/ca.key", "wb") as key_file:
+        key_file.write(cakey_pem)
+
     """ Build node certificate (overwrite if it exist) """
     hostname = subprocess.check_output(['hostname']).strip().decode('utf-8')
-    # TODO give ca_cert and ca_key
-    _, _ = mk_signed_cert_files(
+    cert_pem, key_pem = mk_signed_cert_files(
         hostname,
         pki["country"],
         pki["state"],
@@ -81,6 +85,15 @@
         cakey_pem
     )
 
+    node_pem = cert_pem + key_pem
+
+    with open("/var/db/pki/node.pem", "wb") as node_file:
+        node_file.write(node_pem)
+    with open("/var/db/pki/node.cert", "wb") as cert_file:
+        cert_file.write(cert_pem)
+    with open("/var/db/pki/node.key", "wb") as key_file:
+        key_file.write(key_pem)
+
     """ Generate Diffie hellman configuration """
     os.system("openssl dhparam -out /var/db/pki/dh2048.pem 2048")
 

requirements.in

@@ -4,6 +4,7 @@ pymongo
 jinja2
 iptools
 django-crontab
+django-environ
 requests
 pyOpenSSL
 redis~=4.5

requirements.txt

@@ -330,6 +330,10 @@ django==4.2.11 \
 django-crontab==0.7.1 \
     --hash=sha256:1201810a212460aaaa48eb6a766738740daf42c1a4f6aafecfb1525036929236
     # via -r requirements.in
+django-environ==0.11.2 \
+    --hash=sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05 \
+    --hash=sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be
+    # via -r requirements.in
 django-jsoneditor==0.2.4 \
     --hash=sha256:1d3dfca28f047feefa6ebc6f9541179eb815fb459b006faf3fb8d0fb2197d2df \
     --hash=sha256:d7a639a7251e376126b5be64ea588c925c7a40d45e0e212f66ef475d2f0f90bb

vulture_os/applications/backend/models.py

@@ -39,6 +39,7 @@
 
 # Extern modules imports
 from jinja2 import Environment, FileSystemLoader
+from os.path import join as path_join
 
 # Required exceptions imports
 from jinja2.exceptions import (TemplateAssertionError, TemplateNotFound, TemplatesNotFound, TemplateRuntimeError,
@@ -118,13 +119,13 @@
 )
 
 # Jinja template for backends rendering
-JINJA_PATH = "/home/vlt-os/vulture_os/applications/backend/config/"
+JINJA_PATH = path_join(settings.BASE_DIR, "applications/backend/config/")
 JINJA_TEMPLATE = "haproxy_backend.conf"
 
 BACKEND_OWNER = HAPROXY_OWNER
 BACKEND_PERMS = HAPROXY_PERMS
 
-UNIX_SOCKET_PATH = "/var/sockets/rsyslog"
+UNIX_SOCKET_PATH = path_join(settings.SOCKETS_PATH, "rsyslog")
 
 
 class Backend(models.Model):

vulture_os/applications/logfwd/models.py

@@ -35,6 +35,7 @@
 
 # Extern modules imports
 from jinja2 import Environment, FileSystemLoader
+from os.path import join as path_join
 import pymongo
 import hashlib
 
@@ -81,8 +82,7 @@
     ('stream', "Stream mode, using xadd"),
 )
 
-CONF_PATH = "/usr/local/etc/rsyslog.d/10-applications.conf"
-JINJA_PATH = "/home/vlt-os/vulture_os/applications/logfwd/config/"
+JINJA_PATH = path_join(settings.BASE_DIR, "applications/logfwd/config/")
 
 
 class LogOM (models.Model):
@@ -256,7 +256,7 @@
             subclass_obj = self.logomrelp
         elif hasattr(self, 'logomkafka'):
             subclass_obj = self.logomkafka
         elif hasattr(self, "logom_ptr") and type(self.logom_ptr) != LogOM:
             subclass_obj = self.logom_ptr
         else:
             raise Exception(f"Cannot find type of LogOM named '{self.name}' !")
@@ -823,7 +823,7 @@
         """  returns the attributes of the class """
         template = super().to_template(**kwargs)
         tpl = Template(self.key)
         key = tpl.render(Context({'ruleset': kwargs.get('ruleset')}))
         template.update({
             'broker': self.broker,
             'topic': self.topic,

vulture_os/applications/reputation_ctx/models.py

@@ -34,6 +34,7 @@
 
 # Extern modules imports
 from gzip import decompress as gzip_decompress
+from os.path import join as path_join
 import requests
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 from re import compile as re_compile
@@ -75,7 +76,7 @@
     'digest': HTTPDigestAuth
 }
 
-DATABASES_PATH = "/var/db/darwin"
+DATABASES_PATH = path_join(settings.DBS_PATH, "darwin")
 DATABASES_OWNER = "vlt-os:vlt-conf"
 DATABASES_PERMS = "644"
 

vulture_os/authentication/openid/models.py

@@ -32,6 +32,7 @@
 # Django project imports
 from authentication.base_repository import BaseRepository
 from authentication.user_scope.models import UserScope
+from os.path import join as path_join
 from system.exceptions import VultureSystemConfigError
 from system.pki.models import CERT_OWNER, CERT_PERMS
 from toolkit.auth.authy_client import AuthyClient
@@ -253,7 +254,7 @@ def to_html_template(self):
         }
 
     def get_jwt_key_filename(self):
-        return f"/var/db/pki/openid-{self.pk}.pub"
+        return path_join(settings.DBS_PATH, "pki/openid-{self.pk}.pub")
 
     @staticmethod
     def jwt_validate_with_certificate(jwt_signature_type):

vulture_os/authentication/portal_template/models.py

@@ -34,6 +34,7 @@
 from system.config.models import write_conf
 
 # Extern modules imports
+from os.path import join as path_join
 
 # Required exceptions imports
 
@@ -308,7 +309,7 @@
                 continue
 
             try:
-                with open("/usr/local/etc/haproxy.d/templates/portal_%s_%s.conf" % (str(self.id), tpl),
+                with open(path_join(settings.LOCALETC_PATH, "haproxy.d/templates/portal_%s_%s.conf" % (str(self.id), tpl)),
                           'w') as f:
                     html = getattr(self, tpl)
                     if tpl not in ["email_subject", "email_body", "email_register_subject", "email_register_body"]:
@@ -417,7 +418,7 @@
         """
         try:
             url = 'portal_statics/{}'.format(self.uid)
         except:
             return None
 
         return url

vulture_os/authentication/user_portal/models.py

@@ -49,6 +49,7 @@
 
 # Extern modules imports
 from jinja2 import Environment, FileSystemLoader
+from os.path import join as path_join
 
 # Required exceptions imports
 from jinja2.exceptions import (TemplateAssertionError, TemplateNotFound, TemplatesNotFound, TemplateRuntimeError,
@@ -61,7 +62,7 @@
 logging.config.dictConfig(settings.LOG_SETTINGS)
 logger = logging.getLogger('gui')
 
-JINJA_PATH = "/home/vlt-os/vulture_os/authentication/user_portal/config/"
+JINJA_PATH = path_join(settings.BASE_DIR, "authentication/user_portal/config/")
 JINJA_TEMPLATE = "haproxy_portal.conf"
 
 

vulture_os/darwin/access_control/models.py

@@ -34,6 +34,7 @@
 from bson import ObjectId
 from hashlib import sha1
 import logging
+from os.path import join as path_join
 import json
 
 logging.config.dictConfig(settings.LOG_SETTINGS)
@@ -114,7 +115,7 @@
 ]
 
 # Jinja template for backends rendering
-JINJA_PATH = "/home/vlt-os/vulture_os/darwin/access_control/config"
+JINJA_PATH = path_join(settings.BASE_DIR, "darwin/access_control/config")
 JINJA_TEST_TEMPLATE = "haproxy_test.conf"
 
 

vulture_os/darwin/policy/models.py

@@ -44,24 +44,24 @@
 logging.config.dictConfig(settings.LOG_SETTINGS)
 logger = logging.getLogger('gui')
 
-JINJA_PATH = "/home/vlt-os/vulture_os/darwin/log_viewer/config/"
+JINJA_PATH = os_path.join(settings.BASE_DIR, "darwin/log_viewer/config/")
 
 
-SOCKETS_PATH = "/var/sockets/darwin"
-FILTERS_PATH = "/home/darwin/filters"
-CONF_PATH = "/home/darwin/conf"
-DATA_PATH = "/var/db/darwin"
+SOCKETS_PATH = os_path.join(settings.SOCKETS_PATH, "darwin")
+FILTERS_PATH = os_path.join(settings.HOMES_PATH, "darwin/filters")
+CONF_PATH = os_path.join(settings.HOMES_PATH, "darwin/conf")
+DATA_PATH = os_path.join(settings.DBS_PATH, "darwin")
 TEMPLATE_OWNER = "darwin:vlt-web"
 TEMPLATE_PERMS = "644"
 
-REDIS_SOCKET_PATH = "/var/sockets/redis/redis.sock"
+REDIS_SOCKET_PATH = os_path.join(settings.SOCKETS_PATH, "redis/redis.sock")
 ALERTS_REDIS_LIST_NAME = "darwin_alerts"
 ALERTS_REDIS_CHANNEL_NAME = "darwin.alerts"
-ALERTS_LOG_FILEPATH = "/var/log/darwin/alerts.log"
+ALERTS_LOG_FILEPATH = os_path.join(settings.LOGS_PATH,"darwin/alerts.log")
 
-DGA_MODELS_PATH = CONF_PATH + '/fdgad/'
-VAST_MODELS_PATH = CONF_PATH + '/fvast/'
-VAML_MODELS_PATH = CONF_PATH + '/fvaml/'
+DGA_MODELS_PATH = os_path.join(CONF_PATH, 'fdgad/')
+VAST_MODELS_PATH = os_path.join(CONF_PATH, 'fvast/')
+VAML_MODELS_PATH = os_path.join(CONF_PATH, 'fvaml/')
 
 DARWIN_LOGLEVEL_CHOICES = (
     ('CRITICAL', 'Critical'),
@@ -777,6 +777,6 @@
     def str_attrs():
         return ['conf_path', 'nb_thread', 'log_level', 'config']
 
     def __str__(self):
         return "[{}] {}".format(self.policy, self.name)
 

vulture_os/gui/context_processors.py

@@ -33,7 +33,6 @@
 from services.service import Service
 from system.cluster.models import Cluster
 from system.system import System
-from toolkit.network.network import get_hostname
 from workflow.workflow import Workflows
 
 import logging
@@ -65,7 +64,7 @@ def admin_media(request):
         'VERSION': settings.VERSION,
         'CURRENT_NODE': node_name,
         'DEV_MODE': settings.DEV_MODE,
-        'TITLE': get_hostname(),
+        'TITLE': settings.HOSTNAME,
         'COLLAPSE': request.session.get('collapse')
     }
 

vulture_os/gui/crontab/feed.py

@@ -32,13 +32,13 @@
 import django
 from django.conf import settings
 django.setup()
 from django.utils.crypto import get_random_string
 
 from django.utils.timezone import now as timezone_now
 from gui.models.rss import RSS
-from toolkit.network.network import get_hostname, get_proxy
+from toolkit.network.network import get_proxy
 from applications.reputation_ctx.models import ReputationContext
 from services.rsyslogd.rsyslog import restart_service as restart_rsyslog_service
 from system.exceptions import VultureSystemError
 
 import subprocess
@@ -109,13 +109,13 @@
             logger.info("Crontab::security_update: No vulnerability found.")
         elif "is vulnerable" in res:
             logger.info("Crontab::security_update: Security problem found : {}".format(res))
-            security_alert("Security problem found on node {}".format(get_hostname()), "danger", res)
+            security_alert("Security problem found on node {}".format(settings.HOSTNAME), "danger", res)
     except subprocess.CalledProcessError as e:
         if e.stdout.decode("utf-8").startswith("0 problem"):
             logger.info("Crontab::security_update: No vulnerability found.")
         elif "is vulnerable" in e.stdout.decode("utf-8"):
             logger.info("Crontab::security_update: Security problem found : {}".format(e.stdout.decode('utf-8')))
-            security_alert("Security problem found on node {}".format(get_hostname()), "danger",
+            security_alert("Security problem found on node {}".format(settings.HOSTNAME), "danger",
                            e.stdout.decode("utf-8"))
         else:
             logger.error("Crontab::security_update: Failed to retrieve vulnerabilities : "

vulture_os/gui/crontab/pki.py

@@ -28,6 +28,9 @@
 import subprocess
 import os.path
 
+from django.conf import settings
+
+CONF_PATH = os.path.join(settings.DBS_PATH, "acme")
 
 def update_crl():
     """
@@ -44,7 +47,7 @@ def acme_update():
     """
     :return: Run acme.sh to automatically renew Let's encrypt certificates
     """
-    subprocess.check_output(["/usr/local/sbin/acme.sh", "--cron", "--home", "/var/db/acme/.acme.sh"])
+    subprocess.check_output(["/usr/local/sbin/acme.sh", "--cron", "--home", os.path.join(CONF_PATH, ".acme.sh")])
 
     """ Now update certificate database"""
     need_restart = False
@@ -53,8 +56,8 @@ def acme_update():
         subject = crypto_cert.subject
         common_name_obj = subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0]
         cn = common_name_obj.value
-        if os.path.isfile("/var/db/acme/.acme.sh/{}/{}.cer".format(cn, cn)):
-            with open("/var/db/acme/.acme.sh/{}/{}.cer".format(cn, cn)) as file_cert:
+        if os.path.isfile(os.path.join(CONF_PATH, ".acme.sh/{}/{}.cer".format(cn, cn))):
+            with open(os.path.join(CONF_PATH, ".acme.sh/{}/{}.cer".format(cn, cn))) as file_cert:
                 pem_cert = file_cert.read()
                 cert.cert = pem_cert
                 cert.save()