P.O.C: Manage Django Settings with Dynaconf

awx/__init__.py

@@ -62,7 +62,8 @@ def version_file():
 
 def prepare_env():
     # Update the default settings environment variable based on current mode.
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings')
+    os.environ.setdefault('AWX_MODE', MODE)
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/api/generics.py

@@ -161,6 +161,7 @@ def get_view_description(view, html=False):
 
 
 def get_default_schema():
+    # TODO: get current_env
     if settings.SETTINGS_MODULE == 'awx.settings.development':
         from awx.api.swagger import schema_view
 

awx/api/views/root.py

@@ -66,6 +66,7 @@ class ApiVersionRootView(APIView):
     def get(self, request, format=None):
         '''List top level resources'''
         data = OrderedDict()
+        data["foo"] = settings.FOO
         data['ping'] = reverse('api:api_v2_ping_view', request=request)
         data['instances'] = reverse('api:instance_list', request=request)
         data['instance_groups'] = reverse('api:instance_group_list', request=request)

awx/main/tests/settings_for_test.py

@@ -1,6 +1,8 @@
 # Python
 import uuid
 
+# TODO: move to settings test
+
 # Load development settings for base variables.
 from awx.settings.development import *  # NOQA
 

awx/settings/__init__.py

@@ -1,2 +1,87 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
+import os
+from ansible_base.lib.dynamic_config import factory, export, load_standard_settings_files
+from .application_name import merge_application_name
+from dynaconf import Validator
+from dynaconf.loaders import env_loader
+
+
+DYNACONF = factory(
+    "AWX",
+    environments=("development", "production", "quiet", "kube"),
+    settings_files=["defaults.py"],
+)
+DYNACONF.validators.register(Validator("FOO", required=True, gt=99))
+
+# Store snapshot before loading any custom config file
+DEFAULTS_SNAPSHOT = {}
+if DYNACONF.is_development_mode:
+    DYNACONF.set(
+        "DEFAULTS_SNAPSHOT",
+        DYNACONF.as_dict(internal=False),  # should use deepcopy here?
+        loader_identifier="awx.settings:DEFAULTS_SNAPSHOT",
+    )
+###############################################################################################
+#
+#  Any settings loaded after this point will be marked as as a read_only database setting
+#
+################################################################################################
+
+# Load extra config
+# Load settings from any .py files in the global conf.d directory specified in
+# the environment, defaulting to /etc/tower/conf.d/.
+settings_dir = os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/')
+settings_files_path = os.path.join(settings_dir, '*.py')
+# Load remaining settings from the global settings file specified in the
+# environment, defaulting to /etc/tower/settings.py.
+settings_file_path = os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py')
+# Attempt to load settings from /etc/tower/settings.py first, followed by
+# /etc/tower/conf.d/*.py.
+DYNACONF.load_file(settings_file_path)
+DYNACONF.load_file(settings_files_path)
+
+if DYNACONF.get_environ("AWX_KUBE_DEVEL"):
+    DYNACONF.load_file("kube_defaults.py")
+else:
+    DYNACONF.load_file("local_*.py")
+
+# Load new standard settings files from /etc/ansible-automation-platform/config/awx/
+load_standard_settings_files(DYNACONF)
+
+# Check at least one required setting file has been loaded
+# NOTE: This potentially could be moved to a validator
+if "production" in DYNACONF.current_env.lower():
+    required_settings_paths = [
+        os.path.dirname(settings_file_path),
+        "/etc/ansible-automation-platform/",
+    ]
+    # check if at least one file has been loaded any of the required paths
+    # use DYNACONF._loaded_files to check any filename inside the paths
+    # if not loaded then raise an ImproperlyConfigured error
+    for path in required_settings_paths:
+        if any(path in f for f in DYNACONF._loaded_files):
+            break
+    else:
+        from django.core.exceptions import ImproperlyConfigured
+
+        msg = 'No AWX configuration found at %s.' % required_settings_paths
+        msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
+        msg += 'specify an alternate path.'
+        raise ImproperlyConfigured(msg)
+
+# Load envvars at the end to allow them to override everything
+env_loader.load(DYNACONF, identifier="awx.settings")
+
+# This must run after all custom settings are imported
+DYNACONF.update(
+    merge_application_name(DYNACONF),
+    loader_identifier="awx.settings:merge_application_name",
+    merge=True,
+)
+
+# Update django.conf.settings with DYNACONF keys.
+export(DYNACONF)
+
+# Validate the settings according to the validators registered
+DYNACONF.validators.validate()

awx/settings/application_name.py

@@ -25,6 +25,7 @@ def get_application_name(CLUSTER_HOST_ID, function=''):
 
 
 def set_application_name(DATABASES, CLUSTER_HOST_ID, function=''):
+    """In place modification of DATABASES to set the application name for the connection."""
     # If settings files were not properly passed DATABASES could be {} at which point we don't need to set the app name.
     if not DATABASES or 'default' not in DATABASES:
         return
@@ -33,3 +34,11 @@ def set_application_name(DATABASES, CLUSTER_HOST_ID, function=''):
         return
     options_dict = DATABASES['default'].setdefault('OPTIONS', dict())
     options_dict['application_name'] = get_application_name(CLUSTER_HOST_ID, function)
+
+
+def merge_application_name(settings):
+    """Return a dynaconf merge dict to set the application name for the connection."""
+    data = {}
+    if "sqlite3" not in settings.get("DATABASES__default__ENGINE", ""):
+        data["DATABASES__default__OPTIONS__application_name"] = get_application_name(settings.CLUSTER_HOST_ID)
+    return data

awx/settings/defaults.py

@@ -11,7 +11,7 @@
 
 from split_settings.tools import include
 
-
+FOO = 0
 DEBUG = True
 SQL_DEBUG = DEBUG
 
@@ -1012,6 +1012,7 @@
 ANSIBLE_BASE_RESOURCE_CONFIG_MODULE = 'awx.resource_api'
 ANSIBLE_BASE_PERMISSION_MODEL = 'main.Permission'
 
+# TODO: Move this to the settings object from DAB
 from ansible_base.lib import dynamic_config  # noqa: E402
 
 include(os.path.join(os.path.dirname(dynamic_config.__file__), 'dynamic_settings.py'))

awx/settings/development.py

@@ -1,121 +1,8 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved.
-
-# Development settings for AWX project.
-
-# Python
-import os
-import socket
-import copy
-import sys
-import traceback
-
-# Centos-7 doesn't include the svg mime type
-# /usr/lib64/python/mimetypes.py
-import mimetypes
-
-# Django Split Settings
-from split_settings.tools import optional, include
-
-# Load default settings.
-from .defaults import *  # NOQA
-
-# awx-manage shell_plus --notebook
-NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
-
-# print SQL queries in shell_plus
-SHELL_PLUS_PRINT_SQL = False
-
-# show colored logs in the dev environment
-# to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
-COLOR_LOGS = True
-LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
-
-ALLOWED_HOSTS = ['*']
-
-mimetypes.add_type("image/svg+xml", ".svg", True)
-mimetypes.add_type("image/svg+xml", ".svgz", True)
-
-# Disallow sending session cookies over insecure connections
-SESSION_COOKIE_SECURE = False
-
-# Disallow sending csrf cookies over insecure connections
-CSRF_COOKIE_SECURE = False
-
-# Disable Pendo on the UI for development/test.
-# Note: This setting may be overridden by database settings.
-PENDO_TRACKING_STATE = "off"
-INSIGHTS_TRACKING_STATE = False
-
-# debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
-
-INSTALLED_APPS += ['drf_yasg', 'debug_toolbar']  # NOQA
-
-MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE  # NOQA
-
-DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
-
-# Configure a default UUID for development only.
-SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
-INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
-
-# Ansible base virtualenv paths and enablement
-# only used for deprecated fields and management commands for them
-BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
-
-CLUSTER_HOST_ID = socket.gethostname()
-
-AWX_CALLBACK_PROFILE = True
-
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-# Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
-# Allows user to trigger task managers directly for debugging and profiling purposes.
-# Only works in combination with settings.SETTINGS_MODULE == 'awx.settings.development'
-AWX_DISABLE_TASK_MANAGERS = False
-
-# Needed for launching runserver in debug mode
-# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
-
-# Store a snapshot of default settings at this point before loading any
-# customizable config files.
-this_module = sys.modules[__name__]
-local_vars = dir(this_module)
-DEFAULTS_SNAPSHOT = {}  # define after we save local_vars so we do not snapshot the snapshot
-for setting in local_vars:
-    if setting.isupper():
-        DEFAULTS_SNAPSHOT[setting] = copy.deepcopy(getattr(this_module, setting))
-
-del local_vars  # avoid temporary variables from showing up in dir(settings)
-del this_module
-#
-###############################################################################################
-#
-#  Any settings defined after this point will be marked as as a read_only database setting
-#
-################################################################################################
-
-# If there is an `/etc/tower/settings.py`, include it.
-# If there is a `/etc/tower/conf.d/*.py`, include them.
-include(optional('/etc/tower/settings.py'), scope=locals())
-include(optional('/etc/tower/conf.d/*.py'), scope=locals())
-
-# If any local_*.py files are present in awx/settings/, use them to override
-# default settings for development.  If not present, we can still run using
-# only the defaults.
-# this needs to stay at the bottom of this file
-try:
-    if os.getenv('AWX_KUBE_DEVEL', False):
-        include(optional('development_kube.py'), scope=locals())
-    else:
-        include(optional('local_*.py'), scope=locals())
-except ImportError:
-    traceback.print_exc()
-    sys.exit(1)
-
-# The below runs AFTER all of the custom settings are imported
-# because conf.d files will define DATABASES and this should modify that
-from .application_name import set_application_name
-
-set_application_name(DATABASES, CLUSTER_HOST_ID)  # NOQA
-
-del set_application_name
+# This file exists for backwards compatibility only
+# the current way of running AWX is to point settings to
+# awx/settings/__init__.py as the entry point for the settings
+# that is done by exporting: export DJANGO_SETTINGS_MODULE=awx.settings
+from ansible_base.lib.dynamic_config import export
+from . import DYNACONF  # noqa
+
+export(DYNACONF)

awx/settings/development_defaults.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+
+# Development settings for AWX project.
+
+# Python
+import os
+import socket
+
+# Centos-7 doesn't include the svg mime type
+# /usr/lib64/python/mimetypes.py
+import mimetypes
+
+# awx-manage shell_plus --notebook
+NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
+
+# print SQL queries in shell_plus
+SHELL_PLUS_PRINT_SQL = False
+
+# show colored logs in the dev environment
+# to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
+COLOR_LOGS = True
+LOGGING__handlers__console = '@merge {"()": "awx.main.utils.handlers.ColorHandler"}'
+
+ALLOWED_HOSTS = ['*']
+
+mimetypes.add_type("image/svg+xml", ".svg", True)
+mimetypes.add_type("image/svg+xml", ".svgz", True)
+
+# Disallow sending session cookies over insecure connections
+SESSION_COOKIE_SECURE = False
+
+# Disallow sending csrf cookies over insecure connections
+CSRF_COOKIE_SECURE = False
+
+# Disable Pendo on the UI for development/test.
+# Note: This setting may be overridden by database settings.
+PENDO_TRACKING_STATE = "off"
+INSIGHTS_TRACKING_STATE = False
+
+# debug toolbar and swagger assume that requirements/requirements_dev.txt are installed
+INSTALLED_APPS = "@merge drf_yasg,debug_toolbar"
+MIDDLEWARE = "@insert 0 debug_toolbar.middleware.DebugToolbarMiddleware"
+
+DEBUG_TOOLBAR_CONFIG = {'ENABLE_STACKTRACES': True}
+
+# Configure a default UUID for development only.
+SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
+INSTALL_UUID = '00000000-0000-0000-0000-000000000000'
+
+# Ansible base virtualenv paths and enablement
+# only used for deprecated fields and management commands for them
+BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")
+
+CLUSTER_HOST_ID = socket.gethostname()
+
+AWX_CALLBACK_PROFILE = True
+
+# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================
+# Disable normal scheduled/triggered task managers (DependencyManager, TaskManager, WorkflowManager).
+# Allows user to trigger task managers directly for debugging and profiling purposes.
+# Only works in combination with settings.SETTINGS_MODULE == 'awx.settings.development'
+AWX_DISABLE_TASK_MANAGERS = False
+
+# Needed for launching runserver in debug mode
+# ======================!!!!!!! FOR DEVELOPMENT ONLY !!!!!!!=================================