[NOTEST][WIP]initial commit with vault+sprout integration via dynaconf

sprout/appliances/models.py

@@ -34,7 +34,7 @@
 from cfme.utils.appliance import Appliance as CFMEAppliance, IPAppliance
 from cfme.utils.bz import Bugzilla
 from cfme.utils.conf import cfme_data
-from cfme.utils.providers import get_mgmt
+from sprout.appliances.utils.providers import get_mgmt
 from cfme.utils.timeutil import nice_seconds
 from cfme.utils.version import Version
 

sprout/appliances/utils/providers.py

@@ -0,0 +1,71 @@
+from collections import Mapping
+
+import six
+
+from cfme.common.provider import all_types
+from cfme.exceptions import UnknownProviderType
+from cfme.utils import conf
+from cfme.utils.log import logger
+from sprout.vault.vault import settings
+
+providers_data = conf.cfme_data.get("management_systems", {})
+
+PROVIDER_MGMT_CACHE = {}
+
+
+def get_class_from_type(prov_type):
+    try:
+        return all_types()[prov_type]
+    except KeyError:
+        raise UnknownProviderType("Unknown provider type: {}!".format(prov_type))
+
+
+def get_mgmt(provider_key, providers=None, credentials=None):
+    """ Provides a ``wrapanapi`` object, based on the request.
+
+    Args:
+        provider_key: The name of a provider, as supplied in the yaml configuration files.
+            You can also use the dictionary if you want to pass the provider data directly.
+        providers: A set of data in the same format as the ``management_systems`` section in the
+            configuration yamls. If ``None`` then the configuration is loaded from the default
+            locations. Expects a dict.
+        credentials: A set of credentials in the same format as the ``credentials`` yamls files.
+            If ``None`` then credentials are loaded from the vault using dynaconf. Expects a dict.
+    Return: A provider instance of the appropriate ``wrapanapi.WrapanapiAPIBase``
+        subclass
+    """
+    if providers is None:
+        providers = providers_data
+    # provider_key can also be provider_data for some reason
+    # TODO rename the parameter; might break things
+    if isinstance(provider_key, Mapping):
+        provider_data = provider_key
+        provider_key = provider_data['name']
+    else:
+        provider_data = providers[provider_key]
+
+    if credentials is None:
+        # create env matching provider_keys in vault to hold credentials
+        with settings.using_env(provider_key):
+            credentials = {key.lower(): val for key, val in settings.as_dict().items()
+            if 'VAULT' not in key}
+
+    # Munge together provider dict and creds,
+    # Let the provider do whatever they need with them
+    provider_kwargs = provider_data.copy()
+    provider_kwargs.update(credentials)
+
+    if not provider_kwargs.get('username') and provider_kwargs.get('principal'):
+        provider_kwargs['username'] = provider_kwargs['principal']
+        provider_kwargs['password'] = provider_kwargs['secret']
+
+    if isinstance(provider_key, six.string_types):
+        provider_kwargs['provider_key'] = provider_key
+    provider_kwargs['logger'] = logger
+
+    if provider_key not in PROVIDER_MGMT_CACHE:
+        mgmt_instance = get_class_from_type(provider_data['type']).mgmt_class(**provider_kwargs)
+        PROVIDER_MGMT_CACHE[provider_key] = mgmt_instance
+    else:
+        logger.debug("returning cached mgmt class for '%s'", provider_key)
+    return PROVIDER_MGMT_CACHE[provider_key]

sprout/appliances/views.py

@@ -26,7 +26,7 @@
 
 from sprout.log import create_logger
 from cfme.utils.bz import Bugzilla
-from cfme.utils.providers import get_mgmt
+from sprout.appliances.utils.providers import get_mgmt
 from cfme.utils.version import Version
 
 from wrapanapi import Openshift

sprout/requirements.txt

@@ -4,6 +4,7 @@ Django<1.10
 celery[redis]<3.2
 django-object-actions
 django-ipware
+dynaconf[vault]
 flower
 gunicorn
 pika

sprout/vault/approle_setup.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# This file briefly describes the steps and can execute those as well - to create Approle Token.
+# make sure to have exported VAULT_ADDR=https://addr:port
+# and VAULT_SKIP_VERIFY=true to disable ssl verification
+
+echo 'login with kerberos - make sure you are admin by reading listed policies'
+vault login -method=ldap -tls-skip-verify=true username=<user>
+
+# =======================================================================================
+# NOTE: we do not need to run this every time, but only when you need a new AppRole Token.
+echo 'write policy'
+vault policy write cfme-qe-infra-ro-policy cfme-qe-infra-ro-policy.json
+
+echo 'enable AppRole auth'
+vault auth enable approle
+
+echo 'create an AppRole called'
+vault write auth/approle/role/cfme-qe-infra secret_id_ttl=10m secret_id_num_uses=0 token_num_uses=20 token_ttl=30m token_max_ttl=60m policies=cfme-qe-infra-ro-policy
+
+echo 'Creating a Limited-Use Token'
+vault policy write cfme-qe-infra-approle-token cfme-qe-infra-approle-token.json
+vault token create -policy=cfme-qe-infra-approle-token
+# =======================================================================================
+
+echo  'Set following env variable.'
+echo  'export VAULT_ENABLED_FOR_DYNACONF=true'
+echo  'export VAULT_URL_FOR_DYNACONF=https://infra-assets.cfme2.lab.eng.rdu2.redhat.com:8201'
+echo  'export VAULT_APPROLE_TOKEN=<token generated in previous step>'
+echo  'export VAULT_VERIFY_FOR_DYNACONF=false'

sprout/vault/cfme-qe-infra-approle-token.json

@@ -0,0 +1,10 @@
+{
+  "path": {
+    "auth/approle/role/cfme-qe-infra/role-id": {
+      "capabilities": ["read"]
+    },
+    "auth/approle/role/cfme-qe-infra/secret-id": {
+      "capabilities": ["create", "update"]
+    }
+  }
+}

sprout/vault/cfme-qe-infra-ro-policy.json

@@ -0,0 +1,90 @@
+{
+  "path": {
+    "secret/*": {
+      "capabilities": [
+        "list"
+      ]
+    },
+    "secret/cfme-qe-sprout/*": {
+      "capabilities": [
+        "read",
+        "list"
+      ]
+    },
+    "secret/data/cfme-qe-sprout/*": {
+      "capabilities": [
+        "read",
+        "list"
+      ]
+    },
+    "auth/token/lookup-self": {
+      "capabilities": [
+        "read"
+      ]
+    },
+    "auth/token/renew-self": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "auth/token/revoke-self": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/capabilities-self": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/renew": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/leases/renew": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/leases/lookup": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "cubbyhole/*": {
+      "capabilities": [
+        "create",
+        "read",
+        "update",
+        "delete",
+        "list"
+      ]
+    },
+    "sys/wrapping/wrap": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/wrapping/lookup": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/wrapping/unwrap": {
+      "capabilities": [
+        "update"
+      ]
+    },
+    "sys/mounts": {
+      "capabilities": [
+        "read"
+      ]
+    },
+    "sys/auth": {
+      "capabilities": [
+        "read"
+      ]
+    }
+  }
+}

sprout/vault/vault.py

@@ -0,0 +1,53 @@
+import os
+
+from dynaconf import LazySettings
+from hvac import Client
+
+VAULT_APPROLE = 'cfme-qe-infra'
+extra_dynaconf_args = {}
+
+# The process to authenticate is to basically create AppRole Token and use that to authenticate
+# with vault and every time you authenticate also renew the token. That token will allow you
+# to create role_id and secret_id which together lets you authenticate to AppRole
+vault_approle_token = os.environ.get("VAULT_APPROLE_TOKEN", None)
+vault_url = os.environ.get("VAULT_URL_FOR_DYNACONF", None)
+
+
+def _login_and_renew_token(url, token):
+    """Log into Vault, renew the token, and return the Vault client"""
+    vault = Client(url=url, token=token, verify=False)
+    if not vault.is_authenticated():
+        return None
+    # Renew the token so that it's valid for another 7 days
+    vault.renew_token()
+    return vault
+
+
+def _get_approle_ids(url, token):
+    vault = _login_and_renew_token(url, token)
+    if not vault:
+        return None
+    role_id = vault.get_role_id(VAULT_APPROLE)
+    secret_id = vault.create_role_secret_id(VAULT_APPROLE).get("data", {}).get("secret_id")
+    return {"role_id": role_id, "secret_id": secret_id}
+
+
+if vault_approle_token and vault_url:
+    # Generate secret id
+    vault_approle_ids = _get_approle_ids(vault_url, vault_approle_token)
+    if not vault_approle_ids:
+        raise Exception(f"Cannot auth with Vault with AppRole token '{vault_approle_token}'")
+    extra_dynaconf_args.update(
+        {
+            "VAULT_ROLE_ID": vault_approle_ids["role_id"],
+            "VAULT_ROLE_ID_FOR_DYNACONF": vault_approle_ids["role_id"],  # Jenkins vault
+            "VAULT_SECRET_ID": vault_approle_ids["secret_id"],
+            "VAULT_SECRET_ID_FOR_DYNACONF": vault_approle_ids["secret_id"],  # Jenkins vault
+        }
+    )
+settings = LazySettings(
+    VAULT_PATH_FOR_DYNACONF="cfme-qe-sprout",
+    VAULT_VERIFY_FOR_DYNACONF=False,
+    VAULT_ENABLED_FOR_DYNACONF=True,
+    **extra_dynaconf_args
+)