Update service and deployment into separate specs

Add CLI entrypoint for Klat deployment
Add method to build Klat deployment configuration

neon_diana_utils/cli.py

@@ -67,6 +67,30 @@ def configure_neon_core(username, password, orchestrator, output_path):
     configure_neon_core(username, password, output_path, orchestrator)
 
 
+# Klat
+@neon_diana_cli.command(help="Configure Klat Chat")
+@click.option("--klat-url", "--url", help="Externally accessible URL")
+@click.option("--mongo-host", help="MongoDB Host Address")
+@click.option("--mongo-user", help="MongoDB Username")
+@click.option("--mongo-pass", help="MongoDB Password")
+@click.option("--mongo-db", help="MongoDB Database", default="klatchat")
+@click.option("--username", "-u", help="RabbitMQ username for Klat Observer")
+@click.option("--password", "-p", help="RabbitMQ password for Klat Observer")
+@click.option("--orchestrator", "-o", default="kubernetes",
+              help="Container orchestrator (`kubernetes` or `docker-compose`")
+@click.argument("output_path", default=None, required=False)
+def configure_klat(klat_url, mongo_host, mongo_user, mongo_pass, mongo_db,
+                   username, password, orchestrator, output_path):
+    from neon_diana_utils.configuration import configure_klat_chat, Orchestrator
+    try:
+        orchestrator = Orchestrator(orchestrator)
+    except ValueError:
+        click.echo(f"{orchestrator} is not a valid orchestrator")
+        return
+    configure_klat_chat(klat_url, mongo_host, mongo_user, mongo_pass, mongo_db,
+                        username, password, output_path, orchestrator, True)
+
+
 # Backend
 @neon_diana_cli.command(help="Configure DIANA Backend")
 @click.option("--username", "-u", help="RabbitMQ username")

neon_diana_utils/configuration.py

@@ -47,6 +47,20 @@ class Orchestrator(Enum):
     COMPOSE = "docker-compose"
 
 
+def _collect_helm_charts(output_path: str, charts_dir: str):
+    """
+    Collect Helm charts in the output directory and remove any leftover build
+    artifacts.
+    """
+    shutil.copytree(join(dirname(__file__), "helm_charts", charts_dir),
+                    join(output_path, charts_dir))
+    # Cleanup any leftover build files
+    for root, _, files in walk(join(output_path, charts_dir)):
+        for file in files:
+            if any((file.endswith(x) for x in (".lock", ".tgz"))):
+                remove(join(root, file))
+
+
 def validate_output_path(output_path: str) -> bool:
     """
     Ensure the requested output path is available to be written
@@ -297,34 +311,34 @@ def update_env_file(env_file: str):
         f.write(contents)
 
 
-def _get_neon_mq_user_config(mq_user: Optional[str], mq_pass: Optional[str],
-                             rmq_config: str) -> dict:
+def _get_mq_service_user_config(mq_user: Optional[str], mq_pass: Optional[str],
+                                mq_tag: str, rmq_config: str) -> dict:
     """
-    Get MQ config for neon core.
-    @param mq_user: RabbitMQ Neon username
-    @param mq_pass: RabbitMQ Neon password
+    Get MQ config for an added service from an existing RabbitMQ configuration.
+    @param mq_user: RabbitMQ service username
+    @param mq_pass: RabbitMQ service password
+    @param mq_tag: RabbitMQ User tag used to identify this service
     @param rmq_config: Path to RabbitMQ configuration file to import
     @returns dict user config to connect Neon Core to an MQ instance
     """
     # Check for passed or previously configured MQ user
     if not all((mq_user, mq_pass)) and isfile(rmq_config):
-        if click.confirm(f"Import Neon MQ user from {rmq_config}?"):
+        if click.confirm(f"Import {mq_tag} MQ user from {rmq_config}?"):
             with open(rmq_config) as f:
                 config = json.load(f)
             for user in config['users']:
-                if "core" in user['tags']:
+                if mq_tag in user['tags']:
                     mq_user = user['name']
                     mq_pass = user['password']
                     break
 
     # Interactively configure MQ authentication
     user_config = {"user": mq_user, "password": mq_pass}
     if not all((mq_user, mq_pass)):
-        if click.confirm("Configure MQ Connection?"):
+        if click.confirm("Configure MQ Connection Manually?"):
             confirmed = False
             while not confirmed:
-                mq_user = click.prompt("MQ Username", type=str,
-                                       default="neon_core")
+                mq_user = click.prompt("MQ Username", type=str)
                 mq_pass = click.prompt("MQ Password", type=str)
                 user_config = {
                     "user": mq_user,
@@ -359,13 +373,7 @@ def configure_backend(username: str = None,
     if orchestrator == Orchestrator.KUBERNETES:
         for path in ("diana-backend", "http-services", "ingress-common",
                      "mq-services", "neon-rabbitmq"):
-            shutil.copytree(join(dirname(__file__), "helm_charts", path),
-                            join(output_path, path))
-        # Cleanup any leftover build files
-        for root, _, files in walk(dirname(output_path)):
-            for file in files:
-                if any((file.endswith(x) for x in (".lock", ".tgz"))):
-                    remove(join(root, file))
+            _collect_helm_charts(output_path, path)
         rmq_file = join(output_path, "diana-backend", "rabbitmq.json")
         diana_config = join(output_path, "diana-backend", "diana.yaml")
 
@@ -444,15 +452,8 @@ def configure_neon_core(mq_user: str = None,
         return
 
     if orchestrator == Orchestrator.KUBERNETES:
-        shutil.copytree(join(dirname(__file__), "helm_charts", "neon-core"),
-                        join(output_path, "neon-core"))
-        # Cleanup any leftover build files
-        for root, _, files in walk(join(output_path, "neon-core")):
-            for file in files:
-                if any((file.endswith(x) for x in (".lock", ".tgz"))):
-                    remove(join(root, file))
+        _collect_helm_charts(output_path, "neon-core")
         neon_config_file = join(output_path, "neon-core", "neon.yaml")
-
     elif orchestrator == Orchestrator.COMPOSE:
         shutil.copytree(join(dirname(__file__), "docker", "neon_core"),
                         output_path)
@@ -464,7 +465,8 @@ def configure_neon_core(mq_user: str = None,
 
     try:
         # Get MQ User Configuration
-        user_config = _get_neon_mq_user_config(mq_user, mq_pass, rmq_config)
+        user_config = _get_mq_service_user_config(mq_user, mq_pass, "core",
+                                                  rmq_config)
         if not all((user_config['user'], user_config['password'])):
             # TODO: Prompt to configure MQ server/port?
             mq_config = dict()
@@ -495,24 +497,123 @@ def configure_neon_core(mq_user: str = None,
         click.echo(e)
 
 
-def configure_klat_chat():
-    internal_url = "http://klat-chat:8010"
-    external_url = ""  # TODO: Get external URL
-    mongo_config = {"host": "",
-                    # "port": 27017,
-                    "database": "klatchat"}  # TODO: Get MongoDB Config
+def configure_klat_chat(external_url: str = None,
+                        mongodb_host: str = None,
+                        mongodb_user: str = None,
+                        mongodb_pass: str = None,
+                        mongo_database: str = None,
+                        mq_user: str = None,
+                        mq_pass: str = None,
+                        output_path: str = None,
+                        orchestrator: Orchestrator = Orchestrator.KUBERNETES,
+                        prompt_update_rmq: bool = True):
+
+    # Validate output paths
+    output_path = expanduser(output_path or join(xdg_config_home(), "diana"))
+    rmq_config = join(output_path, "diana-backend", "rabbitmq.json")
+    # Output to `core` subdirectory
+    if not validate_output_path(join(output_path, "klat-chat")):
+        click.echo(f"Path exists: {output_path}")
+        return
+
+    # Get MQ User Configuration
+    if prompt_update_rmq and click.confirm("Configure RabbitMQ for Klat?"):
+        update_rmq_config(rmq_config)
+        click.echo(f"Updated RabbitMQ config file: {rmq_config}")
+    user_config = _get_mq_service_user_config(mq_user, mq_pass, "klat",
+                                              rmq_config)
+    # Get configuration
+    internal_url = "http://klat-chat-server:8010"
+    mongodb_port = 27017
+    mongo_config = dict()
+    sftp_config = dict()
+
+    # Confirm URL
+    while not external_url:
+        external_url = click.prompt("Klat URL", type=str)
+        if not click.confirm(f"Is '{external_url}' correct?"):
+            external_url = None
+    if not external_url.startswith("http"):
+        external_url = f"https://{external_url}"
+
+    https = external_url.startswith("https")
+    libretranslate_url = "https://libretranslate.2022.us"
+
+    # Confirm MongoDB host/port
+    confirmed = False
+    while not confirmed:
+        mongodb_host = click.prompt("MongoDB host address", type=str,
+                                    default=mongodb_host)
+        mongodb_user = click.prompt("MongoDB username", type=str,
+                                    default=mongodb_user)
+        mongodb_pass = click.prompt("MongoDB password", type=str,
+                                    default=mongodb_pass)
+        mongo_database = click.prompt("MongoDB database", type=str,
+                                      default=mongo_database)
+        if ':' in mongodb_host:
+            mongodb_host, mongodb_port = mongodb_host.split(':')
+            mongodb_port = int(mongodb_port)
+
+        mongo_config = {"host": mongodb_host,
+                        "port": mongodb_port,
+                        "username": mongodb_user,
+                        "password": mongodb_pass,
+                        "database": mongo_database}
+        click.echo(pformat(mongo_config))
+        confirmed = click.confirm("Is this configuration correct?")
+    mongo_config['dialect'] = 'mongo'
+
+    # Configure SFTP
+    confirmed = False
+    while not confirmed:
+        sftp_host = click.prompt("SFTP host URL/IP address", type=str)
+        sftp_port = click.prompt("SFTP port", type=int, default=22)
+        sftp_user = click.prompt("SFTP auth username", type=str)
+        sftp_pass = click.prompt("SFTP auth password", type=str)
+        sftp_root = click.prompt("SFTP root path", type=str,
+                                 default="/files/klat/")
+        sftp_config = {"HOST": sftp_host,
+                       "PORT": sftp_port,
+                       "USERNAME": sftp_user,
+                       "PASSWORD": sftp_pass,
+                       "ROOT_PATH": sftp_root}
+        click.echo(pformat(sftp_config))
+        confirmed = click.confirm("Is this configuration correct?")
+
     config = {"SIO_URL": internal_url,
-              "SOCKET_IO_SERVER_URL": external_url,
-              # TODO: Get chat_observer MQ config
-              "MQ": {"users": {},
+              "MQ": {"users": {"chat_observer": user_config},
                      "server": "neon-rabbitmq",
                      "port": 5672},
-              "CHAT_CLIENT": {"PROD": {
-                  "SERVER_URL": internal_url,
-                  "RUNTIME_CONFIG": {"CHAT_SERVER_URL_BASE": external_url}
-              }},
-              "CHAT_SERVER": {"PROD": {
-                  "connection_properties": mongo_config,
-                  "SERVER_IP": "klat-chat"
-              }},
-              "DATABASE_CONFIG": {"PROD": {"pyklatchat_3333": mongo_config}}}
+              "CHAT_CLIENT": {"SERVER_URL": internal_url,
+                              "FORCE_HTTPS": https,
+                              "RUNTIME_CONFIG": {
+                                  "CHAT_SERVER_URL_BASE": external_url}},
+              "CHAT_SERVER": {"DEBUG": True,
+                              "MINIFY": False,
+                              "SERVER_IP": "klat-chat-server",
+                              "COOKIES": {
+                                  "LIFETIME": 3600,
+                                  "REFRESH_RATE": 300,
+                                  "SECRET": "775115fdecb9b4971193b919d27d410a",
+                                  "JWT_ALGO": "HS256"},
+                              "LIBRE_TRANSLATE_URL": libretranslate_url,
+                              "SFTP": sftp_config
+                              },
+              "DATABASE_CONFIG": mongo_config}
+
+    if orchestrator == Orchestrator.KUBERNETES:
+        _collect_helm_charts(output_path, "klat-chat")
+        klat_config_file = join(output_path, "klat-chat", "klat.yaml")
+        # Update Helm values with configured URL
+        with open(join(output_path, "klat-chat", "values.yaml"), 'r') as f:
+            helm_values = yaml.safe_load(f)
+        helm_values['domain'] = external_url.split('://', 1)[1].split('.', 1)[1]
+        with open(join(output_path, "klat-chat", "values.yaml"), 'w') as f:
+            yaml.safe_dump(helm_values, f)
+    else:
+        raise RuntimeError(f"{orchestrator} is not yet supported")
+
+    # Write Klat configuration
+    with open(klat_config_file, 'w+') as f:
+        yaml.safe_dump(config, f)
+    click.echo(f"Wrote Klat configuration to {klat_config_file}")

neon_diana_utils/helm_charts/klat-chat/templates/config-map.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.configMap }}
+data:
+  "{{ .Values.configFilename }}": |
+    {{ .Files.Get "klat.yaml" | nindent 4}}

neon_diana_utils/helm_charts/klat-chat/templates/deployment-client.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}-client
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      neon.diana.service: {{ .Chart.Name }}-client
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      annotations:
+        releaseTime: {{ dateInZone "2006-01-02 15:04:05Z" (now) "UTC"| quote }}
+      labels:
+        neon.diana.service: {{ .Chart.Name }}-client
+        neon.project.name: klat
+        neon.service.class: klat-chat
+    spec:
+      restartPolicy: Always
+      imagePullSecrets:
+        - name: {{ .Values.imagePullSecret }}
+      volumes:
+        - name: config
+          projected:
+            sources:
+              - configMap:
+                  name: {{ .Values.configMap }}
+      containers:
+        - image: {{ .Values.images.client.image }}:{{ .Values.images.tag }}
+          imagePullPolicy: {{ $.Values.images.pullPolicy }}
+          name: {{ .Values.images.client.name }}
+          env:
+            - name: OVOS_CONFIG_FILENAME
+              value: klat.yaml
+            - name: OVOS_CONFIG_BASE_FOLDER
+              value: neon
+            - name: XDG_CONFIG_HOME
+              value: /config
+            - name: CORS_ALLOWED_ORIGINS
+              value: https://klat.{{ .Values.domain }}
+            - name: LOG_LEVEL
+              value: DEBUG
+            - name: HOST
+              value: 0.0.0.0
+            - name: PORT
+              value: "8001"
+            - name: INCLUDED_LANGUAGES
+              value: en,es,de,uk,fr,pl,pt
+          volumeMounts:
+            - name: config
+              mountPath: /config/neon/{{ $.Values.configFilename }}
+              subPath: {{ $.Values.configFilename }}
+          {{- if $.Values.resources }}
+          resources:
+          {{- toYaml $.Values.resources | nindent 12 -}}
+        {{ end }}