diff --git a/neon_diana_utils/cli.py b/neon_diana_utils/cli.py index 1bf31de1..a931d18c 100644 --- a/neon_diana_utils/cli.py +++ b/neon_diana_utils/cli.py @@ -67,6 +67,30 @@ def configure_neon_core(username, password, orchestrator, output_path): configure_neon_core(username, password, output_path, orchestrator) +# Klat +@neon_diana_cli.command(help="Configure Klat Chat") +@click.option("--klat-url", "--url", help="Externally accessible URL") +@click.option("--mongo-host", help="MongoDB Host Address") +@click.option("--mongo-user", help="MongoDB Username") +@click.option("--mongo-pass", help="MongoDB Password") +@click.option("--mongo-db", help="MongoDB Database", default="klatchat") +@click.option("--username", "-u", help="RabbitMQ username for Klat Observer") +@click.option("--password", "-p", help="RabbitMQ password for Klat Observer") +@click.option("--orchestrator", "-o", default="kubernetes", + help="Container orchestrator (`kubernetes` or `docker-compose`") +@click.argument("output_path", default=None, required=False) +def configure_klat(klat_url, mongo_host, mongo_user, mongo_pass, mongo_db, + username, password, orchestrator, output_path): + from neon_diana_utils.configuration import configure_klat_chat, Orchestrator + try: + orchestrator = Orchestrator(orchestrator) + except ValueError: + click.echo(f"{orchestrator} is not a valid orchestrator") + return + configure_klat_chat(klat_url, mongo_host, mongo_user, mongo_pass, mongo_db, + username, password, output_path, orchestrator, True) + + # Backend @neon_diana_cli.command(help="Configure DIANA Backend") @click.option("--username", "-u", help="RabbitMQ username") diff --git a/neon_diana_utils/configuration.py b/neon_diana_utils/configuration.py index 15fae0c3..c80101d0 100644 --- a/neon_diana_utils/configuration.py +++ b/neon_diana_utils/configuration.py @@ -47,6 +47,20 @@ class Orchestrator(Enum): COMPOSE = "docker-compose" +def _collect_helm_charts(output_path: str, charts_dir: str): + """ + Collect Helm charts in the output directory and remove any leftover build + artifacts. + """ + shutil.copytree(join(dirname(__file__), "helm_charts", charts_dir), + join(output_path, charts_dir)) + # Cleanup any leftover build files + for root, _, files in walk(join(output_path, charts_dir)): + for file in files: + if any((file.endswith(x) for x in (".lock", ".tgz"))): + remove(join(root, file)) + + def validate_output_path(output_path: str) -> bool: """ Ensure the requested output path is available to be written @@ -297,22 +311,23 @@ def update_env_file(env_file: str): f.write(contents) -def _get_neon_mq_user_config(mq_user: Optional[str], mq_pass: Optional[str], - rmq_config: str) -> dict: +def _get_mq_service_user_config(mq_user: Optional[str], mq_pass: Optional[str], + mq_tag: str, rmq_config: str) -> dict: """ - Get MQ config for neon core. - @param mq_user: RabbitMQ Neon username - @param mq_pass: RabbitMQ Neon password + Get MQ config for an added service from an existing RabbitMQ configuration. + @param mq_user: RabbitMQ service username + @param mq_pass: RabbitMQ service password + @param mq_tag: RabbitMQ User tag used to identify this service @param rmq_config: Path to RabbitMQ configuration file to import @returns dict user config to connect Neon Core to an MQ instance """ # Check for passed or previously configured MQ user if not all((mq_user, mq_pass)) and isfile(rmq_config): - if click.confirm(f"Import Neon MQ user from {rmq_config}?"): + if click.confirm(f"Import {mq_tag} MQ user from {rmq_config}?"): with open(rmq_config) as f: config = json.load(f) for user in config['users']: - if "core" in user['tags']: + if mq_tag in user['tags']: mq_user = user['name'] mq_pass = user['password'] break @@ -320,11 +335,10 @@ def _get_neon_mq_user_config(mq_user: Optional[str], mq_pass: Optional[str], # Interactively configure MQ authentication user_config = {"user": mq_user, "password": mq_pass} if not all((mq_user, mq_pass)): - if click.confirm("Configure MQ Connection?"): + if click.confirm("Configure MQ Connection Manually?"): confirmed = False while not confirmed: - mq_user = click.prompt("MQ Username", type=str, - default="neon_core") + mq_user = click.prompt("MQ Username", type=str) mq_pass = click.prompt("MQ Password", type=str) user_config = { "user": mq_user, @@ -359,13 +373,7 @@ def configure_backend(username: str = None, if orchestrator == Orchestrator.KUBERNETES: for path in ("diana-backend", "http-services", "ingress-common", "mq-services", "neon-rabbitmq"): - shutil.copytree(join(dirname(__file__), "helm_charts", path), - join(output_path, path)) - # Cleanup any leftover build files - for root, _, files in walk(dirname(output_path)): - for file in files: - if any((file.endswith(x) for x in (".lock", ".tgz"))): - remove(join(root, file)) + _collect_helm_charts(output_path, path) rmq_file = join(output_path, "diana-backend", "rabbitmq.json") diana_config = join(output_path, "diana-backend", "diana.yaml") @@ -444,15 +452,8 @@ def configure_neon_core(mq_user: str = None, return if orchestrator == Orchestrator.KUBERNETES: - shutil.copytree(join(dirname(__file__), "helm_charts", "neon-core"), - join(output_path, "neon-core")) - # Cleanup any leftover build files - for root, _, files in walk(join(output_path, "neon-core")): - for file in files: - if any((file.endswith(x) for x in (".lock", ".tgz"))): - remove(join(root, file)) + _collect_helm_charts(output_path, "neon-core") neon_config_file = join(output_path, "neon-core", "neon.yaml") - elif orchestrator == Orchestrator.COMPOSE: shutil.copytree(join(dirname(__file__), "docker", "neon_core"), output_path) @@ -464,7 +465,8 @@ def configure_neon_core(mq_user: str = None, try: # Get MQ User Configuration - user_config = _get_neon_mq_user_config(mq_user, mq_pass, rmq_config) + user_config = _get_mq_service_user_config(mq_user, mq_pass, "core", + rmq_config) if not all((user_config['user'], user_config['password'])): # TODO: Prompt to configure MQ server/port? mq_config = dict() @@ -495,24 +497,123 @@ def configure_neon_core(mq_user: str = None, click.echo(e) -def configure_klat_chat(): - internal_url = "http://klat-chat:8010" - external_url = "" # TODO: Get external URL - mongo_config = {"host": "", - # "port": 27017, - "database": "klatchat"} # TODO: Get MongoDB Config +def configure_klat_chat(external_url: str = None, + mongodb_host: str = None, + mongodb_user: str = None, + mongodb_pass: str = None, + mongo_database: str = None, + mq_user: str = None, + mq_pass: str = None, + output_path: str = None, + orchestrator: Orchestrator = Orchestrator.KUBERNETES, + prompt_update_rmq: bool = True): + + # Validate output paths + output_path = expanduser(output_path or join(xdg_config_home(), "diana")) + rmq_config = join(output_path, "diana-backend", "rabbitmq.json") + # Output to `core` subdirectory + if not validate_output_path(join(output_path, "klat-chat")): + click.echo(f"Path exists: {output_path}") + return + + # Get MQ User Configuration + if prompt_update_rmq and click.confirm("Configure RabbitMQ for Klat?"): + update_rmq_config(rmq_config) + click.echo(f"Updated RabbitMQ config file: {rmq_config}") + user_config = _get_mq_service_user_config(mq_user, mq_pass, "klat", + rmq_config) + # Get configuration + internal_url = "http://klat-chat-server:8010" + mongodb_port = 27017 + mongo_config = dict() + sftp_config = dict() + + # Confirm URL + while not external_url: + external_url = click.prompt("Klat URL", type=str) + if not click.confirm(f"Is '{external_url}' correct?"): + external_url = None + if not external_url.startswith("http"): + external_url = f"https://{external_url}" + + https = external_url.startswith("https") + libretranslate_url = "https://libretranslate.2022.us" + + # Confirm MongoDB host/port + confirmed = False + while not confirmed: + mongodb_host = click.prompt("MongoDB host address", type=str, + default=mongodb_host) + mongodb_user = click.prompt("MongoDB username", type=str, + default=mongodb_user) + mongodb_pass = click.prompt("MongoDB password", type=str, + default=mongodb_pass) + mongo_database = click.prompt("MongoDB database", type=str, + default=mongo_database) + if ':' in mongodb_host: + mongodb_host, mongodb_port = mongodb_host.split(':') + mongodb_port = int(mongodb_port) + + mongo_config = {"host": mongodb_host, + "port": mongodb_port, + "username": mongodb_user, + "password": mongodb_pass, + "database": mongo_database} + click.echo(pformat(mongo_config)) + confirmed = click.confirm("Is this configuration correct?") + mongo_config['dialect'] = 'mongo' + + # Configure SFTP + confirmed = False + while not confirmed: + sftp_host = click.prompt("SFTP host URL/IP address", type=str) + sftp_port = click.prompt("SFTP port", type=int, default=22) + sftp_user = click.prompt("SFTP auth username", type=str) + sftp_pass = click.prompt("SFTP auth password", type=str) + sftp_root = click.prompt("SFTP root path", type=str, + default="/files/klat/") + sftp_config = {"HOST": sftp_host, + "PORT": sftp_port, + "USERNAME": sftp_user, + "PASSWORD": sftp_pass, + "ROOT_PATH": sftp_root} + click.echo(pformat(sftp_config)) + confirmed = click.confirm("Is this configuration correct?") + config = {"SIO_URL": internal_url, - "SOCKET_IO_SERVER_URL": external_url, - # TODO: Get chat_observer MQ config - "MQ": {"users": {}, + "MQ": {"users": {"chat_observer": user_config}, "server": "neon-rabbitmq", "port": 5672}, - "CHAT_CLIENT": {"PROD": { - "SERVER_URL": internal_url, - "RUNTIME_CONFIG": {"CHAT_SERVER_URL_BASE": external_url} - }}, - "CHAT_SERVER": {"PROD": { - "connection_properties": mongo_config, - "SERVER_IP": "klat-chat" - }}, - "DATABASE_CONFIG": {"PROD": {"pyklatchat_3333": mongo_config}}} + "CHAT_CLIENT": {"SERVER_URL": internal_url, + "FORCE_HTTPS": https, + "RUNTIME_CONFIG": { + "CHAT_SERVER_URL_BASE": external_url}}, + "CHAT_SERVER": {"DEBUG": True, + "MINIFY": False, + "SERVER_IP": "klat-chat-server", + "COOKIES": { + "LIFETIME": 3600, + "REFRESH_RATE": 300, + "SECRET": "775115fdecb9b4971193b919d27d410a", + "JWT_ALGO": "HS256"}, + "LIBRE_TRANSLATE_URL": libretranslate_url, + "SFTP": sftp_config + }, + "DATABASE_CONFIG": mongo_config} + + if orchestrator == Orchestrator.KUBERNETES: + _collect_helm_charts(output_path, "klat-chat") + klat_config_file = join(output_path, "klat-chat", "klat.yaml") + # Update Helm values with configured URL + with open(join(output_path, "klat-chat", "values.yaml"), 'r') as f: + helm_values = yaml.safe_load(f) + helm_values['domain'] = external_url.split('://', 1)[1].split('.', 1)[1] + with open(join(output_path, "klat-chat", "values.yaml"), 'w') as f: + yaml.safe_dump(helm_values, f) + else: + raise RuntimeError(f"{orchestrator} is not yet supported") + + # Write Klat configuration + with open(klat_config_file, 'w+') as f: + yaml.safe_dump(config, f) + click.echo(f"Wrote Klat configuration to {klat_config_file}") diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/config-map.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/config-map.yaml new file mode 100644 index 00000000..f75040a5 --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/config-map.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.configMap }} +data: + "{{ .Values.configFilename }}": | + {{ .Files.Get "klat.yaml" | nindent 4}} \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/deployment-client.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-client.yaml new file mode 100644 index 00000000..7c5ed77b --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-client.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }}-client +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + neon.diana.service: {{ .Chart.Name }}-client + strategy: + type: Recreate + template: + metadata: + annotations: + releaseTime: {{ dateInZone "2006-01-02 15:04:05Z" (now) "UTC"| quote }} + labels: + neon.diana.service: {{ .Chart.Name }}-client + neon.project.name: klat + neon.service.class: klat-chat + spec: + restartPolicy: Always + imagePullSecrets: + - name: {{ .Values.imagePullSecret }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ .Values.configMap }} + containers: + - image: {{ .Values.images.client.image }}:{{ .Values.images.tag }} + imagePullPolicy: {{ $.Values.images.pullPolicy }} + name: {{ .Values.images.client.name }} + env: + - name: OVOS_CONFIG_FILENAME + value: klat.yaml + - name: OVOS_CONFIG_BASE_FOLDER + value: neon + - name: XDG_CONFIG_HOME + value: /config + - name: CORS_ALLOWED_ORIGINS + value: https://klat.{{ .Values.domain }} + - name: LOG_LEVEL + value: DEBUG + - name: HOST + value: 0.0.0.0 + - name: PORT + value: "8001" + - name: INCLUDED_LANGUAGES + value: en,es,de,uk,fr,pl,pt + volumeMounts: + - name: config + mountPath: /config/neon/{{ $.Values.configFilename }} + subPath: {{ $.Values.configFilename }} + {{- if $.Values.resources }} + resources: + {{- toYaml $.Values.resources | nindent 12 -}} + {{ end }} \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/deployment.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-observer.yaml similarity index 57% rename from neon_diana_utils/helm_charts/klat-chat/templates/deployment.yaml rename to neon_diana_utils/helm_charts/klat-chat/templates/deployment-observer.yaml index e042b260..b69ef3de 100644 --- a/neon_diana_utils/helm_charts/klat-chat/templates/deployment.yaml +++ b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-observer.yaml @@ -1,12 +1,12 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Chart.Name }} + name: {{ .Chart.Name }}-observer spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - neon.diana.service: {{ .Chart.Name }} + neon.diana.service: {{ .Chart.Name }}-observer strategy: type: Recreate template: @@ -14,21 +14,30 @@ spec: annotations: releaseTime: {{ dateInZone "2006-01-02 15:04:05Z" (now) "UTC"| quote }} labels: - neon.diana.service: {{ .Chart.Name }} + neon.diana.service: {{ .Chart.Name }}-observer neon.project.name: klat neon.service.class: klat-chat spec: restartPolicy: Always imagePullSecrets: - name: {{ .Values.imagePullSecret }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ .Values.configMap }} containers: - {{- range .Values.images.containers }} - - image: {{ .image }}:{{ $.Values.images.tag }} + - image: {{ .Values.images.observer.image }}:{{ .Values.images.tag }} imagePullPolicy: {{ $.Values.images.pullPolicy }} - name: {{ .name }} + name: {{ .Values.images.observer.name }} env: - - name: {{ $.Values.configEnvVar }} - value: /config/neon/{{ $.Values.configFilename }} + - name: OVOS_CONFIG_FILENAME + value: klat.yaml + - name: OVOS_CONFIG_BASE_FOLDER + value: neon + - name: XDG_CONFIG_HOME + value: /config volumeMounts: - name: config mountPath: /config/neon/{{ $.Values.configFilename }} @@ -36,5 +45,4 @@ spec: {{- if $.Values.resources }} resources: {{- toYaml $.Values.resources | nindent 12 -}} - {{ end }} {{ end }} \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/deployment-server.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-server.yaml new file mode 100644 index 00000000..868aa078 --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/deployment-server.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }}-server +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + neon.diana.service: {{ .Chart.Name }}-server + strategy: + type: Recreate + template: + metadata: + annotations: + releaseTime: {{ dateInZone "2006-01-02 15:04:05Z" (now) "UTC"| quote }} + labels: + neon.diana.service: {{ .Chart.Name }}-server + neon.project.name: klat + neon.service.class: klat-chat + spec: + restartPolicy: Always + imagePullSecrets: + - name: {{ .Values.imagePullSecret }} + volumes: + - name: config + projected: + sources: + - configMap: + name: {{ .Values.configMap }} + containers: + - image: {{ .Values.images.server.image }}:{{ .Values.images.tag }} + imagePullPolicy: {{ $.Values.images.pullPolicy }} + name: {{ .Values.images.server.name }} + env: + - name: OVOS_CONFIG_FILENAME + value: klat.yaml + - name: OVOS_CONFIG_BASE_FOLDER + value: neon + - name: XDG_CONFIG_HOME + value: /config + - name: CORS_ALLOWED_ORIGINS + value: https://klat.{{ .Values.domain }} + - name: LOG_LEVEL + value: DEBUG + - name: HOST + value: 0.0.0.0 + - name: PORT + value: "8010" + - name: INCLUDED_LANGUAGES + value: en,es,de,uk,fr,pl,pt + volumeMounts: + - name: config + mountPath: /config/neon/{{ $.Values.configFilename }} + subPath: {{ $.Values.configFilename }} + {{- if $.Values.resources }} + resources: + {{- toYaml $.Values.resources | nindent 12 -}} + {{ end }} \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/ingress.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/ingress.yaml index c054bc4f..7213aa2c 100644 --- a/neon_diana_utils/helm_charts/klat-chat/templates/ingress.yaml +++ b/neon_diana_utils/helm_charts/klat-chat/templates/ingress.yaml @@ -17,12 +17,12 @@ spec: tls: - secretName: {{ .Values.ingress.tlsSecretName }} hosts: - {{- range $rules }} + {{- range .Values.ingress.rules }} - {{ printf "%s.%s " .host $.Values.domain }} {{- end }} rules: - {{- range $rules }} + {{- range .Values.ingress.rules }} - host: {{ printf "%s.%s " .host $.Values.domain }} http: paths: diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/service-client.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/service-client.yaml new file mode 100644 index 00000000..a99b8510 --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/service-client.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + neon.project.name: klat + neon.diana.service: {{ .Chart.Name }} + neon.service.class: klat-client + name: {{ .Chart.Name }}-client +spec: + type: ClusterIP + selector: + neon.diana.service: {{ .Chart.Name }}-client + ports: + - name: klat-client + port: 8001 + targetPort: 8001 + protocol: TCP diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/service-observer.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/service-observer.yaml new file mode 100644 index 00000000..12634123 --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/service-observer.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + neon.project.name: klat + neon.diana.service: {{ .Chart.Name }} + neon.service.class: klat-observer + name: {{ .Chart.Name }}-observer +spec: + clusterIP: None + selector: + neon.diana.service: {{ .Chart.Name }}-observer + ports: + - name: headless + port: 55555 + targetPort: 0 \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/service-server.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/service-server.yaml new file mode 100644 index 00000000..6a0dff78 --- /dev/null +++ b/neon_diana_utils/helm_charts/klat-chat/templates/service-server.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + neon.project.name: klat + neon.diana.service: {{ .Chart.Name }} + neon.service.class: klat-server + name: {{ .Chart.Name }}-server +spec: + type: ClusterIP + selector: + neon.diana.service: {{ .Chart.Name }}-server + ports: + - name: klat-server + port: 8010 + targetPort: 8010 + protocol: TCP diff --git a/neon_diana_utils/helm_charts/klat-chat/templates/service.yaml b/neon_diana_utils/helm_charts/klat-chat/templates/service.yaml deleted file mode 100644 index 0861d75e..00000000 --- a/neon_diana_utils/helm_charts/klat-chat/templates/service.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- apiVersion: v1 - kind: Service - metadata: - labels: - neon.project.name: klat - neon.diana.service: {{ .Chart.Name }} - neon.service.class: klat-client - name: {{ .Chart.Name }} - spec: - clusterIP: None - selector: - neon.diana.service: {{ .Chart.Name }}-client - ports: - - name: klat-client - port: 8001 - targetPort: 8001 - protocol: TCP -- apiVersion: v1 - kind: Service - metadata: - labels: - neon.project.name: klat - neon.diana.service: {{ .Chart.Name }} - neon.service.class: klat-server - name: {{ .Chart.Name }} - spec: - clusterIP: None - selector: - neon.diana.service: {{ .Chart.Name }}-server - ports: - - name: klat-server - port: 8010 - targetPort: 8010 - protocol: TCP -- apiVersion: v1 - kind: Service - metadata: - labels: - neon.project.name: klat - neon.diana.service: {{ .Chart.Name }} - neon.service.class: klat-observer - name: {{ .Chart.Name }} - spec: - clusterIP: None - selector: - neon.diana.service: {{ .Chart.Name }}-observer - ports: - - name: headless - port: 55555 - targetPort: 0 \ No newline at end of file diff --git a/neon_diana_utils/helm_charts/klat-chat/values.yaml b/neon_diana_utils/helm_charts/klat-chat/values.yaml index eb62bd1c..a05416b3 100644 --- a/neon_diana_utils/helm_charts/klat-chat/values.yaml +++ b/neon_diana_utils/helm_charts/klat-chat/values.yaml @@ -2,17 +2,20 @@ replicaCount: 1 imagePullSecret: github-auth configFilename: klat.yaml domain: diana.k8s +configMap: klat-config images: tag: dev pullPolicy: Always - containers: - - image: ghcr.io/neongeckocom/klatchat_observer - name: klat-observer - - image: ghcr.io/neongeckocom/chat_server - name: klat-server - - image: ghcr.io/neongeckocom/chat_client - name: klat-client + client: + image: ghcr.io/neondaniel/chat_client + name: klat-chat-client + server: + image: ghcr.io/neondaniel/chat_server + name: klat-chat-server + observer: + image: ghcr.io/neondaniel/klatchat_observer + name: klat-chat-observer ingress: enabled: True @@ -21,5 +24,8 @@ ingress: certIssuer: letsencrypt-private-key rules: - host: klat - serviceName: klat + serviceName: klat-chat-client + servicePort: 8001 + - host: klatapi + serviceName: klat-chat-server servicePort: 8010 \ No newline at end of file