Adding Conda-Store to QHub (#967)

* Adding support for nix flakes

* Adding dependencies

* Adding .direnv

* Adding conda-store terraform charts along with postgres/minio charts

* Adding changes to ensure postgresql and minio deployment

* Make easy connection to s3 and database storage

* Adding url for conda-store in qhub

* Adding documentation on Conda-Store environment management

* Fixing all refrences to old conda-store image

* Remove use of fstring flake8 formatting

* FEAT: Add additional port to handle Minio object storage (#988)

* FEAT: Add additional port to handle Minio object storage

* Add missing vars

* Update minio entrypoint and disable tls

* Exclude traitlets config from pre-commit

* Update conda-store minio service eternal endpoint

Co-authored-by: viniciusdc <[email protected]>

Co-authored-by: Harsh Mishra <[email protected]>
Co-authored-by: viniciusdc <[email protected]>

.github/workflows/image.yaml

@@ -34,7 +34,6 @@ jobs:
           - dask-worker
           - dask-worker-centos
           - dask-gateway
-          - conda-store
     steps:
       - name: 'Checkout Infrastructure'
         uses: actions/checkout@main

.gitignore

@@ -20,6 +20,7 @@ result
 shell.nix
 .envrc
 build.nix
+.direnv
 
 # ignore deploments
 do-jupyterhub

.pre-commit-config.yaml

@@ -14,6 +14,7 @@ repos:
     rev: 3.8.4
     hooks:
     -   id: flake8
+        exclude: 'qhub/template/\{\{ cookiecutter\.repo_directory \}\}/infrastructure/modules/kubernetes/services/conda-store/config/conda_store_config.py'
   - repo: https://github.com/hadolint/hadolint.git
     rev: v2.3.0
     hooks:

docs/source/dev_guide/minikube.md

@@ -102,7 +102,6 @@ docker pull quansight/qhub-jupyterhub:v0.x.x
 docker pull quansight/qhub-jupyterlab:v0.x.x
 docker pull quansight/qhub-dask-worker:v0.x.x
 docker pull quansight/qhub-dask-gateway:v0.x.x
-docker pull quansight/qhub-conda-store:v0.x.x
 ```
 
 Replacing `v0.x.x` with the current version that's listed. Note this may take several minutes. Once the process is complete, the user can copy them to the Minikube cache.
@@ -112,7 +111,6 @@ minikube image load quansight/qhub-jupyterhub:v0.x.x
 minikube image load quansight/qhub-jupyterlab:v0.x.x
 minikube image load quansight/qhub-dask-worker:v0.x.x
 minikube image load quansight/qhub-dask-gateway:v0.x.x
-minikube image load quansight/qhub-conda-store:v0.x.x
 ```
 
 Again, adding the correct version. With this completed local Minikube deployment no longer requires pulling the preceding docker images.

docs/source/installation/configuration.md

@@ -534,7 +534,7 @@ default_images:
   jupyterlab: "quansight/qhub-jupyterlab:v||QHUB_VERSION||"
   dask_worker: "quansight/qhub-dask-worker:v||QHUB_VERSION||"
   dask_gateway: "quansight/qhub-dask-gateway:v||QHUB_VERSION||"
-  conda_store: "quansight/qhub-conda-store:v||QHUB_VERSION||"
+  conda_store: "quansight/conda-store-server:v0.3.3"
 ```
 
 ## Storage
@@ -856,7 +856,7 @@ default_images:
   jupyterhub: "quansight/qhub-jupyterhub:v||QHUB_VERSION||"
   jupyterlab: "quansight/qhub-jupyterlab:v||QHUB_VERSION||"
   dask_worker: "quansight/qhub-dask-worker:v||QHUB_VERSION||"
-  conda_store: "quansight/qhub-conda-store:v||QHUB_VERSION||"
+  conda_store: "quansight/conda-store-server:v0.3.3"
 
 theme:
   jupyterhub:

docs/source/installation/existing.md

@@ -140,7 +140,7 @@ default_images:
   jupyterlab: quansight/qhub-jupyterlab:v0.3.13
   dask_worker: quansight/qhub-dask-worker:v0.3.13
   dask_gateway: quansight/qhub-dask-gateway:v0.3.13
-  conda_store: quansight/qhub-conda-store:v0.3.13
+  conda_store: quansight/conda-store-server:v0.3.3
 storage:
   conda_store: 60Gi
   shared_filesystem: 100Gi

docs/source/user_guide/environments.md

@@ -0,0 +1,44 @@
+# Managing Conda Environments
+
+QHub has several ways to manage environments for users. The
+traditional approach available in older QHub deployments is still
+available by editing the `qhub-config.yaml` `environments:` key within
+the configuration file. An example would be 
+
+```yaml
+environments:
+  "environment-default.yaml":
+    name: default
+    channels:
+      - conda-forge
+      - defaults
+    dependencies:
+      - python=3.7
+      - ipykernel
+      - ipywidgets
+      - qhub-dask==||QHUB_VERSION||
+      - numpy
+      - numba
+      - pandas
+```
+
+When the environments are updated in this file and an automated `qhub
+deploy` is kicked off the environments are updated for all users. An
+additional method is allowed which enables users to easily create
+ad-hoc environments. Visiting `https://<qhub-domain>/conda-store/`
+will take you to
+[Conda-Store](https://conda-store.readthedocs.io/en/latest/) an open
+source tool for managing conda environments within enterprise
+environments. For now the username is anything with a password of
+`password`. Soon this will be integrated with central authentication
+via keycloak. The [create environment
+endpoint](https://conda-store.readthedocs.io/en/latest/user_guide.html#create-create-environment)
+will allow you to easily create a new environment. Additionally you
+can update existing environments via [visiting the
+environment](https://conda-store.readthedocs.io/en/latest/user_guide.html#environment-namespace-name-environments)
+and clicking edit. 
+
+We are working towards developing an extension within jupyterlab for
+editing these environments but it is not complete at the
+moment. Follow [gator](https://github.com/mamba-org/gator) for
+progress on this extension.

docs/source/user_guide/experimental.md

@@ -14,7 +14,7 @@ default_images:
   jupyterlab: quansight/qhub-jupyterlab-centos:main     <--- here
   dask_worker: quansight/qhub-dask-worker-centos:main   <--- here
   dask_gateway: quansight/qhub-dask-gateway:main
-  conda_store: quansight/qhub-conda-store:main
+  conda_store: quansight/conda-store-server:v0.3.3
 ...
 profiles:
   jupyterlab:

docs/source/user_guide/index.md

@@ -4,6 +4,7 @@
 :maxdepth: 2
 
 getting_started.md
+environments.md
 code_server.md
 ssh.md
 dashboard.md

flake.nix

@@ -0,0 +1,46 @@
+{
+  description = "QHub";
+
+  inputs = {
+    nixpkgs = { url = "github:nixos/nixpkgs/nixpkgs-unstable"; };
+    nixpkgs-keycloak = { url = "github:costrouc/nixpkgs/python-keycloak"; };
+  };
+
+  outputs = inputs@{ self, nixpkgs, nixpkgs-keycloak, ... }: {
+    devShell.x86_64-linux =
+      let
+        pkgs = import nixpkgs { system = "x86_64-linux"; };
+        pythonPackages = pkgs.python3Packages;
+
+        keycloak = (import nixpkgs-keycloak { system = "x86_64-linux"; }).python3Packages.python-keycloak;
+      in pkgs.mkShell {
+        buildInputs = [
+          pythonPackages.cookiecutter
+          pythonPackages.ruamel-yaml
+          pythonPackages.cloudflare
+          pythonPackages.auth0-python
+          pythonPackages.pydantic
+          pythonPackages.pynacl
+          pythonPackages.bcrypt
+          pythonPackages.kubernetes
+          pythonPackages.packaging
+          keycloak
+
+          # cloud packages
+          pythonPackages.azure-mgmt-containerservice
+          pythonPackages.azure-identity
+          pythonPackages.boto3
+
+          # development
+          pythonPackages.pytest
+          pythonPackages.black
+          pythonPackages.flake8
+          pythonPackages.sphinx
+
+          # additional
+          pkgs.minikube
+          pkgs.k9s
+        ];
+      };
+  };
+}

qhub/initialize.py

@@ -38,7 +38,7 @@
         "jupyterlab": f"quansight/qhub-jupyterlab:{qhub_image_tag}",
         "dask_worker": f"quansight/qhub-dask-worker:{qhub_image_tag}",
         "dask_gateway": f"quansight/qhub-dask-gateway:{qhub_image_tag}",
-        "conda_store": f"quansight/qhub-conda-store:{qhub_image_tag}",
+        "conda_store": "quansight/conda-store-server:v0.3.3",
     },
     "storage": {"conda_store": "60Gi", "shared_filesystem": "100Gi"},
     "theme": {

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/kubernetes.tf

@@ -102,6 +102,13 @@ module "kubernetes-conda-store-server" {
 
   name              = "qhub"
   namespace         = var.environment
+
+  external-url = var.endpoint
+
+{% if cookiecutter.certificate.type == "existing" %}
+  certificate-secret-name = "{{ cookiecutter.certificate.secret_name }}"
+{% endif %}
+
   nfs_capacity      = "{{ cookiecutter.storage.conda_store }}"
   node-group        = local.node_groups.general
   conda-store-image = var.conda-store-image

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/ingress/main.tf

@@ -96,6 +96,13 @@ resource "kubernetes_service" "main" {
       target_port = 8023
     }
 
+    port {
+      name        = "minio"
+      protocol    = "TCP"
+      port        = 9080
+      target_port = 9080
+    }
+
     port {
       name        = "tcp"
       protocol    = "TCP"
@@ -219,6 +226,8 @@ resource "kubernetes_deployment" "main" {
             "--entrypoints.sftp.address=:8023",
             "--entryPoints.tcp.address=:8786",
             "--entryPoints.traefik.address=:9000",
+            # Define the entrypoint port for Minio
+            "--entryPoints.minio.address=:9080",
             "--entrypoints.web.http.redirections.entryPoint.to=websecure",
             "--entrypoints.web.http.redirections.entryPoint.scheme=https",
             # Enable Prometheus Monitoring of Traefik
@@ -263,6 +272,11 @@ resource "kubernetes_deployment" "main" {
             container_port = 9000
           }
 
+          port {
+            name           = "minio"
+            container_port = 9080
+          }
+
           liveness_probe {
             http_get {
               path = "/ping"

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/services/conda-store/config/conda_store_config.py

@@ -0,0 +1,48 @@
+import logging
+
+from conda_store_server.storage import S3Storage
+from conda_store_server.server.auth import DummyAuthentication
+
+# ==================================
+#      conda-store settings
+# ==================================
+c.CondaStore.storage_class = S3Storage
+c.CondaStore.store_directory = "/home/conda/"
+c.CondaStore.environment_directory = "/home/conda/environments/"
+c.CondaStore.database_url = "postgresql+psycopg2://${postgres-username}:${postgres-password}@${postgres-service}/conda-store"
+c.CondaStore.default_uid = 1000
+c.CondaStore.default_gid = 100
+c.CondaStore.default_permissions = "775"
+
+c.S3Storage.internal_endpoint = "${minio-service}:9000"
+c.S3Storage.external_endpoint = "${external-url}:9080"
+c.S3Storage.access_key = "${minio-username}"
+c.S3Storage.secret_key = "${minio-password}"
+c.S3Storage.region = "us-east-1"  # minio region default
+c.S3Storage.bucket_name = "conda-store"
+c.S3Storage.secure = False
+
+# ==================================
+#        server settings
+# ==================================
+c.CondaStoreServer.log_level = logging.INFO
+c.CondaStoreServer.enable_ui = True
+c.CondaStoreServer.enable_api = True
+c.CondaStoreServer.enable_registry = True
+c.CondaStoreServer.enable_metrics = True
+c.CondaStoreServer.address = "0.0.0.0"
+c.CondaStoreServer.port = 5000
+# This MUST start with `/`
+c.CondaStoreServer.url_prefix = "/conda-store"
+
+
+# ==================================
+#         auth settings
+# ==================================
+c.CondaStoreServer.authentication_class = DummyAuthentication
+
+# ==================================
+#         worker settings
+# ==================================
+c.CondaStoreWorker.log_level = logging.INFO
+c.CondaStoreWorker.watch_paths = ["/opt/environments"]

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/services/conda-store/locals.tf

@@ -0,0 +1,7 @@
+locals {
+  tls = (var.certificate-secret-name != "") ? ({
+    secretName = var.certificate-secret-name
+    }) : ({
+    certResolver = "default"
+  })
+}

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/services/conda-store/output.tf

@@ -1,9 +1,9 @@
 output "endpoint" {
   description = "Endpoint dns name of conda-store nfs server"
-  value       = "${var.name}-nfs.${var.namespace}.svc.cluster.local"
+  value       = "${kubernetes_service.nfs.metadata.0.name}.${var.namespace}.svc.cluster.local"
 }
 
 output "endpoint_ip" {
   description = "IP Address of conda-store nfs server"
-  value       = kubernetes_service.main.spec.0.cluster_ip
+  value       = kubernetes_service.nfs.spec.0.cluster_ip
 }