Add support for persistent NFS storage type in production deployments

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
       - name: teardown
         run: |
           ./unmake.sh
-          sudo rm -fr ./FS
+          sudo rm -fr ./CHRIS_REMOTE_FS
           docker swarm leave --force
   test-cube:
     name: CUBE tests

README.rst

@@ -138,20 +138,23 @@ Production deployments
 Docker Swarm-based deployment
 =============================
 
+A single-machine deployment is provided.
+
 Configure pfcon services
 ------------------------
 
-Modify the ``.env`` files in the ``swarm/prod_deployments/secrets`` directory appropriately.
+Modify the ``.env`` files in the ``swarm/prod/base/secrets`` directory appropriately.
+
+Single-machine deployment
+-------------------------
 
-Start production pfcon
-----------------------
+Start production pfcon:
 
 .. code-block:: bash
 
     $> ./deploy.sh up
 
-Tear down production pfcon
---------------------------
+Tear down production pfcon:
 
 .. code-block:: bash
 
@@ -160,21 +163,41 @@ Tear down production pfcon
 Kubernetes-based deployment
 ===========================
 
+A single-machine deployment using Kubernetes' "hostPath" storage is provided. In addition
+a multi-machine deployment for an external NFS drive is provided using NFS persistent volume.
+
 Configure pfcon services
 ------------------------
 
-Modify the ``.env`` files in the ``kubernetes/prod_deployments/secrets`` directory appropriately.
+Modify the ``.env`` files in the ``kubernetes/prod/base/secrets`` directory appropriately.
 
-Start production pfcon
-----------------------
+Single-machine deployment
+-------------------------
+
+Start production pfcon:
 
 .. code-block:: bash
 
     $> ./deploy.sh -O kubernetes up
 
 Tear down production pfcon
---------------------------
 
 .. code-block:: bash
 
     $> ./deploy.sh -O kubernetes down
+
+Multi-machine deployment
+-------------------------
+
+Start production pfcon:
+
+.. code-block:: bash
+
+    $> ./deploy.sh -O kubernetes -T nfs -S <NFS export dir> -P <NFS server IP addr> up
+
+Tear down production pfcon
+
+.. code-block:: bash
+
+    $> ./deploy.sh -O kubernetes -T nfs -S <NFS export dir> -P <NFS server IP addr> down
+

deploy.sh

@@ -8,8 +8,10 @@
 #
 #   deploy.sh                   [-h]
 #                               [-O <swarm|kubernetes>] \
-#                               [-S <storeBase>]        \
 #                               [-N <namespace>]        \
+#                               [-T <host|nfs>]         \
+#                               [-P <nfsServerIp>]      \
+#                               [-S <storeBase>]        \
 #                               [up|down]
 #
 # DESC
@@ -44,6 +46,16 @@
 #       Explicitly set the kubernetes namespace to <namespace>. Default is chris.
 #       Not used for swarm.
 #
+#   -T <host|nfs>
+#
+#       Explicitly set the storage type for the STOREBASE dir. Default is host.
+#       Note: The nfs storage type is not implemented for swarm orchestrator yet.
+#
+#   -P <nfsServerIp>
+#
+#       Set the IP address of the NFS server. Required when storage type is set to 'nfs'.
+#       Not used for 'host' storage type.
+#
 #   -S <storeBase>
 #
 #       Explicitly set the STOREBASE dir to <storeBase>. This is the remote ChRIS
@@ -62,14 +74,16 @@ source ./cparse.sh
 declare -i STEP=0
 ORCHESTRATOR=swarm
 NAMESPACE=chris
+STORAGE_TYPE=host
 HERE=$(pwd)
 
 print_usage () {
-    echo "Usage: ./deploy.sh [-h] [-O <swarm|kubernetes>] [-N <namespace>] [-S <storeBase>] [up|down]"
+    echo "Usage: ./deploy.sh [-h] [-O <swarm|kubernetes>] [-N <namespace>] [-T <host|nfs>]
+         [-P <nfsServerIp>] [-S <storeBase>] [up|down]"
     exit 1
 }
 
-while getopts ":hO:N:S:" opt; do
+while getopts ":hO:N:T:P:S:" opt; do
     case $opt in
         h) print_usage
            ;;
@@ -81,6 +95,14 @@ while getopts ":hO:N:S:" opt; do
            ;;
         N) NAMESPACE=$OPTARG
            ;;
+        T) STORAGE_TYPE=$OPTARG
+           if ! [[ "$STORAGE_TYPE" =~ ^(host|nfs)$ ]]; then
+              echo "Invalid value for option -- T"
+              print_usage
+           fi
+           ;;
+        P) NFS_SERVER=$OPTARG
+           ;;
         S) STOREBASE=$OPTARG
            ;;
         \?) echo "Invalid option -- $OPTARG"
@@ -93,6 +115,23 @@ while getopts ":hO:N:S:" opt; do
 done
 shift $(($OPTIND - 1))
 
+if [[ $STORAGE_TYPE == nfs ]]; then
+    if [[ $ORCHESTRATOR == swarm ]]; then
+        echo -e "Sorry, nfs storage type is not supported for swarm orchestrator yet"  | ./boxes.sh
+        exit 1
+    fi
+    if [ -z ${NFS_SERVER+x} ]; then
+        echo "-P <NFS_SERVER> (the NFS server ip address) must be specified or the shell
+             environment variable NFS_SERVER must be set when using nfs storage type"
+        print_usage
+    fi
+    if [ -z ${STOREBASE+x} ]; then
+        echo "-S <storeBase> must be specified or the shell environment variable STOREBASE
+             must be set when using nfs storage type"
+        print_usage
+    fi
+fi
+
 COMMAND=up
 if (( $# == 1 )) ; then
     COMMAND=$1
@@ -103,16 +142,26 @@ if (( $# == 1 )) ; then
 fi
 
 title -d 1 "Setting global exports..."
-    if [ -z ${STOREBASE+x} ]; then
-        if [[ ! -d CHRIS_REMOTE_FS ]] ; then
-            mkdir CHRIS_REMOTE_FS
-        fi
-        STOREBASE=$HERE/CHRIS_REMOTE_FS
-    else
-        if [[ ! -d $STOREBASE ]] ; then
-            mkdir -p $STOREBASE
+    if [[ $STORAGE_TYPE == host ]]; then
+        if [ -z ${STOREBASE+x} ]; then
+            if [[ ! -d CHRIS_REMOTE_FS ]] ; then
+                mkdir CHRIS_REMOTE_FS
+            fi
+            STOREBASE=$HERE/CHRIS_REMOTE_FS
+        else
+            if [[ ! -d $STOREBASE ]] ; then
+                mkdir -p $STOREBASE
+            fi
         fi
     fi
+    echo -e "exporting ORCHESTRATOR=$ORCHESTRATOR"                | ./boxes.sh
+    export ORCHESTRATOR=$ORCHESTRATOR
+    echo -e "exporting STORAGE_TYPE=$STORAGE_TYPE"                  | ./boxes.sh
+    export STORAGE_TYPE=$STORAGE_TYPE
+    if [[ $STORAGE_TYPE == nfs ]]; then
+        echo -e "exporting NFS_SERVER=$NFS_SERVER"              | ./boxes.sh
+        export NFS_SERVER=$NFS_SERVER
+    fi
     echo -e "exporting STOREBASE=$STOREBASE"                      | ./boxes.sh
     export STOREBASE=$STOREBASE
     if [[ $ORCHESTRATOR == kubernetes ]]; then
@@ -125,8 +174,8 @@ if [[ "$COMMAND" == 'up' ]]; then
 
     title -d 1 "Starting pfcon containerized prod environment on $ORCHESTRATOR"
     if [[ $ORCHESTRATOR == swarm ]]; then
-        echo "docker stack deploy -c swarm/prod_deployments/docker-compose.yml pfcon_stack"   | ./boxes.sh ${LightCyan}
-        docker stack deploy -c swarm/prod_deployments/docker-compose.yml pfcon_stack
+        echo "docker stack deploy -c swarm/prod/docker-compose.yml pfcon_stack"   | ./boxes.sh ${LightCyan}
+        docker stack deploy -c swarm/prod/docker-compose.yml pfcon_stack
     elif [[ $ORCHESTRATOR == kubernetes ]]; then
         echo "kubectl create namespace $NAMESPACE"   | ./boxes.sh ${LightCyan}
         namespace=$(kubectl get namespaces $NAMESPACE --no-headers -o custom-columns=:metadata.name 2> /dev/null)
@@ -135,8 +184,13 @@ if [[ "$COMMAND" == 'up' ]]; then
         else
             echo "$NAMESPACE namespace already exists, skipping creation"
         fi
-        echo "kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl apply -f -"  | ./boxes.sh ${LightCyan}
-        kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl apply -f -
+        if [[ $STORAGE_TYPE == host ]]; then
+            echo "kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl apply -f -"  | ./boxes.sh ${LightCyan}
+            kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl apply -f -
+        else
+            echo "kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl apply -f -"  | ./boxes.sh ${LightCyan}
+            kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl apply -f -
+        fi
     fi
     windowBottom
 fi
@@ -148,10 +202,13 @@ if [[ "$COMMAND" == 'down' ]]; then
         echo "docker stack rm pfcon_stack"                               | ./boxes.sh ${LightCyan}
         docker stack rm pfcon_stack
     elif [[ $ORCHESTRATOR == kubernetes ]]; then
-        echo "kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl delete -f -"  | ./boxes.sh ${LightCyan}
-        kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl delete -f -
+        if [[ $STORAGE_TYPE == host ]]; then
+            echo "kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl delete -f -"  | ./boxes.sh ${LightCyan}
+            kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl delete -f -
+        else
+            echo "kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl delete -f -"  | ./boxes.sh ${LightCyan}
+            kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl delete -f -
+        fi
     fi
-    echo "Removing STOREBASE tree $STOREBASE"                            | ./boxes.sh
-    rm -fr $STOREBASE
     windowBottom
 fi

kubernetes/pfcon_dev.yaml

@@ -58,15 +58,15 @@ spec:
           args: ["--ip", "0.0.0.0", "--port", "30006", "--storeBase", "/home/localuser/storeBase", "--verbosity", "1"]
           volumeMounts:
             - mountPath: "/home/localuser/storeBase"
-              name: "store-base"
+              name: "storebase"
             - mountPath: "/home/localuser/pfcon/pfcon"
               name: "pfcon-source"
             - mountPath: "/home/localuser/pfcon/bin"
               name: "pfcon-bin"
             - mountPath: "/home/localuser/pfcon/tests"
               name: "pfcon-tests"
       volumes:
-        - name: "store-base"
+        - name: "storebase"
           hostPath:
             path: ${STOREBASE}
         - name: "pfcon-source"

kubernetes/prod_deployments/resources/pfcon.yaml → kubernetes/prod/base/resources/pfcon.yaml

@@ -42,8 +42,8 @@ spec:
           image: busybox:1.32
           command: [ "sh", "-c", "until wget --spider -S -T 2 http://$(PMAN_SERVICE_NAME):5010/api/v1/ 2>&1 | grep '200 OK'; do echo waiting for pman; done" ]
       containers:
-        - image: fnndsc/pfcon
-          name: pfcon
+        - name: pfcon
+          image: fnndsc/pfcon
           ports:
             - containerPort: 30005
           env:
@@ -55,11 +55,10 @@ spec:
           command: ["gunicorn"]
           args: ["-w", "5", "-b", "0.0.0.0:30005", "-t",  "200", "pfcon.wsgi:application"]
           volumeMounts:
-            - mountPath: "/home/localuser/storeBase"
-              name: "store-base"
-      # We need to mount a physical dir in the HOST onto the key store in pfcon. This dir
-      # is given by the STOREBASE env variable substitution.
+            - name: storebase
+              mountPath: "/home/localuser/storeBase"
+      # Mount a persistent volume onto pfcon's key store (shared data volume with plugins)
       volumes:
-        - name: "store-base"
-          hostPath:
-            path: ${STOREBASE}
+        - name: storebase
+          persistentVolumeClaim:
+            claimName: storebase

kubernetes/prod_deployments/resources/pman.yaml → kubernetes/prod/base/resources/pman.yaml

@@ -36,8 +36,8 @@ spec:
         env: production
     spec:
       containers:
-        - image: fnndsc/pman
-          name: pman
+        - name: pman
+          image: fnndsc/pman
           ports:
             - containerPort: 5010
           # Since pman spins off containers of its own it needs to mount storeBase dir
@@ -49,7 +49,7 @@ spec:
             - name: CONTAINER_ENV
               value: kubernetes
             - name: JOB_NAMESPACE
-              value: chris
+              value: ${NAMESPACE}
           envFrom:
             - configMapRef:
                 name: pman-config

kubernetes/prod/overlays/host/kustomization.yaml

@@ -0,0 +1,10 @@
+kind: Kustomization
+
+namespace: ${NAMESPACE}
+
+bases:
+- ../../base
+
+resources:
+- resources/storebase-pvc.yaml
+- resources/storebase-pv.yaml

kubernetes/prod/overlays/host/resources/storebase-pv.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: storebase
+  labels:
+    volume: storebase
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  # Mount a physical dir in the HOST onto the key store in pfcon. This dir
+  # is given by the STOREBASE env variable substitution.
+  hostPath:
+    path: ${STOREBASE}

kubernetes/prod/overlays/host/resources/storebase-pvc.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: storebase
+spec:
+  accessModes:
+  - ReadWriteOnce
+  storageClassName: ""
+  resources:
+    requests:
+      storage: 10Gi
+  selector:
+    matchLabels:
+      volume: storebase

kubernetes/prod/overlays/nfs/kustomization.yaml

@@ -0,0 +1,13 @@
+kind: Kustomization
+
+namespace: ${NAMESPACE}
+
+bases:
+- ../../base
+
+resources:
+- resources/storebase-pvc.yaml
+- resources/storebase-pv.yaml
+
+patches:
+- patches/pman-env.yaml

kubernetes/prod/overlays/nfs/patches/pman-env.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pman
+spec:
+  template:
+    spec:
+      containers:
+        - name: pman
+          # NFS needs additional env variables
+          env:
+            - name: STORAGE_TYPE
+              value: ${STORAGE_TYPE}
+            - name: NFS_SERVER
+              value: ${NFS_SERVER}