diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c13918e..cf2908b 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: - name: teardown run: | ./unmake.sh - sudo rm -fr ./FS + sudo rm -fr ./CHRIS_REMOTE_FS docker swarm leave --force test-cube: name: CUBE tests diff --git a/README.rst b/README.rst index 4cde9bc..d64bcf7 100755 --- a/README.rst +++ b/README.rst @@ -138,20 +138,23 @@ Production deployments Docker Swarm-based deployment ============================= +A single-machine deployment is provided. + Configure pfcon services ------------------------ -Modify the ``.env`` files in the ``swarm/prod_deployments/secrets`` directory appropriately. +Modify the ``.env`` files in the ``swarm/prod/base/secrets`` directory appropriately. + +Single-machine deployment +------------------------- -Start production pfcon ----------------------- +Start production pfcon: .. code-block:: bash $> ./deploy.sh up -Tear down production pfcon --------------------------- +Tear down production pfcon: .. code-block:: bash @@ -160,21 +163,41 @@ Tear down production pfcon Kubernetes-based deployment =========================== +A single-machine deployment using Kubernetes' "hostPath" storage is provided. In addition +a multi-machine deployment for an external NFS drive is provided using NFS persistent volume. + Configure pfcon services ------------------------ -Modify the ``.env`` files in the ``kubernetes/prod_deployments/secrets`` directory appropriately. +Modify the ``.env`` files in the ``kubernetes/prod/base/secrets`` directory appropriately. -Start production pfcon ----------------------- +Single-machine deployment +------------------------- + +Start production pfcon: .. code-block:: bash $> ./deploy.sh -O kubernetes up Tear down production pfcon --------------------------- .. code-block:: bash $> ./deploy.sh -O kubernetes down + +Multi-machine deployment +------------------------- + +Start production pfcon: + +.. code-block:: bash + + $> ./deploy.sh -O kubernetes -T nfs -S -P up + +Tear down production pfcon + +.. code-block:: bash + + $> ./deploy.sh -O kubernetes -T nfs -S -P down + diff --git a/deploy.sh b/deploy.sh index c222089..7b5056d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -8,8 +8,10 @@ # # deploy.sh [-h] # [-O ] \ -# [-S ] \ # [-N ] \ +# [-T ] \ +# [-P ] \ +# [-S ] \ # [up|down] # # DESC @@ -44,6 +46,16 @@ # Explicitly set the kubernetes namespace to . Default is chris. # Not used for swarm. # +# -T +# +# Explicitly set the storage type for the STOREBASE dir. Default is host. +# Note: The nfs storage type is not implemented for swarm orchestrator yet. +# +# -P +# +# Set the IP address of the NFS server. Required when storage type is set to 'nfs'. +# Not used for 'host' storage type. +# # -S # # Explicitly set the STOREBASE dir to . This is the remote ChRIS @@ -62,14 +74,16 @@ source ./cparse.sh declare -i STEP=0 ORCHESTRATOR=swarm NAMESPACE=chris +STORAGE_TYPE=host HERE=$(pwd) print_usage () { - echo "Usage: ./deploy.sh [-h] [-O ] [-N ] [-S ] [up|down]" + echo "Usage: ./deploy.sh [-h] [-O ] [-N ] [-T ] + [-P ] [-S ] [up|down]" exit 1 } -while getopts ":hO:N:S:" opt; do +while getopts ":hO:N:T:P:S:" opt; do case $opt in h) print_usage ;; @@ -81,6 +95,14 @@ while getopts ":hO:N:S:" opt; do ;; N) NAMESPACE=$OPTARG ;; + T) STORAGE_TYPE=$OPTARG + if ! [[ "$STORAGE_TYPE" =~ ^(host|nfs)$ ]]; then + echo "Invalid value for option -- T" + print_usage + fi + ;; + P) NFS_SERVER=$OPTARG + ;; S) STOREBASE=$OPTARG ;; \?) echo "Invalid option -- $OPTARG" @@ -93,6 +115,23 @@ while getopts ":hO:N:S:" opt; do done shift $(($OPTIND - 1)) +if [[ $STORAGE_TYPE == nfs ]]; then + if [[ $ORCHESTRATOR == swarm ]]; then + echo -e "Sorry, nfs storage type is not supported for swarm orchestrator yet" | ./boxes.sh + exit 1 + fi + if [ -z ${NFS_SERVER+x} ]; then + echo "-P (the NFS server ip address) must be specified or the shell + environment variable NFS_SERVER must be set when using nfs storage type" + print_usage + fi + if [ -z ${STOREBASE+x} ]; then + echo "-S must be specified or the shell environment variable STOREBASE + must be set when using nfs storage type" + print_usage + fi +fi + COMMAND=up if (( $# == 1 )) ; then COMMAND=$1 @@ -103,16 +142,26 @@ if (( $# == 1 )) ; then fi title -d 1 "Setting global exports..." - if [ -z ${STOREBASE+x} ]; then - if [[ ! -d CHRIS_REMOTE_FS ]] ; then - mkdir CHRIS_REMOTE_FS - fi - STOREBASE=$HERE/CHRIS_REMOTE_FS - else - if [[ ! -d $STOREBASE ]] ; then - mkdir -p $STOREBASE + if [[ $STORAGE_TYPE == host ]]; then + if [ -z ${STOREBASE+x} ]; then + if [[ ! -d CHRIS_REMOTE_FS ]] ; then + mkdir CHRIS_REMOTE_FS + fi + STOREBASE=$HERE/CHRIS_REMOTE_FS + else + if [[ ! -d $STOREBASE ]] ; then + mkdir -p $STOREBASE + fi fi fi + echo -e "exporting ORCHESTRATOR=$ORCHESTRATOR" | ./boxes.sh + export ORCHESTRATOR=$ORCHESTRATOR + echo -e "exporting STORAGE_TYPE=$STORAGE_TYPE" | ./boxes.sh + export STORAGE_TYPE=$STORAGE_TYPE + if [[ $STORAGE_TYPE == nfs ]]; then + echo -e "exporting NFS_SERVER=$NFS_SERVER" | ./boxes.sh + export NFS_SERVER=$NFS_SERVER + fi echo -e "exporting STOREBASE=$STOREBASE" | ./boxes.sh export STOREBASE=$STOREBASE if [[ $ORCHESTRATOR == kubernetes ]]; then @@ -125,8 +174,8 @@ if [[ "$COMMAND" == 'up' ]]; then title -d 1 "Starting pfcon containerized prod environment on $ORCHESTRATOR" if [[ $ORCHESTRATOR == swarm ]]; then - echo "docker stack deploy -c swarm/prod_deployments/docker-compose.yml pfcon_stack" | ./boxes.sh ${LightCyan} - docker stack deploy -c swarm/prod_deployments/docker-compose.yml pfcon_stack + echo "docker stack deploy -c swarm/prod/docker-compose.yml pfcon_stack" | ./boxes.sh ${LightCyan} + docker stack deploy -c swarm/prod/docker-compose.yml pfcon_stack elif [[ $ORCHESTRATOR == kubernetes ]]; then echo "kubectl create namespace $NAMESPACE" | ./boxes.sh ${LightCyan} namespace=$(kubectl get namespaces $NAMESPACE --no-headers -o custom-columns=:metadata.name 2> /dev/null) @@ -135,8 +184,13 @@ if [[ "$COMMAND" == 'up' ]]; then else echo "$NAMESPACE namespace already exists, skipping creation" fi - echo "kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl apply -f -" | ./boxes.sh ${LightCyan} - kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl apply -f - + if [[ $STORAGE_TYPE == host ]]; then + echo "kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl apply -f -" | ./boxes.sh ${LightCyan} + kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl apply -f - + else + echo "kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl apply -f -" | ./boxes.sh ${LightCyan} + kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl apply -f - + fi fi windowBottom fi @@ -148,10 +202,13 @@ if [[ "$COMMAND" == 'down' ]]; then echo "docker stack rm pfcon_stack" | ./boxes.sh ${LightCyan} docker stack rm pfcon_stack elif [[ $ORCHESTRATOR == kubernetes ]]; then - echo "kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl delete -f -" | ./boxes.sh ${LightCyan} - kubectl kustomize kubernetes/prod_deployments | envsubst | kubectl delete -f - + if [[ $STORAGE_TYPE == host ]]; then + echo "kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl delete -f -" | ./boxes.sh ${LightCyan} + kubectl kustomize kubernetes/prod/overlays/host | envsubst | kubectl delete -f - + else + echo "kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl delete -f -" | ./boxes.sh ${LightCyan} + kubectl kustomize kubernetes/prod/overlays/nfs | envsubst | kubectl delete -f - + fi fi - echo "Removing STOREBASE tree $STOREBASE" | ./boxes.sh - rm -fr $STOREBASE windowBottom fi diff --git a/kubernetes/pfcon_dev.yaml b/kubernetes/pfcon_dev.yaml index 7c7762f..28e6bab 100755 --- a/kubernetes/pfcon_dev.yaml +++ b/kubernetes/pfcon_dev.yaml @@ -58,7 +58,7 @@ spec: args: ["--ip", "0.0.0.0", "--port", "30006", "--storeBase", "/home/localuser/storeBase", "--verbosity", "1"] volumeMounts: - mountPath: "/home/localuser/storeBase" - name: "store-base" + name: "storebase" - mountPath: "/home/localuser/pfcon/pfcon" name: "pfcon-source" - mountPath: "/home/localuser/pfcon/bin" @@ -66,7 +66,7 @@ spec: - mountPath: "/home/localuser/pfcon/tests" name: "pfcon-tests" volumes: - - name: "store-base" + - name: "storebase" hostPath: path: ${STOREBASE} - name: "pfcon-source" diff --git a/kubernetes/prod_deployments/kustomization.yaml b/kubernetes/prod/base/kustomization.yaml similarity index 100% rename from kubernetes/prod_deployments/kustomization.yaml rename to kubernetes/prod/base/kustomization.yaml diff --git a/kubernetes/prod_deployments/resources/pfcon.yaml b/kubernetes/prod/base/resources/pfcon.yaml similarity index 76% rename from kubernetes/prod_deployments/resources/pfcon.yaml rename to kubernetes/prod/base/resources/pfcon.yaml index ccb923d..4785b28 100755 --- a/kubernetes/prod_deployments/resources/pfcon.yaml +++ b/kubernetes/prod/base/resources/pfcon.yaml @@ -42,8 +42,8 @@ spec: image: busybox:1.32 command: [ "sh", "-c", "until wget --spider -S -T 2 http://$(PMAN_SERVICE_NAME):5010/api/v1/ 2>&1 | grep '200 OK'; do echo waiting for pman; done" ] containers: - - image: fnndsc/pfcon - name: pfcon + - name: pfcon + image: fnndsc/pfcon ports: - containerPort: 30005 env: @@ -55,11 +55,10 @@ spec: command: ["gunicorn"] args: ["-w", "5", "-b", "0.0.0.0:30005", "-t", "200", "pfcon.wsgi:application"] volumeMounts: - - mountPath: "/home/localuser/storeBase" - name: "store-base" - # We need to mount a physical dir in the HOST onto the key store in pfcon. This dir - # is given by the STOREBASE env variable substitution. + - name: storebase + mountPath: "/home/localuser/storeBase" + # Mount a persistent volume onto pfcon's key store (shared data volume with plugins) volumes: - - name: "store-base" - hostPath: - path: ${STOREBASE} + - name: storebase + persistentVolumeClaim: + claimName: storebase diff --git a/kubernetes/prod_deployments/resources/pman.yaml b/kubernetes/prod/base/resources/pman.yaml similarity index 92% rename from kubernetes/prod_deployments/resources/pman.yaml rename to kubernetes/prod/base/resources/pman.yaml index f90f120..fe2e81e 100755 --- a/kubernetes/prod_deployments/resources/pman.yaml +++ b/kubernetes/prod/base/resources/pman.yaml @@ -36,8 +36,8 @@ spec: env: production spec: containers: - - image: fnndsc/pman - name: pman + - name: pman + image: fnndsc/pman ports: - containerPort: 5010 # Since pman spins off containers of its own it needs to mount storeBase dir @@ -49,7 +49,7 @@ spec: - name: CONTAINER_ENV value: kubernetes - name: JOB_NAMESPACE - value: chris + value: ${NAMESPACE} envFrom: - configMapRef: name: pman-config diff --git a/kubernetes/prod_deployments/secrets/.pfcon.env b/kubernetes/prod/base/secrets/.pfcon.env similarity index 100% rename from kubernetes/prod_deployments/secrets/.pfcon.env rename to kubernetes/prod/base/secrets/.pfcon.env diff --git a/kubernetes/prod_deployments/secrets/.pman.env b/kubernetes/prod/base/secrets/.pman.env similarity index 100% rename from kubernetes/prod_deployments/secrets/.pman.env rename to kubernetes/prod/base/secrets/.pman.env diff --git a/kubernetes/prod/overlays/host/kustomization.yaml b/kubernetes/prod/overlays/host/kustomization.yaml new file mode 100755 index 0000000..3d445dc --- /dev/null +++ b/kubernetes/prod/overlays/host/kustomization.yaml @@ -0,0 +1,10 @@ +kind: Kustomization + +namespace: ${NAMESPACE} + +bases: +- ../../base + +resources: +- resources/storebase-pvc.yaml +- resources/storebase-pv.yaml diff --git a/kubernetes/prod/overlays/host/resources/storebase-pv.yaml b/kubernetes/prod/overlays/host/resources/storebase-pv.yaml new file mode 100755 index 0000000..820fde4 --- /dev/null +++ b/kubernetes/prod/overlays/host/resources/storebase-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storebase + labels: + volume: storebase +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + # Mount a physical dir in the HOST onto the key store in pfcon. This dir + # is given by the STOREBASE env variable substitution. + hostPath: + path: ${STOREBASE} diff --git a/kubernetes/prod/overlays/host/resources/storebase-pvc.yaml b/kubernetes/prod/overlays/host/resources/storebase-pvc.yaml new file mode 100755 index 0000000..ff51d11 --- /dev/null +++ b/kubernetes/prod/overlays/host/resources/storebase-pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storebase +spec: + accessModes: + - ReadWriteOnce + storageClassName: "" + resources: + requests: + storage: 10Gi + selector: + matchLabels: + volume: storebase diff --git a/kubernetes/prod/overlays/nfs/kustomization.yaml b/kubernetes/prod/overlays/nfs/kustomization.yaml new file mode 100755 index 0000000..fa281cf --- /dev/null +++ b/kubernetes/prod/overlays/nfs/kustomization.yaml @@ -0,0 +1,13 @@ +kind: Kustomization + +namespace: ${NAMESPACE} + +bases: +- ../../base + +resources: +- resources/storebase-pvc.yaml +- resources/storebase-pv.yaml + +patches: +- patches/pman-env.yaml diff --git a/kubernetes/prod/overlays/nfs/patches/pman-env.yaml b/kubernetes/prod/overlays/nfs/patches/pman-env.yaml new file mode 100755 index 0000000..b9891be --- /dev/null +++ b/kubernetes/prod/overlays/nfs/patches/pman-env.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pman +spec: + template: + spec: + containers: + - name: pman + # NFS needs additional env variables + env: + - name: STORAGE_TYPE + value: ${STORAGE_TYPE} + - name: NFS_SERVER + value: ${NFS_SERVER} diff --git a/kubernetes/prod/overlays/nfs/resources/storebase-pv.yaml b/kubernetes/prod/overlays/nfs/resources/storebase-pv.yaml new file mode 100755 index 0000000..409ab9a --- /dev/null +++ b/kubernetes/prod/overlays/nfs/resources/storebase-pv.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: storebase + labels: + volume: storebase +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 10Gi + nfs: + # The key store in pfcon beacomes a path in the NFS drive + server: ${NFS_SERVER} + path: ${STOREBASE} diff --git a/kubernetes/prod/overlays/nfs/resources/storebase-pvc.yaml b/kubernetes/prod/overlays/nfs/resources/storebase-pvc.yaml new file mode 100755 index 0000000..2afb342 --- /dev/null +++ b/kubernetes/prod/overlays/nfs/resources/storebase-pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storebase +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 10Gi + selector: + matchLabels: + volume: storebase diff --git a/make.sh b/make.sh index 0076441..93963a8 100755 --- a/make.sh +++ b/make.sh @@ -147,6 +147,8 @@ title -d 1 "Setting global exports..." mkdir -p $STOREBASE fi fi + echo -e "exporting ORCHESTRATOR=$ORCHESTRATOR" | ./boxes.sh + export ORCHESTRATOR=$ORCHESTRATOR echo -e "exporting STOREBASE=$STOREBASE " | ./boxes.sh export STOREBASE=$STOREBASE export SOURCEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/swarm/prod_deployments/docker-compose.yml b/swarm/prod/docker-compose.yml similarity index 100% rename from swarm/prod_deployments/docker-compose.yml rename to swarm/prod/docker-compose.yml diff --git a/swarm/prod_deployments/secrets/.pfcon.env b/swarm/prod/secrets/.pfcon.env similarity index 100% rename from swarm/prod_deployments/secrets/.pfcon.env rename to swarm/prod/secrets/.pfcon.env diff --git a/swarm/prod_deployments/secrets/.pman.env b/swarm/prod/secrets/.pman.env similarity index 100% rename from swarm/prod_deployments/secrets/.pman.env rename to swarm/prod/secrets/.pman.env diff --git a/unmake.sh b/unmake.sh index 4838b80..c4ac792 100755 --- a/unmake.sh +++ b/unmake.sh @@ -79,6 +79,8 @@ title -d 1 "Setting global exports..." if [ -z ${STOREBASE+x} ]; then STOREBASE=$(pwd)/CHRIS_REMOTE_FS fi + echo -e "exporting ORCHESTRATOR=$ORCHESTRATOR" | ./boxes.sh + export ORCHESTRATOR=$ORCHESTRATOR echo -e "exporting STOREBASE=$STOREBASE " | ./boxes.sh export STOREBASE=$STOREBASE windowBottom