From a3b8e8148f72fc06aa692191425ef9650f7c1317 Mon Sep 17 00:00:00 2001 From: Chirayu Kapoor Date: Tue, 27 Aug 2024 17:28:13 +0530 Subject: [PATCH] * Add support for installing OpenCost alongside KubePlus * Create a script for installing the KubePlus operator and KubePlus kubectl plugin * Update README.md to include a "Quick Installation" section and move the getting-started example to a separate file to streamline the README * Add an OpenCost example to the getting-started file * Update GitHub Actions PR workflow to use the install.sh script for installing the KubePlus operator and kubectl plugin; modularized into distinct steps --- .github/workflows/pr.yaml | 155 +++++++++++++++++++------------ README.md | 156 +++---------------------------- examples/getting-started.md | 150 ++++++++++++++++++++++++++++++ install.sh | 178 ++++++++++++++++++++++++++++++++++++ 4 files changed, 440 insertions(+), 199 deletions(-) create mode 100644 examples/getting-started.md create mode 100755 install.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index db9d763a..b4badfd8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,116 +4,159 @@ on: env: ACTIONS_RUNNER_DEBUG: true KUBEPLUS_TEST_OUTPUT: yes + KUBEPLUS_CI: true jobs: job1: runs-on: ubuntu-20.04 name: Deploy to minikube steps: - - uses: actions/checkout@v2 - - name: Start minikube - uses: medyagh/setup-minikube@master - - name: Try the cluster ! + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1 + + - name: Verify Cluster run: kubectl get pods -A - - name: Deploy KubePlus and application to minikube + + - name: Display Glibc Version + run: ldd --version + + - name: Install Helm, Python3 LXML and Golang run: | - export SHELL=/bin/bash - eval $(minikube -p minikube docker-env) - echo "Glibc version" - ldd --version sudo apt-get install python3-lxml - echo "Installing helm..." + echo "Installing Helm..." wget https://get.helm.sh/helm-v3.12.1-linux-amd64.tar.gz gunzip helm-v3.12.1-linux-amd64.tar.gz tar -xvf helm-v3.12.1-linux-amd64.tar sudo mv linux-amd64/helm /usr/local/bin/. - echo "Installing golang..." - rm -rf /usr/local/go + echo "Installing Golang..." + rm -rf /usr/local/go wget https://go.dev/dl/go1.22.4.linux-amd64.tar.gz sudo tar -C /usr/local -xzf go1.22.4.linux-amd64.tar.gz export PATH=$PATH:/usr/local/go/bin go version + + - name: Prepare KubePlus Environment + run: | + echo "Setting up KubePlus environment..." echo "Current directory:`pwd`" - echo "Folders:`ls`" - kubeplus_folder="$(basename `pwd`)" - echo "KubePlus folder name:$kubeplus_folder" mkdir -p $HOME/go/src/github.com/cloud-ark cd .. - runner_dir=`pwd` echo "Current directory:`pwd`" - mv $kubeplus_folder $HOME/go/src/github.com/cloud-ark/kubeplus + kubeplus_folder="$(basename `pwd`)" + echo "KubePlus folder name:$kubeplus_folder" + cp -R $kubeplus_folder $HOME/go/src/github.com/cloud-ark/kubeplus cd $HOME/go/src/github.com/cloud-ark/kubeplus + export KUBEPLUS_NS=default + echo "KUBEPLUS_NS=default" >> $GITHUB_ENV export KUBEPLUS_HOME=`pwd` + echo "KUBEPLUS_HOME=$KUBEPLUS_HOME" >> $GITHUB_ENV export PATH=$KUBEPLUS_HOME/plugins:$PATH - echo "PATH:$PATH" - echo "KUBEPLUS_HOME:$KUBEPLUS_HOME" - kubectl kubeplus commands - export KUBEPLUS_NS=default - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - apiserver=`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` - echo "API_SERVER_URL:$apiserver" - python3 provider-kubeconfig.py -s $apiserver create $KUBEPLUS_NS - deactivate + echo "PATH=$PATH" >> $GITHUB_ENV - echo "Building mutating-webhook..." + - name: Build Mutating Webhook + run: | + echo "KUBEPLUS_HOME:$KUBEPLUS_HOME" cd $KUBEPLUS_HOME/mutating-webhook - export GO111MODULE=on; go get github.com/googleapis/gnostic@v0.4.0 + export GO111MODULE=on + go get github.com/googleapis/gnostic@v0.4.0 ./build-artifact.sh latest - echo "Building helmer..." + - name: Build Helmer + run: | cd $KUBEPLUS_HOME/platform-operator/helm-pod/ go mod vendor ./build-artifact.sh latest - echo "Building platform-operator..." + - name: Build Platform Operator + run: | cd $KUBEPLUS_HOME/platform-operator ./build-artifact.sh latest - echo "Building kubeconfiggenerator..." + - name: Build KubeConfig Generator + run: | cd $KUBEPLUS_HOME/deploy ./build-artifact-kubeconfiggenerator.sh latest - #echo "Building webhook_init_container..." - #./build-artifact.sh latest - #echo "Building resource cleaner..." - #./build-artifact-clean.sh latest - - #cd $KUBEPLUS_HOME/consumerui - #echo "Building consumer ui..." - #./build-artifact.sh latest + - name: List Docker Images + run: | cd $KUBEPLUS_HOME - ls docker images - echo "Installing KubePlus..." - helm install kubeplus ./deploy/kubeplus-chart --kubeconfig=kubeplus-saas-provider.json --set MUTATING_WEBHOOK=gcr.io/cloudark-kubeplus/pac-mutating-admission-webhook:latest --set PLATFORM_OPERATOR=gcr.io/cloudark-kubeplus/platform-operator:latest --set HELMER=gcr.io/cloudark-kubeplus/helm-pod:latest --set CRD_REGISTRATION_HELPER=gcr.io/cloudark-kubeplus/kubeconfiggenerator:latest -n $KUBEPLUS_NS + - name: Deploy KubePlus, Prometheus, and OpenCost to minikube + run: | + echo "Deploying KubePlus, Prometheus, and OpenCost..." + wget https://raw.githubusercontent.com/opencost/opencost/develop/configs/default.json + ./install.sh --prometheus --opencost default.json --kubeplus-plugin --kubeplus $KUBEPLUS_NS + + - name: Verify Prometheus Installation + run: | + echo "Verifying Prometheus installation..." + kubectl get pods -n prometheus-system | grep prometheus - kubectl get pods -A + - name: Verify OpenCost Installation + run: | + echo "Verifying OpenCost installation..." + kubectl get pods -n opencost | grep opencost + + - name: Verify KubePlus Installation + run: | + echo "Verifying KubePlus installation..." + kubectl get pods -n $KUBEPLUS_NS | grep kubeplus - until kubectl get pods -A | grep kubeplus | grep -i Running; do echo "Waiting for KubePlus to start.."; sleep 1; kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'`; kubectl get pods $kubeplus_pod; done + - name: Retrieve KubePlus Pod Logs + run: | kubeplus_pod=`kubectl get pods | grep kubeplus | awk '{print $1}'` - echo "helmer logs..." + echo "Helmer logs..." kubectl logs $kubeplus_pod -c helmer - echo "platform-operator logs..." + echo "Platform Operator logs..." kubectl logs $kubeplus_pod -c platform-operator - echo "crd-hook logs..." + echo "CRD Hook logs..." kubectl logs $kubeplus_pod -c crd-hook - kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json - echo "Sleeping for 10 seconds before continuing..." - sleep 10 + + - name: Upload Example Chart + run: kubectl upload chart ./examples/multitenancy/application-hosting/wordpress/wordpress-chart-0.0.3.tgz kubeplus-saas-provider.json + + - name: Sleep Before Continuing + run: sleep 10 + + - name: Deploy WordPress Service Composition + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml --kubeconfig=kubeplus-saas-provider.json - until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do echo "Waiting for CRD to be registered.."; sleep 1; done + + - name: Wait for CRD Registration + run: | + until kubectl get crds | grep wordpressservices.platformapi.kubeplus; do + echo "Waiting for CRD to be registered..." + sleep 1 + done + + - name: Deploy Tenant Example + run: | kubectl create -f ./examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json kubectl get resourcecompositions kubectl describe resourcecomposition wordpress-service-composition - until kubectl get pods -n wp-tenant1 | grep Running; do echo "Waiting for Application Pods to start.."; sleep 1; done + + - name: Wait for Application Pods to Start + run: | + until kubectl get pods -n wp-tenant1 | grep Running; do + echo "Waiting for Application Pods to start..." + sleep 1 + done + + - name: Interact with Deployed Application + run: | kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json + + - name: Cleanup Deployed Resources + run: | kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - echo "Running tests..starting in 5 seconds" - sleep 5 + + - name: Run Unit Tests + run: | cd tests python3 -m venv venv source venv/bin/activate diff --git a/README.md b/README.md index 386e7091..62585f4e 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ KubePlus takes an application Helm chart and wraps it under a Kubernetes API (CR

+## Key Features + ### Isolation KubePlus takes an application Helm chart and wraps it in a Kubernetes API (CRD). This API is used to provision application instances on a cluster. KubePlus isolates each application instance in a separate namespace. It adds a safety perimeter around such namespaces using Kubernetes network policies and non-shared persistent volumes ensuring that each application instance is appropriately isolated from other instances. Additionally, it provides controls for application providers to deploy different tenant application instances on different worker nodes for node isolation. @@ -37,155 +39,23 @@ A new version of an application can be deployed by updating the application Helm ### Customization -The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. - - -## Demo - -https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 - -## Getting Started with an example - -Let’s look at an example of creating a multi-instance WordPress Service using KubePlus. The WordPress service provider goes through the following steps towards this on their cluster: - -1. Create cluster or use an existing cluster. For testing purposes you can create a [minikube](https://minikube.sigs.k8s.io/docs/) or [kind](https://kind.sigs.k8s.io/) cluster: - - `minikube start` - - or - - `kind create cluster` - -2. Unzip KubePlus plugins and set up the PATH - - ``` - wget https://github.com/cloud-ark/kubeplus/raw/master/kubeplus-kubectl-plugins.tar.gz - tar -zxvf kubeplus-kubectl-plugins.tar.gz - export KUBEPLUS_HOME=`pwd` - export PATH=$KUBEPLUS_HOME/plugins:$PATH - kubectl kubeplus commands - ``` - -3. Set the Namespace in which to deploy KubePlus - - `export KUBEPLUS_NS=default` - -4. Create provider kubeconfig using provider-kubeconfig.py - - ``` - wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/requirements.txt - wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/provider-kubeconfig.py - python3 -m venv venv - source venv/bin/activate - pip3 install -r requirements.txt - apiserver=`kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'` - python3 provider-kubeconfig.py -s $apiserver create $KUBEPLUS_NS - deactivate - ``` - -5. Install KubePlus Operator using the generated provider kubeconfig - - ``` - helm install kubeplus "https://github.com/cloud-ark/operatorcharts/blob/master/kubeplus-chart-4.0.0.tgz?raw=true" --kubeconfig=kubeplus-saas-provider.json -n $KUBEPLUS_NS - until kubectl get pods -A | grep kubeplus | grep Running; do echo "Waiting for KubePlus to start.."; sleep 1; done - ``` - -6. Create Kubernetes CRD representing WordPress Helm chart. - - *The WordPress Helm chart can be specified as a [public url](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml) or can be [available locally](./examples/multitenancy/application-hosting/wordpress/wordpress-service-composition-localchart.yaml).* +The spec properties of the Kubernetes CRD wrapping the application Helm chart are the fields defined in the chart’s values.yaml file. Application deployments can be customized by specifying different values for these spec properties. - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/wordpress-service-composition.yaml --kubeconfig=kubeplus-saas-provider.json - kubectl get resourcecompositions - kubectl describe resourcecomposition wordpress-service-composition - ``` - - If the status of the `wordpress-service-composition` indicates that the new CRD has been created successfully, verify it: - - ``` - kubectl get crds - ``` - - You should see `wordpressservices.platformapi.kubeplus` CRD registered. +## Quick Installation -7. Create WordpressService instance `wp-tenant1` +To install KubePlus and its kubectl plugin, use the following commands: - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant1.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -8. Create WordpressService instance `wp-tenant2` - - ``` - kubectl create -f https://raw.githubusercontent.com/cloud-ark/kubeplus/master/examples/multitenancy/application-hosting/wordpress/tenant2.yaml --kubeconfig=kubeplus-saas-provider.json - ``` - -9. Check created WordpressService instances - - ``` - kubectl get wordpressservices - - NAME AGE - wp-tenant1 86s - wp-tenant2 26s - ``` - -10. Check the details of created instance: - - ``` - kubectl describe wordpressservices wp-tenant1 - ``` - -11. Check created application resources. Notice that the `WordpressService` instance resources are deployed in a Namespace `wp-tenant1`, which was created by KubePlus. - - ``` - kubectl appresources WordpressService wp-tenant1 –k kubeplus-saas-provider.json - - NAMESPACE KIND NAME - default WordpressService wp-tenant1 - wp-tenant1 PersistentVolumeClaim mysql-pv-claim - wp-tenant1 PersistentVolumeClaim wp-for-tenant1 - wp-tenant1 Service wordpress-mysql - wp-tenant1 Service wp-for-tenant1 - wp-tenant1 Deployment mysql - wp-tenant1 Deployment wp-for-tenant1 - wp-tenant1 Pod mysql-76d6d9bdfd-2wl2p - wp-tenant1 Pod wp-for-tenant1-87c4c954-s2cct - wp-tenant1 NetworkPolicy allow-external-traffic - wp-tenant1 NetworkPolicy restrict-cross-ns-traffic - wp-tenant1 ResourceQuota wordpressservice-wp-tenant1 - ``` +```sh +wget https://raw.githubusercontent.com/cloud-ark/kubeplus/master/install.sh +chmod +x install.sh +./install.sh --kubeplus --kubeplus-plugin +``` -12. Check application resource consumption - - ``` - kubectl metrics WordpressService wp-tenant1 $KUBEPLUS_NS -k kubeplus-saas-provider.json - - ---------------------------------------------------------- - Kubernetes Resources created: - Number of Sub-resources: - - Number of Pods: 2 - Number of Containers: 2 - Number of Nodes: 1 - Number of Not Running Pods: 0 - Underlying Physical Resoures consumed: - Total CPU(cores): 0.773497m - Total MEMORY(bytes): 516.30859375Mi - Total Storage(bytes): 40Gi - Total Network bytes received: 0 - Total Network bytes transferred: 0 - ---------------------------------------------------------- - ``` +## Demo -13. Cleanup +https://github.com/cloud-ark/kubeplus/assets/732525/efb255ff-fc73-446b-a583-4b89dbf61638 - ``` - kubectl delete wordpressservice wp-tenant1 --kubeconfig=kubeplus-saas-provider.json - kubectl delete wordpressservice wp-tenant2 --kubeconfig=kubeplus-saas-provider.json - kubectl delete resourcecomposition wordpress-service-composition --kubeconfig=kubeplus-saas-provider.json - helm delete kubeplus -n $KUBEPLUS_NS - python3 provider-kubeconfig.py delete $KUBEPLUS_NS - ``` +To get started with an example, follow: [kubeplus/examples/getting-started.md](kubeplus/examples/getting-started.md)