Added longhorn via fleet

suse-edge · Jun 23, 2023 · aef8221 · aef8221
1 parent 8ecca3f
commit aef8221
Show file tree

Hide file tree

Showing 8 changed files with 166 additions and 2 deletions.
diff --git a/fleet-examples/README.md b/fleet-examples/README.md
@@ -16,4 +16,107 @@ kubectl apply -f https://raw.githubusercontent.com/suse-edge/misc/main/fleet-exa
 
 ```
 kubectl apply -f https://raw.githubusercontent.com/suse-edge/misc/main/fleet-examples/gitrepos/elemental-gitrepo.yaml
+```
+
+* [Longhorn](./fleets/longhorn) - [Longhorn](https://longhorn.io/):
+
+```
+kubectl apply -f https://raw.githubusercontent.com/suse-edge/misc/main/fleet-examples/gitrepos/longhorn-gitrepo.yaml
+```
+
+A few notes about this example:
+
+* The Longhorn UI is not exposed by default. If you want to expose it, you need to specify a couple of Helm values such as:
+
+```
+ingress:
+  enabled: true
+  host: "longhorn-example.com"
+```
+
+You can modify the Longhorn's [fleet.yaml](./fleets/longhorn/fleet.yaml) file to fit your needs.
+
+* You can configure Fleet to read Helm custom values in a configmap created somewhere in the cluster such as:
+
+```
+  valuesFrom:
+  - configMapKeyRef:
+      name: longhorn-chart-values
+      # default to namespace of bundle
+      namespace: fleet-local
+      key: values.yaml
+```
+
+Basically you can create a configmap there with the `values.yaml` content you want to provide. This is not restricted to ingress but anything included in the [Longhorn Helm Chart values.yaml](https://github.com/longhorn/longhorn/blob/master/chart/values.yaml) can be used:
+
+```
+cat <<- EOF | kubectl apply -f -
+apiVersion: v1
+data:
+  values.yaml: |
+    ingress:
+      enabled: true
+      host: "longhorn-example.com"
+kind: ConfigMap
+metadata:
+  name: longhorn-chart-values
+  namespace: fleet-local
+EOF
+```
+
+* The Fleet included here contains a customization such as:
+
+```
+targetCustomizations:
+  # Customization Name
+- name: local
+  # If the local cluster is used
+  clusterSelector:
+    matchLabels:
+      management.cattle.io/cluster-display-name: local
+  helm:
+    values:
+      ingress:
+        # Use this custom Helm values
+        enabled: true
+        # This is a manual annotation that needs to be set in the clusters.fleet.cattle.io/local object
+        host: longhorn-${ .ClusterAnnotations.ingressip }.sslip.io
+        # This annotation will enable user/password authentication for the Longhorn UI
+        annotations:
+          traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-longhorn-basic-auth@kubernetescrd
+  # This kustomization will create the required objects for the user/password authentication
+  kustomize:
+    dir: overlays/local
+```
+
+This means:
+  * If using a local cluster
+  * If the Traefik Ingress controller is deployed
+  * If the Traefik Ingress uses sslip.io
+  * If the local cluster has been annotated with the Ingress IP:
+
+`kubectl annotate clusters.fleet.cattle.io/local -n fleet-local  "ingressip=$(kubectl get svc -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[*].ip}')"`
+
+It will enable the Longhorn UI protected via user/password using a [kustomization overlay](./fleets/longhorn/longhorn/overlays/kustomization.yaml)
+
+This is basically intended to be used with the [create-vm.sh](../slemicro/create_vm.sh) script as:
+
+```
+create_vm.sh -f myvm
+export KUBECONFIG=$(get_kubeconfig.sh -f myvm -w)
+kubectl annotate clusters.fleet.cattle.io/local -n fleet-local  "ingressip=$(kubectl get svc -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[*].ip}')"
+kubectl apply -f https://raw.githubusercontent.com/suse-edge/misc/main/fleet-examples/gitrepos/longhorn-gitrepo.yaml
+```
+
+**NOTE:** Due to https://github.com/rancher/fleet/issues/1507, this needs to be done before applying the longhorn gitrepo:
+
+```
+helm -n cattle-fleet-system upgrade --create-namespace fleet-crd https://github.com/rancher/fleet/releases/download/v0.7.0/fleet-crd-0.7.0.tgz
+helm -n cattle-fleet-system upgrade --create-namespace fleet https://github.com/rancher/fleet/releases/download/v0.7.0/fleet-0.7.0.tgz
+```
+
+* To uninstall the application, it is required to set the `deleting-confirmation-flag` to true as per [the instructions](https://longhorn.io/docs/1.4.2/deploy/uninstall/#prerequisite) before removing the Helm chart or the `gitrepo` object:
+
+```
+kubectl -n longhorn-system patch -p '{"value": "true"}' --type=merge lhs deleting-confirmation-flag
 ```
diff --git a/fleet-examples/fleets/longhorn/longhorn-crd/fleet.yaml b/fleet-examples/fleets/longhorn/longhorn-crd/fleet.yaml
@@ -0,0 +1,5 @@
+defaultNamespace: longhorn-system
+helm:
+  releaseName: longhorn-crd
+  chart: longhorn-crd
+  repo: https://charts.rancher.io/
diff --git a/fleet-examples/fleets/longhorn/longhorn/fleet.yaml b/fleet-examples/fleets/longhorn/longhorn/fleet.yaml
@@ -0,0 +1,20 @@
+defaultNamespace: longhorn-system
+helm:
+  releaseName: longhorn
+  chart: longhorn
+  repo: https://charts.rancher.io/
+targetCustomizations:
+- name: local
+  clusterSelector:
+    matchLabels:
+      management.cattle.io/cluster-display-name: local
+  helm:
+    values:
+      ingress:
+        enabled: true
+        host: longhorn-${ .ClusterAnnotations.ingressip }.sslip.io
+        annotations:
+          traefik.ingress.kubernetes.io/router.middlewares: longhorn-system-longhorn-basic-auth@kubernetescrd
+  kustomize:
+    dir: overlays/local
+
diff --git a/fleet-examples/fleets/longhorn/longhorn/overlays/local/kustomization.yaml b/fleet-examples/fleets/longhorn/longhorn/overlays/local/kustomization.yaml
@@ -0,0 +1,4 @@
+resources:
+- longhorn-basic-auth-middleware.yaml
+- longhorn-ui-auth-secret.yaml
+
diff --git a/fleet-examples/fleets/longhorn/longhorn/overlays/local/longhorn-basic-auth-middleware.yaml b/fleet-examples/fleets/longhorn/longhorn/overlays/local/longhorn-basic-auth-middleware.yaml
@@ -0,0 +1,8 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: longhorn-basic-auth
+  namespace: longhorn-system
+spec:
+  basicAuth:
+    secret: longhorn-ui-auth
diff --git a/fleet-examples/fleets/longhorn/longhorn/overlays/local/longhorn-ui-auth-secret.yaml b/fleet-examples/fleets/longhorn/longhorn/overlays/local/longhorn-ui-auth-secret.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: longhorn-ui-auth
+  namespace: longhorn-system
+data:
+  # admin:admin
+  users: |2
+    YWRtaW46JGFwcjEkYXpXL0J4czMkWHFoNDYyUGdZS251Y3ljT0JjeVJCMQoK
diff --git a/fleet-examples/gitrepos/longhorn-gitrepo.yaml b/fleet-examples/gitrepos/longhorn-gitrepo.yaml
@@ -0,0 +1,10 @@
+kind: GitRepo
+apiVersion: fleet.cattle.io/v1alpha1
+metadata:
+  name: longhorn
+  namespace: fleet-local
+spec:
+  repo: https://github.com/suse-edge/misc.git
+  branch: main
+  paths:
+  - fleet-examples/fleets/longhorn/
diff --git a/slemicro/get_kubeconfig.sh b/slemicro/get_kubeconfig.sh
@@ -126,8 +126,13 @@ else
 		while ! [ "$(curl -s -w '%{http_code}' -k -o /dev/null https://${RANCHERHOSTNAME}/healthz)" -eq 200 ]; do sleep 10; done
 	fi
 
-	# Login
-	TOKEN=$(curl -sk -X POST https://${RANCHERHOSTNAME}/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d "{\"username\":\"admin\",\"responseType\": \"token\", \"password\": \"${RANCHERFINALPASSWORD}\"}" | jq -r .token)
+	TOKEN="null"
+	# Prevent a race condition while the rancher bootstrap is being applied
+	while [ ${TOKEN} == "null" ]; do
+		sleep 5
+		# Login
+		TOKEN=$(curl -sk -X POST https://${RANCHERHOSTNAME}/v3-public/localProviders/local?action=login -H 'content-type: application/json' -d "{\"username\":\"admin\",\"responseType\": \"token\", \"password\": \"${RANCHERFINALPASSWORD}\"}" | jq -r .token)
+	done
 
 	# Create a temp file
 	TMPKUBECONFIG=$(mktemp)