From f92480c51bfba042ca23eb14ed9b6332e78e91a0 Mon Sep 17 00:00:00 2001 From: Daniel Paul Date: Thu, 8 Feb 2024 15:35:51 +0100 Subject: [PATCH] unify kubevirt deployment for ocp / vanilla --- assets/kubevirt/migration.yml | 7 -- assets/kubevirt/ocp/pxbbq-route.yml | 16 ++++ assets/kubevirt/{ocp => }/pxbbq-freebsd.yml | 17 ----- assets/kubevirt/{ocp => }/pxbbq.yml | 17 ----- assets/kubevirt/vm.yml | 81 --------------------- docs/templates/ocp-kubevirt/readme.MD | 4 +- scripts/kubevirt | 44 ++++++----- scripts/kubevirt-apps | 17 +++++ scripts/kubevirt-apps-freebsd | 17 +++++ scripts/ocp-kubevirt-apps | 6 -- scripts/ocp-kubevirt-apps-freebsd | 8 -- templates/kubevirt.yml | 2 +- templates/ocp-kubevirt.yml | 2 +- vsphere.go | 2 +- 14 files changed, 80 insertions(+), 160 deletions(-) delete mode 100644 assets/kubevirt/migration.yml create mode 100644 assets/kubevirt/ocp/pxbbq-route.yml rename assets/kubevirt/{ocp => }/pxbbq-freebsd.yml (94%) rename assets/kubevirt/{ocp => }/pxbbq.yml (95%) delete mode 100644 assets/kubevirt/vm.yml create mode 100644 scripts/kubevirt-apps create mode 100644 scripts/kubevirt-apps-freebsd delete mode 100644 scripts/ocp-kubevirt-apps delete mode 100644 scripts/ocp-kubevirt-apps-freebsd diff --git a/assets/kubevirt/migration.yml b/assets/kubevirt/migration.yml deleted file mode 100644 index f132aabd..00000000 --- a/assets/kubevirt/migration.yml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachineInstanceMigration -metadata: - name: testvm1-migration -spec: - vmiName: testvm1 - diff --git a/assets/kubevirt/ocp/pxbbq-route.yml b/assets/kubevirt/ocp/pxbbq-route.yml new file mode 100644 index 00000000..d431dedc --- /dev/null +++ b/assets/kubevirt/ocp/pxbbq-route.yml @@ -0,0 +1,16 @@ +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: pxbbq + namespace: pxbbq + labels: + app: pxbbq-web + px-dr: 'false' +spec: + to: + kind: Service + name: pxbbq-svc + weight: 100 + port: + targetPort: 8080 + wildcardPolicy: None \ No newline at end of file diff --git a/assets/kubevirt/ocp/pxbbq-freebsd.yml b/assets/kubevirt/pxbbq-freebsd.yml similarity index 94% rename from assets/kubevirt/ocp/pxbbq-freebsd.yml rename to assets/kubevirt/pxbbq-freebsd.yml index 3390592f..b0dadf34 100644 --- a/assets/kubevirt/ocp/pxbbq-freebsd.yml +++ b/assets/kubevirt/pxbbq-freebsd.yml @@ -64,23 +64,6 @@ spec: selector: app: pxbbq-web --- -kind: Route -apiVersion: route.openshift.io/v1 -metadata: - name: pxbbq - namespace: pxbbq - labels: - app: pxbbq-web - px-dr: 'false' -spec: - to: - kind: Service - name: pxbbq-svc - weight: 100 - port: - targetPort: 8080 - wildcardPolicy: None ---- apiVersion: kubevirt.io/v1 kind: VirtualMachine metadata: diff --git a/assets/kubevirt/ocp/pxbbq.yml b/assets/kubevirt/pxbbq.yml similarity index 95% rename from assets/kubevirt/ocp/pxbbq.yml rename to assets/kubevirt/pxbbq.yml index c8ffe4a7..e3d2c240 100644 --- a/assets/kubevirt/ocp/pxbbq.yml +++ b/assets/kubevirt/pxbbq.yml @@ -64,23 +64,6 @@ spec: selector: app: pxbbq-web --- -kind: Route -apiVersion: route.openshift.io/v1 -metadata: - name: pxbbq - namespace: pxbbq - labels: - app: pxbbq-web - px-dr: 'false' -spec: - to: - kind: Service - name: pxbbq-svc - weight: 100 - port: - targetPort: 8080 - wildcardPolicy: None ---- apiVersion: kubevirt.io/v1 kind: VirtualMachine metadata: diff --git a/assets/kubevirt/vm.yml b/assets/kubevirt/vm.yml deleted file mode 100644 index e61fd307..00000000 --- a/assets/kubevirt/vm.yml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: testvm1-ssh -spec: - ports: - - port: 22 - protocol: TCP - targetPort: 22 - selector: - kubevirt.io/vm: testvm1 - type: NodePort ---- -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - labels: - kubevirt.io/vm: testvm1 - name: testvm1 -spec: - dataVolumeTemplates: - - metadata: - name: ubuntu-dv - spec: - pvc: - storageClassName: px-virtualization - accessModes: - - ReadWriteMany - resources: - requests: - storage: 5Gi - source: - http: - url: "https://cloud-images.ubuntu.com/minimal/releases/mantic/release/ubuntu-23.10-minimal-cloudimg-amd64.img" - running: true - template: - metadata: - labels: - kubevirt.io/vm: testvm1 - spec: - domain: - cpu: - cores: 1 - devices: - disks: - - bootOrder: 1 - disk: - bus: virtio - name: test-datavolume - - bootOrder: 2 - disk: - bus: virtio - name: cloudinitvolume - interfaces: - - name: default - masquerade: {} - resources: - requests: - memory: 2Gi - networks: - - name: default - pod: {} - volumes: - - dataVolume: - name: ubuntu-dv - name: test-datavolume - - cloudInitNoCloud: - userData: | - #cloud-config - hostname: testvm1 - users: - - name: kubevirt - sudo: ALL=(ALL) NOPASSWD:ALL - passwd: $6$Q1dGl.LfuMeS8RPP$b4xynn.Z3n2/h.YqOV90H7GzoAfjLKPMAKP1rzCeeBxiYNhROkxOXAC4rmQNbQf3oZ.Om8/Q7W8XmTgHsdoLw. - shell: /bin/bash - home: /home/kubevirt - lock_passwd: false - runcmd: - - sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/60-cloudimg-settings.conf - - systemctl restart ssh - name: cloudinitvolume diff --git a/docs/templates/ocp-kubevirt/readme.MD b/docs/templates/ocp-kubevirt/readme.MD index 1e8e60d9..b79dae64 100644 --- a/docs/templates/ocp-kubevirt/readme.MD +++ b/docs/templates/ocp-kubevirt/readme.MD @@ -89,8 +89,8 @@ env: * in case pxbbq deployment fails you can delete the default (FreeBSD) Deployment and try Ubuntu -* **c1cli**: `$ kubectl delete -f /assets/kubevirt/ocp/pxbbq-freebsd.yml` +* **c1cli**: `$ kubectl delete -f /assets/kubevirt/pxbbq-freebsd.yml` * `$ kubectl apply -f /assets/kubevirt/ubuntu-dv.yml` -* `$ kubectl apply -f /assets/kubevirt/ocp/pxbbq-freebsd.yml` \ No newline at end of file +* `$ kubectl apply -f /assets/kubevirt/pxbbq.yml` \ No newline at end of file diff --git a/scripts/kubevirt b/scripts/kubevirt index de8a5832..79cee371 100644 --- a/scripts/kubevirt +++ b/scripts/kubevirt @@ -1,32 +1,32 @@ -export KUBEVIRTVERSION="v1.1.0" -export CDIVERSION="v1.57.0" +export KUBEVIRTVERSION="v1.1.1" +export CDIVERSION="v1.58.1" #export KUBEVIRTVERSION=$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases | grep tag_name | grep -v -- '-rc' | sort -r | head -1 | awk -F': ' '{print $2}' | sed 's/,//' | xargs) echo "downloading kubevirt $KUBEVIRTVERSION" -kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRTVERSION}/kubevirt-operator.yaml +kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRTVERSION}/kubevirt-operator.yaml echo "waiting for kubevirt operator readiness" while ! kubectl wait --for=condition=ready pod --timeout=300s -lname=virt-operator -n kubevirt; do sleep 2 done -# check if emulation (slow!!!) needed -# hint: check only done on master node... -egrep '^flags.*(vmx|svm)' /proc/cpuinfo >/dev/null -if [ $? = 1 ]; then - echo "No virtualizazion capabilities found on master node cpu" - echo "switching on software emulation (slow)" - kubectl -n kubevirt patch kubevirts kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' -fi - -kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRTVERSION}/kubevirt-cr.yaml +kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRTVERSION}/kubevirt-cr.yaml echo "waiting for kubevirt CRD readiness" while ! kubectl wait kubevirt.kubevirt.io/kubevirt -n kubevirt --timeout=300s --for=jsonpath='{.status.phase}'='Deployed'; do sleep 2 done +# check node-1-1 if emulation (slow!!!) needed +echo "checking for virtualizazion capabilities on node-1-1 cpu" +ssh node-1-1 egrep '^flags.*\(vmx\|svm\)' /proc/cpuinfo >/dev/null +if [ $? = 1 ]; then + echo "No virtualizazion capabilities found on node-1-1 cpu" + echo "enabling software emulation (slow)" + kubectl -n kubevirt patch kubevirts kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}' +fi + echo "installing virtctl" ARCH=$(uname -s | tr A-Z a-z)-$(uname -m | sed 's/x86_64/amd64/') || windows-amd64.exe echo ${ARCH} @@ -35,12 +35,18 @@ chmod +x /tmp/virtctl sudo install /tmp/virtctl /usr/local/bin ln -s /usr/local/bin/virtctl /usr/local/bin/kubectl-virt -kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${CDIVERSION}/cdi-operator.yaml -kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${CDIVERSION}/cdi-cr.yaml - -kubectl apply -f /assets/kubevirt/px-virt-sc.yml +kubectl apply -f https://github.com/kubevirt/containerized-data-importer/releases/download/${CDIVERSION}/cdi-operator.yaml +kubectl apply -f https://github.com/kubevirt/containerized-data-importer/releases/download/${CDIVERSION}/cdi-cr.yaml -while ! kubectl patch storageprofile --type merge px-virtualization -p '{ "spec": { "claimPropertySets": [ { "accessModes": [ "ReadWriteMany" ], "volumeMode": "Filesystem" } ], "cloneStrategy": "csi-clone" } }'; do - echo waiting for storageprofile +echo "waiting for cdi operator readiness" +while ! kubectl wait cdis.cdi.kubevirt.io cdi -n cdi --for=jsonpath='{.status.phase}'='Deployed'; do sleep 2 done + + +#kubectl apply -f /assets/kubevirt/px-virt-sc.yml + +#while ! kubectl patch storageprofile --type merge px-virtualization -p '{ "spec": { "claimPropertySets": [ { "accessModes": [ "ReadWriteMany" ], "volumeMode": "Filesystem" } ], "cloneStrategy": "csi-clone" } }'; do +# echo waiting for storageprofile +# sleep 2 +#done diff --git a/scripts/kubevirt-apps b/scripts/kubevirt-apps new file mode 100644 index 00000000..352a6a3c --- /dev/null +++ b/scripts/kubevirt-apps @@ -0,0 +1,17 @@ +kubectl create ns pxbbq +kubectl label ns pxbbq app=pxbbq +kubectl label ns pxbbq backup=true + +kubectl apply -f /assets/kubevirt/dv-ubuntu.yml +kubectl apply -f /assets/kubevirt/pxbbq.yml + +if [ "$platform" = ocp4 ]; then + kubectl apply -f /assets/kubevirt/ocp/pxbbq-route.yml +else + kubectl patch svc pxbbq-svc -n pxbbq --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"}]' + pubIP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) + appPort=$(kubectl get svc pxbbq-svc -n pxbbq -o=jsonpath='{.spec.ports[?(@.port==80)].nodePort}') + echo "Access PXBBQ on http://$pubIP:$appPort" >> /etc/motd +fi + + diff --git a/scripts/kubevirt-apps-freebsd b/scripts/kubevirt-apps-freebsd new file mode 100644 index 00000000..36c9d2f2 --- /dev/null +++ b/scripts/kubevirt-apps-freebsd @@ -0,0 +1,17 @@ +kubectl create ns pxbbq +kubectl label ns pxbbq app=pxbbq +kubectl label ns pxbbq backup=true + +curl -O https://object-storage.public.mtl1.vexxhost.net/swift/v1/1dbafeefbd4f4c80864414a441e72dd2/bsd-cloud-image.org/images/freebsd/13.2/2023-04-21/zfs/freebsd-13.2-zfs-2023-04-21.qcow2 +virtctl image-upload pvc freebsd-image -n pxbbq --size 5Gi --insecure --storage-class px-csi-db --image-path=./freebsd-13.2-zfs-2023-04-21.qcow2 + +kubectl apply -f /assets/kubevirt/pxbbq-freebsd.yml + +if [ "$platform" = ocp4 ]; then + kubectl apply -f /assets/kubevirt/ocp/pxbbq-route.yml +else + kubectl patch svc pxbbq-svc -n pxbbq --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"}]' + pubIP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) + appPort=$(kubectl get svc pxbbq-svc -n pxbbq -o=jsonpath='{.spec.ports[?(@.port==80)].nodePort}') + echo "Access PXBBQ on http://$pubIP:$appPort" >> /etc/motd +fi diff --git a/scripts/ocp-kubevirt-apps b/scripts/ocp-kubevirt-apps deleted file mode 100644 index 45f82f02..00000000 --- a/scripts/ocp-kubevirt-apps +++ /dev/null @@ -1,6 +0,0 @@ -kubectl create ns pxbbq -kubectl label ns pxbbq app=pxbbq -kubectl label ns pxbbq backup=true - -kubectl apply -f /assets/kubevirt/dv-ubuntu.yml -kubectl apply -f /assets/kubevirt/ocp/pxbbq.yml diff --git a/scripts/ocp-kubevirt-apps-freebsd b/scripts/ocp-kubevirt-apps-freebsd deleted file mode 100644 index 96491b8d..00000000 --- a/scripts/ocp-kubevirt-apps-freebsd +++ /dev/null @@ -1,8 +0,0 @@ -kubectl create ns pxbbq -kubectl label ns pxbbq app=pxbbq -kubectl label ns pxbbq backup=true - -curl -O https://object-storage.public.mtl1.vexxhost.net/swift/v1/1dbafeefbd4f4c80864414a441e72dd2/bsd-cloud-image.org/images/freebsd/13.2/2023-04-21/zfs/freebsd-13.2-zfs-2023-04-21.qcow2 -virtctl image-upload pvc freebsd-image -n pxbbq --size 5Gi --insecure --storage-class px-csi-db --image-path=./freebsd-13.2-zfs-2023-04-21.qcow2 - -kubectl apply -f /assets/kubevirt/ocp/pxbbq-freebsd.yml diff --git a/templates/kubevirt.yml b/templates/kubevirt.yml index c5a9ddfb..4b1f3844 100644 --- a/templates/kubevirt.yml +++ b/templates/kubevirt.yml @@ -1,4 +1,4 @@ description: Install kubevirt on a bare-metal cluster -scripts: ["install-px","kubevirt"] +scripts: ["install-px","kubevirt","kubevirt-apps"] #aws_type: "c5n.metal" diff --git a/templates/ocp-kubevirt.yml b/templates/ocp-kubevirt.yml index 3e8e5e6e..368e600b 100644 --- a/templates/ocp-kubevirt.yml +++ b/templates/ocp-kubevirt.yml @@ -6,7 +6,7 @@ cloud: "aws" clusters: 2 cluster: - id: 1 - scripts: [ "ocp-kubevirt-apps-freebsd", "helm-backup", "helm-backup-ocp4-kubevirt"] + scripts: [ "kubevirt-apps-freebsd", "helm-backup", "helm-backup-ocp4-kubevirt"] - id: 2 scripts: ["clusterpair-dr"] env: diff --git a/vsphere.go b/vsphere.go index 0bed93da..9bcf9edb 100644 --- a/vsphere.go +++ b/vsphere.go @@ -697,7 +697,7 @@ func vsphere_init() { var govc_opts []string config := parse_yaml("defaults.yml") - fmt.Printf("Hint: there is a way faster way to deploy the base template. \n Please follow this documentation:\n https://github.com/andrewh1978/px-deploy/tree/master/docs/vsphere/README.md \n Or get a coffee now\n") + fmt.Printf("Hint: there is a way faster way to deploy the base template. \n Please follow this documentation:\n https://github.com/andrewh1978/px-deploy/tree/master/docs/cloud/vsphere/README.md \n Or get a coffee now\n") checkvar := []string{"vsphere_compute_resource", "vsphere_datacenter", "vsphere_datastore", "vsphere_host", "vsphere_network", "vsphere_resource_pool", "vsphere_template", "vsphere_user", "vsphere_password", "vsphere_repo"} emptyVars := isEmpty(config.Vsphere_Compute_Resource, config.Vsphere_Datacenter, config.Vsphere_Datastore, config.Vsphere_Host, config.Vsphere_Network, config.Vsphere_Resource_Pool, config.Vsphere_Template, config.Vsphere_User, config.Vsphere_Password, config.Vsphere_Repo) if len(emptyVars) > 0 {