From 4088a4d510821d474bf3d295aa24d08570caa831 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 26 Nov 2024 19:59:16 +0900 Subject: [PATCH 1/9] feat: add sidecar for snapshot Signed-off-by: hlts2 --- deploy/install.yaml | 16 +++++++++++++++- deploy/kubernetes/04_controller.yaml | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/deploy/install.yaml b/deploy/install.yaml index a3639c3..53f5610 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -225,7 +225,7 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/csi.civo.com - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -237,6 +237,20 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/kubelet/plugins/csi.civo.com + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v8.1.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--timeout=30s" + - "--retry-interval-max=10" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi.civo.com - name: civo-csi-plugin image: gcr.io/consummate-yew-302509/csi:latest env: diff --git a/deploy/kubernetes/04_controller.yaml b/deploy/kubernetes/04_controller.yaml index ce03016..3be178f 100644 --- a/deploy/kubernetes/04_controller.yaml +++ b/deploy/kubernetes/04_controller.yaml @@ -58,7 +58,7 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/csi.civo.com - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -70,6 +70,20 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/kubelet/plugins/csi.civo.com + - name: csi-snapshotter + image: k8s.gcr.io/sig-storage/csi-snapshotter:v8.1.0 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--timeout=30s" + - "--retry-interval-max=10" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/csi.civo.com - name: civo-csi-plugin image: gcr.io/consummate-yew-302509/csi:latest env: From 4742af8da0e6eba08b3c905137bdb93491f8d798 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 26 Nov 2024 22:59:56 +0900 Subject: [PATCH 2/9] feat: add cluster role and binding for snapshotter Signed-off-by: hlts2 --- deploy/install.yaml | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/deploy/install.yaml b/deploy/install.yaml index 53f5610..dd8d9a3 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -148,6 +148,46 @@ roleRef: name: external-resizer-runner apiGroup: rbac.authorization.k8s.io --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-snapshotter-runner + rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +subjects: + - kind: ServiceAccount + name: civo-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-snapshotter-runner + apiGroup: rbac.authorization.k8s.io +--- apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: @@ -243,7 +283,6 @@ spec: - "--v=5" - "--csi-address=$(ADDRESS)" - "--timeout=30s" - - "--retry-interval-max=10" env: - name: ADDRESS value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock From f5c752f5facbcf8530a9422eacaf3f5de237493e Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 26 Nov 2024 23:06:43 +0900 Subject: [PATCH 3/9] fix: indent and add cluster role and binding Signed-off-by: hlts2 --- deploy/install.yaml | 45 +++++++++++++++++----------------- deploy/kubernetes/01_rbac.yaml | 40 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/deploy/install.yaml b/deploy/install.yaml index dd8d9a3..8d14c6e 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -152,28 +152,28 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: external-snapshotter-runner - rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -283,6 +283,7 @@ spec: - "--v=5" - "--csi-address=$(ADDRESS)" - "--timeout=30s" + - "--retry-interval-max=10" env: - name: ADDRESS value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock diff --git a/deploy/kubernetes/01_rbac.yaml b/deploy/kubernetes/01_rbac.yaml index cfbaaa7..11ff404 100644 --- a/deploy/kubernetes/01_rbac.yaml +++ b/deploy/kubernetes/01_rbac.yaml @@ -147,3 +147,43 @@ roleRef: kind: ClusterRole name: external-resizer-runner apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-snapshotter-runner +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +subjects: + - kind: ServiceAccount + name: civo-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: external-snapshotter-runner + apiGroup: rbac.authorization.k8s.io From be87b1f6e2d9b467db9e7b775f42a237321cf62d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 28 Nov 2024 15:36:17 +0900 Subject: [PATCH 4/9] feat: add snapshot controller manifest Signed-off-by: hlts2 --- deploy/kubernetes/kustomization.yaml | 8 ++++++++ deploy/kubernetes/snapshot/kustomization.yaml | 17 +++++++++++++++++ .../snapshot/volume_snapshot_class.yaml | 8 ++++++++ 3 files changed, 33 insertions(+) create mode 100644 deploy/kubernetes/kustomization.yaml create mode 100644 deploy/kubernetes/snapshot/kustomization.yaml create mode 100644 deploy/kubernetes/snapshot/volume_snapshot_class.yaml diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 0000000..c0bb909 --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 01_rbac.yaml + - 02_csi_driver.yaml + - 03_storage_class.yaml + - 04_controller.yaml + - 05_node.yaml diff --git a/deploy/kubernetes/snapshot/kustomization.yaml b/deploy/kubernetes/snapshot/kustomization.yaml new file mode 100644 index 0000000..d100453 --- /dev/null +++ b/deploy/kubernetes/snapshot/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # For CRD + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml + + ## For snapshot-controller + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.0.1/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.0.1/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + + # For SCI driver + - ../../kubernetes + + # For VolumeSnapshotClass + - volume_snapshot_class.yaml diff --git a/deploy/kubernetes/snapshot/volume_snapshot_class.yaml b/deploy/kubernetes/snapshot/volume_snapshot_class.yaml new file mode 100644 index 0000000..6f02d85 --- /dev/null +++ b/deploy/kubernetes/snapshot/volume_snapshot_class.yaml @@ -0,0 +1,8 @@ +# Usage of the v1 API implies that the cluster must have +# external-snapshotter v4.x installed.↴ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: civo-volume-snapshot +driver: csi.civo.com +deletionPolicy: Delete From 116848ea12a473df33156d56121675ad8059702f Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 28 Nov 2024 15:54:46 +0900 Subject: [PATCH 5/9] fix: tweak Signed-off-by: hlts2 --- deploy/kubernetes/kustomization.yaml | 3 +++ .../{snapshot => snapshot-controller}/kustomization.yaml | 3 --- .../volume_snapshot_class.yaml | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename deploy/kubernetes/{snapshot => snapshot-controller}/kustomization.yaml (95%) rename deploy/kubernetes/{snapshot => snapshot-controller}/volume_snapshot_class.yaml (100%) diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml index c0bb909..1ca78ce 100644 --- a/deploy/kubernetes/kustomization.yaml +++ b/deploy/kubernetes/kustomization.yaml @@ -1,6 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - snapshot-controller + + # For CSI driver - 01_rbac.yaml - 02_csi_driver.yaml - 03_storage_class.yaml diff --git a/deploy/kubernetes/snapshot/kustomization.yaml b/deploy/kubernetes/snapshot-controller/kustomization.yaml similarity index 95% rename from deploy/kubernetes/snapshot/kustomization.yaml rename to deploy/kubernetes/snapshot-controller/kustomization.yaml index d100453..0b43e70 100644 --- a/deploy/kubernetes/snapshot/kustomization.yaml +++ b/deploy/kubernetes/snapshot-controller/kustomization.yaml @@ -10,8 +10,5 @@ resources: - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.0.1/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml - https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.0.1/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml - # For SCI driver - - ../../kubernetes - # For VolumeSnapshotClass - volume_snapshot_class.yaml diff --git a/deploy/kubernetes/snapshot/volume_snapshot_class.yaml b/deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml similarity index 100% rename from deploy/kubernetes/snapshot/volume_snapshot_class.yaml rename to deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml From 9570dd46cb07fffd1bb20fd7c1d6b1f58aa7bf9d Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 28 Nov 2024 16:12:20 +0900 Subject: [PATCH 6/9] fix: override image tag name Signed-off-by: hlts2 --- deploy/kubernetes/snapshot-controller/kustomization.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/kubernetes/snapshot-controller/kustomization.yaml b/deploy/kubernetes/snapshot-controller/kustomization.yaml index 0b43e70..851242c 100644 --- a/deploy/kubernetes/snapshot-controller/kustomization.yaml +++ b/deploy/kubernetes/snapshot-controller/kustomization.yaml @@ -12,3 +12,7 @@ resources: # For VolumeSnapshotClass - volume_snapshot_class.yaml + +images: + - name: registry.k8s.io/sig-storage/snapshot-controller:v7.0.1 + newTag: v8.0.1 From b4edc495539af506a7968d63cd3b2a6afa82952b Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 28 Nov 2024 16:16:23 +0900 Subject: [PATCH 7/9] fix: snapshot controller image name Signed-off-by: hlts2 --- deploy/kubernetes/snapshot-controller/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/kubernetes/snapshot-controller/kustomization.yaml b/deploy/kubernetes/snapshot-controller/kustomization.yaml index 851242c..31efddf 100644 --- a/deploy/kubernetes/snapshot-controller/kustomization.yaml +++ b/deploy/kubernetes/snapshot-controller/kustomization.yaml @@ -15,4 +15,5 @@ resources: images: - name: registry.k8s.io/sig-storage/snapshot-controller:v7.0.1 + newName: k8s.gcr.io/sig-storage/snapshot-controller newTag: v8.0.1 From f3e2f866c7946220923127b52dfd64b37db14b99 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Thu, 28 Nov 2024 16:18:11 +0900 Subject: [PATCH 8/9] fix: deleted comment Signed-off-by: hlts2 --- .../kubernetes/snapshot-controller/volume_snapshot_class.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml b/deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml index 6f02d85..4bc7a5a 100644 --- a/deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml +++ b/deploy/kubernetes/snapshot-controller/volume_snapshot_class.yaml @@ -1,5 +1,3 @@ -# Usage of the v1 API implies that the cluster must have -# external-snapshotter v4.x installed.↴ apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: From c2151b8655f52f2873f01c1847c9b5b76dd7c0b8 Mon Sep 17 00:00:00 2001 From: hlts2 Date: Tue, 3 Dec 2024 15:45:02 +0900 Subject: [PATCH 9/9] fix: deleted unnecessary option Signed-off-by: hlts2 --- deploy/install.yaml | 1 - deploy/kubernetes/04_controller.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/deploy/install.yaml b/deploy/install.yaml index 8d14c6e..de3ff3e 100644 --- a/deploy/install.yaml +++ b/deploy/install.yaml @@ -283,7 +283,6 @@ spec: - "--v=5" - "--csi-address=$(ADDRESS)" - "--timeout=30s" - - "--retry-interval-max=10" env: - name: ADDRESS value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock diff --git a/deploy/kubernetes/04_controller.yaml b/deploy/kubernetes/04_controller.yaml index 3be178f..237394d 100644 --- a/deploy/kubernetes/04_controller.yaml +++ b/deploy/kubernetes/04_controller.yaml @@ -76,7 +76,6 @@ spec: - "--v=5" - "--csi-address=$(ADDRESS)" - "--timeout=30s" - - "--retry-interval-max=10" env: - name: ADDRESS value: /var/lib/kubelet/plugins/csi.civo.com/csi.sock