From 407e29124677932bab420aca2a6d2cb23e8d22c2 Mon Sep 17 00:00:00 2001 From: leonliao Date: Fri, 24 Jan 2025 11:12:18 +0800 Subject: [PATCH 1/2] fix: Supports path-style s3 access. #8867 --- .../storageprovider/s3-compatible.yaml | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 deploy/helm/templates/storageprovider/s3-compatible.yaml diff --git a/deploy/helm/templates/storageprovider/s3-compatible.yaml b/deploy/helm/templates/storageprovider/s3-compatible.yaml new file mode 100644 index 00000000000..3dfe697afd0 --- /dev/null +++ b/deploy/helm/templates/storageprovider/s3-compatible.yaml @@ -0,0 +1,87 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: StorageProvider +metadata: + name: s3-compatible + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} +spec: + csiDriverName: ru.yandex.s3.csi + csiDriverSecretTemplate: | + accessKeyID: {{ `{{ index .Parameters "accessKeyId" }}` }} + secretAccessKey: {{ `{{ index .Parameters "secretAccessKey" }}` }} + endpoint: {{ `{{ index .Parameters "endpoint" }}` }} + + storageClassTemplate: | + provisioner: ru.yandex.s3.csi + parameters: + mounter: geesefs + # you can set mount options here, for example limit memory cache size (recommended) + options: {{ `{{ printf "--memory-limit %s --read-ahead-large %s --dir-mode 0777 --file-mode 0666 %s" .Parameters.geesefsMemoryLimit .Parameters.geesefsReadAheadLarge .Parameters.mountOptions }}` }} + bucket: {{ `{{ index .Parameters "bucket" }}` }} + csi.storage.k8s.io/provisioner-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} + csi.storage.k8s.io/controller-publish-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} + csi.storage.k8s.io/controller-publish-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} + csi.storage.k8s.io/node-stage-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} + csi.storage.k8s.io/node-publish-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} + csi.storage.k8s.io/node-publish-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} + + datasafedConfigTemplate: | + [storage] + type = s3 + provider = Other + env_auth = false + access_key_id = {{ `{{ index .Parameters "accessKeyId" }}` }} + secret_access_key = {{ `{{ index .Parameters "secretAccessKey" }}` }} + endpoint = {{ `{{ index .Parameters "endpoint" }}` }} + root = {{ `{{ index .Parameters "bucket" }}` }} + no_check_certificate = {{ `{{ index .Parameters "insecure" }}` }} + chunk_size = 50Mi + force_path_style = {{ `{{ index .Parameters "usePathStyle" }}` }} + + parametersSchema: + openAPIV3Schema: + type: "object" + properties: + bucket: + type: string + description: "S3 bucket" + endpoint: + type: string + description: "S3 endpoint (optional)" + mountOptions: + type: string + description: "Extra mount options for geesefs" + geesefsMemoryLimit: + type: integer + default: 512 + description: "The value of --memory-limit parameter for geesefs, in MB" + geesefsReadAheadLarge: + type: integer + default: 20480 + description: "The value of --read-ahead-large parameter for geesefs, in KB" + accessKeyId: + type: string + description: "AWS access key" + secretAccessKey: + type: string + description: "AWS secret key" + insecure: + type: boolean + default: false + description: "Do not verify the server SSL certificate" + usePathStyle: + type: boolean + default: false + description: "If set to false, use virtual hosted-style access. If set to true, use path-style access." + + required: + - bucket + - region + - accessKeyId + - secretAccessKey + + credentialFields: + - accessKeyId + - secretAccessKey From e2726c43202bbcc0b6566cc4a13317a80c4fadf8 Mon Sep 17 00:00:00 2001 From: leonliao Date: Fri, 24 Jan 2025 18:09:46 +0800 Subject: [PATCH 2/2] fix: Refine the name of parameters. #8867 --- .../storageprovider/s3-compatible.yaml | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/deploy/helm/templates/storageprovider/s3-compatible.yaml b/deploy/helm/templates/storageprovider/s3-compatible.yaml index 3dfe697afd0..5d23d270b17 100644 --- a/deploy/helm/templates/storageprovider/s3-compatible.yaml +++ b/deploy/helm/templates/storageprovider/s3-compatible.yaml @@ -2,21 +2,22 @@ apiVersion: dataprotection.kubeblocks.io/v1alpha1 kind: StorageProvider metadata: name: s3-compatible - labels: - {{- include "kubeblocks.labels" . | nindent 4 }} spec: csiDriverName: ru.yandex.s3.csi csiDriverSecretTemplate: | accessKeyID: {{ `{{ index .Parameters "accessKeyId" }}` }} secretAccessKey: {{ `{{ index .Parameters "secretAccessKey" }}` }} endpoint: {{ `{{ index .Parameters "endpoint" }}` }} - storageClassTemplate: | provisioner: ru.yandex.s3.csi parameters: mounter: geesefs + {{ `{{ $regionOption := "" }}` }} + {{ `{{ if .Parameters.region }}` }} + {{ `{{ $regionOption = (printf "--region %s" .Parameters.region) }}` }} + {{ `{{ end }}` }} # you can set mount options here, for example limit memory cache size (recommended) - options: {{ `{{ printf "--memory-limit %s --read-ahead-large %s --dir-mode 0777 --file-mode 0666 %s" .Parameters.geesefsMemoryLimit .Parameters.geesefsReadAheadLarge .Parameters.mountOptions }}` }} + options: {{ `{{ printf "--memory-limit %s --read-ahead-large %s --dir-mode 0777 --file-mode 0666 %s %s" .Parameters.geesefsMemoryLimit .Parameters.geesefsReadAheadLarge .Parameters.mountOptions $regionOption }}` }} bucket: {{ `{{ index .Parameters "bucket" }}` }} csi.storage.k8s.io/provisioner-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} csi.storage.k8s.io/provisioner-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} @@ -26,30 +27,32 @@ spec: csi.storage.k8s.io/node-stage-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} csi.storage.k8s.io/node-publish-secret-name: {{ `{{ .CSIDriverSecretRef.Name }}` }} csi.storage.k8s.io/node-publish-secret-namespace: {{ `{{ .CSIDriverSecretRef.Namespace }}` }} - datasafedConfigTemplate: | [storage] type = s3 - provider = Other + provider = {{ `{{ index .Parameters "serviceProvider" | default "Other" }}` }} env_auth = false access_key_id = {{ `{{ index .Parameters "accessKeyId" }}` }} secret_access_key = {{ `{{ index .Parameters "secretAccessKey" }}` }} + region = {{ `{{ index .Parameters "region" }}` }} endpoint = {{ `{{ index .Parameters "endpoint" }}` }} root = {{ `{{ index .Parameters "bucket" }}` }} no_check_certificate = {{ `{{ index .Parameters "insecure" }}` }} chunk_size = 50Mi - force_path_style = {{ `{{ index .Parameters "usePathStyle" }}` }} - + force_path_style = {{ `{{ index .Parameters "forcePathStyle" | default "false" }}` }} parametersSchema: openAPIV3Schema: type: "object" properties: + region: + type: string + description: "Storage region (optional), e.g. us-west-1" bucket: type: string - description: "S3 bucket" + description: "Storage bucket" endpoint: type: string - description: "S3 endpoint (optional)" + description: "Storage endpoint" mountOptions: type: string description: "Extra mount options for geesefs" @@ -63,25 +66,27 @@ spec: description: "The value of --read-ahead-large parameter for geesefs, in KB" accessKeyId: type: string - description: "AWS access key" + description: "Storage access key" secretAccessKey: type: string - description: "AWS secret key" + description: "Storage secret key" insecure: type: boolean default: false description: "Do not verify the server SSL certificate" - usePathStyle: + serviceProvider: + type: string + default: "Other" + description: "Provider name supported by rclone (https://rclone.org/s3/#configuration), e.g. DigitalOcean" + forcePathStyle: type: boolean default: false - description: "If set to false, use virtual hosted-style access. If set to true, use path-style access." - + description: "Force path-style access" required: - bucket - - region + - endpoint - accessKeyId - secretAccessKey - credentialFields: - accessKeyId - - secretAccessKey + - secretAccessKey \ No newline at end of file