From 03efcb426d03afbc2d3e4ae0227bde15976e8f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cgifi-siby=E2=80=9D?= <“gifi.s@ibm.com“> Date: Mon, 3 Mar 2025 04:14:41 -0800 Subject: [PATCH 1/2] gcpnative support for bacup tool Signed-off-by: gifi-siby Signed-off-by: gifi-siby --- README.md | 9 +- cmd/backup_yaml.go | 28 +- configs/backup.yaml | 49 ++-- core/backup_context.go | 2 + core/paramtable/base_table.go | 11 + core/paramtable/params.go | 58 ++-- core/storage/chunk_manager.go | 2 + core/storage/gcp_native_chunk_manager.go | 340 +++++++++++++++++++++++ core/storage/options.go | 8 + go.mod | 22 ++ go.sum | 59 ++++ 11 files changed, 529 insertions(+), 59 deletions(-) create mode 100644 core/storage/gcp_native_chunk_manager.go diff --git a/README.md b/README.md index ab69b5f8..fa2c88ee 100644 --- a/README.md +++ b/README.md @@ -107,20 +107,22 @@ Below is a summary of the configurations supported in `backup.yaml`: | | `serverName ` | Server name | `localhost` | | | `mtlsCertPath` | Path to your mtls certificate file | `/path/to/certificate` | | | `mtlsKeyPath ` | Path to your mtls key file | `/path/to/key` | -| `minio` | `storageType` | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` | +| `minio` | `storageType` | Storage type for Milvus (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider. | `minio` | | | `address` | MinIO/S3 address. | `localhost` | | | `port` | MinIO/S3 port. | `9000` | | | `accessKeyID` | MinIO/S3 access key ID. | `minioadmin` | | | `secretAccessKey` | MinIO/S3 secret access key. | `minioadmin` | +| | `gcpCredentialJSON` | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider. | `/path/to/json-key-file` | | | `useSSL` | Whether to use SSL for MinIO/S3. | `false` | | | `bucketName` | Bucket name in MinIO/S3. | `a-bucket` | | | `rootPath` | Storage root path in MinIO/S3. | `files` | -| `minio (backup)` | `backupStorageType` | Backup storage type (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`). | `minio` | -| | `backupAddress` | Address of backup storage. | `localhost` | +| `minio (backup)` | `backupStorageType` | Backup storage type (e.g., `local`, `minio`, `s3`, `aws`, `gcp`, `ali(aliyun)`, `azure`, `tc(tencent)`, `gcpnative`). Use `gcpnative` for the Google Cloud Platform provider. | `minio` | +| | `backupAddress` | Address of backup storagße. | `localhost` | | | `backupPort` | Port of backup storage. | `9000` | | | `backupUseSSL` | Whether to use SSL for backup storage. | `false` | | | `backupAccessKeyID` | Backup storage access key ID. | `minioadmin` | | | `backupSecretAccessKey` | Backup storage secret access key. | `minioadmin` | +| | `backupGcpCredentialJSON` | Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider. | `/path/to/json-key-file` | | | `backupBucketName` | Bucket name for backups. | `a-bucket` | | | `backupRootPath` | Root path to store backup data. | `backup` | | | `crossStorage` | Enable cross-storage backups (e.g., MinIO to AWS S3). | `false` | @@ -156,6 +158,7 @@ backupAddress: s3.{your-aws-region}.amazonaws.com # Address of AWS S3 (replace backupPort: 443 # Default port for AWS S3 backupAccessKeyID: # Access key ID for your AWS S3 backupSecretAccessKey: # Secret access key for AWS S3 +backupGcpCredentialJSON: "/path/to/file" # Path to your GCP credential JSON key file. Used only for the "gcpnative" cloud provider. backupBucketName: "your-bucket-name" # Bucket name where the backups will be stored backupRootPath: "backups" # Root path inside the bucket to store backups backupUseSSL: true # Use SSL for secure connections (Required) diff --git a/cmd/backup_yaml.go b/cmd/backup_yaml.go index 329c9b9c..4c9e08ea 100644 --- a/cmd/backup_yaml.go +++ b/cmd/backup_yaml.go @@ -46,18 +46,20 @@ type YAMLConFig struct { mtlsKeyPath string `yaml:"mtlsKeyPath"` } `yaml:"milvus"` Minio struct { - Address string `yaml:"address"` - Port int `yaml:"port"` - AccessKeyID string `yaml:"accessKeyID"` - secretAccessKey string `yaml:"secretAccessKey"` - UseSSL bool `yaml:"useSSL"` - UseIAM bool `yaml:"useIAM"` - CloudProvider string `yaml:"cloudProvider"` - IamEndpoint string `yaml:"iamEndpoint"` - BucketName string `yaml:"bucketName"` - RootPath string `yaml:"rootPath"` - BackupBucketName string `yaml:"backupBucketName"` - BackupRootPath string `yaml:"backupRootPath"` + Address string `yaml:"address"` + Port int `yaml:"port"` + AccessKeyID string `yaml:"accessKeyID"` + secretAccessKey string `yaml:"secretAccessKey"` + GcpCredentialJSON string `yaml:"gcpCredentialJSON"` + UseSSL bool `yaml:"useSSL"` + UseIAM bool `yaml:"useIAM"` + CloudProvider string `yaml:"cloudProvider"` + IamEndpoint string `yaml:"iamEndpoint"` + BucketName string `yaml:"bucketName"` + RootPath string `yaml:"rootPath"` + BackupGcpCredentialJSON string `yaml:"backupGcpCredentialJSON"` + BackupBucketName string `yaml:"backupBucketName"` + BackupRootPath string `yaml:"backupRootPath"` } `yaml:"minio"` Backup struct { MaxSegmentGroupSize string `yaml:"maxSegmentGroupSize"` @@ -89,12 +91,14 @@ func printParams(base *paramtable.BackupParams) { yml.Minio.Port = base.ParseIntWithDefault("minio.port", 9000) yml.Minio.AccessKeyID = base.BaseTable.LoadWithDefault("minio.accessKeyID", "") yml.Minio.secretAccessKey = base.BaseTable.LoadWithDefault("minio.secretAccessKey", "") + yml.Minio.GcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.gcpCredentialJSON", "") yml.Minio.UseSSL = base.ParseBool("minio.useSSL", false) yml.Minio.UseIAM = base.ParseBool("minio.useIAM", false) yml.Minio.CloudProvider = base.BaseTable.LoadWithDefault("minio.cloudProvider", "aws") yml.Minio.IamEndpoint = base.BaseTable.LoadWithDefault("minio.iamEndpoint", "") yml.Minio.BucketName = base.BaseTable.LoadWithDefault("minio.bucketName", "") yml.Minio.RootPath = base.LoadWithDefault("minio.rootPath", "") + yml.Minio.BackupGcpCredentialJSON = base.BaseTable.LoadWithDefault("minio.backupGcpCredentialJSON", "") yml.Minio.BackupBucketName = base.LoadWithDefault("minio.backupBucketName", "") yml.Minio.BackupRootPath = base.LoadWithDefault("minio.backupRootPath", "") diff --git a/configs/backup.yaml b/configs/backup.yaml index 27a354bf..841c1dc7 100644 --- a/configs/backup.yaml +++ b/configs/backup.yaml @@ -10,17 +10,17 @@ http: # milvus proxy address, compatible to milvus.yaml milvus: - address: localhost - port: 19530 - user: "root" - password: "Milvus" + address: ibm-lh-lakehouse-milvus727.milvus.apps.enginecluster-aurora.cp.fyre.ibm.com + port: 443 + user: "admin" + password: "6ihRVTx0gDEP" # tls mode values [0, 1, 2] # 0 is close, 1 is one-way authentication, 2 is mutual authentication - tlsMode: 0 + tlsMode: 1 # tls cert path for validate server, will be used when tlsMode is 1 or 2 - caCertPath: "" - serverName: "" + caCertPath: "/root/hello_milvus/aura.crt" + serverName: "ibm-lh-lakehouse-milvus727.milvus.apps.enginecluster-aurora.cp.fyre.ibm.com" # mutual tls cert path, for server to validate client. # Will be used when tlsMode is 2 # for backward compatibility, if not set, will use tlsmode 1. @@ -31,26 +31,29 @@ milvus: # Related configuration of minio, which is responsible for data persistence for Milvus. minio: # Milvus storage configs, make them the same with milvus config - storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent) - address: localhost # Address of MinIO/S3 - port: 9000 # Port of MinIO/S3 - accessKeyID: minioadmin # accessKeyID of MinIO/S3 - secretAccessKey: minioadmin # MinIO/S3 encryption string - useSSL: false # Access to MinIO/S3 with SSL + storageType: "gcpnative" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative + # You can use "gcpnative" for the Google Cloud Platform provider. Uses service account credentials for authentication. + address: storage.googleapis.com # Address of MinIO/S3 + port: 443 # Port of MinIO/S3 + accessKeyID: # accessKeyID of MinIO/S3 + secretAccessKey: # MinIO/S3 encryption string + gcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider. + useSSL: true # Access to MinIO/S3 with SSL useIAM: false iamEndpoint: "" - bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance - rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance + bucketName: "gcs-nv1" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance + rootPath: "GifiGcpNative/milvus727" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance # Backup storage configs, the storage you want to put the backup data - backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent) - backupAddress: localhost # Address of MinIO/S3 - backupPort: 9000 # Port of MinIO/S3 - backupAccessKeyID: minioadmin # accessKeyID of MinIO/S3 - backupSecretAccessKey: minioadmin # MinIO/S3 encryption string - backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath - backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath - backupUseSSL: false # Access to MinIO/S3 with SSL + backupStorageType: "gcpnative" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative + backupAddress: storage.googleapis.com # Address of MinIO/S3 + backupPort: 443 # Port of MinIO/S3 + backupAccessKeyID: # accessKeyID of MinIO/S3 + backupSecretAccessKey: # MinIO/S3 encryption string + backupGcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider. + backupBucketName: "gcs-nv1" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath + backupRootPath: "giffBackk" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath + backupUseSSL: true # Access to MinIO/S3 with SSL # If you need to back up or restore data between two different storage systems, direct client-side copying is not supported. # Set this option to true to enable data transfer through Milvus Backup. diff --git a/core/backup_context.go b/core/backup_context.go index 3a85645b..69d24d8c 100644 --- a/core/backup_context.go +++ b/core/backup_context.go @@ -149,6 +149,7 @@ func (b *BackupContext) getMilvusStorageClient() storage.ChunkManager { BucketName: b.params.MinioCfg.BucketName, AccessKeyID: b.params.MinioCfg.AccessKeyID, SecretAccessKeyID: b.params.MinioCfg.SecretAccessKey, + GcpCredentialJSON: b.params.MinioCfg.GcpCredentialJSON, UseSSL: b.params.MinioCfg.UseSSL, UseIAM: b.params.MinioCfg.UseIAM, IAMEndpoint: b.params.MinioCfg.IAMEndpoint, @@ -180,6 +181,7 @@ func (b *BackupContext) getBackupStorageClient() storage.ChunkManager { BucketName: b.params.MinioCfg.BackupBucketName, AccessKeyID: b.params.MinioCfg.BackupAccessKeyID, SecretAccessKeyID: b.params.MinioCfg.BackupSecretAccessKey, + GcpCredentialJSON: b.params.MinioCfg.BackupGcpCredentialJSON, UseSSL: b.params.MinioCfg.BackupUseSSL, UseIAM: b.params.MinioCfg.BackupUseIAM, IAMEndpoint: b.params.MinioCfg.BackupIAMEndpoint, diff --git a/core/paramtable/base_table.go b/core/paramtable/base_table.go index 2d278af4..1ff143b3 100644 --- a/core/paramtable/base_table.go +++ b/core/paramtable/base_table.go @@ -37,6 +37,7 @@ const ( DefaultMinioPort = "9000" DefaultMinioAccessKey = "minioadmin" DefaultMinioSecretAccessKey = "minioadmin" + DefaultGcpCredentialJSON = "" DefaultMinioUseSSL = "false" DefaultMinioBucketName = "a-bucket" DefaultMinioRootPath = "files" @@ -429,6 +430,11 @@ func (gp *BaseTable) loadMinioConfig() { _ = gp.Save("minio.secretAccessKey", minioSecretKey) } + gcpCredentialJSON := os.Getenv("GCP_KEY_JSON") + if gcpCredentialJSON != "" { + _ = gp.Save("minio.gcpCredentialJSON", gcpCredentialJSON) + } + minioUseSSL := os.Getenv("MINIO_USE_SSL") if minioUseSSL != "" { _ = gp.Save("minio.useSSL", minioUseSSL) @@ -484,6 +490,11 @@ func (gp *BaseTable) loadMinioConfig() { _ = gp.Save("minio.backupSecretAccessKey", minioBackupSecretKey) } + backupGcpCredentialJSON := os.Getenv("BACKUP_GCP_KEY_JSON") + if backupGcpCredentialJSON != "" { + _ = gp.Save("minio.backupGcpCredentialJSON", backupGcpCredentialJSON) + } + minioBackupUseSSL := os.Getenv("MINIO_BACKUP_USE_SSL") if minioBackupUseSSL != "" { _ = gp.Save("minio.backupUseSSL", minioBackupUseSSL) diff --git a/core/paramtable/params.go b/core/paramtable/params.go index 8be55fa6..fd9589d5 100644 --- a/core/paramtable/params.go +++ b/core/paramtable/params.go @@ -210,6 +210,7 @@ const ( S3 = "s3" CloudProviderAWS = "aws" CloudProviderGCP = "gcp" + CloudProviderGCPNative = "gcpnative" CloudProviderAli = "ali" CloudProviderAliyun = "aliyun" CloudProviderAzure = "azure" @@ -223,6 +224,7 @@ var supportedStorageType = map[string]bool{ S3: true, CloudProviderAWS: true, CloudProviderGCP: true, + CloudProviderGCPNative: true, CloudProviderAli: true, CloudProviderAliyun: true, CloudProviderAzure: true, @@ -235,27 +237,29 @@ type MinioConfig struct { StorageType string // Deprecated - CloudProvider string - Address string - Port string - AccessKeyID string - SecretAccessKey string - UseSSL bool - BucketName string - RootPath string - UseIAM bool - IAMEndpoint string - - BackupStorageType string - BackupAddress string - BackupPort string - BackupAccessKeyID string - BackupSecretAccessKey string - BackupUseSSL bool - BackupBucketName string - BackupRootPath string - BackupUseIAM bool - BackupIAMEndpoint string + CloudProvider string + Address string + Port string + AccessKeyID string + SecretAccessKey string + GcpCredentialJSON string + UseSSL bool + BucketName string + RootPath string + UseIAM bool + IAMEndpoint string + + BackupStorageType string + BackupAddress string + BackupPort string + BackupAccessKeyID string + BackupSecretAccessKey string + BackupGcpCredentialJSON string + BackupUseSSL bool + BackupBucketName string + BackupRootPath string + BackupUseIAM bool + BackupIAMEndpoint string CrossStorage bool } @@ -268,6 +272,7 @@ func (p *MinioConfig) init(base *BaseTable) { p.initPort() p.initAccessKeyID() p.initSecretAccessKey() + p.initGcpCredentialJSON() p.initUseSSL() p.initBucketName() p.initRootPath() @@ -280,6 +285,7 @@ func (p *MinioConfig) init(base *BaseTable) { p.initBackupPort() p.initBackupAccessKeyID() p.initBackupSecretAccessKey() + p.initBackupGcpCredentialJSON() p.initBackupUseSSL() p.initBackupBucketName() p.initBackupRootPath() @@ -309,6 +315,11 @@ func (p *MinioConfig) initSecretAccessKey() { p.SecretAccessKey = key } +func (p *MinioConfig) initGcpCredentialJSON() { + gcpCredentialJSON := p.Base.LoadWithDefault("minio.gcpCredentialJSON", DefaultGcpCredentialJSON) + p.GcpCredentialJSON = gcpCredentialJSON +} + func (p *MinioConfig) initUseSSL() { usessl := p.Base.LoadWithDefault("minio.useSSL", DefaultMinioUseSSL) var err error @@ -419,6 +430,11 @@ func (p *MinioConfig) initBackupSecretAccessKey() { p.BackupSecretAccessKey = key } +func (p *MinioConfig) initBackupGcpCredentialJSON() { + gcpCredentialJSON := p.Base.LoadWithDefault("minio.backupGcpCredentialJSON", DefaultGcpCredentialJSON) + p.BackupGcpCredentialJSON = gcpCredentialJSON +} + func (p *MinioConfig) initBackupBucketName() { bucketName := p.Base.LoadWithDefault("minio.backupBucketName", p.Base.LoadWithDefault("minio.bucketName", DefaultMinioBackupBucketName)) diff --git a/core/storage/chunk_manager.go b/core/storage/chunk_manager.go index 4408b474..146ba2fe 100644 --- a/core/storage/chunk_manager.go +++ b/core/storage/chunk_manager.go @@ -42,6 +42,8 @@ func NewChunkManager(ctx context.Context, params *paramtable.BackupParams, confi case paramtable.CloudProviderAzure: // todo @wayblink return newAzureChunkManagerWithParams(ctx, params, config) + case paramtable.CloudProviderGCPNative: + return newGCPNativeChunkManagerWithConfig(ctx, config) default: return NewMinioChunkManagerWithConfig(ctx, config) } diff --git a/core/storage/gcp_native_chunk_manager.go b/core/storage/gcp_native_chunk_manager.go new file mode 100644 index 00000000..6c3303ac --- /dev/null +++ b/core/storage/gcp_native_chunk_manager.go @@ -0,0 +1,340 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "cloud.google.com/go/storage" + "github.com/cockroachdb/errors" + "github.com/zilliztech/milvus-backup/internal/log" + "github.com/zilliztech/milvus-backup/internal/util/retry" + "go.uber.org/zap" + "golang.org/x/oauth2/google" + "golang.org/x/sync/errgroup" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCPNativeChunkManager struct { + client *storage.Client + config *StorageConfig +} + +func (gcm *GCPNativeChunkManager) Config() *StorageConfig { + return gcm.config +} + +func newGCPNativeChunkManagerWithConfig(ctx context.Context, config *StorageConfig) (*GCPNativeChunkManager, error) { + var client *storage.Client + var err error + var opts []option.ClientOption + var projectId string + if config.Address != "" { + complete_address := "https://" + if !config.UseSSL { + complete_address = "http://" + } + complete_address = complete_address + config.Address + "/storage/v1/" + opts = append(opts, option.WithEndpoint(complete_address)) + } + // Read the credentials file + jsonData, err := os.ReadFile(config.GcpCredentialJSON) + if err != nil { + return nil, fmt.Errorf("unable to read credentials file:: %v", err) + } + + creds, err := google.CredentialsFromJSON(ctx, jsonData, storage.ScopeReadWrite) + if err != nil { + return nil, fmt.Errorf("failed to create credentials from JSON: %v", err) + + } + projectId, err = getProjectId(config.GcpCredentialJSON) + if err != nil { + return nil, fmt.Errorf("failed to get project ID: %v", err) + } + opts = append(opts, option.WithCredentials(creds)) + + client, err = storage.NewClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("failed to create GCS client: %v", err) + } + + if config.BucketName == "" { + return nil, fmt.Errorf("invalid empty bucket name") + } + // Check bucket validity + checkBucketFn := func() error { + bucket := client.Bucket(config.BucketName) + _, err := bucket.Attrs(ctx) + if err == storage.ErrBucketNotExist && config.CreateBucket { + log.Info("gcs bucket does not exist, create bucket.", zap.String("bucket name", config.BucketName)) + err = client.Bucket(config.BucketName).Create(ctx, projectId, nil) + if err != nil { + return fmt.Errorf("failed to create bucket: %v", err) + } + return nil + } + return err + } + err = retry.Do(ctx, checkBucketFn, retry.Attempts(CheckBucketRetryAttempts)) + if err != nil { + return nil, fmt.Errorf("bucket check failed: %v", err) + } + + log.Info("GCP native chunk manager init success.", zap.String("bucketname", config.BucketName)) + return &GCPNativeChunkManager{ + client: client, + config: config, + }, nil +} + +func getProjectId(credentialPath string) (string, error) { + if credentialPath == "" { + return "", errors.New("the path to the JSON file is empty") + } + + // Read the credentials file + jsonData, err := os.ReadFile(credentialPath) + if err != nil { + return "", fmt.Errorf("failed to read credentials file: %v", err) + } + + var data map[string]interface{} + if err := json.Unmarshal(jsonData, &data); err != nil { + return "", errors.New("failed to parse Google Cloud credentials as JSON") + } + + propertyValue, ok := data["project_id"] + if !ok { + return "", errors.New("project_id doesn't exist") + } + + projectId := fmt.Sprintf("%v", propertyValue) + return projectId, nil +} + +// Write writes data to filePath in the specified GCS bucket +func (gcm *GCPNativeChunkManager) Write(ctx context.Context, bucketName string, filePath string, content []byte) error { + wc := gcm.client.Bucket(bucketName).Object(filePath).NewWriter(ctx) + defer func() { + if err := wc.Close(); err != nil { + log.Error("Failed to close writer", zap.Error(err)) + } + }() + + _, err := wc.Write(content) + if err != nil { + return err + } + return nil +} + +// Exist checks whether chunk is saved to GCS storage. +func (gcm *GCPNativeChunkManager) Exist(ctx context.Context, bucketName string, filePath string) (bool, error) { + objectKeys, _, err := gcm.ListWithPrefix(ctx, bucketName, filePath, false) + if err != nil { + return false, err + } + + if len(objectKeys) > 0 { + return true, nil + } + + return false, nil +} + +// Read reads the storage data if exists. +func (gcm *GCPNativeChunkManager) Read(ctx context.Context, bucketName string, filePath string) ([]byte, error) { + rc, err := gcm.client.Bucket(bucketName).Object(filePath).NewReader(ctx) + if err != nil { + return nil, err + } + defer func() { + if err := rc.Close(); err != nil { + log.Error("Failed to close reader", zap.Error(err)) + } + }() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + + return data, nil +} + +// ListWithPrefix returns objects with provided prefix. +func (gcm *GCPNativeChunkManager) ListWithPrefix(ctx context.Context, bucketName string, prefix string, recursive bool) ([]string, []int64, error) { + var objectKeys []string + var sizes []int64 + delimiter := "" + if !recursive { + delimiter = "/" + } + + it := gcm.client.Bucket(bucketName).Objects(ctx, &storage.Query{ + Prefix: prefix, + Delimiter: delimiter, + }) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, nil, fmt.Errorf("failed to list objects: %w", err) + } + // Check if it's an object (not a directory) + if attrs.Name != "" { + objectKeys = append(objectKeys, attrs.Name) + sizes = append(sizes, attrs.Size) + } else if attrs.Prefix != "" { + // If recursive is false, directories are handled via attrs.Prefix (not attrs.Name) + // For directories, we only add the directory name (attrs.Prefix) once + objectKeys = append(objectKeys, attrs.Prefix) + sizes = append(sizes, 0) // No size for directories + } + } + return objectKeys, sizes, nil +} + +// Remove deletes an object with @key. +func (gcm *GCPNativeChunkManager) Remove(ctx context.Context, bucketName string, filePath string) error { + err := gcm.client.Bucket(bucketName).Object(filePath).Delete(ctx) + if err != nil { + return err + } + return nil +} + +func (gcm *GCPNativeChunkManager) RemoveWithPrefix(ctx context.Context, bucketName string, prefix string) error { + objectKeys, _, err := gcm.ListWithPrefix(ctx, bucketName, prefix, false) + if err != nil { + return err + } + // Group objects by their depth (number of / in the key) + groupedByLevel := make(map[int][]string) + var maxLevel int + for _, key := range objectKeys { + level := strings.Count(key, "/") + groupedByLevel[level] = append(groupedByLevel[level], key) + if level > maxLevel { + maxLevel = level + } + } + + for level := maxLevel; level >= 0; level-- { + // Get the objects at this level + keysAtLevel, exists := groupedByLevel[level] + if !exists || len(keysAtLevel) == 0 { + continue + } + + // Dynamically adjust maxGoroutines based on the number of objects at this level + maxGoroutines := 10 + if len(keysAtLevel) < maxGoroutines { + maxGoroutines = len(keysAtLevel) + } + i := 0 + for i < len(keysAtLevel) { + runningGroup, groupCtx := errgroup.WithContext(context.Background()) + for j := 0; j < maxGoroutines && i < len(keysAtLevel); j++ { + key := keysAtLevel[i] + runningGroup.Go(func(key string) func() error { + return func() error { + err := gcm.Remove(groupCtx, bucketName, key) + if err != nil { + log.Warn("failed to remove object", zap.String("bucket", bucketName), zap.String("path", key), zap.Error(err)) + return err + } + return nil + } + }(key)) + i++ + } + if err := runningGroup.Wait(); err != nil { + return err + } + } + } + err = gcm.Remove(ctx, bucketName, strings.TrimSuffix(prefix, "/")) + if err != nil { + log.Warn("failed to remove object", zap.String("bucket", bucketName), zap.String("path", prefix), zap.Error(err)) + return err + } + return nil +} + +// Copy files from fromPath into toPath recursively +func (gcm *GCPNativeChunkManager) Copy(ctx context.Context, fromBucketName string, toBucketName string, fromPath string, toPath string) error { + objectKeys, _, err := gcm.ListWithPrefix(ctx, fromBucketName, fromPath, true) + if err != nil { + log.Error("Error listing objects", zap.Error(err)) + return err + } + for _, srcObjectKey := range objectKeys { + dstObjectKey := strings.Replace(srcObjectKey, fromPath, toPath, 1) + + srcReader, err := gcm.client.Bucket(fromBucketName).Object(srcObjectKey).NewReader(ctx) + if err != nil { + log.Error("Error creating reader for object", zap.String("srcObjectKey", srcObjectKey), zap.Error(err)) + return err + } + + defer func(srcObjectKey string, srcReader *storage.Reader) { + if err := srcReader.Close(); err != nil { + log.Error("Error closing source reader", zap.String("srcObjectKey", srcObjectKey), zap.Error(err)) + } + }(srcObjectKey, srcReader) + + dstWriter := gcm.client.Bucket(toBucketName).Object(dstObjectKey).NewWriter(ctx) + defer func(dstObjectKey string, dstWriter *storage.Writer) { + if err := dstWriter.Close(); err != nil { + log.Error("Error closing destination writer", zap.String("dstObjectKey", dstObjectKey), zap.Error(err)) + } + }(dstObjectKey, dstWriter) + + if _, err := io.Copy(dstWriter, srcReader); err != nil { + log.Error("Error copying object", zap.String("from", srcObjectKey), zap.String("to", dstObjectKey), zap.Error(err)) + return err + } + } + return nil +} + +func (gcm *GCPNativeChunkManager) ListObjectsPage(ctx context.Context, bucket, prefix string) (ListObjectsPaginator, error) { + panic("not implemented") +} + +func (gcm *GCPNativeChunkManager) HeadObject(ctx context.Context, bucket, key string) (ObjectAttr, error) { + panic("not implemented") +} + +func (gcm *GCPNativeChunkManager) GetObject(ctx context.Context, bucketName, key string) (*Object, error) { + panic("not implemented") +} + +func (gcm *GCPNativeChunkManager) UploadObject(ctx context.Context, i UploadObjectInput) error { + panic("not implemented") +} diff --git a/core/storage/options.go b/core/storage/options.go index db976f82..cc24ef26 100644 --- a/core/storage/options.go +++ b/core/storage/options.go @@ -7,6 +7,7 @@ type StorageConfig struct { BucketName string AccessKeyID string SecretAccessKeyID string + GcpCredentialJSON string UseSSL bool CreateBucket bool RootPath string @@ -15,6 +16,7 @@ type StorageConfig struct { backupAccessKeyID string backupSecretAccessKeyID string + BackupGcpCredentialJSON string backupBucketName string backupRootPath string } @@ -49,6 +51,12 @@ func SecretAccessKeyID(secretAccessKeyID string) Option { } } +func GcpCredentialJSON(gcpCredentialJSON string) Option { + return func(c *StorageConfig) { + c.GcpCredentialJSON = gcpCredentialJSON + } +} + func UseSSL(useSSL bool) Option { return func(c *StorageConfig) { c.UseSSL = useSSL diff --git a/go.mod b/go.mod index 08df6d2e..bbe9c384 100644 --- a/go.mod +++ b/go.mod @@ -37,14 +37,35 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.10.0 golang.org/x/time v0.8.0 + google.golang.org/api v0.171.0 google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) +require ( + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/iam v1.1.6 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect +) + require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/storage v1.38.0 github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/KyleBanks/depth v1.2.1 // indirect @@ -77,6 +98,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3a571672..2ba68624 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/storage v1.38.0 h1:Az68ZRGlnNTpIBbLjSMIV2BDcwwXYlRlQzis0llkpJg= +cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= @@ -68,6 +74,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= @@ -88,6 +96,7 @@ github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -118,22 +127,46 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -287,8 +320,14 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= @@ -344,6 +383,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -414,11 +454,20 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/api v0.171.0 h1:w174hnBPqut76FzW5Qaupt7zY8Kql6fiVjgys4f58sU= +google.golang.org/api v0.171.0/go.mod h1:Hnq5AHm4OTMt2BUVjael2CWZFD6vksJdWCWiUAmjC9o= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= +google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 h1:TqExAhdPaB60Ux47Cn0oLV07rGnxZzIsaRhQaqS666A= google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -426,8 +475,18 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7a0712dfbdfb1744355894dedbb0550bb3e6e9cd Mon Sep 17 00:00:00 2001 From: gifi-siby Date: Mon, 3 Mar 2025 22:23:49 -0800 Subject: [PATCH 2/2] GCS support Signed-off-by: gifi-siby Signed-off-by: gifi-siby --- configs/backup.yaml | 54 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/configs/backup.yaml b/configs/backup.yaml index 841c1dc7..bb0d7956 100644 --- a/configs/backup.yaml +++ b/configs/backup.yaml @@ -10,17 +10,17 @@ http: # milvus proxy address, compatible to milvus.yaml milvus: - address: ibm-lh-lakehouse-milvus727.milvus.apps.enginecluster-aurora.cp.fyre.ibm.com - port: 443 - user: "admin" - password: "6ihRVTx0gDEP" + address: localhost + port: 19530 + user: "root" + password: "Milvus" # tls mode values [0, 1, 2] # 0 is close, 1 is one-way authentication, 2 is mutual authentication - tlsMode: 1 + tlsMode: 0 # tls cert path for validate server, will be used when tlsMode is 1 or 2 - caCertPath: "/root/hello_milvus/aura.crt" - serverName: "ibm-lh-lakehouse-milvus727.milvus.apps.enginecluster-aurora.cp.fyre.ibm.com" + caCertPath: "" + serverName: "" # mutual tls cert path, for server to validate client. # Will be used when tlsMode is 2 # for backward compatibility, if not set, will use tlsmode 1. @@ -31,29 +31,31 @@ milvus: # Related configuration of minio, which is responsible for data persistence for Milvus. minio: # Milvus storage configs, make them the same with milvus config - storageType: "gcpnative" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative + storageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative # You can use "gcpnative" for the Google Cloud Platform provider. Uses service account credentials for authentication. - address: storage.googleapis.com # Address of MinIO/S3 - port: 443 # Port of MinIO/S3 - accessKeyID: # accessKeyID of MinIO/S3 - secretAccessKey: # MinIO/S3 encryption string - gcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider. - useSSL: true # Access to MinIO/S3 with SSL + address: localhost # Address of MinIO/S3 + port: 9000 # Port of MinIO/S3 + accessKeyID: minioadmin # accessKeyID of MinIO/S3 + secretAccessKey: minioadmin # MinIO/S3 encryption string + gcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. + # Used only for the "gcpnative" cloud provider. + useSSL: false # Access to MinIO/S3 with SSL useIAM: false iamEndpoint: "" - bucketName: "gcs-nv1" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance - rootPath: "GifiGcpNative/milvus727" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance + bucketName: "a-bucket" # Milvus Bucket name in MinIO/S3, make it the same as your milvus instance + rootPath: "files" # Milvus storage root path in MinIO/S3, make it the same as your milvus instance # Backup storage configs, the storage you want to put the backup data - backupStorageType: "gcpnative" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent), gcpnative - backupAddress: storage.googleapis.com # Address of MinIO/S3 - backupPort: 443 # Port of MinIO/S3 - backupAccessKeyID: # accessKeyID of MinIO/S3 - backupSecretAccessKey: # MinIO/S3 encryption string - backupGcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. Used only for the "gcpnative" cloud provider. - backupBucketName: "gcs-nv1" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath - backupRootPath: "giffBackk" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath - backupUseSSL: true # Access to MinIO/S3 with SSL + backupStorageType: "minio" # support storage type: local, minio, s3, aws, gcp, ali(aliyun), azure, tc(tencent) + backupAddress: localhost # Address of MinIO/S3 + backupPort: 9000 # Port of MinIO/S3 + backupAccessKeyID: minioadmin # accessKeyID of MinIO/S3 + backupSecretAccessKey: minioadmin # MinIO/S3 encryption string + backupGcpCredentialJSON: "/root/gcs/gcpnative_creds.json" # The JSON content contains the gcs service account credentials. + # Used only for the "gcpnative" cloud provider. + backupBucketName: "a-bucket" # Bucket name to store backup data. Backup data will store to backupBucketName/backupRootPath + backupRootPath: "backup" # Rootpath to store backup data. Backup data will store to backupBucketName/backupRootPath + backupUseSSL: false # Access to MinIO/S3 with SSL # If you need to back up or restore data between two different storage systems, direct client-side copying is not supported. # Set this option to true to enable data transfer through Milvus Backup. @@ -79,4 +81,4 @@ backup: gcPause: enable: true seconds: 7200 - address: http://localhost:9091 + address: http://localhost:9091 \ No newline at end of file