From 1c0728cb26980759401eef7be7036f2c0791361a Mon Sep 17 00:00:00 2001 From: zekaifeng <37854724+dormanze@users.noreply.github.com> Date: Wed, 23 Oct 2024 18:50:09 +0800 Subject: [PATCH 1/2] mirror:support one mirror process to back up specified buckets support one mirror process to back up specified buckets --- cmd/mirror-main.go | 28 +++++++++++++++++---------- cmd/mirror-url.go | 48 +++++++++++++++++++++++++++++----------------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/cmd/mirror-main.go b/cmd/mirror-main.go index a3ae4965fe..134198e868 100644 --- a/cmd/mirror-main.go +++ b/cmd/mirror-main.go @@ -108,6 +108,10 @@ var ( Name: "exclude-bucket", Usage: "exclude bucket(s) that match specified bucket name pattern", }, + cli.StringSliceFlag{ + Name: "include-bucket", + Usage: "mirror bucket(s) that match specified bucket name pattern", + }, cli.StringSliceFlag{ Name: "exclude-storageclass", Usage: "exclude object(s) that match the specified storage class", @@ -205,22 +209,25 @@ EXAMPLES: Exclude test* buckets and backup* buckets when mirroring. {{.Prompt}} {{.HelpName}} --exclude-bucket 'test*' --exclude 'backup*' s3 ~/test - 11. Mirror objects newer than 10 days from bucket test to a local folder. + 11. Mirror test* buckets from aliased Amazon S3 cloud storage to a local folder. + {{.Prompt}} {{.HelpName}} --include-bucket 'test*' s3 ~/test + + 12. Mirror objects newer than 10 days from bucket test to a local folder. {{.Prompt}} {{.HelpName}} --newer-than 10d s3/test ~/localfolder - 12. Mirror objects older than 30 days from Amazon S3 bucket test to a local folder. + 13. Mirror objects older than 30 days from Amazon S3 bucket test to a local folder. {{.Prompt}} {{.HelpName}} --older-than 30d s3/test ~/test - 13. Mirror server encrypted objects from Amazon S3 cloud storage to a bucket on Amazon S3 cloud storage + 14. Mirror server encrypted objects from Amazon S3 cloud storage to a bucket on Amazon S3 cloud storage {{.Prompt}} {{.HelpName}} --enc-c "minio/archive=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" --enc-c "s3/archive=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5BBB" s3/archive/ minio/archive/ - 14. Update 'Cache-Control' header on all existing objects recursively. + 15. Update 'Cache-Control' header on all existing objects recursively. {{.Prompt}} {{.HelpName}} --attr "Cache-Control=max-age=90000,min-fresh=9000" myminio/video-files myminio/video-files - 15. Mirror a local folder recursively to Amazon S3 cloud storage and preserve all local file attributes. + 16. Mirror a local folder recursively to Amazon S3 cloud storage and preserve all local file attributes. {{.Prompt}} {{.HelpName}} -a backup/ s3/archive - 16. Cross mirror between sites in a active-active deployment. + 17. Cross mirror between sites in a active-active deployment. Site-A: {{.Prompt}} {{.HelpName}} --active-active siteA siteB Site-B: {{.Prompt}} {{.HelpName}} --active-active siteB siteA `, @@ -657,8 +664,8 @@ func (mj *mirrorJob) watchMirrorEvents(ctx context.Context, events []EventInfo) if matchExcludeOptions(mj.opts.excludeOptions, sourceSuffix, sourceURL.Type) { continue } - // Skip the bucket, if it matches the Exclude options provided - if matchExcludeBucketOptions(mj.opts.excludeBuckets, sourceSuffix) { + // Skip the bucket, if it matches the provided exclude options or does not match the included options + if matchBucketOptions(mj.opts.excludeBuckets, mj.opts.includeBuckets, sourceSuffix) { continue } @@ -1002,6 +1009,7 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc skipErrors: cli.Bool("skip-errors"), excludeOptions: cli.StringSlice("exclude"), excludeBuckets: cli.StringSlice("exclude-bucket"), + includeBuckets: cli.StringSlice("include-bucket"), excludeStorageClasses: cli.StringSlice("exclude-storageclass"), olderThan: cli.String("older-than"), newerThan: cli.String("newer-than"), @@ -1073,8 +1081,8 @@ func runMirror(ctx context.Context, srcURL, dstURL string, cli *cli.Context, enc continue } - // Skip create bucket, if it matches the Exclude options provided - if matchExcludeBucketOptions(mopts.excludeBuckets, sourceSuffix) { + // Skips bucket creation if it matches the provided exclusion options or does not match the included options + if matchBucketOptions(mopts.excludeBuckets, mj.opts.includeBuckets, sourceSuffix) { continue } diff --git a/cmd/mirror-url.go b/cmd/mirror-url.go index f3410d8df6..59fd2e94b1 100644 --- a/cmd/mirror-url.go +++ b/cmd/mirror-url.go @@ -108,7 +108,7 @@ func matchExcludeOptions(excludeOptions []string, srcSuffix string, typ ClientUR return false } -func matchExcludeBucketOptions(excludeBuckets []string, srcSuffix string) bool { +func matchBucketOptions(excludeBuckets, includeBuckets []string, srcSuffix string) bool { if strings.HasPrefix(srcSuffix, "/") { srcSuffix = srcSuffix[1:] } else if runtime.GOOS == "windows" && strings.HasPrefix(srcSuffix, `\`) { @@ -120,12 +120,24 @@ func matchExcludeBucketOptions(excludeBuckets []string, srcSuffix string) bool { } else { bucketName = strings.Split(srcSuffix, "/")[0] } + // mirror all bucket + if bucketName == "" { + return false + } for _, pattern := range excludeBuckets { if wildcard.Match(pattern, bucketName) { return true } } - return false + if len(includeBuckets) <= 0 { + return false + } + for _, pattern := range includeBuckets { + if wildcard.Match(pattern, bucketName) { + return false + } + } + return true } func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mirrorOptions, URLsCh chan<- URLs) { @@ -180,8 +192,8 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi continue } - // Skip the source bucket if it matches the Exclude options provided - if matchExcludeBucketOptions(opts.excludeBuckets, srcSuffix) { + // Skip the source bucket if it matches the provided exclude options or does not match the included options + if matchBucketOptions(opts.excludeBuckets, opts.includeBuckets, srcSuffix) { continue } @@ -191,8 +203,8 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi continue } - // Skip the target bucket if it matches the Exclude options provided - if matchExcludeBucketOptions(opts.excludeBuckets, tgtSuffix) { + // Skip the target bucket if it matches the provided exclude options or does not match the included options + if matchBucketOptions(opts.excludeBuckets, opts.includeBuckets, tgtSuffix) { continue } @@ -265,18 +277,18 @@ func deltaSourceTarget(ctx context.Context, sourceURL, targetURL string, opts mi } type mirrorOptions struct { - isFake, isOverwrite, activeActive bool - isWatch, isRemove, isMetadata bool - isRetriable bool - isSummary bool - skipErrors bool - excludeOptions, excludeStorageClasses, excludeBuckets []string - encKeyDB map[string][]prefixSSEPair - md5, disableMultipart bool - olderThan, newerThan string - storageClass string - userMetadata map[string]string - checksum minio.ChecksumType + isFake, isOverwrite, activeActive bool + isWatch, isRemove, isMetadata bool + isRetriable bool + isSummary bool + skipErrors bool + excludeOptions, excludeStorageClasses, excludeBuckets, includeBuckets []string + encKeyDB map[string][]prefixSSEPair + md5, disableMultipart bool + olderThan, newerThan string + storageClass string + userMetadata map[string]string + checksum minio.ChecksumType } // Prepares urls that need to be copied or removed based on requested options. From b5edaa5746b89ebc3fc4b461ede73790234f3478 Mon Sep 17 00:00:00 2001 From: zekaifeng <37854724+dormanze@users.noreply.github.com> Date: Wed, 23 Oct 2024 18:51:41 +0800 Subject: [PATCH 2/2] Update mirror-url.go --- cmd/mirror-url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mirror-url.go b/cmd/mirror-url.go index 59fd2e94b1..0150f3a6e9 100644 --- a/cmd/mirror-url.go +++ b/cmd/mirror-url.go @@ -120,7 +120,7 @@ func matchBucketOptions(excludeBuckets, includeBuckets []string, srcSuffix strin } else { bucketName = strings.Split(srcSuffix, "/")[0] } - // mirror all bucket + // mirror all buckets if bucketName == "" { return false }