From d930d6bb77f3691c0f75cadb2b8768b13a3f47d5 Mon Sep 17 00:00:00 2001 From: kannappanr <30541348+kannappanr@users.noreply.github.com> Date: Wed, 20 Nov 2019 10:11:05 -0800 Subject: [PATCH] Add object retention support (#1187) --- api-object-lock.go | 6 +- api-object-retention.go | 168 +++++++++++++++++++++++++++++ api-put-object.go | 16 +++ api-remove.go | 29 +++++ docs/API.md | 93 ++++++++++++++++ examples/s3/getobjectretention.go | 48 +++++++++ examples/s3/putobjectretention.go | 54 ++++++++++ examples/s3/removeobjectoptions.go | 49 +++++++++ utils.go | 2 + 9 files changed, 464 insertions(+), 1 deletion(-) create mode 100644 api-object-retention.go create mode 100644 examples/s3/getobjectretention.go create mode 100644 examples/s3/putobjectretention.go create mode 100644 examples/s3/removeobjectoptions.go diff --git a/api-object-lock.go b/api-object-lock.go index 15f13a543..c30ab3258 100644 --- a/api-object-lock.go +++ b/api-object-lock.go @@ -203,7 +203,11 @@ func (c Client) GetBucketObjectLockConfig(bucketName string) (mode *RetentionMod if err != nil { return nil, nil, nil, err } - + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") + } + } config := &objectLockConfig{} if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { return nil, nil, nil, err diff --git a/api-object-retention.go b/api-object-retention.go new file mode 100644 index 000000000..8aa08f073 --- /dev/null +++ b/api-object-retention.go @@ -0,0 +1,168 @@ +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v6/pkg/s3utils" +) + +// objectRetention - object retention specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type objectRetention struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"Retention"` + Mode RetentionMode `xml:"Mode"` + RetainUntilDate time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate"` +} + +func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { + if mode == nil { + return nil, fmt.Errorf("Mode not set") + } + + if date == nil { + return nil, fmt.Errorf("RetainUntilDate not set") + } + + if !mode.IsValid() { + return nil, fmt.Errorf("invalid retention mode `%v`", mode) + } + objectRetention := &objectRetention{ + Mode: *mode, + RetainUntilDate: *date, + } + return objectRetention, nil +} + +// PutObjectRetentionOptions represents options specified by user for PutObject call +type PutObjectRetentionOptions struct { + GovernanceBypass bool + Mode *RetentionMode + RetainUntilDate *time.Time + VersionID string +} + +// PutObjectRetention : sets object retention for a given object and versionID. +func (c Client) PutObjectRetention(bucketName, objectName string, opts PutObjectRetentionOptions) error { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return err + } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + urlValues.Set("retention", "") + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + retention, err := newObjectRetention(opts.Mode, opts.RetainUntilDate) + if err != nil { + return err + } + + retentionData, err := xml.Marshal(retention) + if err != nil { + return err + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } + + reqMetadata := requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentBody: bytes.NewReader(retentionData), + contentLength: int64(len(retentionData)), + contentMD5Base64: sumMD5Base64(retentionData), + contentSHA256Hex: sum256Hex(retentionData), + customHeader: headers, + } + + // Execute PUT Object Retention. + resp, err := c.executeMethod(context.Background(), "PUT", reqMetadata) + defer closeResponse(resp) + if err != nil { + return err + } + if resp != nil { + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + return httpRespToErrorResponse(resp, bucketName, objectName) + } + } + return nil +} + +// GetObjectRetention gets retention of given object. +func (c Client) GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { + // Input validation. + if err := s3utils.CheckValidBucketName(bucketName); err != nil { + return nil, nil, err + } + + if err := s3utils.CheckValidObjectName(objectName); err != nil { + return nil, nil, err + } + urlValues := make(url.Values) + urlValues.Set("retention", "") + if versionID != "" { + urlValues.Set("versionId", versionID) + } + // Execute GET on bucket to list objects. + resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{ + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Hex: emptySHA256Hex, + }) + defer closeResponse(resp) + if err != nil { + return nil, nil, err + } + if resp != nil { + if resp.StatusCode != http.StatusOK { + return nil, nil, httpRespToErrorResponse(resp, bucketName, objectName) + } + } + retention := &objectRetention{} + if err = xml.NewDecoder(resp.Body).Decode(retention); err != nil { + return nil, nil, err + } + + return &retention.Mode, &retention.RetainUntilDate, nil +} diff --git a/api-put-object.go b/api-put-object.go index 8f33fca6c..bc0848bbf 100644 --- a/api-put-object.go +++ b/api-put-object.go @@ -25,6 +25,7 @@ import ( "net/http" "runtime/debug" "sort" + "time" "github.com/minio/minio-go/v6/pkg/encrypt" "github.com/minio/minio-go/v6/pkg/s3utils" @@ -40,6 +41,8 @@ type PutObjectOptions struct { ContentDisposition string ContentLanguage string CacheControl string + Mode *RetentionMode + RetainUntilDate *time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string @@ -80,6 +83,14 @@ func (opts PutObjectOptions) Header() (header http.Header) { if opts.CacheControl != "" { header["Cache-Control"] = []string{opts.CacheControl} } + + if opts.Mode != nil { + header["x-amz-object-lock-mode"] = []string{opts.Mode.String()} + } + if opts.RetainUntilDate != nil { + header["x-amz-object-lock-retain-until-date"] = []string{opts.RetainUntilDate.Format(time.RFC3339)} + } + if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } @@ -109,6 +120,11 @@ func (opts PutObjectOptions) validate() (err error) { return ErrInvalidArgument(v + " unsupported user defined metadata value") } } + if opts.Mode != nil { + if !opts.Mode.IsValid() { + return ErrInvalidArgument(opts.Mode.String() + " unsupported retention mode") + } + } return nil } diff --git a/api-remove.go b/api-remove.go index e919be17d..4c8c335c3 100644 --- a/api-remove.go +++ b/api-remove.go @@ -60,6 +60,17 @@ func (c Client) RemoveBucket(bucketName string) error { // RemoveObject remove an object from a bucket. func (c Client) RemoveObject(bucketName, objectName string) error { + return c.RemoveObjectWithOptions(bucketName, objectName, RemoveObjectOptions{}) +} + +// RemoveObjectOptions represents options specified by user for PutObject call +type RemoveObjectOptions struct { + GovernanceBypass bool + VersionID string +} + +// RemoveObjectWithOptions removes an object from a bucket. +func (c Client) RemoveObjectWithOptions(bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err @@ -67,11 +78,29 @@ func (c Client) RemoveObject(bucketName, objectName string) error { if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } + + // Get resources properly escaped and lined up before + // using them in http request. + urlValues := make(url.Values) + + if opts.VersionID != "" { + urlValues.Set("versionId", opts.VersionID) + } + + // Build headers. + headers := make(http.Header) + + if opts.GovernanceBypass { + // Set the bypass goverenance retention header + headers.Set("x-amz-bypass-governance-retention", "True") + } // Execute DELETE on objectName. resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, + queryValues: urlValues, + customHeader: headers, }) defer closeResponse(resp) if err != nil { diff --git a/docs/API.md b/docs/API.md index beb684d63..f17487de9 100644 --- a/docs/API.md +++ b/docs/API.md @@ -69,6 +69,9 @@ func main() { | | [`FPutObjectWithContext`](#FPutObjectWithContext) | [`FPutObjectWithContext`](#FPutObjectWithContext) | | | | | [`FGetObjectWithContext`](#FGetObjectWithContext) | [`FGetObjectWithContext`](#FGetObjectWithContext) | | | | | [`RemoveObjectsWithContext`](#RemoveObjectsWithContext) | | | | +| | [`RemoveObjectWithOptions`](#RemoveObjectWithOptions) | | | | +| | [`PutObjectRetention`](#PutObjectRetention) | | | | +| | [`GetObjectRetention`](#GetObjectRetention) | | | | | | [`SelectObjectContent`](#SelectObjectContent) | | ## 1. Constructor @@ -585,6 +588,8 @@ __minio.PutObjectOptions__ | `opts.ContentDisposition` | _string_ | Content disposition of object, "inline" | | `opts.ContentLanguage` | _string_ | Content language of object, e.g "French" | | `opts.CacheControl` | _string_ | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600"| +| `opts.Mode` | _*minio.RetentionMode_ | Retention mode to be set, e.g "COMPLIANCE" | +| `opts.RetainUntilDate` | _*time.Time_ | Time until which the retention applied is valid| | `opts.ServerSideEncryption` | _encrypt.ServerSide_ | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v6) | | `opts.StorageClass` | _string_ | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | _string_ | Specify a redirect for the object, to another object in the same bucket or to a external URL. | @@ -1102,6 +1107,94 @@ for rErr := range minioClient.RemoveObjects(ctx, "my-bucketname", objectsCh) { fmt.Println("Error detected during deletion: ", rErr) } ``` + +### RemoveObjectWithOptions(bucketName, objectName string, opts minio.RemoveObjectOptions) error +Removes an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.RemoveObjectOptions_ |Allows user to set options | + +__minio.RemoveObjectOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to delete an object locked with GOVERNANCE mode| +| `opts.VersionID` | _string_ |Version ID of the object to delete| + + +```go +opts := minio.RemoveObjectOptions { + GovernanceBypass: true, + VersionID: "myversionid", + } +err = minioClient.RemoveObjectWithOptions("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` + +### PutObjectRetention(bucketName, objectName string, opts minio.PutObjectRetentionOptions) error +Applies object retention lock onto an object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`opts` |_minio.PutObjectRetentionOptions_ |Allows user to set options like retention mode, expiry date and version id | + +__minio.PutObjectRetentionOptions__ + +|Field | Type | Description | +|:--- |:--- | :--- | +| `opts.GovernanceBypass` | _bool_ |Set the bypass governance header to overwrite object retention if the existing retention mode is set to GOVERNANCE| +| `opts.Mode` | _*minio.RetentionMode_ |Retention mode to be set| +| `opts.RetainUntilDate` | _*time.Time_ |Time until which the retention applied is valid| +| `opts.VersionID` | _string_ |Version ID of the object to apply retention on| + +```go +t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) +m := minio.RetentionMode(minio.Compliance) +opts := minio.PutObjectRetentionOptions { + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } +err = minioClient.PutObjectRetention("mybucket", "myobject", opts) +if err != nil { + fmt.Println(err) + return +} +``` + +### GetObjectRetention(bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) +Returns retention set on a given object. + +__Parameters__ + + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`versionID` |_string_ |Version ID of the object | + +```go +err = minioClient.PutObjectRetention("mybucket", "myobject", "") +if err != nil { + fmt.Println(err) + return +} +``` ### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults Parameters diff --git a/examples/s3/getobjectretention.go b/examples/s3/getobjectretention.go new file mode 100644 index 000000000..527f8a2ec --- /dev/null +++ b/examples/s3/getobjectretention.go @@ -0,0 +1,48 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + m, t, err = s3Client.GetObjectRetention("my-bucket", "my-object", "") + if err != nil { + log.Fatalln(err) + } + log.Println("Get object retention successful, Mode: ", m.String(), " Retainuntil Date ", t.String()) +} diff --git a/examples/s3/putobjectretention.go b/examples/s3/putobjectretention.go new file mode 100644 index 000000000..b0db21a5b --- /dev/null +++ b/examples/s3/putobjectretention.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "time" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) + m := minio.RetentionMode(minio.Governance) + opts := minio.PutObjectRetentionOptions{ + GovernanceBypass: true, + RetainUntilDate: &t, + Mode: &m, + } + err = s3Client.PutObjectRetention("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Set object retention on my-object successfully.") +} diff --git a/examples/s3/removeobjectoptions.go b/examples/s3/removeobjectoptions.go new file mode 100644 index 000000000..926d80eed --- /dev/null +++ b/examples/s3/removeobjectoptions.go @@ -0,0 +1,49 @@ +// +build ignore + +/* + * MinIO Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2019 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + + "github.com/minio/minio-go/v6" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + opts := minio.RemoveObjectOptions{ + GovernanceBypass: true, + } + err = s3Client.RemoveObjectWithOptions("my-bucket", "my-object", opts) + if err != nil { + log.Fatalln(err) + } + log.Println("Remove object successful") +} diff --git a/utils.go b/utils.go index 2a743a834..d24cfb5c7 100644 --- a/utils.go +++ b/utils.go @@ -224,6 +224,8 @@ var supportedHeaders = []string{ "content-disposition", "content-language", "x-amz-website-redirect-location", + "x-amz-object-lock-mode", + "x-amz-object-lock-retain-until-date", "expires", // Add more supported headers here. }