Skip to content

Commit

Permalink
add Client.SetS3EnableDualstack (#1945)
Browse files Browse the repository at this point in the history 
Closes #1766

This PR adds a new configuration method, that enables users to switch off the use of dual-stack endpoints, when they use the client against Amazon S3.

I wanted to keep the name of the config aligned with how AWS SDKs and CLI call it, even though the default value is reversed. That is, AWS SDKs call it like `UseDualStack bool` or `withDualstackEnabled(bool)`.
  • Loading branch information
@narqo
narqo authored Mar 10, 2024
1 parent 4a223cc commit f823450
Show file tree
Hide file tree
Showing 3 changed files with 172 additions and 46 deletions.
21 changes: 18 additions & 3 deletions api.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2023 MinIO, Inc.
* Copyright 2015-2024 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -80,6 +80,8 @@ type Client struct {

// S3 specific accelerated endpoint.
s3AccelerateEndpoint string
// S3 dual-stack endpoints are enabled by default.
s3DualstackEnabled bool

// Region endpoint
region string
Expand Down Expand Up @@ -158,9 +160,12 @@ func New(endpoint string, opts *Options) (*Client, error) {
if err != nil {
return nil, err
}
// If Amazon S3 set to signature v4.
if s3utils.IsAmazonEndpoint(*clnt.endpointURL) {
// If Amazon S3 set to signature v4.
clnt.overrideSignerType = credentials.SignatureV4
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
// for backwards compatibility.
clnt.s3DualstackEnabled = true
}

return clnt, nil
Expand Down Expand Up @@ -330,6 +335,16 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
}
}

// SetS3EnableDualstack turns s3 dual-stack endpoints on or off for all requests.
// The feature is only specific to S3 and is on by default. To read more about
// Amazon S3 dual-stack endpoints visit -
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html
func (c *Client) SetS3EnableDualstack(enabled bool) {
if s3utils.IsAmazonEndpoint(*c.endpointURL) {
c.s3DualstackEnabled = enabled
}
}

// Hash materials provides relevant initialized hash algo writers
// based on the expected signature type.
//
Expand Down Expand Up @@ -926,7 +941,7 @@ func (c *Client) makeTargetURL(bucketName, objectName, bucketLocation string, is
// Do not change the host if the endpoint URL is a FIPS S3 endpoint or a S3 PrivateLink interface endpoint
if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) {
// Fetch new host based on the bucket location.
host = getS3Endpoint(bucketLocation)
host = getS3Endpoint(bucketLocation, c.s3DualstackEnabled)
}
}
}
Expand Down
14 changes: 9 additions & 5 deletions api_unit_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2024 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -29,14 +29,18 @@ import (
func TestValidBucketLocation(t *testing.T) {
s3Hosts := []struct {
bucketLocation string
useDualstack bool
endpoint string
}{
{"us-east-1", "s3.dualstack.us-east-1.amazonaws.com"},
{"unknown", "s3.dualstack.us-east-1.amazonaws.com"},
{"ap-southeast-1", "s3.dualstack.ap-southeast-1.amazonaws.com"},
{"us-east-1", true, "s3.dualstack.us-east-1.amazonaws.com"},
{"us-east-1", false, "s3.us-east-1.amazonaws.com"},
{"unknown", true, "s3.dualstack.us-east-1.amazonaws.com"},
{"unknown", false, "s3.us-east-1.amazonaws.com"},
{"ap-southeast-1", true, "s3.dualstack.ap-southeast-1.amazonaws.com"},
{"ap-southeast-1", false, "s3.ap-southeast-1.amazonaws.com"},
}
for _, s3Host := range s3Hosts {
endpoint := getS3Endpoint(s3Host.bucketLocation)
endpoint := getS3Endpoint(s3Host.bucketLocation, s3Host.useDualstack)
if endpoint != s3Host.endpoint {
t.Fatal("Error: invalid bucket location", endpoint)
}
Expand Down
183 changes: 145 additions & 38 deletions s3-endpoints.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2017 MinIO, Inc.
* Copyright 2015-2024 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,48 +17,155 @@

package minio

type awsS3Endpoint struct {
endpoint string
dualstackEndpoint string
}

// awsS3EndpointMap Amazon S3 endpoint map.
var awsS3EndpointMap = map[string]string{
"us-east-1": "s3.dualstack.us-east-1.amazonaws.com",
"us-east-2": "s3.dualstack.us-east-2.amazonaws.com",
"us-west-2": "s3.dualstack.us-west-2.amazonaws.com",
"us-west-1": "s3.dualstack.us-west-1.amazonaws.com",
"ca-central-1": "s3.dualstack.ca-central-1.amazonaws.com",
"eu-west-1": "s3.dualstack.eu-west-1.amazonaws.com",
"eu-west-2": "s3.dualstack.eu-west-2.amazonaws.com",
"eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com",
"eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com",
"eu-central-2": "s3.dualstack.eu-central-2.amazonaws.com",
"eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com",
"eu-south-1": "s3.dualstack.eu-south-1.amazonaws.com",
"eu-south-2": "s3.dualstack.eu-south-2.amazonaws.com",
"ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com",
"ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com",
"ap-south-2": "s3.dualstack.ap-south-2.amazonaws.com",
"ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com",
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",
"ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com",
"ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com",
"af-south-1": "s3.dualstack.af-south-1.amazonaws.com",
"me-central-1": "s3.dualstack.me-central-1.amazonaws.com",
"me-south-1": "s3.dualstack.me-south-1.amazonaws.com",
"sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com",
"us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com",
"us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com",
"cn-north-1": "s3.dualstack.cn-north-1.amazonaws.com.cn",
"cn-northwest-1": "s3.dualstack.cn-northwest-1.amazonaws.com.cn",
"ap-southeast-3": "s3.dualstack.ap-southeast-3.amazonaws.com",
"ap-southeast-4": "s3.dualstack.ap-southeast-4.amazonaws.com",
"il-central-1": "s3.dualstack.il-central-1.amazonaws.com",
var awsS3EndpointMap = map[string]awsS3Endpoint{
"us-east-1": {
"s3.us-east-1.amazonaws.com",
"s3.dualstack.us-east-1.amazonaws.com",
},
"us-east-2": {
"s3.us-east-2.amazonaws.com",
"s3.dualstack.us-east-2.amazonaws.com",
},
"us-west-2": {
"s3.us-west-2.amazonaws.com",
"s3.dualstack.us-west-2.amazonaws.com",
},
"us-west-1": {
"s3.us-west-1.amazonaws.com",
"s3.dualstack.us-west-1.amazonaws.com",
},
"ca-central-1": {
"s3.ca-central-1.amazonaws.com",
"s3.dualstack.ca-central-1.amazonaws.com",
},
"eu-west-1": {
"s3.eu-west-1.amazonaws.com",
"s3.dualstack.eu-west-1.amazonaws.com",
},
"eu-west-2": {
"s3.eu-west-2.amazonaws.com",
"s3.dualstack.eu-west-2.amazonaws.com",
},
"eu-west-3": {
"s3.eu-west-3.amazonaws.com",
"s3.dualstack.eu-west-3.amazonaws.com",
},
"eu-central-1": {
"s3.eu-central-1.amazonaws.com",
"s3.dualstack.eu-central-1.amazonaws.com",
},
"eu-central-2": {
"s3.eu-central-2.amazonaws.com",
"s3.dualstack.eu-central-2.amazonaws.com",
},
"eu-north-1": {
"s3.eu-north-1.amazonaws.com",
"s3.dualstack.eu-north-1.amazonaws.com",
},
"eu-south-1": {
"s3.eu-south-1.amazonaws.com",
"s3.dualstack.eu-south-1.amazonaws.com",
},
"eu-south-2": {
"s3.eu-south-2.amazonaws.com",
"s3.dualstack.eu-south-2.amazonaws.com",
},
"ap-east-1": {
"s3.ap-east-1.amazonaws.com",
"s3.dualstack.ap-east-1.amazonaws.com",
},
"ap-south-1": {
"s3.ap-south-1.amazonaws.com",
"s3.dualstack.ap-south-1.amazonaws.com",
},
"ap-south-2": {
"s3.ap-south-2.amazonaws.com",
"s3.dualstack.ap-south-2.amazonaws.com",
},
"ap-southeast-1": {
"s3.ap-southeast-1.amazonaws.com",
"s3.dualstack.ap-southeast-1.amazonaws.com",
},
"ap-southeast-2": {
"s3.ap-southeast-2.amazonaws.com",
"s3.dualstack.ap-southeast-2.amazonaws.com",
},
"ap-southeast-3": {
"s3.ap-southeast-3.amazonaws.com",
"s3.dualstack.ap-southeast-3.amazonaws.com",
},
"ap-southeast-4": {
"s3.ap-southeast-4.amazonaws.com",
"s3.dualstack.ap-southeast-4.amazonaws.com",
},
"ap-northeast-1": {
"s3.ap-northeast-1.amazonaws.com",
"s3.dualstack.ap-northeast-1.amazonaws.com",
},
"ap-northeast-2": {
"s3.ap-northeast-2.amazonaws.com",
"s3.dualstack.ap-northeast-2.amazonaws.com",
},
"ap-northeast-3": {
"s3.ap-northeast-3.amazonaws.com",
"s3.dualstack.ap-northeast-3.amazonaws.com",
},
"af-south-1": {
"s3.af-south-1.amazonaws.com",
"s3.dualstack.af-south-1.amazonaws.com",
},
"me-central-1": {
"s3.me-central-1.amazonaws.com",
"s3.dualstack.me-central-1.amazonaws.com",
},
"me-south-1": {
"s3.me-south-1.amazonaws.com",
"s3.dualstack.me-south-1.amazonaws.com",
},
"sa-east-1": {
"s3.sa-east-1.amazonaws.com",
"s3.dualstack.sa-east-1.amazonaws.com",
},
"us-gov-west-1": {
"s3.us-gov-west-1.amazonaws.com",
"s3.dualstack.us-gov-west-1.amazonaws.com",
},
"us-gov-east-1": {
"s3.us-gov-east-1.amazonaws.com",
"s3.dualstack.us-gov-east-1.amazonaws.com",
},
"cn-north-1": {
"s3.cn-north-1.amazonaws.com.cn",
"s3.dualstack.cn-north-1.amazonaws.com.cn",
},
"cn-northwest-1": {
"s3.cn-northwest-1.amazonaws.com.cn",
"s3.dualstack.cn-northwest-1.amazonaws.com.cn",
},
"il-central-1": {
"s3.il-central-1.amazonaws.com",
"s3.dualstack.il-central-1.amazonaws.com",
},
}

// getS3Endpoint get Amazon S3 endpoint based on the bucket location.
func getS3Endpoint(bucketLocation string) (s3Endpoint string) {
func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
s3Endpoint, ok := awsS3EndpointMap[bucketLocation]
if !ok {
// Default to 's3.dualstack.us-east-1.amazonaws.com' endpoint.
s3Endpoint = "s3.dualstack.us-east-1.amazonaws.com"
// Default to 's3.us-east-1.amazonaws.com' endpoint.
if useDualstack {
return "s3.dualstack.us-east-1.amazonaws.com"
}
return "s3.us-east-1.amazonaws.com"
}
if useDualstack {
return s3Endpoint.dualstackEndpoint
}
return s3Endpoint
return s3Endpoint.endpoint
}

0 comments on commit f823450

Please sign in to comment.