Merge pull request #54 from buildkite/keithduncan/add-additional-regi…

…on-fallback

Add additional region configuration and fallback

git-credential-s3-secrets

@@ -52,44 +52,12 @@ parse_url() {
   done
 }
 
-s3_bucket_region() {
-  local bucket="$1"
-
-  local guess_region="${AWS_DEFAULT_REGION:-}"
-  if [ -z "${guess_region}" ]
-  then
-    # This plug-in may not be executing in an AWS VPC or have access to the IDMS
-    # Fail fast with the --connect-timeout flag
-    local token=$(curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" --fail --silent --show-error --location --connect-timeout 5 "http://169.254.169.254/latest/api/token")
-    if [ -n "${token}" ]
-    then
-      guess_region=$(curl -H "X-aws-ec2-metadata-token: $token" --fail --silent --show-error --location "http://169.254.169.254/latest/meta-data/placement/region")
-    fi
-  fi
-  if [ -z "${guess_region}" ]
-  then
-    guess_region="us-east-1"
-  fi
-
-  # Buckets in us-east-1 have a LocationConstraint of null
-  # https://docs.aws.amazon.com/cli/latest/reference/s3api/get-bucket-location.html
-  local bucket_region="$(aws s3api get-bucket-location --bucket "${bucket}" --region "${guess_region}" --output text --query "LocationConstraint || 'us-east-1'")"
-
-  echo "${bucket_region}"
-}
-
 s3_download() {
   local bucket="$1"
-  local key="$2"
-
-  local bucket_region="$(s3_bucket_region "${bucket}")"
-  if [ -z "${bucket_region}" ]
-  then
-    echo "Could not determine the bucket region for ${bucket}" >&2
-    exit 2
-  fi
+  local region="$2"
+  local key="$3"
 
-  local aws_s3_args=("--quiet" "--region=${bucket_region}")
+  local aws_s3_args=("--quiet" "--region=${region}")
 
   if [[ "${BUILDKITE_USE_KMS:-true}" =~ ^(true|1)$ ]] ; then
     aws_s3_args+=("--sse" "aws:kms")
@@ -103,14 +71,15 @@ s3_download() {
 }
 
 bucket="$1"
-key="$2"
-action="${3:-get}"
+region="$2"
+key="$3"
+action="${4:-get}"
 
 # we only support get and we don't parse the stdin params
 if [ "$action" == "get" ] ; then
 
   # read git-credentials, which is a list of uris
-  s3_download "$bucket" "$key" | while read -r uri ; do
+  s3_download "$bucket" "$region" "$key" | while read -r uri ; do
     if ! parse_url "$uri" ; then
       echo "Failed to parse uri $uri" >&2
       exit 1

s3secrets-helper/env/env.go

@@ -0,0 +1,10 @@
+package env
+
+const (
+	EnvBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
+	EnvRegion     = "BUILDKITE_PLUGIN_S3_SECRETS_REGION"
+	EnvPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
+	EnvPipeline   = "BUILDKITE_PIPELINE_SLUG"
+	EnvRepo       = "BUILDKITE_REPO"
+	EnvCredHelper = "BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER"
+)

s3secrets-helper/main.go

@@ -5,19 +5,12 @@ import (
 	"log"
 	"os"
 
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/env"
 	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/s3"
 	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/secrets"
 	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sshagent"
 )
 
-const (
-	envBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
-	envPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
-	envPipeline   = "BUILDKITE_PIPELINE_SLUG"
-	envRepo       = "BUILDKITE_REPO"
-	envCredHelper = "BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER"
-)
-
 func main() {
 	log := log.New(os.Stderr, "", log.Lmsgprefix)
 	if err := mainWithError(log); err != nil {
@@ -26,33 +19,36 @@ func main() {
 }
 
 func mainWithError(log *log.Logger) error {
-	bucket := os.Getenv(envBucket)
+	bucket := os.Getenv(env.EnvBucket)
 	if bucket == "" {
 		return nil
 	}
 
-	prefix := os.Getenv(envPrefix)
+	// May be empty string
+	regionHint := os.Getenv(env.EnvRegion)
+
+	prefix := os.Getenv(env.EnvPrefix)
 	if prefix == "" {
-		prefix = os.Getenv(envPipeline)
+		prefix = os.Getenv(env.EnvPipeline)
 	}
 	if prefix == "" {
-		return fmt.Errorf("%s or %s required", envPrefix, envPipeline)
+		return fmt.Errorf("One of the %s or %s environment variables is required, set one to configure the bucket key prefix that is scanned for secrets.", env.EnvPrefix, env.EnvPipeline)
 	}
 
-	client, err := s3.New(log, bucket)
+	client, err := s3.New(log, bucket, regionHint)
 	if err != nil {
 		return err
 	}
 
 	agent := &sshagent.Agent{}
 
-	credHelper := os.Getenv(envCredHelper)
+	credHelper := os.Getenv(env.EnvCredHelper)
 	if credHelper == "" {
-		return fmt.Errorf("%s required", envCredHelper)
+		return fmt.Errorf("The %s environment variable is required, set it to the path of the git-credential-s3-secrets script.", env.EnvCredHelper)
 	}
 
 	return secrets.Run(secrets.Config{
-		Repo:                os.Getenv(envRepo),
+		Repo:                os.Getenv(env.EnvRepo),
 		Bucket:              bucket,
 		Prefix:              prefix,
 		Client:              client,

s3secrets-helper/s3/s3.go

@@ -8,21 +8,24 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go"
+	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/env"
 	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
 )
 
 type Client struct {
 	s3 *s3.Client
 	bucket string
+	region string
 }
 
-func getRegion(ctx context.Context) (string, error) {
+func getCurrentRegion(ctx context.Context) (string, error) {
 	if region := os.Getenv("AWS_DEFAULT_REGION"); len(region) > 0 {
 		return region, nil
 	}
@@ -37,45 +40,62 @@ func getRegion(ctx context.Context) (string, error) {
 	return "", errors.New("Unknown current region")
 }
 
-func New(log *log.Logger, bucket string) (*Client, error) {
+func New(log *log.Logger, bucket string, regionHint string) (*Client, error) {
 	ctx := context.Background()
 
-	// Using the current region (or a guess) find where the bucket lives
+	var awsConfig aws.Config
+	var err error
 
-	region, err := getRegion(ctx)
-	if err != nil {
-		// Ignore error and fallback to us-east-1 for bucket lookup
-		region = "us-east-1"
-	}
+	if regionHint != "" {
+		// If there is a region hint provided, we use it unconditionally
+		awsConfig, err = config.LoadDefaultConfig(ctx,
+			config.WithRegion(regionHint),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
+		}
+	} else {
+		// Otherwise, use the current region (or a guess) to dynamically find
+		// where the bucket lives.
+		region, err := getCurrentRegion(ctx)
+		if err != nil {
+			// Ignore error and fallback to us-east-1 for bucket lookup
+			region = "us-east-1"
+		}
 
-	config, err := config.LoadDefaultConfig(ctx,
-		config.WithRegion(region),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
-	}
+		awsConfig, err = config.LoadDefaultConfig(ctx,
+			config.WithRegion(region),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err)
+		}
 
-	log.Printf("Discovered current region as %q\n", config.Region)
+		log.Printf("Discovered current region as %q\n", awsConfig.Region)
 
-	bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(config), bucket)
-	if err != nil {
-		return nil, fmt.Errorf("Could not discover the region for bucket %q: (%v)", bucket, err)
+		bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(awsConfig), bucket)
+		if err == nil && bucketRegion != "" {
+			log.Printf("Discovered bucket region as %q\n", bucketRegion)
+			awsConfig.Region = bucketRegion
+		} else {
+			log.Printf("Could not discover region for bucket %q. Using the %q region as a fallback, if this is not correct configure a bucket region using the %q environment variable. (%v)\n", bucket, awsConfig.Region, env.EnvRegion, err)
+		}
 	}
 
-	log.Printf("Discovered bucket region as %q\n", bucketRegion)
-
-	config.Region = bucketRegion
-
 	return &Client{
-		s3: s3.NewFromConfig(config),
+		s3: s3.NewFromConfig(awsConfig),
 		bucket: bucket,
+		region: awsConfig.Region,
 	}, nil
 }
 
 func (c *Client) Bucket() (string) {
 	return c.bucket
 }
 
+func (c *Client) Region() (string) {
+	return c.region
+}
+
 // Get downloads an object from S3.
 // Intended for small files; object is fully read into memory.
 // sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases.

s3secrets-helper/secrets/secrets.go

@@ -13,6 +13,7 @@ import (
 // Client represents interaction with AWS S3
 type Client interface {
 	Bucket() (string)
+	Region() (string)
 	Get(key string) ([]byte, error)
 	BucketExists() (bool, error)
 }
@@ -209,7 +210,7 @@ func handleGitCredentials(conf Config, results <-chan getResult) error {
 		// Replace spaces ' ' in the helper path with an escaped space '\ '
 		escapedCredentialHelper := strings.ReplaceAll(conf.GitCredentialHelper, " ", "\\ ")
 
-		helper := fmt.Sprintf("credential.helper=%s %s %s", escapedCredentialHelper, r.bucket, r.key)
+		helper := fmt.Sprintf("credential.helper=%s %s %s %s", escapedCredentialHelper, r.bucket, conf.Client.Region(), r.key)
 
 		helpers = append(helpers, helper)
 	}