Develop

.github/workflows/cd.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692
-        go-version: ['1.21.1']
+        go-version: ['1.22']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
@@ -109,10 +109,14 @@ jobs:
         tag_name: ${{ github.ref_name }}
         files: |
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_windows_amd64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_darwin_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}_darwin_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_linux_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}_linux_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_windows_amd64.zip
 
     - name: Publish to DockerHub

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692
-        go-version: ['1.21.1']
+        go-version: ['1.22']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
@@ -70,7 +70,7 @@ jobs:
       env:
         SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
-    - name: Run Snyk to check for vulnerabilities in aws-only Docker
+    - name: Run Snyk to check for vulnerabilities in aws-only Docker image
       run: snyk container test snowplow/snowbridge:${{steps.version.outputs.VERSION}}-aws-only --severity-threshold=high
       env:
         SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

CHANGELOG

@@ -1,3 +1,11 @@
+Version 2.3.0 (2024-04-15)
+--------------------------
+Make kinesis source scaling checks configurable (#313)
+Add arm64 support, thanks @vbhamidipati! (#302)
+Bump dependencies (#310)
+Add GTM SS preview header transformation (#312)
+Add dynamic headers (#311)
+
 Version 2.2.3 (2024-02-05)
 --------------------------
 Patch kafka docs (#306)

Dockerfile.aws

@@ -1,7 +1,9 @@
-FROM alpine:3.18.5
+FROM --platform=$TARGETPLATFORM alpine:3.18.5
 LABEL org.opencontainers.image.authors="Snowplow Analytics <[email protected]>"
 
-ADD build/output/linux/aws/cli/snowbridge /opt/snowplow/
+ARG TARGETARCH
+
+ADD build/output/linux/aws/cli/$TARGETARCH/snowbridge /opt/snowplow/
 
 RUN adduser -D snowplow
 USER snowplow

Dockerfile.main

@@ -1,7 +1,9 @@
-FROM alpine:3.18.5
+FROM --platform=$TARGETPLATFORM alpine:3.18.5
 LABEL org.opencontainers.image.authors="Snowplow Analytics <[email protected]>"
 
-ADD build/output/linux/main/cli/snowbridge /opt/snowplow/
+ARG TARGETARCH
+
+ADD build/output/linux/main/cli/$TARGETARCH/snowbridge /opt/snowplow/
 
 RUN adduser -D snowplow
 USER snowplow

Makefile

@@ -28,8 +28,9 @@ linux_out_dir   = $(output_dir)/linux
 darwin_out_dir  = $(output_dir)/darwin
 windows_out_dir = $(output_dir)/windows
 
-container_name = snowplow/snowbridge
+linux_container_image_out_dir = $(output_dir)/container/linux
 
+container_name = snowplow/snowbridge
 
 # -----------------------------------------------------------------------------
 #  BUILDING
@@ -48,44 +49,67 @@ cli: gox cli-linux cli-darwin cli-windows
 	cp assets/awslicense/AMAZON_LICENSE AMAZON_LICENSE
 # linux aws:
 # Zip up the binaries
-	(cd $(linux_out_dir)/aws/cli/ && zip -r staging.zip snowbridge)
+	(cd $(linux_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge)
 # Add the readme, and relevant licence(s)
-	zip -u $(linux_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	zip -u $(linux_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
 # Move to its compiled_dir location, with appropriate long form name
-	mv $(linux_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip
+	mv $(linux_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip
 # Rinse and repeat for each distribution
+# linux arm aws:
+	(cd $(linux_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(linux_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_arm64.zip
 # darwin aws:
-	(cd $(darwin_out_dir)/aws/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(darwin_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
-	mv $(darwin_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip
+	(cd $(darwin_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(darwin_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip
+# darwin arm aws:
+	(cd $(darwin_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(darwin_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_arm64.zip
 # Windows aws:
-	(cd $(windows_out_dir)/aws/cli/ && zip -r staging.zip snowbridge.exe)
-	zip -u $(windows_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
-	mv $(windows_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip
+	(cd $(windows_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge.exe)
+	zip -u $(windows_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(windows_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip
 # linux main:
-	(cd $(linux_out_dir)/main/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(linux_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(linux_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip
+	(cd $(linux_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(linux_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip
+# linux arm main:
+	(cd $(linux_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md
+	mv $(linux_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_arm64.zip
 # darwin main:
-	(cd $(darwin_out_dir)/main/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(darwin_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(darwin_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip
+	(cd $(darwin_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(darwin_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip
+# darwin arm main:
+	(cd $(darwin_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md
+	mv $(darwin_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_arm64.zip	
 # windows main:
-	(cd $(windows_out_dir)/main/cli/ && zip -r staging.zip snowbridge.exe)
-	zip -u $(windows_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(windows_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip
+	(cd $(windows_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge.exe)
+	zip -u $(windows_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(windows_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip
 
+# Build CLI binaries for each distro
 cli-linux: gox
-	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
+
+	CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/	
 
 cli-darwin: gox
-	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
+
+	CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/	
 
 cli-windows: gox
-	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
 
 container: cli-linux
 	docker build -t $(container_name):$(aws_only_version) -f Dockerfile.aws .
@@ -122,7 +146,7 @@ test: test-setup
 integration-test: test-setup
 	go test $(integration_test_dirs) -v -covermode=count -coverprofile=$(coverage_out)
 	go tool cover -html=$(coverage_out) -o $(coverage_html)
-	go tool cover -func=$(coverage_out)
+	go tool cover -func=$(coverage_out)	
 
 # e2e-test covers only the e2e release tests, in preparation for when these will rely on deployed assets
 e2e-test: test-setup
@@ -141,7 +165,8 @@ e2e-down:
 
 integration-reset: integration-down integration-up
 
-# For integration tests we need localstack and pubsub, but not kafka (yet)
+# For integration tests we need localstack, pubsub kafka and http server
+# To run on mac M1, for example, set the default docker platform: export DOCKER_DEFAULT_PLATFORM=linux/arm64
 integration-up: http-up
 	(cd $(integration_dir) && docker compose up -d)
 	sleep 5
@@ -168,8 +193,10 @@ http-down:
 # Make & push docker assets, don't tag as latest if there's a `-` in the version (eg. 0.1.0-rc1)
 container-release:
 	@-docker login --username $(DOCKER_USERNAME) --password $(DOCKER_PASSWORD)
-	docker push $(container_name):$(aws_only_version)
-	docker push $(container_name):$(version)
+	docker buildx create --name multi-arch-builder --driver=docker-container --platform linux/amd64,linux/arm64 --use
+	docker buildx build -t $(container_name):$(aws_only_version) -f Dockerfile.aws --platform=linux/amd64,linux/arm64 --push .
+	docker buildx build -t $(container_name):$(version) -f Dockerfile.main --platform=linux/amd64,linux/arm64 --push .
+
 	if ! [[ $(version) =~ "-" ]]; then docker tag ${container_name}:${version} ${container_name}:latest; docker push $(container_name):latest; fi;
 
 # -----------------------------------------------------------------------------

README.md

@@ -18,5 +18,5 @@ The AWS-specific distribution contains everything, including the Kinesis source,
 
 To comply with the [Amazon Software License](https://github.com/twitchscience/kinsumer/blob/master/LICENSE), you may only use this distribution of Snowbridge _“with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc.”_
 
-[release-image]: http://img.shields.io/badge/golang-2.2.3-6ad7e5.svg?style=flat
+[release-image]: http://img.shields.io/badge/golang-2.3.0-6ad7e5.svg?style=flat
 [releases]: https://github.com/snowplow/snowbridge/releases/

VERSION

@@ -1 +1 @@
-2.2.3
+2.3.0

assets/docs/configuration/sources/kinesis-full-example.hcl

@@ -28,7 +28,13 @@ source {
     # Default is 250, cannot be set to lower than 200.
     read_throttle_delay_ms = 500
 
+    # Optional configures how often each kinesis consumer checks for whether it needs to change which shards it owns
+    shard_check_freq_seconds = 15
+
+    # Optional configures how often the kinesis client checks the stream for shard count changes, which triggers consumer ownership changes
+    leader_action_freq_seconds = 305
+
     # Maximum concurrent goroutines (lightweight threads) for message processing (default: 50)
     concurrent_writes = 15
   }
-}
+}

assets/docs/configuration/targets/http-full-example.hcl

@@ -38,6 +38,8 @@ target {
     # Whether to skip verifying ssl certificates chain (default: false)
     # If tls_cert and tls_key are not provided, this setting is not applied.
     skip_verify_tls            = true
+
+    # Whether to enable setting headers dynamically
+    dynamic_headers            = true
   }
 }
-

assets/docs/configuration/transformations/custom-scripts/create-a-script-header-example.js

@@ -0,0 +1,7 @@
+function main(x) {
+    var httpHeaders = {
+      foo: 'bar'
+    };
+    x.HTTPHeaders = httpHeaders;
+    return x;
+  }

assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-full-example .hcl

@@ -0,0 +1,4 @@
+transform {
+  use "spGtmssPreview" {
+  }
+}

assets/docs/configuration/transformations/snowplow-builtin/spGtmssPreview-minimal-example.hcl

@@ -0,0 +1,4 @@
+transform {
+  use "spGtmssPreview" {
+  }
+}

cmd/constants.go

@@ -9,7 +9,7 @@ package cmd
 
 const (
 	// AppVersion is the current version of the app
-	AppVersion = "2.2.3"
+	AppVersion = "2.3.0"
 
 	// AppName is the name of the application to use in logging / places that require the artifact
 	AppName = "snowbridge"

docs/configuration_transformations_docs_test.go

@@ -160,6 +160,8 @@ func testTransformationConfig(t *testing.T, filepath string, fullExample bool) {
 			configObject = &transform.Base64DecodeConfig{}
 		case "base64Encode":
 			configObject = &transform.Base64EncodeConfig{}
+		case "spGtmssPreview":
+			configObject = &transform.GTMSSPreviewConfig{}
 		case "js":
 			configObject = &engine.JSEngineConfig{}
 		case "lua":