[WIP] Add RabbitMQ source + target support

.github/workflows/cd.yml

@@ -10,29 +10,23 @@ jobs:
     name: Release
     strategy:
       matrix:
-        go-version: [1.17.0]
+        go-version: [1.18]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
     steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.go-version }}
-
-    - name: Checkout code
-      uses: actions/checkout@v2
-
-    - name: Cache go modules
-      uses: actions/cache@v2
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
+        check-latest: true
+        cache: true
 
     - name: Run linter
       run: make lint
@@ -49,6 +43,14 @@ jobs:
         echo "VERSION file ${{steps.version.outputs.VERSION_FILE}} does not match tagged version ${{ github.ref }}"
         exit 1
 
+    - name: Snyk Setup
+      uses: snyk/actions/setup@master
+
+    - name: Run Snyk to check for vulnerabilities
+      run: snyk test --project-name=stream-replicator --severity-threshold=high
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+
     - name: Compile
       run: make all
 

.github/workflows/ci.yml

@@ -12,28 +12,30 @@ jobs:
     name: Compile & Test
     strategy:
       matrix:
-        go-version: [1.17.0]
+        go-version: [1.18]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
       NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }}
 
     steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.go-version }}
+        check-latest: true
+        cache: true
 
-    - name: Checkout code
-      uses: actions/checkout@v2
+    - name: Snyk Setup
+      uses: snyk/actions/setup@master
 
-    - name: Cache go modules
-      uses: actions/cache@v2
-      with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
+    - name: Run Snyk to check for vulnerabilities
+      run: snyk test --project-name=stream-replicator --severity-threshold=high
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
     - name: Block Concurrent Executions of Integration Tests
       if: ${{ env.NGROK_TOKEN != '' }} 
@@ -60,4 +62,4 @@ jobs:
       run: make lint
 
     - name: Compile all targets
-      run: make all
+      run: make all

.github/workflows/snyk.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Run Snyk to check for vulnerabilities
       uses: snyk/actions/golang@master

.gitignore

@@ -12,3 +12,6 @@ vendor/
 build/
 dist/
 .localstack/
+
+#temporary directory created by tests
+tmp_replicator/

CHANGELOG

@@ -1,3 +1,28 @@
+Version 1.0.0 (2022-XX-XX)
+--------------------------
+Bump dependencies (#120)
+Fix import order (#175)
+Allow env var configuration of updated transformation config (#174)
+Fix typo in statsd reporting (#158)
+Cleanup Makefile (#112)
+Make setting of EventHub Partition Key configurable (#148)
+Fix latency reporting with no transformations (#108)
+Rationalise transformations and transformation config (#169)
+Resolve CI caching errors (#164)
+Bump to Go 1.18 (#163)
+Bump analytics SDK version to v0.3.0 (#131)
+Fix bug in makefile that prevents integration-down from completing (#162)
+Revamp unit testing project-wide (#129)
+Make anything that doesn't need to be exported private (#111)
+Add custom transformation layer (#146)
+Fail tests on NewConfig error (#145)
+Remove AWS Lambda and GCP Cloudfunctions builds (#140)
+Add telemetry (#124)
+Extend filtering to use custom data (#176)
+Use Snyk test to block release if there are vulnerabilities (#119)
+Change tls configurations to b64 env vars (#177)
+Allow configuration from a file (#105)
+
 Version 0.8.1 (2022-06-07)
 --------------------------
 Update Sarama package to 1.34 for kafka v3 (#133)

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all gox aws-lambda gcp-cloudfunctions cli cli-linux cli-darwin cli-windows container format lint tidy test-setup test integration-reset integration-up integration-down integration-test container-release clean
+.PHONY: all gox cli cli-linux cli-darwin cli-windows container format lint tidy test-setup test integration-reset integration-up integration-down integration-test container-release clean
 
 # -----------------------------------------------------------------------------
 #  CONSTANTS
@@ -11,8 +11,6 @@ go_dirs = `go list ./... | grep -v /build/ | grep -v /vendor/`
 build_dir       = build
 vendor_dir      = vendor
 integration_dir = integration
-cert_dir 		= $(integration_dir)/http
-abs_cert_dir	= $$(pwd)/$(cert_dir)
 ngrok_path		= ${NGROK_DIR}ngrok # Set NGROK_DIR to `/path/to/directory/` for local setup
 
 coverage_dir  = $(build_dir)/coverage
@@ -34,57 +32,12 @@ gcp_container_name = snowplow/stream-replicator-gcp
 #  BUILDING
 # -----------------------------------------------------------------------------
 
-all: aws-lambda gcp-cloudfunctions cli container
+all: cli container
 
 gox:
-	GO111MODULE=on go install github.com/mitchellh/gox@latest
+	go install github.com/mitchellh/gox@latest
 	mkdir -p $(compiled_dir)
 
-aws-lambda: gox
-	# WARNING: Binary must be called 'main' to work in Lambda
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/lambda/main ./cmd/aws/lambda/
-
-	# Create ZIP file for upload to Lambda
-	(cd $(linux_out_dir)/aws/lambda/ && zip -r staging.zip main)
-	mv $(linux_out_dir)/aws/lambda/staging.zip $(compiled_dir)/aws_lambda_stream_replicator_$(version)_linux_amd64.zip
-
-gcp-cloudfunctions: gox
-	mkdir -p $(staging_dir)/gcp/cloudfunctions
-
-	# Copy dependencies into staging area
-	cp ./cmd/gcp/cloudfunctions/function.go $(staging_dir)/gcp/cloudfunctions/function.go
-
-	# Get module dependencies in a vendor directory
-	GO111MODULE=on go mod vendor
-	cp -R ./$(vendor_dir)/ $(staging_dir)/gcp/cloudfunctions/vendor/
-
-	# Copy local packages into staging area
-	mkdir -p $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/cmd/
-	cp ./cmd/constants.go $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/cmd/constants.go
-	cp ./cmd/init.go $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/cmd/init.go
-	cp ./cmd/serverless.go $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/cmd/serverless.go
-
-	mkdir -p $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/config/
-	cp ./config/config.go $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/config/config.go
-
-	mkdir -p $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/
-	cp -R ./pkg/ $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/pkg/
-
-	mkdir -p $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/third_party/snowplow/
-	cp -R ./third_party/snowplow/badrows/ $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/third_party/snowplow/badrows
-	cp -R ./third_party/snowplow/iglu/ $(staging_dir)/gcp/cloudfunctions/vendor/github.com/snowplow-devops/stream-replicator/third_party/snowplow/iglu
-
-	echo "# github.com/snowplow-devops/stream-replicator v$(version)" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-	echo "github.com/snowplow-devops/stream-replicator/config" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-	echo "github.com/snowplow-devops/stream-replicator/cmd" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-	echo "github.com/snowplow-devops/stream-replicator/pkg" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-	echo "github.com/snowplow-devops/stream-replicator/third_party/snowplow/badrows" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-	echo "github.com/snowplow-devops/stream-replicator/third_party/snowplow/iglu" >> $(staging_dir)/gcp/cloudfunctions/vendor/modules.txt
-
-	# Create ZIP file for upload to CloudFunctions
-	(cd $(staging_dir)/gcp/cloudfunctions/ && zip -r staging.zip .)
-	mv $(staging_dir)/gcp/cloudfunctions/staging.zip $(compiled_dir)/gcp_cloudfunctions_stream_replicator_$(version)_linux_amd64.zip
-
 cli: gox cli-linux cli-darwin cli-windows
 	(cd $(linux_out_dir)/aws/cli/ && zip -r staging.zip stream-replicator)
 	mv $(linux_out_dir)/aws/cli/staging.zip $(compiled_dir)/aws_cli_stream_replicator_$(version)_linux_amd64.zip
@@ -100,16 +53,16 @@ cli: gox cli-linux cli-darwin cli-windows
 	mv $(windows_out_dir)/gcp/cli/staging.zip $(compiled_dir)/gcp_cli_stream_replicator_$(version)_windows_amd64.zip
 
 cli-linux: gox
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
 
 cli-darwin: gox
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
 
 cli-windows: gox
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
-	GO111MODULE=on CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/stream-replicator ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/gcp/cli/stream-replicator ./cmd/gcp/cli/
 
 container: cli-linux
 	docker build -t $(aws_container_name):$(version) -f Dockerfile.aws .
@@ -120,34 +73,33 @@ container: cli-linux
 # -----------------------------------------------------------------------------
 
 format:
-	GO111MODULE=on go fmt $(go_dirs)
-	GO111MODULE=on gofmt -s -w .
+	go fmt $(go_dirs)
+	gofmt -s -w .
 
 lint:
-	GO111MODULE=on go install golang.org/x/lint/golint@latest
+	go install golang.org/x/lint/golint@latest
 	LINTRESULT=$$(golint $(go_dirs)); echo "$$LINTRESULT"; [ -z "$$LINTRESULT" ];
 
 tidy:
-	GO111MODULE=on go mod tidy
+	go mod tidy
 
 # -----------------------------------------------------------------------------
 #  TESTING
 # -----------------------------------------------------------------------------
 
 test-setup:
 	mkdir -p $(coverage_dir)
-	GO111MODULE=on go install golang.org/x/tools/cmd/cover@latest
+	go install golang.org/x/tools/cmd/cover@latest
 
 test: test-setup
-	GO111MODULE=on go test $(go_dirs) -v -short -covermode=count -coverprofile=$(coverage_out)
-	GO111MODULE=on go tool cover -html=$(coverage_out) -o $(coverage_html)
-	GO111MODULE=on go tool cover -func=$(coverage_out)
+	go test $(go_dirs) -v -short -covermode=count -coverprofile=$(coverage_out)
+	go tool cover -html=$(coverage_out) -o $(coverage_html)
+	go tool cover -func=$(coverage_out)
 
 integration-test: test-setup
-	export CERT_DIR=$(abs_cert_dir); \
-	GO111MODULE=on go test $(go_dirs) -v -covermode=count -coverprofile=$(coverage_out)
-	GO111MODULE=on go tool cover -html=$(coverage_out) -o $(coverage_html)
-	GO111MODULE=on go tool cover -func=$(coverage_out)
+	go test $(go_dirs) -v -covermode=count -coverprofile=$(coverage_out)
+	go tool cover -html=$(coverage_out) -o $(coverage_html)
+	go tool cover -func=$(coverage_out)
 
 integration-reset: integration-down integration-up
 
@@ -159,14 +111,15 @@ integration-down: http-down
 	(cd $(integration_dir) && docker-compose -f ./docker-compose.yml down)
 	rm -rf $(integration_dir)/.localstack
 
+# ngrok needs to be installed and auth token must be configured for this if running locally
 http-up:
 	(cd "$(integration_dir)/http/server" && go run server.go &)
 	sleep 5
 	($(ngrok_path) http https://localhost:8999 &>/dev/null &)
 
 http-down:
 	(cd "$(integration_dir)/http/shutdown" && go run shutdownRequest.go)
-	killall ngrok
+	killall ngrok || true 
 
 # -----------------------------------------------------------------------------
 #  RELEASE

README.md

@@ -4,7 +4,7 @@
 
 ## Overview
 
-Easily replicate data streams wherever you need them to be!  This application is available in three different runtimes to facilitate different needs - AWS Lambda, GCP CloudFunctions and as a standalone application.
+Easily replicate data streams wherever you need them to be!  This application is available as a standalone application.
 
 See the [wiki documention](https://github.com/snowplow-devops/stream-replicator/wiki) for details on how to configure and run the application.
 
@@ -60,5 +60,5 @@ Unauthorized copying of this project via any medium is strictly prohibited.
 
 Copyright (c) 2020-2022 Snowplow Analytics Ltd. All rights reserved.
 
-[release-image]: http://img.shields.io/badge/golang-0.8.1-6ad7e5.svg?style=flat
+[release-image]: http://img.shields.io/badge/golang-1.0.0-6ad7e5.svg?style=flat
 [releases]: https://github.com/snowplow-devops/stream-replicator/releases/

VERSION

@@ -1 +1 @@
-0.8.1
+1.0.0

cmd/aws/cli/main.go

@@ -10,14 +10,15 @@ import (
 	"github.com/snowplow-devops/stream-replicator/cmd/cli"
 	kinesissource "github.com/snowplow-devops/stream-replicator/pkg/source/kinesis"
 	pubsubsource "github.com/snowplow-devops/stream-replicator/pkg/source/pubsub"
+	rabbitmqsource "github.com/snowplow-devops/stream-replicator/pkg/source/rabbitmq"
 	"github.com/snowplow-devops/stream-replicator/pkg/source/sourceconfig"
 	sqssource "github.com/snowplow-devops/stream-replicator/pkg/source/sqs"
 	stdinsource "github.com/snowplow-devops/stream-replicator/pkg/source/stdin"
 )
 
 func main() {
 	// Make a slice of SourceConfigPairs supported for this build
-	sourceConfigPairs := []sourceconfig.ConfigPair{stdinsource.ConfigPair, sqssource.ConfigPair, pubsubsource.ConfigPair, kinesissource.ConfigPair}
+	sourceConfigPairs := []sourceconfig.ConfigPair{stdinsource.ConfigPair, sqssource.ConfigPair, pubsubsource.ConfigPair, kinesissource.ConfigPair, rabbitmqsource.ConfigPair}
 
 	cli.RunCli(sourceConfigPairs)
 }