Skip to content

Commit

Permalink
Bump Go and Sarama version on server package (#29)
Browse files Browse the repository at this point in the history
* Bump Go version to 1.22

* Fix producer and load-test commands to run in docker

* Upgrade ginkgo library

* Update ginkgo on Makefile

* Upgrade multiple dependencies

* Remove Open Telemetry Sarama dependency

* Bump Sarama to IBM package latest version

* Add docker volume to load-test and producer commands

* Fix makefile

* Trying to fix action not handling error code

* Force Server tests to fail for testing github action error handling

* Fixing CI

* Revert force fail server tests

* Added healthcheck to eventsgateway api on docker compose

* Testing tests on docker

* Fix docker run client tests

* Debugging CI

* Debugging CI

* Debugging CI

* Debugging CI

* Add custom test methods for CI

* Debugging CI

* Decouple CI and local test commands
  • Loading branch information
helder-junior authored Aug 1, 2024
1 parent 5930965 commit 0148951
Show file tree
Hide file tree
Showing 13 changed files with 353 additions and 759 deletions.
6 changes: 2 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@ jobs:
steps:
- name: Check out the repo
uses: actions/checkout@v4
- run: make build-dev
- run: make test
- run: make test-ci
server-tests:
name: Run server tests
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v4
- run: cd server && make build-dev
- run: cd server && make test
- run: cd server && make test-ci
24 changes: 12 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,33 +16,33 @@ build-dev:
@make -f server/Makefile build-dev

test:
docker compose up client-tests
@make deps-start
@docker run -i -v ./:/app --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c 'make test-go'

spark-notebook:
@docker compose up jupyter

producer:
@echo "Will connect to server at ${MY_IP}:5000"
@go run main.go producer -d
@docker run -it -v ./:/app --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c "go run main.go producer -d"

load-test:
@echo "Will connect to server at ${MY_IP}:5000"
@go run main.go load-test -d
@docker run -it -v ./:/app --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c "go run main.go load-test -d"

deps-start:
@docker compose up -d eventsgateway-api
@docker compose up -d eventsgateway-api --wait

setup:
@go install github.com/onsi/ginkgo/v2/ginkgo@v2.1.4
@go install github.com/onsi/ginkgo/v2/ginkgo@v2.19.1
@go install github.com/wadey/[email protected]
@go mod tidy
@cd .git/hooks && ln -sf ./hooks/pre-commit.sh pre-commit

setup-ci:
@go install github.com/mattn/[email protected]
@go install github.com/onsi/ginkgo/v2/[email protected]
@go install github.com/wadey/[email protected]
@go mod tidy
# Run all CI commands inside a single Make target to make easier debugging.
test-ci:
@docker build -t eventsgateway-client-dev -f dev.Dockerfile .
@docker build -t eventsgateway-server -f server/Dockerfile server
@docker compose -f docker-compose-ci.yaml up -d eventsgateway-api --wait
@docker run -i --network eventsgateway_eventsgateway eventsgateway-client-dev sh -c 'make test-go'

test-go: unit integration test-coverage-func

Expand Down
4 changes: 2 additions & 2 deletions dev.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ FROM golang:1.17-alpine

WORKDIR /app

RUN apk add make build-base

ADD .. /app

RUN apk add make build-base

RUN go install github.com/wadey/[email protected] && \
go install github.com/onsi/ginkgo/v2/[email protected] && \
go mod tidy
67 changes: 67 additions & 0 deletions docker-compose-ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
services:
zookeeper:
image: confluentinc/cp-zookeeper:6.2.15
ports:
- 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
networks:
- eventsgateway

kafka:
image: confluentinc/cp-kafka:6.2.15 #Kafka 2.8
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_CREATE_TOPICS: uploads-au_default:1:0
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
depends_on:
zookeeper:
condition: service_started
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "9092" ]
interval: 5s
timeout: 10s
retries: 10
networks:
- eventsgateway

jaeger:
image: jaegertracing/all-in-one:latest
environment:
- COLLECTOR_OTLP_ENABLED=true
ports:
- "5775:5775/udp"
- "6831:6831/udp"
- "6832:6832/udp"
- "5778:5778"
- "16686:16686"
- "4317:4317"
networks:
- eventsgateway

eventsgateway-api:
image: eventsgateway-server
ports:
- "5000:5000"
- "6060:6060"
depends_on:
kafka:
condition: service_healthy
jaeger:
condition: service_started
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "5000" ]
interval: 5s
timeout: 10s
retries: 10

networks:
- eventsgateway

networks:
eventsgateway:
29 changes: 12 additions & 17 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ services:
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
depends_on:
- zookeeper
zookeeper:
condition: service_started
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "9092" ]
interval: 5s
Expand Down Expand Up @@ -61,9 +62,16 @@ services:
- "5000:5000"
- "6060:6060"
depends_on:
- zookeeper
- kafka
- jaeger
kafka:
condition: service_healthy
jaeger:
condition: service_started
healthcheck:
test: [ "CMD", "nc", "-z", "localhost", "5000" ]
interval: 5s
timeout: 10s
retries: 10

volumes:
- ./server:/app
command:
Expand All @@ -73,19 +81,6 @@ services:
networks:
- eventsgateway

client-tests:
image: eventsgateway-client-dev
volumes:
- ./:/app
command:
- sh
- -c
- 'make test-go'
depends_on:
- eventsgateway-api
networks:
- eventsgateway

jupyter:
image: quay.io/jupyter/all-spark-notebook
ports:
Expand Down
25 changes: 3 additions & 22 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,44 +5,31 @@ go 1.17
require (
github.com/Shopify/sarama v1.35.0
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.5.2
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
github.com/mmcloughlin/professor v0.0.0-20170922221822-6b97112ab8b3
github.com/onsi/ginkgo/v2 v2.1.4
github.com/onsi/gomega v1.20.0
github.com/opentracing/opentracing-go v1.1.0
github.com/prometheus/client_golang v1.12.2
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/satori/go.uuid v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v0.0.1
github.com/spf13/viper v1.3.2
github.com/topfreegames/avro v1.0.2
github.com/topfreegames/protos v1.6.1
go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.34.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0
go.opentelemetry.io/otel v1.9.0
go.opentelemetry.io/otel/sdk v1.9.0
google.golang.org/grpc v1.48.0
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/eapache/go-resiliency v1.3.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
Expand All @@ -61,14 +48,11 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/spf13/afero v1.1.2 // indirect
github.com/spf13/cast v1.3.0 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/pflag v1.0.3 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.9.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.9.0 // indirect
go.opentelemetry.io/otel/trace v1.9.0 // indirect
go.opentelemetry.io/proto/otlp v0.18.0 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
Expand All @@ -80,7 +64,4 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.9.0
)
require github.com/inconshreveable/mousetrap v1.0.0 // indirect
Loading

0 comments on commit 0148951

Please sign in to comment.