From 1177488ba5b69513817d93d121424995506ba0ed Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Wed, 5 Jul 2023 23:19:41 -0500 Subject: [PATCH] chore: rename to raystack (#57) --- .github/workflows/package.yaml | 4 +- Makefile | 6 +- README.md | 36 ++--- app/proc.go | 4 +- app/server.go | 14 +- clients/go/Makefile | 2 +- clients/go/README.md | 33 ++++- clients/go/doc.go | 2 +- clients/go/examples/grpc/main.go | 6 +- clients/go/examples/rest/main.go | 8 +- clients/go/examples/ws/main.go | 6 +- clients/go/go.mod | 4 +- clients/go/go.sum | 8 +- clients/go/grpc/grpc.go | 10 +- clients/go/grpc/grpc_test.go | 6 +- clients/go/grpc/option.go | 6 +- clients/go/rest/option.go | 8 +- clients/go/rest/option_test.go | 2 +- clients/go/rest/rest.go | 12 +- clients/go/rest/rest_test.go | 6 +- clients/go/retry/retry.go | 2 +- clients/go/serializer/serializer_test.go | 2 +- clients/go/testdata/sample.proto | 2 +- clients/go/wire/wire_test.go | 2 +- clients/go/ws/option.go | 8 +- clients/go/ws/ws.go | 12 +- clients/go/ws/ws_test.go | 8 +- clients/java/Makefile | 6 +- clients/java/README.md | 9 +- clients/java/build.gradle | 10 +- collection/collector.go | 4 +- config/acks.go | 2 +- config/distribution.go | 2 +- config/log.go | 2 +- config/metric.go | 2 +- config/publisher.go | 2 +- config/server.go | 2 +- config/worker.go | 4 +- deserialization/proto_test.go | 2 +- docs/docs/concepts/architecture.md | 14 +- docs/docs/concepts/structure.md | 2 +- docs/docs/contribute/contribution.md | 31 ++-- docs/docs/guides/deployment.md | 99 +++++++------ docs/docs/guides/monitoring.md | 11 +- docs/docs/guides/overview.md | 7 +- docs/docs/guides/publishing.md | 12 +- docs/docs/guides/troubleshooting.md | 15 +- docs/docs/introduction.md | 10 +- docs/docs/quickstart.md | 7 +- docs/docs/reference/configurations.md | 136 +++++++++--------- docs/docs/roadmap.md | 4 +- docs/docs/support.md | 6 +- docs/docusaurus.config.js | 10 +- example/example.proto | 2 +- example/main.go | 2 +- example/readme.md | 20 ++- example/sample.pb.go | 2 +- go.mod | 4 +- go.sum | 4 +- integration/integration_test.go | 2 +- main.go | 8 +- metrics/statsd.go | 4 +- publisher/kafka.go | 8 +- publisher/kafka_test.go | 4 +- serialization/json_test.go | 2 +- services/grpc/handler.go | 12 +- services/grpc/handler_test.go | 10 +- services/grpc/service.go | 6 +- services/rest/handler.go | 16 +-- services/rest/response.go | 4 +- services/rest/response_test.go | 4 +- services/rest/service.go | 10 +- services/rest/websocket/ack.go | 6 +- services/rest/websocket/connection/conn.go | 6 +- services/rest/websocket/connection/table.go | 2 +- .../rest/websocket/connection/table_test.go | 2 +- .../rest/websocket/connection/upgrader.go | 8 +- .../websocket/connection/upgrader_test.go | 4 +- services/rest/websocket/handler.go | 16 +-- services/rest/websocket/handler_test.go | 10 +- services/rest/websocket/pinger.go | 10 +- services/services.go | 10 +- worker/init_test.go | 2 +- worker/mocks.go | 2 +- worker/worker.go | 8 +- worker/worker_test.go | 6 +- 86 files changed, 435 insertions(+), 411 deletions(-) diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index f390d5a2..bebf4602 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -33,5 +33,5 @@ jobs: tags: | docker.pkg.github.com/${{ github.repository }}/raccoon:latest docker.pkg.github.com/${{ github.repository }}/raccoon:${{ steps.get_version.outputs.version-without-v }} - odpf/raccoon:latest - odpf/raccoon:${{ steps.get_version.outputs.version-without-v }} + raystack/raccoon:latest + raystack/raccoon:${{ steps.get_version.outputs.version-without-v }} diff --git a/Makefile b/Makefile index 2f7e5174..eb3c0a1a 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,9 @@ PROTO_PACKAGE=/proto generate-proto: rm -rf .temp mkdir -p .temp - curl -o .temp/proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 - protoc --proto_path=.temp/ .temp/odpf/raccoon/v1beta1/raccoon.proto --go_out=./ --go_opt=paths=import --go_opt=Modpf/raccoon/v1beta1/raccoon.proto=$(PROTO_PACKAGE) - protoc --proto_path=.temp/ .temp/odpf/raccoon/v1beta1/raccoon.proto --go-grpc_opt=paths=import --go-grpc_opt=Modpf/raccoon/v1beta1/raccoon.proto=$(PROTO_PACKAGE) --go-grpc_out=./ + curl -o .temp/proton.tar.gz -L http://api.github.com/repos/raystack/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 + protoc --proto_path=.temp/ .temp/raystack/raccoon/v1beta1/raccoon.proto --go_out=./ --go_opt=paths=import --go_opt=Mraystack/raccoon/v1beta1/raccoon.proto=$(PROTO_PACKAGE) + protoc --proto_path=.temp/ .temp/raystack/raccoon/v1beta1/raccoon.proto --go-grpc_opt=paths=import --go-grpc_opt=Mraystack/raccoon/v1beta1/raccoon.proto=$(PROTO_PACKAGE) --go-grpc_out=./ # Build Lifecycle compile: diff --git a/README.md b/README.md index e84c441a..351d24d9 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # Raccoon -![build workflow](https://github.com/odpf/raccoon/actions/workflows/build.yaml/badge.svg) -![package workflow](https://github.com/odpf/raccoon/actions/workflows/package.yaml/badge.svg) +![build workflow](https://github.com/raystack/raccoon/actions/workflows/build.yaml/badge.svg) +![package workflow](https://github.com/raystack/raccoon/actions/workflows/package.yaml/badge.svg) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg?logo=apache)](LICENSE) -[![Version](https://img.shields.io/github/v/release/odpf/raccoon?logo=semantic-release)](Version) +[![Version](https://img.shields.io/github/v/release/raystack/raccoon?logo=semantic-release)](Version) -Raccoon is high throughput, low-latency service that provides an API to ingest clickstream data from mobile apps, sites and publish it to Kafka. Raccoon uses the Websocket protocol for peer-to-peer communication and protobuf as the serialization format. It provides an event type agnostic API that accepts a batch (array) of events in protobuf format. Refer [here](https://github.com/odpf/proton/tree/main/odpf/raccoon) for proto definition format that Raccoon accepts. +Raccoon is high throughput, low-latency service that provides an API to ingest clickstream data from mobile apps, sites and publish it to Kafka. Raccoon uses the Websocket protocol for peer-to-peer communication and protobuf as the serialization format. It provides an event type agnostic API that accepts a batch (array) of events in protobuf format. Refer [here](https://github.com/raystack/proton/tree/main/raystack/raccoon) for proto definition format that Raccoon accepts.

@@ -18,7 +18,7 @@ Raccoon is high throughput, low-latency service that provides an API to ingest c - **Reduced payload sizes** - Protobuf based - **Metrics:** - Built-in monitoring includes latency and active connections. -To know more, follow the detailed [documentation](https://odpf.github.io/raccoon/) +To know more, follow the detailed [documentation](https://raystack.github.io/raccoon/) ## Use cases @@ -33,10 +33,10 @@ Raccoon can be used as an event collector, event distributor and as a forwarder Explore the following resources to get started with Raccoon: -- [Guides](https://odpf.github.io/raccoon/guides/overview) provides guidance on deployment and client sample. -- [Concepts](https://odpf.github.io/raccoon/concepts/architecture) describes all important Raccoon concepts. -- [Reference](https://odpf.github.io/raccoon//reference/configurations) contains details about configurations, metrics and other aspects of Raccoon. -- [Contribute](https://odpf.github.io/raccoon/contribute/contribution) contains resources for anyone who wants to contribute to Raccoon. +- [Guides](https://raystack.github.io/raccoon/guides/overview) provides guidance on deployment and client sample. +- [Concepts](https://raystack.github.io/raccoon/concepts/architecture) describes all important Raccoon concepts. +- [Reference](https://raystack.github.io/raccoon//reference/configurations) contains details about configurations, metrics and other aspects of Raccoon. +- [Contribute](https://raystack.github.io/raccoon/contribute/contribution) contains resources for anyone who wants to contribute to Raccoon. ## Run with Docker @@ -46,11 +46,11 @@ Explore the following resources to get started with Raccoon: **Run Docker Image** -Raccoon provides Docker [image](https://hub.docker.com/r/odpf/raccoon) as part of the release. Make sure you have Kafka running on your local and run the following. +Raccoon provides Docker [image](https://hub.docker.com/r/raystack/raccoon) as part of the release. Make sure you have Kafka running on your local and run the following. ```sh # Download docker image from docker hub -$ docker pull odpf/raccoon +$ docker pull raystack/raccoon # Run the following docker command with minimal config. $ docker run -p 8080:8080 \ @@ -58,7 +58,7 @@ $ docker run -p 8080:8080 \ -e SERVER_WEBSOCKET_CONN_ID_HEADER=X-User-ID \ -e PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS=host.docker.internal:9093 \ -e EVENT_DISTRIBUTION_PUBLISHER_PATTERN=clickstream-%s-log \ - odpf/raccoon + raystack/raccoon ``` **Run Docker Compose** @@ -71,7 +71,7 @@ $ make docker-run $ make docker-stop ``` -You can consume the published events from the host machine by using `localhost:9094` as kafka broker server. Mind the [topic routing](https://odpf.github.io/raccoon/concepts/architecture#event-distribution) when you consume the events. +You can consume the published events from the host machine by using `localhost:9094` as kafka broker server. Mind the [topic routing](https://raystack.github.io/raccoon/concepts/architecture#event-distribution) when you consume the events. ## Running locally @@ -82,7 +82,7 @@ Prerequisite: ```sh # Clone the repo -$ git clone https://github.com/odpf/raccoon.git +$ git clone https://github.com/raystack/raccoon.git # Build the executable $ make @@ -94,7 +94,7 @@ $ vim .env $ ./out/raccoon ``` -**Note:** Read the detail of each configurations [here](https://odpf.github.io/raccoon/reference/configuration). +**Note:** Read the detail of each configurations [here](https://raystack.github.io/raccoon/reference/configuration). ## Running tests @@ -112,11 +112,11 @@ $ INTEGTEST_BOOTSTRAP_SERVER=localhost:9094 INTEGTEST_HOST=localhost:8080 INTEGT Development of Raccoon happens in the open on GitHub, and we are grateful to the community for contributing bugfixes and improvements. Read below to learn how you can take part in improving Raccoon. -Read our [contributing guide](https://odpf.github.io/raccoon/contribute/contribution) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Raccoon. +Read our [contributing guide](https://raystack.github.io/raccoon/contribute/contribution) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Raccoon. -To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/odpf/raccoon/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started. +To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/raystack/raccoon/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started. -This project exists thanks to all the [contributors](https://github.com/odpf/raccoon/graphs/contributors). +This project exists thanks to all the [contributors](https://github.com/raystack/raccoon/graphs/contributors). ## License diff --git a/app/proc.go b/app/proc.go index 3fa4af8c..9f340218 100644 --- a/app/proc.go +++ b/app/proc.go @@ -3,10 +3,10 @@ package app import ( "context" - "github.com/odpf/raccoon/logger" + "github.com/raystack/raccoon/logger" ) -//Run the server +// Run the server func Run() error { ctx := context.Background() ctx, cancel := context.WithCancel(ctx) diff --git a/app/server.go b/app/server.go index 26df59a9..b99b3df4 100644 --- a/app/server.go +++ b/app/server.go @@ -9,13 +9,13 @@ import ( "syscall" "time" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - "github.com/odpf/raccoon/publisher" - "github.com/odpf/raccoon/services" - "github.com/odpf/raccoon/worker" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + "github.com/raystack/raccoon/publisher" + "github.com/raystack/raccoon/services" + "github.com/raystack/raccoon/worker" ) // StartServer starts the server diff --git a/clients/go/Makefile b/clients/go/Makefile index 4905f8a6..b0b8b67b 100644 --- a/clients/go/Makefile +++ b/clients/go/Makefile @@ -3,7 +3,7 @@ all : setup fmt staticcheck test setup: @echo "installing the dependencies..." - go get go.buf.build/odpf/gw/odpf/proton + go get go.buf.build/raystack/gw/raystack/proton go get golang.org/x/lint go get google.golang.org/protobuf go get github.com/gorilla/websocket diff --git a/clients/go/README.md b/clients/go/README.md index c96f1452..02528292 100644 --- a/clients/go/README.md +++ b/clients/go/README.md @@ -1,21 +1,27 @@ -# Raccoon Client Library for Go # +# Raccoon Client Library for Go [![Go Reference](https://pkg.go.dev/badge/google.golang.org/api.svg)](https://pkg.go.dev/google.golang.org/api) ## Requirements + go 1.16 or above ## Install + ```bash -go get github.com/odpf/raccoon/clients/go +go get github.com/raystack/raccoon/clients/go ``` + ## Usage #### Construct a new REST client, then use the various options on the client. + For example: + ```go -import "github.com/odpf/raccoon/clients/go/rest" +import "github.com/raystack/raccoon/clients/go/rest" ``` + ```go client, err := rest.New( rest.WithUrl("http://localhost:8080/api/v1/events"), @@ -27,10 +33,13 @@ client, err := rest.New( see example: [examples/rest](examples/rest/main.go) #### Construct a new GRPC client, then use the various options on the client. + For example: + ```go -import "github.com/odpf/raccoon/clients/go/grpc" +import "github.com/raystack/raccoon/clients/go/grpc" ``` + ```go client, err := grpc.New( grpc.WithAddr("http://localhost:8080"), @@ -45,10 +54,13 @@ client, err := grpc.New( see example: [examples/grpc](examples/grpc/main.go) #### Construct a new Websocket client, then use the various options on the client. + For example: + ```go -import "github.com/odpf/raccoon/clients/go/ws" +import "github.com/raystack/raccoon/clients/go/ws" ``` + ```go client, err := ws.New( ws.WithUrl("ws://localhost:8080/api/v1/events"), @@ -56,13 +68,17 @@ client, err := ws.New( ws.WithHeader("x-user-type", "gojek")) // default serializer is proto. ``` + Reading the message acknowledgements + ```go resp := <-client.EventAcks() ``` + see example: [examples/websocket](examples/ws/main.go) #### Sending the request to raccoon + ```go reqGuid, resp, err := client.Send([]*raccoon.Event{ { @@ -76,9 +92,8 @@ reqGuid, resp, err := client.Send([]*raccoon.Event{ }) ``` - - #### Retry Configuration + Default settings, wait = `1 second`, and maximum attempts = `3`. The following options can override it. ```go @@ -88,20 +103,24 @@ grpc.WithRetry(time.Second*2, 5) ``` ### Custom Logging Configuration + The default logger logs the information to the standard output, and the default logger can be diabled by the following settings + ```go rest.WithLogger(nil) grpc.WithLogger(nil) ``` The client provide the logger interface that can be implemented by any logger. + ```go type Logger interface { Infof(msg string, keysAndValues ...interface{}) Errorf(msg string, keysAndValues ...interface{}) } ``` + And cutomer logger can set for the client with the following options. ```go diff --git a/clients/go/doc.go b/clients/go/doc.go index 3d170932..4abf99b8 100644 --- a/clients/go/doc.go +++ b/clients/go/doc.go @@ -1,6 +1,6 @@ /* Package raccoon provides grpc, http and websocket client for the raccoon service. -see https://github.com/odpf/raccoon +see https://github.com/raystack/raccoon */ package raccoon diff --git a/clients/go/examples/grpc/main.go b/clients/go/examples/grpc/main.go index 84d765b3..8cf13bd3 100644 --- a/clients/go/examples/grpc/main.go +++ b/clients/go/examples/grpc/main.go @@ -12,9 +12,9 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/grpc" - "github.com/odpf/raccoon/clients/go/testdata" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/grpc" + "github.com/raystack/raccoon/clients/go/testdata" ) func main() { diff --git a/clients/go/examples/rest/main.go b/clients/go/examples/rest/main.go index 31bd6896..97b7d005 100644 --- a/clients/go/examples/rest/main.go +++ b/clients/go/examples/rest/main.go @@ -8,10 +8,10 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/rest" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/testdata" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/rest" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/testdata" ) func main() { diff --git a/clients/go/examples/ws/main.go b/clients/go/examples/ws/main.go index 84d7c1bb..4d10dc3f 100644 --- a/clients/go/examples/ws/main.go +++ b/clients/go/examples/ws/main.go @@ -4,12 +4,12 @@ import ( "fmt" "log" - raccoon "github.com/odpf/raccoon/clients/go" + raccoon "github.com/raystack/raccoon/clients/go" "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" - "github.com/odpf/raccoon/clients/go/testdata" - "github.com/odpf/raccoon/clients/go/ws" + "github.com/raystack/raccoon/clients/go/testdata" + "github.com/raystack/raccoon/clients/go/ws" ) func main() { diff --git a/clients/go/go.mod b/clients/go/go.mod index 265924be..9eb341f5 100644 --- a/clients/go/go.mod +++ b/clients/go/go.mod @@ -1,10 +1,10 @@ -module github.com/odpf/raccoon/clients/go +module github.com/raystack/raccoon/clients/go go 1.16 require ( github.com/stretchr/testify v1.8.0 - go.buf.build/odpf/gw/odpf/proton v1.1.131 + go.buf.build/raystack/gw/raystack/proton v1.1.131 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 google.golang.org/grpc v1.49.0 google.golang.org/protobuf v1.28.1 diff --git a/clients/go/go.sum b/clients/go/go.sum index 1e1108f9..7ae0639b 100644 --- a/clients/go/go.sum +++ b/clients/go/go.sum @@ -84,10 +84,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.buf.build/odpf/gw/envoyproxy/protoc-gen-validate v1.1.7/go.mod h1:Z+auCoGBL8lg1aQQgGe7R3HLnYKf6nbtTawTt0WmFCo= -go.buf.build/odpf/gw/grpc-ecosystem/grpc-gateway v1.1.46/go.mod h1:/LuddrGPi0fwj7ay6Orutt8oFfPz8Y3c8qdBkacJq1A= -go.buf.build/odpf/gw/odpf/proton v1.1.131 h1:ufVq/8R9zkYjLPUq2xKqi/InJPTmh7MjPfNtdb5U8nE= -go.buf.build/odpf/gw/odpf/proton v1.1.131/go.mod h1:/TrDvy+6bIydxZAEliK+Wo3JuMRYFz1L+d9UsYZT44w= +go.buf.build/raystack/gw/envoyproxy/protoc-gen-validate v1.1.7/go.mod h1:Z+auCoGBL8lg1aQQgGe7R3HLnYKf6nbtTawTt0WmFCo= +go.buf.build/raystack/gw/grpc-ecosystem/grpc-gateway v1.1.46/go.mod h1:/LuddrGPi0fwj7ay6Orutt8oFfPz8Y3c8qdBkacJq1A= +go.buf.build/raystack/gw/raystack/proton v1.1.131 h1:ufVq/8R9zkYjLPUq2xKqi/InJPTmh7MjPfNtdb5U8nE= +go.buf.build/raystack/gw/raystack/proton v1.1.131/go.mod h1:/TrDvy+6bIydxZAEliK+Wo3JuMRYFz1L+d9UsYZT44w= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/clients/go/grpc/grpc.go b/clients/go/grpc/grpc.go index 9c2f8e53..d16dfdd6 100644 --- a/clients/go/grpc/grpc.go +++ b/clients/go/grpc/grpc.go @@ -4,18 +4,18 @@ import ( "context" "fmt" - "github.com/odpf/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/log" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/retry" - "github.com/odpf/raccoon/clients/go/serializer" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/retry" + "github.com/raystack/raccoon/clients/go/serializer" ) // New creates the new grpc client with provided options. diff --git a/clients/go/grpc/grpc_test.go b/clients/go/grpc/grpc_test.go index 75994c04..0ca2a87b 100644 --- a/clients/go/grpc/grpc_test.go +++ b/clients/go/grpc/grpc_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" @@ -16,8 +16,8 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/testdata" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/testdata" "github.com/stretchr/testify/assert" ) diff --git a/clients/go/grpc/option.go b/clients/go/grpc/option.go index 52adfda3..7ab7d1d6 100644 --- a/clients/go/grpc/option.go +++ b/clients/go/grpc/option.go @@ -3,12 +3,12 @@ package grpc import ( "time" - "github.com/odpf/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/log" "google.golang.org/grpc" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/serializer" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/serializer" ) // Grpc is the http implementation diff --git a/clients/go/rest/option.go b/clients/go/rest/option.go index 8ac8365a..6e14d4d2 100644 --- a/clients/go/rest/option.go +++ b/clients/go/rest/option.go @@ -5,10 +5,10 @@ import ( "time" "github.com/gojek/heimdall/v7" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/log" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/wire" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/wire" ) // Rest is the http implementation diff --git a/clients/go/rest/option_test.go b/clients/go/rest/option_test.go index 6b9dbb2d..e00d5a22 100644 --- a/clients/go/rest/option_test.go +++ b/clients/go/rest/option_test.go @@ -3,7 +3,7 @@ package rest import ( "testing" - "github.com/odpf/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/serializer" "github.com/stretchr/testify/assert" ) diff --git a/clients/go/rest/rest.go b/clients/go/rest/rest.go index ac868dbc..ded6f4c3 100644 --- a/clients/go/rest/rest.go +++ b/clients/go/rest/rest.go @@ -7,16 +7,16 @@ import ( "net/http" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/protobuf/types/known/timestamppb" "github.com/gojek/heimdall/v7/httpclient" "github.com/google/uuid" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/log" - "github.com/odpf/raccoon/clients/go/retry" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/wire" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/retry" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/wire" ) // New creates the new rest client with provided options. diff --git a/clients/go/rest/rest_test.go b/clients/go/rest/rest_test.go index 7393d9c6..5252e138 100644 --- a/clients/go/rest/rest_test.go +++ b/clients/go/rest/rest_test.go @@ -9,10 +9,10 @@ import ( "net/http" "net/http/httptest" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/serializer" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/serializer" "github.com/stretchr/testify/assert" ) diff --git a/clients/go/retry/retry.go b/clients/go/retry/retry.go index 0e7b6ce9..289a36cf 100644 --- a/clients/go/retry/retry.go +++ b/clients/go/retry/retry.go @@ -3,7 +3,7 @@ package retry import ( "time" - "github.com/odpf/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/log" ) var ( diff --git a/clients/go/serializer/serializer_test.go b/clients/go/serializer/serializer_test.go index 8aa44f8a..0d9c9840 100644 --- a/clients/go/serializer/serializer_test.go +++ b/clients/go/serializer/serializer_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/protobuf/proto" "github.com/stretchr/testify/assert" diff --git a/clients/go/testdata/sample.proto b/clients/go/testdata/sample.proto index ef1184e5..60a331dd 100644 --- a/clients/go/testdata/sample.proto +++ b/clients/go/testdata/sample.proto @@ -2,7 +2,7 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/odpf/raccoon/clients/go/testdata"; +option go_package = "github.com/raystack/raccoon/clients/go/testdata"; message PageEvent { string event_guid = 1; diff --git a/clients/go/wire/wire_test.go b/clients/go/wire/wire_test.go index c04a3933..a70c1ab1 100644 --- a/clients/go/wire/wire_test.go +++ b/clients/go/wire/wire_test.go @@ -3,7 +3,7 @@ package wire import ( "testing" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "github.com/stretchr/testify/assert" ) diff --git a/clients/go/ws/option.go b/clients/go/ws/option.go index 3222451b..1c084408 100644 --- a/clients/go/ws/option.go +++ b/clients/go/ws/option.go @@ -6,10 +6,10 @@ import ( "net/http" "github.com/gorilla/websocket" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/log" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/wire" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/wire" ) // Rest is the http implementation diff --git a/clients/go/ws/ws.go b/clients/go/ws/ws.go index e4315440..883f7c40 100644 --- a/clients/go/ws/ws.go +++ b/clients/go/ws/ws.go @@ -5,16 +5,16 @@ import ( "net/http" - raccoon "github.com/odpf/raccoon/clients/go" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + raccoon "github.com/raystack/raccoon/clients/go" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/clients/go/log" - "github.com/odpf/raccoon/clients/go/retry" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/wire" + "github.com/raystack/raccoon/clients/go/log" + "github.com/raystack/raccoon/clients/go/retry" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/wire" ) // New creates the new websocket client with provided options. diff --git a/clients/go/ws/ws_test.go b/clients/go/ws/ws_test.go index 7206609d..f39b7b2a 100644 --- a/clients/go/ws/ws_test.go +++ b/clients/go/ws/ws_test.go @@ -7,15 +7,15 @@ import ( "net/http" "net/http/httptest" - pb "go.buf.build/odpf/gw/odpf/proton/odpf/raccoon/v1beta1" + pb "go.buf.build/raystack/gw/raystack/proton/raystack/raccoon/v1beta1" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" "github.com/google/uuid" "github.com/gorilla/websocket" - raccoon "github.com/odpf/raccoon/clients/go" - "github.com/odpf/raccoon/clients/go/serializer" - "github.com/odpf/raccoon/clients/go/testdata" + raccoon "github.com/raystack/raccoon/clients/go" + "github.com/raystack/raccoon/clients/go/serializer" + "github.com/raystack/raccoon/clients/go/testdata" "github.com/stretchr/testify/assert" ) diff --git a/clients/java/Makefile b/clients/java/Makefile index c2fe7748..9d144d0e 100644 --- a/clients/java/Makefile +++ b/clients/java/Makefile @@ -2,12 +2,12 @@ all: generate-proto build -# generates the raccoon protos from the https://github.com/odpf/proton using the buf +# generates the raccoon protos from the https://github.com/raystack/proton using the buf generate-proto: rm -rf .temp mkdir -p .temp - curl -o .temp/proton.tar.gz -L http://api.github.com/repos/odpf/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 - buf generate --path=.temp/odpf/raccoon + curl -o .temp/proton.tar.gz -L http://api.github.com/repos/raystack/proton/tarball/main; tar xvf .temp/proton.tar.gz -C .temp/ --strip-components 1 + buf generate --path=.temp/raystack/raccoon clean: rm -rf .temp diff --git a/clients/java/README.md b/clients/java/README.md index 3eab690f..eef61212 100644 --- a/clients/java/README.md +++ b/clients/java/README.md @@ -1,25 +1,23 @@ # Java Client for Raccoon - ## Requirements - [Gradle v7+](https://gradle.org/) - [JDK 8+](https://openjdk.java.net/projects/jdk8/) - ### Add raccoon as dependency #### Gradle ```groovy - implementation group: 'io.odpf', name: 'raccoon', version: '0.1.5' + implementation group: 'io.raystack', name: 'raccoon', version: '0.1.5' ``` #### Maven ```xml - io.odpf + io.raystack racoon 0.1.5 @@ -28,6 +26,7 @@ ## Usage #### Construct a new REST client, then use the various options to build the client. + For example: ```java @@ -61,4 +60,4 @@ For example: if (response.isSuccess() && response.getStatus() == ResponseStatus.STATUS_SUCCESS) { System.out.println("The event sent successfully"); } -``` \ No newline at end of file +``` diff --git a/clients/java/build.gradle b/clients/java/build.gradle index 15db80b4..ac76787a 100644 --- a/clients/java/build.gradle +++ b/clients/java/build.gradle @@ -9,7 +9,7 @@ plugins { id "io.github.gradle-nexus.publish-plugin" version "1.1.0" } -group 'io.odpf' +group 'io.raystack' version '0.1.5-rc' repositories { @@ -50,11 +50,11 @@ checkstyle { } checkstyleMain { - source = 'src/main/java/io/odpf/raccoon' + source = 'src/main/java/io/raystack/raccoon' } checkstyleTest { - source = 'src/test/java/io/odpf/raccoon' + source = 'src/test/java/io/raystack/raccoon' } java { @@ -82,10 +82,10 @@ publishing { version = project.version name = 'Raccoon' description = 'Java client library for sending events to the Raccoon' - url = 'https://github.com/odpf/raccoon' + url = 'https://github.com/raystack/raccoon' scm { - url = 'https://github.com/odpf/raccoon.git' + url = 'https://github.com/raystack/raccoon.git' } licenses { diff --git a/collection/collector.go b/collection/collector.go index 1b821a82..716637b2 100644 --- a/collection/collector.go +++ b/collection/collector.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/odpf/raccoon/identification" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/identification" + pb "github.com/raystack/raccoon/proto" ) type AckFunc func(err error) diff --git a/config/acks.go b/config/acks.go index df113498..400e9061 100644 --- a/config/acks.go +++ b/config/acks.go @@ -1,7 +1,7 @@ package config import ( - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) diff --git a/config/distribution.go b/config/distribution.go index 5334ce04..22257974 100644 --- a/config/distribution.go +++ b/config/distribution.go @@ -1,7 +1,7 @@ package config import ( - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) diff --git a/config/log.go b/config/log.go index 6424a8a7..240fd348 100644 --- a/config/log.go +++ b/config/log.go @@ -1,7 +1,7 @@ package config import ( - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) diff --git a/config/metric.go b/config/metric.go index 8ed877a6..81c071be 100644 --- a/config/metric.go +++ b/config/metric.go @@ -3,7 +3,7 @@ package config import ( "time" - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) diff --git a/config/publisher.go b/config/publisher.go index 569227fe..4290ad7b 100644 --- a/config/publisher.go +++ b/config/publisher.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" confluent "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) diff --git a/config/server.go b/config/server.go index 9c7d1c74..237914f4 100644 --- a/config/server.go +++ b/config/server.go @@ -3,7 +3,7 @@ package config import ( "time" - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) diff --git a/config/worker.go b/config/worker.go index dcfc472c..a7f4a6e6 100644 --- a/config/worker.go +++ b/config/worker.go @@ -3,7 +3,7 @@ package config import ( "time" - "github.com/odpf/raccoon/config/util" + "github.com/raystack/raccoon/config/util" "github.com/spf13/viper" ) @@ -22,7 +22,7 @@ type worker struct { WorkerFlushTimeout time.Duration } -//workerConfigLoader constructs a singleton instance of the worker pool config +// workerConfigLoader constructs a singleton instance of the worker pool config func workerConfigLoader() { viper.SetDefault("WORKER_POOL_SIZE", 5) viper.SetDefault("WORKER_BUFFER_CHANNEL_SIZE", 100) diff --git a/deserialization/proto_test.go b/deserialization/proto_test.go index 2fd79a85..25463ff9 100644 --- a/deserialization/proto_test.go +++ b/deserialization/proto_test.go @@ -3,7 +3,7 @@ package deserialization import ( "testing" - pb "github.com/odpf/raccoon/proto" + pb "github.com/raystack/raccoon/proto" ) func TestProtoDeserilizer_Deserialize(t *testing.T) { diff --git a/docs/docs/concepts/architecture.md b/docs/docs/concepts/architecture.md index 77a19e41..58237b6f 100644 --- a/docs/docs/concepts/architecture.md +++ b/docs/docs/concepts/architecture.md @@ -1,6 +1,6 @@ # Architecture -Raccoon written in [GO](https://github.com/golang) is a high throughput, low-latency service that provides an API to ingest streaming data from mobile apps, sites and publish it to Kafka. Raccoon supports websockets, REST and gRPC protocols for clients to send events. With wesockets it provides long persistent connections, with no overhead of additional headers sizes as in http protocol. Racoon supports protocol buffers and JSON as serialization formats. Websockets and REST API support both whereas with gRPC only protocol buffers are supported. It provides an event type agnostic API that accepts a batch \(array\) of events in protobuf format. Refer [here](https://odpf.gitbook.io/raccoon/guides/publishing#data-formatters) for data definitions format that Raccoon accepts. +Raccoon written in [GO](https://github.com/golang) is a high throughput, low-latency service that provides an API to ingest streaming data from mobile apps, sites and publish it to Kafka. Raccoon supports websockets, REST and gRPC protocols for clients to send events. With wesockets it provides long persistent connections, with no overhead of additional headers sizes as in http protocol. Racoon supports protocol buffers and JSON as serialization formats. Websockets and REST API support both whereas with gRPC only protocol buffers are supported. It provides an event type agnostic API that accepts a batch \(array\) of events in protobuf format. Refer [here](https://raystack.gitbook.io/raccoon/guides/publishing#data-formatters) for data definitions format that Raccoon accepts. Raccoon was built with a primary purpose to source or collect user behaviour data in near-real time. User behaviour data is a stream of events that occur when users traverse through a mobile app or website. Raccoon powers analytics systems, big data pipelines and other disparate consumers by providing high volume, high throughput ingestion APIs consuming real time data. Raccoon’s key architecture principle is a realization of an event agnostic backend \(accepts events of any type without the type awareness\). It is this capability that enables Raccoon to evolve into a strong player in the ingestion/collector ecosystem that has real time streaming/analytical needs. @@ -44,7 +44,7 @@ Client connects to the server with the same endpoint but with POST HTTP method. ### gRPC -It is recommended to generate the gRPC client for Raccoon's [EventService](https://github.com/odpf/proton/blob/main/odpf/raccoon/EventService.proto) and use that client to do gRPC request. Currently only unary requests are supported. +It is recommended to generate the gRPC client for Raccoon's [EventService](https://github.com/raystack/proton/blob/main/raystack/raccoon/EventService.proto) and use that client to do gRPC request. Currently only unary requests are supported. - Client's `SendEvent` method is called to send the event. - Connection identifier is constructed from the values of `SERVER_WEBSOCKET_CONN_ID_HEADER` and `SERVER_WEBSOCKET_CONN_GROUP_HEADER` in gRPC metadata. @@ -64,7 +64,7 @@ Event data loss happens in the following scenarios: ## Acknowledging events -Event acknowledgements was designed to signify if the events batch is received and sent to Kafka successfully. This will enable the clients to retry on failed event delivery. Raccoon chooses when to send event acknowledgement based on the configuration parameter `EVENT_ACK`. +Event acknowledgements was designed to signify if the events batch is received and sent to Kafka successfully. This will enable the clients to retry on failed event delivery. Raccoon chooses when to send event acknowledgement based on the configuration parameter `EVENT_ACK`. ### EVENT_ACK = 0 @@ -86,7 +86,7 @@ Raccoon sends the acknowledgments after the events are acknowledged successfully ![](/assets/raccoon_async.png) -Pros: +Pros: - Minimal data loss, clients can retry/resend events in case of downtime/broker failures. @@ -112,7 +112,7 @@ When an SendEventRequest is sent to Raccoon over any connection be it Websocket/ ### Protobufs -When an [SendEventRequest](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) proto below containing events are sent over the wire +When an [SendEventRequest](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) proto below containing events are sent over the wire ```text message SendEventRequest { @@ -125,7 +125,7 @@ message SendEventRequest { } ``` -a corresponding [SendEventResponse](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) is sent by the server. +a corresponding [SendEventResponse](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) is sent by the server. ```text message SendEventResponse { @@ -191,7 +191,7 @@ For eg. setting the EVENT_DISTRIBUTION_PUBLISHER_PATTERN=topic-%s-log ``` -and a type such as `type=viewed` in the [event](https://github.com/odpf/proton/blob/main/odpf/raccoon/Event.proto) format +and a type such as `type=viewed` in the [event](https://github.com/raystack/proton/blob/main/raystack/raccoon/Event.proto) format and a type such as `type=viewed` in the event format diff --git a/docs/docs/concepts/structure.md b/docs/docs/concepts/structure.md index 2c5560ce..63cbd9e0 100644 --- a/docs/docs/concepts/structure.md +++ b/docs/docs/concepts/structure.md @@ -58,4 +58,4 @@ Contains the code for connection identification. ### Request, Response, and Events Proto -Raccoon depends on [Proton](https://github.com/odpf/proton/tree/main/odpf/raccoon) repository. Proton is a repository to store all ODPF Protobuf files. Code to serde the request and response are generated using Protobuf. You can check how the code is generated on `Makefile`. +Raccoon depends on [Proton](https://github.com/raystack/proton/tree/main/raystack/raccoon) repository. Proton is a repository to store all Raystack Protobuf files. Code to serde the request and response are generated using Protobuf. You can check how the code is generated on `Makefile`. diff --git a/docs/docs/contribute/contribution.md b/docs/docs/contribute/contribution.md index 0cf7f057..164904d1 100644 --- a/docs/docs/contribute/contribution.md +++ b/docs/docs/contribute/contribution.md @@ -2,18 +2,18 @@ The following is a set of guidelines for contributing to Raccoon. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. Here are some important resources: -* [Concepts](contribution.md) section will explain you about Raccoon architecture, -* Our [roadmap](https://github.com/odpf/raccoon/docs/roadmap.md) is the 10k foot view of how we envision Raccoon to evolve -* Github [issues](https://github.com/odpf/raccoon/issues) track the ongoing and reported issues. +- [Concepts](contribution.md) section will explain you about Raccoon architecture, +- Our [roadmap](https://github.com/raystack/raccoon/docs/roadmap.md) is the 10k foot view of how we envision Raccoon to evolve +- Github [issues](https://github.com/raystack/raccoon/issues) track the ongoing and reported issues. ## How can I contribute? We use RFCS and GitHub issues to communicate ideas. -* You can report a bug or suggest a feature enhancement or can just ask questions. Reach out on Github discussions for this purpose. -* You are also welcome to add new features, improve monitoring,logging and code quality. -* You can help with documenting new features or improve existing documentation. -* You can also review and accept other contributions if you are a maintainer. +- You can report a bug or suggest a feature enhancement or can just ask questions. Reach out on Github discussions for this purpose. +- You are also welcome to add new features, improve monitoring,logging and code quality. +- You can help with documenting new features or improve existing documentation. +- You can also review and accept other contributions if you are a maintainer. Please submit a PR to the master branch of the Raccoon repository once you are ready to submit your contribution. Code submission to Raccoon, including submission from project maintainers, require review and approval from maintainers or code owners. PRs that are submitted by the general public need to pass the build. Once build is passed community members will help to review the pull request. @@ -21,17 +21,16 @@ Please submit a PR to the master branch of the Raccoon repository once you are r We are always interested in adding new maintainers. What we look for is a series of contributions, good taste, and an ongoing interest in the project. -* maintainers will have write access to the Raccoon repositories. -* There is no strict protocol for becoming a maintainer or PMC member. Candidates for new maintainers are typically people that are active contributors and community members. -* Candidates for new maintainers can also be suggested by current maintainers or PMC members. -* If you would like to become a maintainer, you should start contributing to Raccoon in any of the ways mentioned. You might also want to talk to other maintainers and ask for their advice and guidance. +- maintainers will have write access to the Raccoon repositories. +- There is no strict protocol for becoming a maintainer or PMC member. Candidates for new maintainers are typically people that are active contributors and community members. +- Candidates for new maintainers can also be suggested by current maintainers or PMC members. +- If you would like to become a maintainer, you should start contributing to Raccoon in any of the ways mentioned. You might also want to talk to other maintainers and ask for their advice and guidance. ## Guidelines Please follow these practices for you change to get merged fast and smoothly: -* Contributions can only be accepted if they contain appropriate testing \(Unit and Integration Tests.\) -* If you are introducing a completely new feature or making any major changes in an existing one, we recommend to start with an RFC and get consensus on the basic design first. -* Make sure your local build is running with all the tests and [golint](https://github.com/golang/lint) checks passing. -* Docs live in the code repo under [`docs`](https://github.com/odpf/raccoon/docs/README.md) so that changes to that can be done in the same PR as changes to the code. - +- Contributions can only be accepted if they contain appropriate testing \(Unit and Integration Tests.\) +- If you are introducing a completely new feature or making any major changes in an existing one, we recommend to start with an RFC and get consensus on the basic design first. +- Make sure your local build is running with all the tests and [golint](https://github.com/golang/lint) checks passing. +- Docs live in the code repo under [`docs`](https://github.com/raystack/raccoon/docs/README.md) so that changes to that can be done in the same PR as changes to the code. diff --git a/docs/docs/guides/deployment.md b/docs/docs/guides/deployment.md index 46fa0dea..f4d6802a 100644 --- a/docs/docs/guides/deployment.md +++ b/docs/docs/guides/deployment.md @@ -4,14 +4,14 @@ This section contains guides and suggestions related to Raccoon deployment. ## Kubernetes -Using [Raccoon docker image](https://hub.docker.com/r/odpf/raccoon), you can deploy Raccoon on [Kubernetes](https://kubernetes.io/) by specifying the image on the [manifest](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#creating-a-deployment). We also provide [Helm chart](https://github.com/odpf/charts/tree/main/stable/raccoon) to ease Kubernetes deployment. In this section we will cover simple deployment on Kubernetes using manifest and Helm. +Using [Raccoon docker image](https://hub.docker.com/r/raystack/raccoon), you can deploy Raccoon on [Kubernetes](https://kubernetes.io/) by specifying the image on the [manifest](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#creating-a-deployment). We also provide [Helm chart](https://github.com/raystack/charts/tree/main/stable/raccoon) to ease Kubernetes deployment. In this section we will cover simple deployment on Kubernetes using manifest and Helm. ### Manifest **Prerequisite** -* Kubernetes cluster setup -* [Kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) installed +- Kubernetes cluster setup +- [Kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) installed **Creating Kubernetes Resources** You need at least 2 manifests for Raccoon. For [deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment) and for [configmap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/). Prepare both manifest as YAML file. You can fill in the configuration as needed. @@ -52,28 +52,28 @@ spec: application: raccoon spec: containers: - - name: raccoon - image: "odpf/raccoon:latest" - imagePullPolicy: IfNotPresent - resources: + - name: raccoon + image: "raystack/raccoon:latest" + imagePullPolicy: IfNotPresent + resources: limits: cpu: 200m memory: 512Mi requests: cpu: 200m memory: 512Mi - envFrom: - - configMapRef: - name: raccoon-config - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName + envFrom: + - configMapRef: + name: raccoon-config + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName volumes: ``` @@ -101,27 +101,27 @@ $ kubectl get configmap raccoon-config `deployment.yaml` ```yaml -... - containers: - - image: telegraf:1.7.4-alpine - imagePullPolicy: IfNotPresent - name: telegrafd - resources: - limits: - cpu: 50m - memory: 64Mi - requests: - cpu: 50m - memory: 64Mi - volumeMounts: - - mountPath: /etc/telegraf - name: telegraf-conf - volumes: - - configMap: - defaultMode: 420 - name: test-raccoon-telegraf-config - name: telegraf-conf -... + +--- +containers: + - image: telegraf:1.7.4-alpine + imagePullPolicy: IfNotPresent + name: telegrafd + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 50m + memory: 64Mi + volumeMounts: + - mountPath: /etc/telegraf + name: telegraf-conf +volumes: + - configMap: + defaultMode: 420 + name: test-raccoon-telegraf-config + name: telegraf-conf ``` `telegraf-conf.yaml` @@ -176,10 +176,10 @@ data: **Prerequisite** -* Kubernetes cluster setup -* Helm installed +- Kubernetes cluster setup +- Helm installed -Raccoon has a Helm chart maintained on [different repository](https://github.com/odpf/charts/tree/main/stable/raccoon). Refer to the repository for a complete deployment guide. +Raccoon has a Helm chart maintained on [different repository](https://github.com/raystack/charts/tree/main/stable/raccoon). Refer to the repository for a complete deployment guide. ## Production Checklist @@ -189,11 +189,11 @@ Before going to production, we recommend the following setup tips. Followings are main configurations closely related to deployment that you need to pay attention: -* [`SERVER_WEBSOCKET_PORT`](https://odpf.gitbook.io/raccoon/reference/configurations#server_websocket_port) -* [`EVENT_DISTRIBUTION_PUBLISHER_PATTERN`](https://odpf.gitbook.io/raccoon/reference/configurations#event_distribution_publisher_pattern) -* [`PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS`](https://odpf.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_bootstrap_servers) -* [`METRIC_STATSD_ADDRESS`](https://odpf.gitbook.io/raccoon/reference/configurations#metric_statsd_address) -* [`SERVER_WEBSOCKET_CONN_ID_HEADER`](https://odpf.gitbook.io/raccoon/reference/configurations#server_websocket_conn_id_header) +- [`SERVER_WEBSOCKET_PORT`](https://raystack.gitbook.io/raccoon/reference/configurations#server_websocket_port) +- [`EVENT_DISTRIBUTION_PUBLISHER_PATTERN`](https://raystack.gitbook.io/raccoon/reference/configurations#event_distribution_publisher_pattern) +- [`PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS`](https://raystack.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_bootstrap_servers) +- [`METRIC_STATSD_ADDRESS`](https://raystack.gitbook.io/raccoon/reference/configurations#metric_statsd_address) +- [`SERVER_WEBSOCKET_CONN_ID_HEADER`](https://raystack.gitbook.io/raccoon/reference/configurations#server_websocket_conn_id_header) **TLS/HTTPS** @@ -205,5 +205,4 @@ Followings are main configurations closely related to deployment that you need t **Test The Setup** - To make sure the deployment can handle the load, you need to test it with the same number of connections and request you are expecting. You can find a guide on how to publish events [here](https://odpf.gitbook.io/raccoon/guides/publishing). You can also check example client [here](https://github.com/odpf/raccoon/tree/main/docs/example). If there is something wrong with Raccon, you can check the [troubleshooting](https://odpf.gitbook.io/raccoon/guides/troubleshooting) section. - + To make sure the deployment can handle the load, you need to test it with the same number of connections and request you are expecting. You can find a guide on how to publish events [here](https://raystack.gitbook.io/raccoon/guides/publishing). You can also check example client [here](https://github.com/raystack/raccoon/tree/main/docs/example). If there is something wrong with Raccon, you can check the [troubleshooting](https://raystack.gitbook.io/raccoon/guides/troubleshooting) section. diff --git a/docs/docs/guides/monitoring.md b/docs/docs/guides/monitoring.md index eca89d5e..44c6d6d1 100644 --- a/docs/docs/guides/monitoring.md +++ b/docs/docs/guides/monitoring.md @@ -48,7 +48,7 @@ This section will cover a setup example using [Telegraf](https://www.influxdata. timeout = "5s" ``` -**Configure Raccoon To Send To The Metric Collector** After you have the collector with the port configured, you need to set [METRIC\_STATSD\_ADDRESS](https://odpf.gitbook.io/raccoon/reference/configurations#metric_statsd_address) to match the metric collector address. Suppose you deploy the telegraf using the default configuration above as sidecar or in localhost, you need to set the value to `:8125`. +**Configure Raccoon To Send To The Metric Collector** After you have the collector with the port configured, you need to set [METRIC_STATSD_ADDRESS](https://raystack.gitbook.io/raccoon/reference/configurations#metric_statsd_address) to match the metric collector address. Suppose you deploy the telegraf using the default configuration above as sidecar or in localhost, you need to set the value to `:8125`. **Visualize And Create Alerting From The Collected Metrics** Now that you have Raccoon and Telegraf as metric collector set, next is to use the metrics reported. You may notice that the Telegraf config above contains `outputs.influxdb`. That config will send the metric received to Influxdb. Make sure you have influx service accessible from the configured URL. You can visualize the metrics using Grafana. To do that, you need to [add influx datasource](https://www.influxdata.com/blog/how-grafana-dashboard-influxdb-flux-influxql/) to make the data available on Grafana. After that, you can use the data to You can visualize the metrics using Grafana. To do that, you need to [add influx datasource](https://www.influxdata.com/blog/how-grafana-dashboard-influxdb-flux-influxql/) to make the data available on Grafana. After that, you can use the data to [make dashboard](https://grafana.com/docs/grafana/latest/datasources/influxdb/#influxql-query-editor). @@ -58,7 +58,7 @@ Following are key monitoring statistics that you can infer from Raccoon metrics. ### Data Loss -To infer data loss, you can count [`kafka_messages_delivered_total`](https://odpf.gitbook.io/raccoon/reference/metrics#kafka_messages_delivered_total) with tag `success=false`. You can also infer the loss rate by calculating the following. +To infer data loss, you can count [`kafka_messages_delivered_total`](https://raystack.gitbook.io/raccoon/reference/metrics#kafka_messages_delivered_total) with tag `success=false`. You can also infer the loss rate by calculating the following. `count(kafka_messages_delivered_total success=false)/count(kafka_messages_delivered_total)` @@ -66,7 +66,6 @@ To infer data loss, you can count [`kafka_messages_delivered_total`](https://odp Raccoon provides fine-grained metrics that denote latency. That gives clues as to where to look in case something goes wrong. In summary, these are key metrics for latency: -* [`event_processing_duration_milliseconds`](https://odpf.gitbook.io/raccoon/reference/metrics#event_processing_duration_milliseconds) This metrics denotes overall latency. You need to look at other latency metrics to find the root cause when this metric is high. -* [`server_processing_latency_milliseconds`](https://odpf.gitbook.io/raccoon/reference/metrics#server_processing_latency_milliseconds) Correlate this metric with `event_processing_duration_milliseconds` to infer whether the issue is with Raccoon itself, or something wrong with the network, or the way [sent\_time](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto#L47) is generated.- -* [`worker_processing_duration_milliseconds`](https://odpf.gitbook.io/raccoon/reference/metrics#worker_processing_duration_milliseconds) High value of this metric indicates that the publisher is slow or can't keep up. - +- [`event_processing_duration_milliseconds`](https://raystack.gitbook.io/raccoon/reference/metrics#event_processing_duration_milliseconds) This metrics denotes overall latency. You need to look at other latency metrics to find the root cause when this metric is high. +- [`server_processing_latency_milliseconds`](https://raystack.gitbook.io/raccoon/reference/metrics#server_processing_latency_milliseconds) Correlate this metric with `event_processing_duration_milliseconds` to infer whether the issue is with Raccoon itself, or something wrong with the network, or the way [sent_time](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto#L47) is generated.- +- [`worker_processing_duration_milliseconds`](https://raystack.gitbook.io/raccoon/reference/metrics#worker_processing_duration_milliseconds) High value of this metric indicates that the publisher is slow or can't keep up. diff --git a/docs/docs/guides/overview.md b/docs/docs/guides/overview.md index 8d4bc2f8..5deb4f16 100644 --- a/docs/docs/guides/overview.md +++ b/docs/docs/guides/overview.md @@ -14,9 +14,9 @@ Raccoon can run inside VMs or containers in a fully managed runtime environment {% page-ref page="deployment.md" %} -### Monitoring Raccoon with exposed metrics +### Monitoring Raccoon with exposed metrics -Always know what’s going on with your deployment with built-in [monitoring](https://github.com/odpf/firehose/blob/main/docs/assets/firehose-grafana-dashboard.json) of throughput, response times, errors and more. This section contains guides, best practices, and advice related to managing Raccoon in production. +Always know what’s going on with your deployment with built-in [monitoring](https://github.com/raystack/firehose/blob/main/docs/assets/firehose-grafana-dashboard.json) of throughput, response times, errors and more. This section contains guides, best practices, and advice related to managing Raccoon in production. {% page-ref page="monitoring.md" %} @@ -25,6 +25,3 @@ Always know what’s going on with your deployment with built-in [monitoring](ht Raccoon scales in an instant, both vertically and horizontally for high-performance event ingestion. This section contains guides, best practices, and advice related to troubleshooting issues with Raccoon in production. {% page-ref page="troubleshooting.md" %} - - - diff --git a/docs/docs/guides/publishing.md b/docs/docs/guides/publishing.md index 05fa0ef4..b5c805e0 100644 --- a/docs/docs/guides/publishing.md +++ b/docs/docs/guides/publishing.md @@ -39,7 +39,7 @@ With a websocket connection the content type is identified based on the mesage t Raccoon accepts an SendEventRequest proto that wraps multiple Event proto. This enables clients to send an event in real-time or multiple events in batches. -Refer to [raccoon.proto](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) for how you can build the request. +Refer to [raccoon.proto](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) for how you can build the request. ```text message SendEventRequest { @@ -51,7 +51,7 @@ message SendEventRequest { Where **req_guid** - A globally unique ID generated by the client denoting this request as unique **sent_time** - This is when the event is sent over the WebSocket in protobuf timestamp format. This time is used to calculate latencies in Raccoon. **events** - are events of type Event proto which is of the format -Refer to the [Event.proto](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) for how the event could be composed of. +Refer to the [Event.proto](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) for how the event could be composed of. ```text message Event { @@ -69,7 +69,7 @@ Clients build the event array and compose the SendEventRequest proto, send them Raccoon also wires response every time a message is read and processed. -Refer to the [raccoon.proto](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) that Raccoon sends for every event. +Refer to the [raccoon.proto](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) that Raccoon sends for every event. ```text message SendEventResponse { @@ -161,7 +161,7 @@ Values of status and codes is same as defined in Protos. ## Headers -Raccoon service accepts headers to identify a user connection uniquely. The header name is made configurable as it enables clients to specify a header name that works for them. For, e.g. for a mobile app having a request header as `X-User-ID` which identifies the user \(client\) connecting to Raccoon, can configure Raccoon service with the config set as below `SERVER_WEBSOCKET_CONN_ID_HEADER=X-User-ID`. Optionally, `SERVER_WEBSOCKET_CONN_GROUP_HEADER` can also be configured to [support multitenancy](https://odpf.gitbook.io/raccoon/concepts/architecture#connections) such as multiple apps connecting to a single Raccoon instance. +Raccoon service accepts headers to identify a user connection uniquely. The header name is made configurable as it enables clients to specify a header name that works for them. For, e.g. for a mobile app having a request header as `X-User-ID` which identifies the user \(client\) connecting to Raccoon, can configure Raccoon service with the config set as below `SERVER_WEBSOCKET_CONN_ID_HEADER=X-User-ID`. Optionally, `SERVER_WEBSOCKET_CONN_GROUP_HEADER` can also be configured to [support multitenancy](https://raystack.gitbook.io/raccoon/concepts/architecture#connections) such as multiple apps connecting to a single Raccoon instance. Raccoon uses the config to fetch the header name and uses the value passed in the request header with this name, as the connection id. This header name uniquely identifies a client. A client, in this case, can be the user in the app. @@ -184,9 +184,9 @@ Following are the supported `Content-Type` headers for various data formats: Events can be sent to Raccoon using gRPC too. -Refer to [EventService.proto](https://github.com/odpf/proton/blob/main/odpf/raccoon/EventService.proto) for the definition of `EventService` which exposes one RPC call `SendEvent`. It is recommended to generate the language specific gRPC client using the proto definition. +Refer to [EventService.proto](https://github.com/raystack/proton/blob/main/raystack/raccoon/EventService.proto) for the definition of `EventService` which exposes one RPC call `SendEvent`. It is recommended to generate the language specific gRPC client using the proto definition. -Input to the RPC call is [SendEventRequest](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto) and the output is [raccoon.proto](https://github.com/odpf/proton/blob/main/odpf/raccoon/v1beta1/raccoon.proto). +Input to the RPC call is [SendEventRequest](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto) and the output is [raccoon.proto](https://github.com/raystack/proton/blob/main/raystack/raccoon/v1beta1/raccoon.proto). To support multi-tenacy while using gRPC, `SERVER_WEBSOCKET_CONN_ID_HEADER` and `SERVER_WEBSOCKET_CONN_GROUP_HEADER` values can be used. The key along with their values if set in grpc metadata while sending the request. Golang client example - diff --git a/docs/docs/guides/troubleshooting.md b/docs/docs/guides/troubleshooting.md index 468de428..43087034 100644 --- a/docs/docs/guides/troubleshooting.md +++ b/docs/docs/guides/troubleshooting.md @@ -14,25 +14,24 @@ Raccoon is using WebSocket as a communication protocol from client to server. We Apart from OS configuration, there are configurations you can tune on Raccoon: -* [SERVER\_WEBSOCKET\_MAX\_CONN](https://odpf.gitbook.io/raccoon/reference/configurations#server_websocket_max_conn) To limit Raccoon resource utilization, we enforce a limit on WebSocket connection. The default value is 30000; adjust it if necessary. +- [SERVER_WEBSOCKET_MAX_CONN](https://raystack.gitbook.io/raccoon/reference/configurations#server_websocket_max_conn) To limit Raccoon resource utilization, we enforce a limit on WebSocket connection. The default value is 30000; adjust it if necessary. ### Worker After the request is deserialized, the server puts the events on the buffer channel. The worker process events from the channel and publish them to Kafka. You can think of the worker and the channel as a buffer in case the publisher slows down temporarily. -* [WORKER\_BUFFER\_CHANNEL\_SIZE](https://odpf.gitbook.io/raccoon/reference/configurations#worker_buffer_channel_size) Buffer before the events get processed. The more the size, the longer it can tolerate a temporary spike or slow down. -* [WORKER\_POOL\_SIZE](https://odpf.gitbook.io/raccoon/reference/configurations#worker_pool_size) The worker will call the publisher client and wait synchronously. Increase this according to the throughput. +- [WORKER_BUFFER_CHANNEL_SIZE](https://raystack.gitbook.io/raccoon/reference/configurations#worker_buffer_channel_size) Buffer before the events get processed. The more the size, the longer it can tolerate a temporary spike or slow down. +- [WORKER_POOL_SIZE](https://raystack.gitbook.io/raccoon/reference/configurations#worker_pool_size) The worker will call the publisher client and wait synchronously. Increase this according to the throughput. ### Publisher Currently, Raccoon is using [Librd Kafka client Go wrapper](https://github.com/confluentinc/confluent-kafka-go) as publisher client. There is plenty of guides out there to tune Kafka producer. Here are some configurations you can tune. -* [PUBLISHER\_KAFKA\_CLIENT\_BATCH\_NUM\_MESSAGES](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) -* [KAFKA\_CLIENT\_ACKS](https://odpf.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_acks) -* KAFKA\_CLIENT\_LINGER\_MS -* [PUBLISHER_KAFKA\_CLIENT_\*](https://odpf.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_) You can put any [librd kafka configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) by replacing \* and change the delimiter to underscore. +- [PUBLISHER_KAFKA_CLIENT_BATCH_NUM_MESSAGES](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) +- [KAFKA_CLIENT_ACKS](https://raystack.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_acks) +- KAFKA_CLIENT_LINGER_MS +- [PUBLISHER*KAFKA_CLIENT*\*](https://raystack.gitbook.io/raccoon/reference/configurations#publisher_kafka_client_) You can put any [librd kafka configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) by replacing \* and change the delimiter to underscore. ## Backpressure You might see the `event_processing_duration_milliseconds` keeps on increasing and `batch_idle_in_channel_milliseconds` is in constant high value. In that case, Raccoon might get backpressure from the publisher. If that happens, you can check the publisher, or you need to tune the publisher configuration on Raccoon. - diff --git a/docs/docs/introduction.md b/docs/docs/introduction.md index 6b886800..2009bd00 100644 --- a/docs/docs/introduction.md +++ b/docs/docs/introduction.md @@ -5,7 +5,7 @@ slug: / # Introduction -Raccoon is high throughput, low-latency service that provides an API to ingest clickstream data from mobile apps; and sites and publish it to Kafka. Raccoon uses the WebSocket protocol for peer-to-peer communication and protobuf as the serialization format. It provides an event-type agnostic API that accepts a batch \(array\) of events in protobuf format. Refer [here](https://github.com/odpf/proton/tree/main/odpf/raccoon) for the protobuf definition format that Raccoon accepts. +Raccoon is high throughput, low-latency service that provides an API to ingest clickstream data from mobile apps; and sites and publish it to Kafka. Raccoon uses the WebSocket protocol for peer-to-peer communication and protobuf as the serialization format. It provides an event-type agnostic API that accepts a batch \(array\) of events in protobuf format. Refer [here](https://github.com/raystack/proton/tree/main/raystack/raccoon) for the protobuf definition format that Raccoon accepts. ![Overiew](/assets/overview.svg) @@ -31,7 +31,7 @@ Raccoon can be used as an event collector, event distributor, and forwarder of e Explore the following resources to get started with Raccoon: -- [Guides](https://github.com/odpf/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/guides/README.md) provide information on deployment and client samples. -- [Concepts](https://github.com/odpf/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/concepts/README.md) describe all important Raccoon concepts. -- [Reference](https://github.com/odpf/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/reference/README.md) contains details about configurations, metrics, and other aspects of Raccoon. -- [Contribute](https://github.com/odpf/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/contribute/contribution.md) contains resources for anyone who wants to contribute to Raccoon. +- [Guides](https://github.com/raystack/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/guides/README.md) provide information on deployment and client samples. +- [Concepts](https://github.com/raystack/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/concepts/README.md) describe all important Raccoon concepts. +- [Reference](https://github.com/raystack/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/reference/README.md) contains details about configurations, metrics, and other aspects of Raccoon. +- [Contribute](https://github.com/raystack/raccoon/tree/48f454ac63a94d7c462d2146f115ba9a1789e1dc/docs/docs/contribute/contribution.md) contains resources for anyone who wants to contribute to Raccoon. diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md index a1b1c902..e369af3b 100644 --- a/docs/docs/quickstart.md +++ b/docs/docs/quickstart.md @@ -11,7 +11,7 @@ $ docker run -p 8080:8080 \ -e SERVER_WEBSOCKET_CONN_ID_HEADER=X-User-ID \ -e PUBLISHER_KAFKA_CLIENT_BOOTSTRAP_SERVERS=host.docker.internal:9092 \ -e EVENT_DISTRIBUTION_PUBLISHER_PATTERN=clickstream-log \ - odpf/raccoon:latest + raystack/raccoon:latest ``` To test whether the service is running or not, you can try to ping the server. @@ -22,7 +22,7 @@ $ curl http://localhost:8080/ping ## Publishing Your First Event -Currently, Raccoon doesn't come with a library client. To start publishing events to Raccoon, we provide you an [example of a go client](https://github.com/odpf/raccoon/tree/main/docs/example) that you can refer to. You can also run the example right away if you have Go installed on your machine. +Currently, Raccoon doesn't come with a library client. To start publishing events to Raccoon, we provide you an [example of a go client](https://github.com/raystack/raccoon/tree/main/docs/example) that you can refer to. You can also run the example right away if you have Go installed on your machine. ```bash # `cd` on the client example directory and run the following @@ -37,5 +37,4 @@ $ kafka-console-consumer --bootstrap-server localhost:9092 --topic clickstream-l ## Where To Go Next -For more detail about publishing events to Raccoon, you can read the [detailed document](https://odpf.gitbook.io/raccoon/guides/publishing) under the guides section. To understand more about how Raccoon work, you can go to the [architecture document](https://odpf.gitbook.io/raccoon/concepts/architecture). - +For more detail about publishing events to Raccoon, you can read the [detailed document](https://raystack.gitbook.io/raccoon/guides/publishing) under the guides section. To understand more about how Raccoon work, you can go to the [architecture document](https://raystack.gitbook.io/raccoon/concepts/architecture). diff --git a/docs/docs/reference/configurations.md b/docs/docs/reference/configurations.md index b1549d7d..4548dc81 100644 --- a/docs/docs/reference/configurations.md +++ b/docs/docs/reference/configurations.md @@ -4,12 +4,12 @@ This page contains reference for all the application configurations for Raccoon. ## Table of Contents -* [Server](configurations.md#server) -* [Worker](configurations.md#worker) -* [Event Distribution](configurations.md#event-distribution) -* [Publisher](configurations.md#publisher) -* [Metric](configurations.md#metric) -* [Log](configurations.md#log) +- [Server](configurations.md#server) +- [Worker](configurations.md#worker) +- [Event Distribution](configurations.md#event-distribution) +- [Publisher](configurations.md#publisher) +- [Metric](configurations.md#metric) +- [Log](configurations.md#log) ## Server @@ -17,92 +17,92 @@ This page contains reference for all the application configurations for Raccoon. Port for the service to listen. -* Type: `Optional` -* Default value: `8080` +- Type: `Optional` +- Default value: `8080` ### `SERVER_WEBSOCKET_MAX_CONN` Maximum connection that can be handled by the server instance. You want to set it according to your resource utilization. You also need to check the [limit of open file descriptor allowed](https://docs.oracle.com/cd/E19623-01/820-6168/file-descriptor-requirements.html#:~:text=Linux%20systems%20limit%20the%20number,worker%20threads%20will%20be%20blocked.) by the OS. -* Type: `Optional` -* Default value: `30000` +- Type: `Optional` +- Default value: `30000` ### `SERVER_WEBSOCKET_READ_BUFFER_SIZE` Specify I/O buffer sizes in bytes: [Refer gorilla websocket API](https://pkg.go.dev/github.com/gorilla/websocket#hdr-Buffers) -* Type: `Optional` -* Default value: `10240` +- Type: `Optional` +- Default value: `10240` ### `SERVER_WEBSOCKET_WRITE_BUFFER_SIZE` Specify I/O buffer sizes in bytes: [Refer gorilla websocket API](https://pkg.go.dev/github.com/gorilla/websocket#hdr-Buffers) -* Type: `Optional` -* Default value: `10240` +- Type: `Optional` +- Default value: `10240` ### `SERVER_WEBSOCKET_CONN_ID_HEADER` Unique identifier for the server to maintain the connection. A single uniq id can only connect once in a session. If, there is a subsequence connection with the same uniq id the connection will be rejected. -* Example value: `X-User-ID` -* Type: `Required` +- Example value: `X-User-ID` +- Type: `Required` ### `SERVER_WEBSOCKET_CONN_GROUP_HEADER` Additional identifier for the server to maintain the connection. Value of the conn group header combined with user id will act as unique identifier instead of only user id. You can use this if you want to differentiate between user groups or clients e.g(mobile, web). The group names is used as conn_group tag in some of the metrics. -* Example value: `X-User-Group` -* Type: `Optional` +- Example value: `X-User-Group` +- Type: `Optional` ### `SERVER_WEBSOCKET_CONN_GROUP_DEFAULT` Default connection group name. The default is fallback when `SERVER_WEBSOCKET_CONN_GROUP_HEADER` is not set or when the value of group header is empty. In case the connection group default is clashing with your actual group name, override this config. -* Default value: `--default--` -* Type: `Optional` +- Default value: `--default--` +- Type: `Optional` ### `SERVER_WEBSOCKET_PING_INTERVAL_MS` Interval of each ping to client. The interval is in seconds. -* Type: `Optional` -* Default value: `30` +- Type: `Optional` +- Default value: `30` ### `SERVER_WEBSOCKET_PONG_WAIT_INTERVAL_MS` Wait time for client to send Pong message back after server sends Ping. When the time exceeded the connection will be dropped. -* Type `Optional` -* Default value: `60` +- Type `Optional` +- Default value: `60` ### `SERVER_WEBSOCKET_WRITE_WAIT_INTERVAL_MS` Timeout Deadline set on the writes. On timeout the websocket state is corrupt and all future writes will return error: [Refer gorilla websocket API](https://pkg.go.dev/github.com/gorilla/websocket#Conn.SetWriteDeadline) -* Type `Optional` -* Default value: `5` +- Type `Optional` +- Default value: `5` ### `SERVER_WEBSOCKET_PINGER_SIZE` Number of goroutine spawn to Ping clients. -* Type `Optional` -* Default value: `1` +- Type `Optional` +- Default value: `1` ### `SERVER_WEBSOCKET_CHECK_ORIGIN` Toggle CORS check function. Set `true` to check each request origin. Set `false` to disable check origin and allow every request. Check origin function check against `Origin` header. -* Type: `Optional` -* Default value: `true` +- Type: `Optional` +- Default value: `true` ### `SERVER_BATCH_DEDUP_IN_CONNECTION_ENABLED` The server decides whether or not to handle duplicate batches for the active connection. If a batch is sent with a duplicate ReqGUID, the server uses best attempts to discard the duplicate batches. Set `true` to enable the setting. -* Type `Optional` -* Default value: `false` +- Type `Optional` +- Default value: `false` ## Worker @@ -110,42 +110,42 @@ The server decides whether or not to handle duplicate batches for the active con Maximum batch that service can handle when workers are busy. When the number of batch is exceeded, the worker will backpressure causing websocket to stop reading new request. -* Type `Optional` -* Default value: `100` +- Type `Optional` +- Default value: `100` ### `WORKER_BUFFER_FLUSH_TIMEOUT_MS` Upon shutdown, the worker try to finish processing events in buffer before the timeout exceeded. When the timeout exceeded, the worker is forcefully closed. -* Type `Optional` -* Default value: `5000` +- Type `Optional` +- Default value: `5000` ### `WORKER_POOL_SIZE` No of workers that processes the events concurrently. -* Type `Optional` -* Default value: `5` +- Type `Optional` +- Default value: `5` ### `WORKER_KAFKA_DELIVERY_CHANNEL_SIZE` Delivery channel is implementation detail where the kafka client asks for channel in the [produce API](https://github.com/confluentinc/confluent-kafka-go/blob/master/examples/producer_example/producer_example.go#L51). The publisher uses the channel to wait for the events to be delivered. The channel contains the status delivery of the events. Normally you won't need to touch this. -* Type `Optional` -* Default value: `10` +- Type `Optional` +- Default value: `10` ## Event Distribution ### `EVENT_DISTRIBUTION_PUBLISHER_PATTERN` -Routes events based on given pattern and [type](https://github.com/odpf/proton/blob/main/odpf/raccoon/Event.proto#L31). The pattern is following [go string format](https://golang.org/pkg/fmt/) with event `type` as second argument. The result of the string format will be the kafka topic target of the event. +Routes events based on given pattern and [type](https://github.com/raystack/proton/blob/main/raystack/raccoon/Event.proto#L31). The pattern is following [go string format](https://golang.org/pkg/fmt/) with event `type` as second argument. The result of the string format will be the kafka topic target of the event. For example, you pass `%s-event` as `EVENT_DISTRIBUTION_PUBLISHER_PATTERN`. If you send event with `click` type, your event will be forwareded to `click-event` kafka topic on the configured broker. If you send event with `buy` type, your event will be forwarded to `buy-event`. You can also route the events to single topic irrespective of the type. To do that you can drop `%s` in the value. For example, provided `mobile-events` as value. All incoming events will be routed to `mobile-events` kafka topic. -* Type `Required` -* Default value: `clickstream-%s-log` +- Type `Required` +- Default value: `clickstream-%s-log` ## Publisher @@ -153,57 +153,57 @@ You can also route the events to single topic irrespective of the type. To do th Kafka brokers IP address where the events are published. -* Example value: localhost:9092 -* Type `Required` +- Example value: localhost:9092 +- Type `Required` ### `PUBLISHER_KAFKA_CLIENT_ACKS` Number of replica acknowledgement before it send ack back to service. -* Type `Optional` -* Default value: `-1` +- Type `Optional` +- Default value: `-1` ### `PUBLISHER_KAFKA_CLIENT_RETRIES` Number of retries in case of failure. -* Type `Optional` -* Default value: `2147483647` +- Type `Optional` +- Default value: `2147483647` ### `PUBLISHER_KAFKA_CLIENT_RETRY_BACKOFF_MS` Backoff time on retry. -* Type `Optional` -* Default value: `100` +- Type `Optional` +- Default value: `100` ### `PUBLISHER_KAFKA_CLIENT_STATISTICS_INTERVAL_MS` -librdkafka statistics emit interval. The application also needs to register a stats callback using rd\_kafka\_conf\_set\_stats\_cb\(\). The granularity is 1000ms. A value of 0 disables statistics. +librdkafka statistics emit interval. The application also needs to register a stats callback using rd_kafka_conf_set_stats_cb\(\). The granularity is 1000ms. A value of 0 disables statistics. -* Type `Optional` -* Default value: `0` +- Type `Optional` +- Default value: `0` ### `PUBLISHER_KAFKA_CLIENT_QUEUE_BUFFERING_MAX_MESSAGES` Maximum number of messages allowed on the producer queue. This queue is shared by all topics and partitions. -* Type `Optional` -* Default value: `100000` +- Type `Optional` +- Default value: `100000` ### `PUBLISHER_KAFKA_CLIENT_*` Kafka client config is dynamically configured. You can see for other configuration [here](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) -* Type `Optional` -* Default value: see the [reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) +- Type `Optional` +- Default value: see the [reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) ### `PUBLISHER_KAFKA_FLUSH_INTERVAL_MS` Upon shutdown, the publisher will try to finish processing events in buffer before the timeout exceeded. When the timeout exceeded, the publisher is forcefully closed. -* Type `Optional` -* Default value: `1000` +- Type `Optional` +- Default value: `1000` ## Metric @@ -211,15 +211,15 @@ Upon shutdown, the publisher will try to finish processing events in buffer befo Address to reports the service metrics. -* Type `Optional` -* Default value: `:8125` +- Type `Optional` +- Default value: `:8125` ### `METRIC_STATSD_FLUSH_PERIOD_MS` Interval for the service to push metrics. -* Type `Optional` -* Default value: `10000` +- Type `Optional` +- Default value: `10000` ## Log @@ -227,8 +227,8 @@ Interval for the service to push metrics. Level available are `info` `panic` `fatal` `error` `warn` `info` `debug` `trace`. -* Type `Optional` -* Default value: `info` +- Type `Optional` +- Default value: `info` ## Event @@ -236,5 +236,5 @@ Level available are `info` `panic` `fatal` `error` `warn` `info` `debug` `trace` Based on this parameter the server decides when to send the acknowledgement to the client. Supported values are `0` and `1`. -* Type `Optional` -* Default value: `0` \ No newline at end of file +- Type `Optional` +- Default value: `0` diff --git a/docs/docs/roadmap.md b/docs/docs/roadmap.md index e337fd6a..bf9f1c03 100644 --- a/docs/docs/roadmap.md +++ b/docs/docs/roadmap.md @@ -1,8 +1,8 @@ # Roadmap -In the following section, you can learn what features we're working on, what stage they're in, and when we expect to bring them to you. Have any questions or comments about items on the roadmap? Join the [discussions](https://github.com/odpf/raccoon/discussions) on the Raccoon Github forum. +In the following section, you can learn what features we're working on, what stage they're in, and when we expect to bring them to you. Have any questions or comments about items on the roadmap? Join the [discussions](https://github.com/raystack/raccoon/discussions) on the Raccoon Github forum. -We’re planning to iterate on the format of the roadmap itself, and we see the potential to engage more in discussions about the future of Raccoon features. If you have feedback about this roadmap section itself, such as how the issues are presented, let us know through [discussions](https://github.com/odpf/raccoon/discussions). +We’re planning to iterate on the format of the roadmap itself, and we see the potential to engage more in discussions about the future of Raccoon features. If you have feedback about this roadmap section itself, such as how the issues are presented, let us know through [discussions](https://github.com/raystack/raccoon/discussions). ## Vision diff --git a/docs/docs/support.md b/docs/docs/support.md index 40a71a37..610bd5b4 100644 --- a/docs/docs/support.md +++ b/docs/docs/support.md @@ -5,14 +5,14 @@ Need a bit of help? We're here for you. Check out our current issues, GitHub dis ### Slack The Raccoon team has an open source slack workspace to discuss development and support. Most of the Raccoon discussions happen in #contribution channel. -[Join us on Slack](https://odpf-community.slack.com/) +[Join us on Slack](https://raystack-community.slack.com/) ### Issues Have a general issue or bug that you've found? We'd love to hear about it in our GitHub issues. This can be feature requests too! -[Go to issues](https://github.com/odpf/raccoon/issues) +[Go to issues](https://github.com/raystack/raccoon/issues) ### Discussions For help and questions about best practices, join our GitHub discussions. Browse and ask questions. -[Go to discussions](https://github.com/orgs/odpf/discussions) +[Go to discussions](https://github.com/orgs/raystack/discussions) diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index ae144324..d110d555 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -6,12 +6,12 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); (module.exports = { title: 'Raccoon', tagline: 'Raccoon is a high-throughput, low-latency service to collect events in real-time from your web, mobile apps, and services using multiple network protocols.', - url: 'https://odpf.github.io/', + url: 'https://raystack.github.io/', baseUrl: '/raccoon/', onBrokenLinks: 'throw', onBrokenMarkdownLinks: 'warn', favicon: 'favicon.ico', - organizationName: 'ODPF', + organizationName: 'Raystack', projectName: 'raccoon', presets: [ @@ -24,7 +24,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); docs: { routeBasePath: '/', sidebarPath: require.resolve('./sidebars.js'), - editUrl: 'https://github.com/odpf/raccoon/edit/master/docs/', + editUrl: 'https://github.com/raystack/raccoon/edit/master/docs/', sidebarCollapsed: false, breadcrumbs: false, }, @@ -63,7 +63,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); className: 'header-slack-link', }, { - href: 'https://github.com/odpf/raccoon', + href: 'https://github.com/raystack/raccoon', className: 'navbar-item-github', position: 'right', }, @@ -78,7 +78,7 @@ const darkCodeTheme = require('prism-react-renderer/themes/dracula'); }, announcementBar: { id: 'star-repo', - content: '⭐️ If you like Raccoon, give it a star on GitHub! ⭐', + content: '⭐️ If you like Raccoon, give it a star on GitHub! ⭐', backgroundColor: '#222', textColor: '#eee', isCloseable: true, diff --git a/example/example.proto b/example/example.proto index c414f318..2312eb9e 100644 --- a/example/example.proto +++ b/example/example.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package odpf.raccoon; +package raystack.raccoon; message SampleEvent { string description = 1; diff --git a/example/main.go b/example/main.go index 1b6e8a04..ace2d210 100644 --- a/example/main.go +++ b/example/main.go @@ -6,7 +6,7 @@ import ( "time" "github.com/gorilla/websocket" - pb "github.com/odpf/raccoon/proto" + pb "github.com/raystack/raccoon/proto" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/example/readme.md b/example/readme.md index 4f18bf32..b58bdc24 100644 --- a/example/readme.md +++ b/example/readme.md @@ -6,10 +6,12 @@ There are couple of things the client to handle to start send events to Raccoon. - [Send The Batch](#send-the-batch) - [Handle The Response](#handle-the-response) -Below are the explanation of sample client in [main.go](https://github.com/odpf/raccoon/tree/main/docs/example/main.go) +Below are the explanation of sample client in [main.go](https://github.com/raystack/raccoon/tree/main/docs/example/main.go) ## Establish Websocket Connection + You are free to use any websocket client as long as it supports passing header. You can connect to `/api/v1/events` endpoint with uniq id header set. You'll also need to handle retry in case Raccon reject the connection because [max connection is reached](). + ```go var ( url = "ws://localhost:8080/api/v1/events" @@ -26,12 +28,16 @@ func main() { ``` ## Set Ping/Pong Handler + Raccoon needs to maintain many alive connections. To clean up dead connection, Raccoon will close client that couldn't response to Ping on time. Because of that, the client need to handle Ping if not handled by the websocket client. + ```go // Gorilla websocket has default PingHandler which sends Pong. No need to explicitly heandle it. go pinger(ws) ``` + You can also check the liveliness of the server by having Pinger function and close the connection if necessary + ```go func pinger(ws *websocket.Conn) { ticker := time.Tick(pingInterval) @@ -44,7 +50,9 @@ func pinger(ws *websocket.Conn) { ``` ## Batch Events + When the connection is set, all you need to do is collect the event and send them in batch. + ```go event1 := generateSampleEvent() event2 := generateSampleEvent() @@ -53,7 +61,9 @@ When the connection is set, all you need to do is collect the event and send the event2, } ``` + Where `generateSampleEvent` is + ```go func generateSampleEvent() *pb.Event { sampleEvent := &SampleEvent{Description: "user_click"} @@ -64,7 +74,9 @@ func generateSampleEvent() *pb.Event { ``` ## Send The Batch + Now you have websocket connection and batch of event ready, all you need is send the batch. Don't forget to fill `send_time` field before sending the request. + ```go sentTime := time.Now() request := &pb.SendEventRequest{ @@ -80,11 +92,13 @@ Now you have websocket connection and batch of event ready, all you need is send ``` ## Handle The Response -Raccoon sends SendEventResponse for every batch of events sent from the client. The ReqGuid in the response identifies the batch that the client sent. The response object could be used for client's telemetry in terms of how may batches succeeded, failed etc., The clients can retry based on failures however server side kafka send failures are not sent as failures due to the [acknowledgement design as explained here](https://github.com/odpf/raccoon/blob/main/docs/concepts/architecture.md#acknowledging-events). + +Raccoon sends SendEventResponse for every batch of events sent from the client. The ReqGuid in the response identifies the batch that the client sent. The response object could be used for client's telemetry in terms of how may batches succeeded, failed etc., The clients can retry based on failures however server side kafka send failures are not sent as failures due to the [acknowledgement design as explained here](https://github.com/raystack/raccoon/blob/main/docs/concepts/architecture.md#acknowledging-events). + ```go _, response, _ := ws.ReadMessage() SendEventResponse := &pb.SendEventResponse{} proto.Unmarshal(response, SendEventResponse) // Handle the response accordingly fmt.Printf("%v", SendEventResponse) -``` \ No newline at end of file +``` diff --git a/example/sample.pb.go b/example/sample.pb.go index 467e3b48..97e79a34 100644 --- a/example/sample.pb.go +++ b/example/sample.pb.go @@ -92,7 +92,7 @@ func file_example_proto_rawDescGZIP() []byte { var file_example_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_example_proto_goTypes = []interface{}{ - (*SampleEvent)(nil), // 0: odpf.raccoon.SampleEvent + (*SampleEvent)(nil), // 0: raystack.raccoon.SampleEvent } var file_example_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type diff --git a/go.mod b/go.mod index 8e4b7a7a..a4d5ea43 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/odpf/raccoon +module github.com/raystack/raccoon go 1.14 @@ -11,7 +11,7 @@ require ( github.com/stretchr/testify v1.7.0 golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3 // indirect google.golang.org/grpc v1.41.0 - google.golang.org/protobuf v1.28.0 + google.golang.org/protobuf v1.31.0 gopkg.in/alexcesaro/statsd.v2 v2.0.0 gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.2 ) diff --git a/go.sum b/go.sum index c2bde8bd..5743f360 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= diff --git a/integration/integration_test.go b/integration/integration_test.go index aaf78774..b996f92f 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -14,7 +14,7 @@ import ( "time" "github.com/gorilla/websocket" - pb "github.com/odpf/raccoon/proto" + pb "github.com/raystack/raccoon/proto" "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/metadata" diff --git a/main.go b/main.go index a271fd12..74b7ee39 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,10 @@ package main import ( - "github.com/odpf/raccoon/app" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" + "github.com/raystack/raccoon/app" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" ) func main() { diff --git a/metrics/statsd.go b/metrics/statsd.go index 818a4e7c..fc154fef 100644 --- a/metrics/statsd.go +++ b/metrics/statsd.go @@ -3,8 +3,8 @@ package metrics import ( "fmt" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/logger" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/logger" client "gopkg.in/alexcesaro/statsd.v2" ) diff --git a/publisher/kafka.go b/publisher/kafka.go index e75155df..b8953950 100644 --- a/publisher/kafka.go +++ b/publisher/kafka.go @@ -9,10 +9,10 @@ import ( // Importing librd to make it work on vendor mode _ "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka/librdkafka" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" ) // KafkaProducer Produce data to kafka synchronously diff --git a/publisher/kafka_test.go b/publisher/kafka_test.go index bfd38522..d60cddd6 100644 --- a/publisher/kafka_test.go +++ b/publisher/kafka_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/odpf/raccoon/logger" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/logger" + pb "github.com/raystack/raccoon/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" diff --git a/serialization/json_test.go b/serialization/json_test.go index 2dcb4cee..098747d0 100644 --- a/serialization/json_test.go +++ b/serialization/json_test.go @@ -5,7 +5,7 @@ import ( "reflect" "testing" - pb "github.com/odpf/raccoon/proto" + pb "github.com/raystack/raccoon/proto" ) func TestJSONSerializer_Serialize(t *testing.T) { diff --git a/services/grpc/handler.go b/services/grpc/handler.go index e1d49bfc..528551a6 100644 --- a/services/grpc/handler.go +++ b/services/grpc/handler.go @@ -6,12 +6,12 @@ import ( "fmt" "time" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/identification" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/identification" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" "google.golang.org/grpc/metadata" ) diff --git a/services/grpc/handler_test.go b/services/grpc/handler_test.go index f9b5579e..996f1a01 100644 --- a/services/grpc/handler_test.go +++ b/services/grpc/handler_test.go @@ -5,11 +5,11 @@ import ( "reflect" "testing" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" "github.com/stretchr/testify/mock" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/services/grpc/service.go b/services/grpc/service.go index a3d1ae9b..cc12dd98 100644 --- a/services/grpc/service.go +++ b/services/grpc/service.go @@ -5,9 +5,9 @@ import ( "fmt" "net" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + pb "github.com/raystack/raccoon/proto" "google.golang.org/grpc" ) diff --git a/services/rest/handler.go b/services/rest/handler.go index ae0f2572..547b8d13 100644 --- a/services/rest/handler.go +++ b/services/rest/handler.go @@ -7,14 +7,14 @@ import ( "net/http" "time" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/deserialization" - "github.com/odpf/raccoon/identification" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" - "github.com/odpf/raccoon/serialization" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/deserialization" + "github.com/raystack/raccoon/identification" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" + "github.com/raystack/raccoon/serialization" ) const ( diff --git a/services/rest/response.go b/services/rest/response.go index ed8e0979..5ebfedac 100644 --- a/services/rest/response.go +++ b/services/rest/response.go @@ -3,8 +3,8 @@ package rest import ( "io" - pb "github.com/odpf/raccoon/proto" - "github.com/odpf/raccoon/serialization" + pb "github.com/raystack/raccoon/proto" + "github.com/raystack/raccoon/serialization" ) type Response struct { diff --git a/services/rest/response_test.go b/services/rest/response_test.go index 6fa79bbe..5c242b80 100644 --- a/services/rest/response_test.go +++ b/services/rest/response_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - pb "github.com/odpf/raccoon/proto" - "github.com/odpf/raccoon/serialization" + pb "github.com/raystack/raccoon/proto" + "github.com/raystack/raccoon/serialization" ) func TestResponse_SetCode(t *testing.T) { diff --git a/services/rest/service.go b/services/rest/service.go index 632c525f..692a7783 100644 --- a/services/rest/service.go +++ b/services/rest/service.go @@ -7,11 +7,11 @@ import ( "time" "github.com/gorilla/mux" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/metrics" - "github.com/odpf/raccoon/services/rest/websocket" - "github.com/odpf/raccoon/services/rest/websocket/connection" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/metrics" + "github.com/raystack/raccoon/services/rest/websocket" + "github.com/raystack/raccoon/services/rest/websocket/connection" ) type Service struct { diff --git a/services/rest/websocket/ack.go b/services/rest/websocket/ack.go index 824f88be..82d0969a 100644 --- a/services/rest/websocket/ack.go +++ b/services/rest/websocket/ack.go @@ -3,9 +3,9 @@ package websocket import ( "time" - "github.com/odpf/raccoon/metrics" - "github.com/odpf/raccoon/serialization" - "github.com/odpf/raccoon/services/rest/websocket/connection" + "github.com/raystack/raccoon/metrics" + "github.com/raystack/raccoon/serialization" + "github.com/raystack/raccoon/services/rest/websocket/connection" ) var AckChan = make(chan AckInfo) diff --git a/services/rest/websocket/connection/conn.go b/services/rest/websocket/connection/conn.go index 54dc63f7..591b341c 100644 --- a/services/rest/websocket/connection/conn.go +++ b/services/rest/websocket/connection/conn.go @@ -5,9 +5,9 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/identification" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" + "github.com/raystack/raccoon/identification" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" ) type Conn struct { diff --git a/services/rest/websocket/connection/table.go b/services/rest/websocket/connection/table.go index bfd9825d..3ed9c892 100644 --- a/services/rest/websocket/connection/table.go +++ b/services/rest/websocket/connection/table.go @@ -4,7 +4,7 @@ import ( "errors" "sync" - "github.com/odpf/raccoon/identification" + "github.com/raystack/raccoon/identification" ) var ( diff --git a/services/rest/websocket/connection/table_test.go b/services/rest/websocket/connection/table_test.go index 03cd49e9..9992639d 100644 --- a/services/rest/websocket/connection/table_test.go +++ b/services/rest/websocket/connection/table_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/odpf/raccoon/identification" + "github.com/raystack/raccoon/identification" "github.com/stretchr/testify/assert" ) diff --git a/services/rest/websocket/connection/upgrader.go b/services/rest/websocket/connection/upgrader.go index e921b8ee..994e95dc 100644 --- a/services/rest/websocket/connection/upgrader.go +++ b/services/rest/websocket/connection/upgrader.go @@ -7,10 +7,10 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/identification" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/identification" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" "google.golang.org/protobuf/proto" ) diff --git a/services/rest/websocket/connection/upgrader_test.go b/services/rest/websocket/connection/upgrader_test.go index b3ca197c..1d4eca0f 100644 --- a/services/rest/websocket/connection/upgrader_test.go +++ b/services/rest/websocket/connection/upgrader_test.go @@ -11,8 +11,8 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" "github.com/stretchr/testify/assert" ) diff --git a/services/rest/websocket/handler.go b/services/rest/websocket/handler.go index 27a2dc36..0b0ba156 100644 --- a/services/rest/websocket/handler.go +++ b/services/rest/websocket/handler.go @@ -6,14 +6,14 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/config" - "github.com/odpf/raccoon/deserialization" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" - "github.com/odpf/raccoon/serialization" - "github.com/odpf/raccoon/services/rest/websocket/connection" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/config" + "github.com/raystack/raccoon/deserialization" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" + "github.com/raystack/raccoon/serialization" + "github.com/raystack/raccoon/services/rest/websocket/connection" ) type serDe struct { diff --git a/services/rest/websocket/handler_test.go b/services/rest/websocket/handler_test.go index ac7cc751..4b9797b6 100644 --- a/services/rest/websocket/handler_test.go +++ b/services/rest/websocket/handler_test.go @@ -10,11 +10,11 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/websocket" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - pb "github.com/odpf/raccoon/proto" - "github.com/odpf/raccoon/services/rest/websocket/connection" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + pb "github.com/raystack/raccoon/proto" + "github.com/raystack/raccoon/services/rest/websocket/connection" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" diff --git a/services/rest/websocket/pinger.go b/services/rest/websocket/pinger.go index 46497180..667ff12f 100644 --- a/services/rest/websocket/pinger.go +++ b/services/rest/websocket/pinger.go @@ -4,13 +4,13 @@ import ( "fmt" "time" - "github.com/odpf/raccoon/identification" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - "github.com/odpf/raccoon/services/rest/websocket/connection" + "github.com/raystack/raccoon/identification" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + "github.com/raystack/raccoon/services/rest/websocket/connection" ) -//Pinger is worker that pings the connected peers based on ping interval. +// Pinger is worker that pings the connected peers based on ping interval. func Pinger(c chan connection.Conn, size int, PingInterval time.Duration, WriteWaitInterval time.Duration) { for i := 0; i < size; i++ { go func() { diff --git a/services/services.go b/services/services.go index 017da96c..94dc8bf3 100644 --- a/services/services.go +++ b/services/services.go @@ -4,11 +4,11 @@ import ( "context" "net/http" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/services/grpc" - "github.com/odpf/raccoon/services/pprof" - "github.com/odpf/raccoon/services/rest" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/services/grpc" + "github.com/raystack/raccoon/services/pprof" + "github.com/raystack/raccoon/services/rest" ) type bootstrapper interface { diff --git a/worker/init_test.go b/worker/init_test.go index 6b4735ad..336231aa 100644 --- a/worker/init_test.go +++ b/worker/init_test.go @@ -4,7 +4,7 @@ import ( "os" "testing" - "github.com/odpf/raccoon/logger" + "github.com/raystack/raccoon/logger" ) type void struct{} diff --git a/worker/mocks.go b/worker/mocks.go index 392a9ed3..05ae5e04 100644 --- a/worker/mocks.go +++ b/worker/mocks.go @@ -1,7 +1,7 @@ package worker import ( - pb "github.com/odpf/raccoon/proto" + pb "github.com/raystack/raccoon/proto" mock "github.com/stretchr/testify/mock" kafka "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) diff --git a/worker/worker.go b/worker/worker.go index 8f639469..e83aae12 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -5,10 +5,10 @@ import ( "sync" "time" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/logger" - "github.com/odpf/raccoon/metrics" - "github.com/odpf/raccoon/publisher" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/logger" + "github.com/raystack/raccoon/metrics" + "github.com/raystack/raccoon/publisher" "gopkg.in/confluentinc/confluent-kafka-go.v1/kafka" ) diff --git a/worker/worker_test.go b/worker/worker_test.go index 50af880d..3937f92a 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -5,9 +5,9 @@ import ( "testing" "time" - "github.com/odpf/raccoon/collection" - "github.com/odpf/raccoon/identification" - pb "github.com/odpf/raccoon/proto" + "github.com/raystack/raccoon/collection" + "github.com/raystack/raccoon/identification" + pb "github.com/raystack/raccoon/proto" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/protobuf/types/known/timestamppb"