From c011761d5ce3af4df0413dae7f3ba5dccd71f8cd Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Mon, 15 Jan 2024 13:09:28 +0100 Subject: [PATCH] feat(examples): add generate example Signed-off-by: Mark Sagi-Kazar --- examples/README.md | 1 + examples/generate/README.md | 75 +++++++++++++++++++++++++++++++++++ examples/generate/config.yaml | 57 ++++++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 examples/generate/README.md create mode 100644 examples/generate/config.yaml diff --git a/examples/README.md b/examples/README.md index 542fefe..9db18c5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,6 +5,7 @@ The following examples demonstrate how to ingest events from various sources int The examples use the custom Benthos distribution in this repository. - [Database](database/) +- [Generate](generate/) random events - [HTTP server](http-server/) (forwarding events to OpenMeter) - [Kubernetes Pod execution time](kubernetes-pod-exec-time/) diff --git a/examples/generate/README.md b/examples/generate/README.md new file mode 100644 index 0000000..5816019 --- /dev/null +++ b/examples/generate/README.md @@ -0,0 +1,75 @@ +# Generate + +This example is based on [our earlier blog post](https://openmeter.io/blog/testing-stream-processing) about testing OpenMeter with Benthos. + +The input in this case generates random events (mimicking API calls). + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Launch the example](#launch-the-example) +- [Checking events](#checking-events) +- [Advanced configuration](#advanced-configuration) +- [Production use](#production-use) + +## Prerequisites + +Go to the [GitHub Releases](https://github.com/openmeterio/benthos-openmeter/releases/latest) page and download the latest `benthos` binary for your platform. + +Check out this repository if you want to run the example locally: + +```shell +git clone https://github.com/openmeterio/benthos-openmeter.git +cd benthos-openmeter/examples/generate +``` + +Create a new `.env` file and add the details of your OpenMeter instance: + +```shell +cp .env.dist .env +# edit .env and fill in the details +``` + +> [!TIP] +> Tweak other options in the `.env` file to change the behavior of the example. + +Create a meter in OpenMeter with the following details: + +- Event type: `api-calls` +- Aggregation: `SUM` +- Value property: `$.duration_ms` +- Group by (optional): + - `method`: `$.method` + - `path`: `$.path` + - `region`: `$.region` + - `zone`: `$.zone` + +> [!TIP] +> Read more about creating a meter in the general examples [README](../README.md#Create-a-meter). + +## Launch the example + +Launch the example: + +```shell +export OPENMETER_TOKEN= +benthos -c config.yaml +``` + +> [!WARNING] +> By default the example generates 1000 events per second. + +## Checking events + +Read more in the general examples [README](../README.md#Checking-events-in-OpenMeter). + +## Advanced configuration + +Check out the configuration files and the [Benthos documentation](https://www.benthos.dev/docs/about) for more details. + +## Production use + +We are actively working on improving the documentation and the examples. +In the meantime, feel free to contact us [in email](https://us10.list-manage.com/contact-form?u=c7d6a96403a0e5e19032ee885&form_id=fe04a7fc4851f8547cfee56763850e95) or [on Discord](https://discord.gg/nYH3ZQ3Xzq). + +We are more than happy to help you set up OpenMeter in your production environment. diff --git a/examples/generate/config.yaml b/examples/generate/config.yaml new file mode 100644 index 0000000..a573724 --- /dev/null +++ b/examples/generate/config.yaml @@ -0,0 +1,57 @@ +input: + generate: + count: ${SEEDER_COUNT:0} + interval: "${SEEDER_INTERVAL:1s}" + batch_size: ${SEEDER_BATCH_SIZE:1} + mapping: | + let max_subjects = ${SEEDER_MAX_SUBJECTS:100} + + let event_type = "api-calls" + let source = "api-gateway" + let methods = ["GET", "POST"] + let paths = ["/", "/about", "/contact", "/pricing", "/docs"] + let regions = ["us-east-1", "us-west-1", "us-east-2", "us-west-2"] + let zoneSuffixes = ["a", "b", "c", "d"] + + let subject = "customer-%d".format(random_int(seed: timestamp_unix_nano()) % $max_subjects) + let time = (now().ts_sub_iso8601("P3D").ts_unix() + random_int(min: 60, max: 60 * 60 * 24 * 3)).ts_format() + + let method = $methods.index(random_int(seed: timestamp_unix_nano()) % $methods.length()) + let path = $paths.index(random_int(seed: timestamp_unix_nano()) % $paths.length()) + let region = $regions.index(random_int(seed: timestamp_unix_nano()) % $regions.length()) + let zone = "%s%s".format($region, $zoneSuffixes.index(random_int(seed: timestamp_unix_nano()) % $zoneSuffixes.length())) + let duration = random_int(seed: timestamp_unix_nano(), max: 1000) + + root = { + "id": uuid_v4(), + "specversion": "1.0", + "type": $event_type, + "source": $source, + "subject": $subject, + "time": $time, + "data": { + "method": $method, + "path": $path, + "region": $region, + "zone": $zone, + "duration_ms": $duration, + }, + } + +output: + switch: + cases: + - check: "" + continue: true + output: + openmeter: + url: "${OPENMETER_URL:https://openmeter.cloud}" + token: "${OPENMETER_TOKEN:}" + batching: + count: ${BATCH_SIZE:20} + period: ${BATCH_PERIOD:} + + - check: '"${SEEDER_LOG:false}" == "true"' + output: + stdout: + codec: lines