From 4a7d3f04f22318c36d4991ec9727e83e09112faf Mon Sep 17 00:00:00 2001 From: Tom Haynes Date: Fri, 3 May 2024 09:44:47 +0100 Subject: [PATCH] various things --- .../lib/lambda_telementry_api-stack.ts | 18 +++++++++--------- loki_extension/README.md | 6 +++--- loki_extension/Taskfile.yml | 4 ++-- loki_extension/loki/promtail.go | 9 ++++++--- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/lambda_telementry_api/lib/lambda_telementry_api-stack.ts b/lambda_telementry_api/lib/lambda_telementry_api-stack.ts index 3021415..5c28bbb 100644 --- a/lambda_telementry_api/lib/lambda_telementry_api-stack.ts +++ b/lambda_telementry_api/lib/lambda_telementry_api-stack.ts @@ -13,18 +13,18 @@ import { join } from 'path'; export class LambdaTelementryApiStack extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { super(scope, id, props); - const account=this.account - const region=this.region - + const account = this.account + const region = this.region + const extensionName = "grafana-loki-extension" - const layerVersion="1" - const loki_ip="3.67.195.192" + const layerVersion = "1" + const LOKI_URL = "3.67.195.192" const distpatch_min_batch_size = "10" - - const lambdatelematryApiLayerArn = "arn:aws:lambda:"+region+":"+account+":layer:"+extensionName+":"+layerVersion + + const lambdatelematryApiLayerArn = "arn:aws:lambda:" + region + ":" + account + ":layer:" + extensionName + ":" + layerVersion const ltaLayer = lambda.LayerVersion.fromLayerVersionArn(this, "ltalayer", lambdatelematryApiLayerArn) - let functions= []; + let functions = []; // GO ********** const fnGO = new lambda.Function(this, 'telemetry-api-starter-go', { @@ -117,7 +117,7 @@ export class LambdaTelementryApiStack extends cdk.Stack { functions[i].addEnvironment("Bucket", bucky.bucketName) functions[i].addEnvironment("TableName", table.tableName) functions[i].addEnvironment("DISPATCH_MIN_BATCH_SIZE", distpatch_min_batch_size) - functions[i].addEnvironment("LOKI_IP", loki_ip) + functions[i].addEnvironment("LOKI_URL", LOKI_URL) bucky.grantRead(functions[i]) table.grantReadWriteData(functions[i]); topic.addSubscription(new subscriptions.LambdaSubscription(functions[i])); diff --git a/loki_extension/README.md b/loki_extension/README.md index 085dbb7..1962957 100644 --- a/loki_extension/README.md +++ b/loki_extension/README.md @@ -5,7 +5,7 @@ Based on the example in https://github.com/aws-samples/aws-lambda-extensions/tre This proof-of-concept code is not production ready. Use it with your own discretion after testing thoroughly. -This sample extension: +This sample extension: 1. Registers the extension with Lambda Extensions API (see `extensionApi/client.go`) 2. Starts a local HTTP server to receive incoming telemetry events from the Telemetry API (see `telemetryApi/listener.go`) 3. Subscribes to the Telemetry API to start receiving incoming telemetry events (see `telemetryApi/client.go`) @@ -30,6 +30,6 @@ Note the ARN and use it as the Lambda Layer. Configure the extension by setting below environment variables -* `LOKI_IP` - the IP of the loki server. This poc runs *without* authentocation. -* `DISPATCH_MIN_BATCH_SIZE` - optimize dispatching telemetry by telling the dispatcher how many log events you want it to batch. On function invoke the telemetry will be dispatched only if number of log events collected so far is greater than `DISPATCH_MIN_BATCH_SIZE`. On function shutdown the telemetry will be dispatched regardless of how many log events were collected so far. +* `LOKI_URL` - the URL of the loki server. This poc runs *without* authentocation. +* `DISPATCH_MIN_BATCH_SIZE` - optimize dispatching telemetry by telling the dispatcher how many log events you want it to batch. On function invoke the telemetry will be dispatched only if number of log events collected so far is greater than `DISPATCH_MIN_BATCH_SIZE`. On function shutdown the telemetry will be dispatched regardless of how many log events were collected so far. diff --git a/loki_extension/Taskfile.yml b/loki_extension/Taskfile.yml index 9615fcc..29f69ba 100644 --- a/loki_extension/Taskfile.yml +++ b/loki_extension/Taskfile.yml @@ -12,9 +12,9 @@ tasks: deps: [build] cmds: - chmod +x extensions/{{.EXTENSION}} - - rm -f extension.zip + - rm -f extension.zip - zip -r extension.zip extensions/ - - ARN=`aws lambda publish-layer-version --layer-name "grafana-loki-extension" --region eu-central-1 --compatible-runtimes nodejs16.x go1.x python3.9 --zip-file "fileb://extension.zip" --query "LayerVersionArn" --output text` && echo $ARN + - ARN=`aws lambda publish-layer-version --layer-name "grafana-loki-extension" --compatible-runtimes nodejs16.x go1.x python3.9 --zip-file "fileb://extension.zip" --query "LayerVersionArn" --output text` && echo $ARN silent: false build: diff --git a/loki_extension/loki/promtail.go b/loki_extension/loki/promtail.go index f5539a0..2ef7899 100644 --- a/loki_extension/loki/promtail.go +++ b/loki_extension/loki/promtail.go @@ -40,17 +40,18 @@ func init() { sendLabels = append(labels, fmt.Sprintf("%s = %s", key, val)) } - lokiIp := os.Getenv("LOKI_IP") + lokiIp := os.Getenv("LOKI_URL") if len(lokiIp) == 0 { panic("LOKI Ip undefined") } + conf = promtail.ClientConfig{ PushURL: fmt.Sprintf("%s/api/v1/push", lokiIp), Labels: fmt.Sprintf("{%s}", strings.Join(labels, ",")), BatchWait: 5 * time.Second, BatchEntriesNumber: 10000, - SendLevel: promtail.INFO, - PrintLevel: promtail.ERROR, + SendLevel: promtail.DEBUG, + PrintLevel: promtail.DEBUG, } loki, err = promtail.NewClientProto(conf) if err != nil { @@ -61,6 +62,8 @@ func init() { func LokiSend(record *string) { tstamp := time.Now().String() + loki.Infof("tomhayn hello") + loki.Infof("%s", conf.PushURL) loki.Infof("%s, time = %s, record = %v\n", strings.Join(sendLabels, ", "), tstamp, *record) }