From 6bdd95200d21e89fc214ae293921aea3f4524053 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Thu, 30 Jan 2025 09:33:41 +0530 Subject: [PATCH 1/6] ksonnet: add support for thanos storage clients --- production/ksonnet/loki/config.libsonnet | 29 +++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index b4e7f1d2abe57..1397d5e69bf81 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -88,6 +88,9 @@ topology_spread_max_skew: 1, }, + // Use thanos object store clients + use_thanos_objstore: false, + // Bigtable variables bigtable_instance: error 'must specify bigtable instance', bigtable_project: error 'must specify bigtable project', @@ -106,6 +109,7 @@ s3_secret_access_key: '', s3_address: error 'must specify s3_address', s3_bucket_name: error 'must specify s3_bucket_name', + s3_bucket_region: '', s3_path_style: false, // Dynamodb variables @@ -153,6 +157,22 @@ }, }, + object_store: + (if std.count($._config.enabledBackends, 'gcs') > 0 then { + gcs: { + bucket_name: $._config.gcs_bucket_name, + }, + } else {}) + + (if std.count($._config.enabledBackends, 's3') > 0 then { + s3: { + bucket_name: $._config.s3_bucket_name, + endpoint: $._config.s3_address, + region: $._config.s3_bucket_region, + access_key_id: $._config.s3_access_key, + secret_access_key: $._config.s3_secret_access_key, + }, + } else {}), + // December 11 is when we first launched to the public. // Assume we can ingest logs that are 5months old. schema_start_date: '2018-07-11', @@ -323,7 +343,14 @@ } else {}) + (if std.count($._config.enabledBackends, 'dynamodb') > 0 then { aws+: $._config.client_configs.dynamo, - } else {}), + } else {}) + + ( + if $._config.use_thanos_objstore then { + use_thanos_objstore: true, + object_store: $._config.object_store, + } else {} + ), + chunk_store_config: { chunk_cache_config: { From 416161ef374cc0c4e2d55330e3e3ad636dbcc78e Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Fri, 31 Jan 2025 13:47:27 +0530 Subject: [PATCH 2/6] reuse client_configs --- production/ksonnet/loki/config.libsonnet | 44 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index ba5dd24ec46a4..84265ece4e3c5 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -129,20 +129,36 @@ }, object_store: - (if std.count($._config.enabledBackends, 'gcs') > 0 then { - gcs: { - bucket_name: $._config.gcs_bucket_name, - }, - } else {}) + - (if std.count($._config.enabledBackends, 's3') > 0 then { - s3: { - bucket_name: $._config.s3_bucket_name, - endpoint: $._config.s3_address, - region: $._config.s3_bucket_region, - access_key_id: $._config.s3_access_key, - secret_access_key: $._config.s3_secret_access_key, - }, - } else {}), + ( + if std.count($._config.enabledBackends, 'gcs') > 0 then { + gcs: $._config.client_configs.gcs, + } + else {} + ) + + ( + if std.count($._config.enabledBackends, 's3') > 0 then { + s3: { + bucket_name: $._config.s3_bucket_name, + endpoint: $._config.s3_address, + } + ( + if $._config.s3_access_key != '' && $._config.s3_secret_access_key != '' then { + access_key_id: $._config.s3_access_key, + secret_access_key: $._config.s3_secret_access_key, + } + else {} + ) + ( + if $._config.s3_bucket_region != '' then { + region: $._config.s3_bucket_region, + } + else {} + ), + } else {} + ) + + ( + if std.count($._config.enabledBackends, 'azure') > 0 then { + azure: $._config.client_configs.azure, + } else {} + ), // December 11 is when we first launched to the public. // Assume we can ingest logs that are 5months old. From a66dcb47564973322f2eac6fc5924c87e3ff0207 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Fri, 31 Jan 2025 13:59:27 +0530 Subject: [PATCH 3/6] s/enabledBackends/storage_backend --- production/ksonnet/loki/config.libsonnet | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index 84265ece4e3c5..de3038aeb5aec 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -130,13 +130,12 @@ object_store: ( - if std.count($._config.enabledBackends, 'gcs') > 0 then { + if $._config.storage_backend == 'gcs' then { gcs: $._config.client_configs.gcs, - } - else {} + } else {} ) + ( - if std.count($._config.enabledBackends, 's3') > 0 then { + if $._config.storage_backend == 's3' then { s3: { bucket_name: $._config.s3_bucket_name, endpoint: $._config.s3_address, @@ -155,7 +154,7 @@ } else {} ) + ( - if std.count($._config.enabledBackends, 'azure') > 0 then { + if $._config.storage_backend == 'azure' then { azure: $._config.client_configs.azure, } else {} ), From 0e0dc437391c28863edb20b9418e69934489f2a7 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Fri, 31 Jan 2025 16:40:48 +0530 Subject: [PATCH 4/6] also update ruler storage --- production/ksonnet/loki/config.libsonnet | 87 ++++++++++++------------ 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index de3038aeb5aec..89d06d3fbab0e 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -105,33 +105,45 @@ // DNS Resolver dns_resolver: 'kube-dns.kube-system.svc.cluster.local', - client_configs: { - s3: { - s3forcepathstyle: $._config.s3_path_style, - } + ( - if $._config.s3_access_key != '' then { - s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name, - } else { - s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name, - } - ), - gcs: { - bucket_name: $._config.gcs_bucket_name, - }, - azure: { - container_name: $._config.azure_container_name, - account_name: $._config.azure_account_name, - } + ( - if $._config.azure_account_key != '' then { - account_key: $._config.azure_account_key, + object_store_config: + ( + if $._config.storage_backend == 'gcs' then { + gcs: { + bucket_name: $._config.gcs_bucket_name, + }, + } else {} + ) + + ( + if $._config.storage_backend == 's3' then { + aws: { + s3forcepathstyle: $._config.s3_path_style, + } + ( + if $._config.s3_access_key != '' then { + s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name, + } else { + s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name, + } + ), + } else {} + ) + + ( + if $._config.storage_backend == 'azure' then { + azure: { + container_name: $._config.azure_container_name, + account_name: $._config.azure_account_name, + } + ( + if $._config.azure_account_key != '' then { + account_key: $._config.azure_account_key, + } else {} + ), } else {} ), - }, - object_store: + // thanos object store config + thanos_object_store_config: ( if $._config.storage_backend == 'gcs' then { - gcs: $._config.client_configs.gcs, + gcs: $._config.object_store_config.gcs, } else {} ) + ( @@ -155,7 +167,7 @@ ) + ( if $._config.storage_backend == 'azure' then { - azure: $._config.client_configs.azure, + azure: $._config.object_store_config.azure, } else {} ), @@ -314,25 +326,11 @@ consistent_hash: true, }, }, - } + ( - if $._config.storage_backend == 'gcs' then { - gcs: $._config.client_configs.gcs, - } else {} - ) + - ( - if $._config.storage_backend == 's3' then { - aws: $._config.client_configs.s3, - } else {} - ) + - ( - if $._config.storage_backend == 'azure' then { - azure: $._config.client_configs.azure, - } else {} - ) + + } + $._config.object_store_config + ( if $._config.use_thanos_objstore then { use_thanos_objstore: true, - object_store: $._config.object_store, + object_store: $._config.thanos_object_store_config, } else {} ), @@ -395,13 +393,14 @@ }, }, storage+: { - type: 'gcs', - gcs+: { - bucket_name: '%(cluster)s-%(namespace)s-ruler' % $._config, - }, - }, + type: $._config.storage_backend, + } + $._config.object_store_config, } else {}, + ruler_storage: if $._config.ruler_enabled then { + backend: $._config.storage_backend, + } + $._config.thanos_object_store_config else {}, + }, }, From 858e4681522fcfd00cd98e44a339aa4714448021 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Fri, 31 Jan 2025 17:22:28 +0530 Subject: [PATCH 5/6] fixup! also update ruler storage --- production/ksonnet/loki/config.libsonnet | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index 89d06d3fbab0e..2242828017b40 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -393,8 +393,11 @@ }, }, storage+: { - type: $._config.storage_backend, - } + $._config.object_store_config, + type: 'gcs', + gcs+: { + bucket_name: '%(cluster)s-%(namespace)s-ruler' % $._config, + }, + }, } else {}, ruler_storage: if $._config.ruler_enabled then { From a29f2e3c1756271d839a165cd18c967ea9f17e38 Mon Sep 17 00:00:00 2001 From: Ashwanth Goli Date: Fri, 31 Jan 2025 17:43:03 +0530 Subject: [PATCH 6/6] tidy --- production/ksonnet/loki/config.libsonnet | 106 ++++++++++------------- 1 file changed, 45 insertions(+), 61 deletions(-) diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index 2242828017b40..545f18dd9401d 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -106,70 +106,54 @@ dns_resolver: 'kube-dns.kube-system.svc.cluster.local', object_store_config: - ( - if $._config.storage_backend == 'gcs' then { - gcs: { - bucket_name: $._config.gcs_bucket_name, - }, - } else {} - ) + - ( - if $._config.storage_backend == 's3' then { - aws: { - s3forcepathstyle: $._config.s3_path_style, - } + ( - if $._config.s3_access_key != '' then { - s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name, - } else { - s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name, - } - ), - } else {} - ) + - ( - if $._config.storage_backend == 'azure' then { - azure: { - container_name: $._config.azure_container_name, - account_name: $._config.azure_account_name, - } + ( - if $._config.azure_account_key != '' then { - account_key: $._config.azure_account_key, - } else {} - ), - } else {} - ), + if $._config.storage_backend == 'gcs' then { + gcs: { + bucket_name: $._config.gcs_bucket_name, + }, + } else if $._config.storage_backend == 's3' then { + aws: { + s3forcepathstyle: $._config.s3_path_style, + } + ( + if $._config.s3_access_key != '' then { + s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name, + } else { + s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name, + } + ), + } else if $._config.storage_backend == 'azure' then { + azure: { + container_name: $._config.azure_container_name, + account_name: $._config.azure_account_name, + } + ( + if $._config.azure_account_key != '' then { + account_key: $._config.azure_account_key, + } else {} + ), + } else {}, // thanos object store config thanos_object_store_config: - ( - if $._config.storage_backend == 'gcs' then { - gcs: $._config.object_store_config.gcs, - } else {} - ) + - ( - if $._config.storage_backend == 's3' then { - s3: { - bucket_name: $._config.s3_bucket_name, - endpoint: $._config.s3_address, - } + ( - if $._config.s3_access_key != '' && $._config.s3_secret_access_key != '' then { - access_key_id: $._config.s3_access_key, - secret_access_key: $._config.s3_secret_access_key, - } - else {} - ) + ( - if $._config.s3_bucket_region != '' then { - region: $._config.s3_bucket_region, - } - else {} - ), - } else {} - ) + - ( - if $._config.storage_backend == 'azure' then { - azure: $._config.object_store_config.azure, - } else {} - ), + if $._config.storage_backend == 'gcs' then { + gcs: $._config.object_store_config.gcs, + } else if $._config.storage_backend == 's3' then { + s3: { + bucket_name: $._config.s3_bucket_name, + endpoint: $._config.s3_address, + } + ( + if $._config.s3_access_key != '' && $._config.s3_secret_access_key != '' then { + access_key_id: $._config.s3_access_key, + secret_access_key: $._config.s3_secret_access_key, + } + else {} + ) + ( + if $._config.s3_bucket_region != '' then { + region: $._config.s3_bucket_region, + } + else {} + ), + } else if $._config.storage_backend == 'azure' then { + azure: $._config.object_store_config.azure, + } else {}, // December 11 is when we first launched to the public. // Assume we can ingest logs that are 5months old.