Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(ksonnet): add configs to use thanos object store #16030

Merged
merged 7 commits into from
Feb 3, 2025
Merged

chore(ksonnet): add configs to use thanos object store #16030

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6bdd952
ksonnet: add support for thanos storage clients
ashwanthgoli Jan 30, 2025
4ecc1a6
Merge branch 'main' into ksonnet-use-thanos
ashwanthgoli Jan 31, 2025
416161e
reuse client_configs
ashwanthgoli Jan 31, 2025
a66dcb4
s/enabledBackends/storage_backend
ashwanthgoli Jan 31, 2025
0e0dc43
also update ruler storage
ashwanthgoli Jan 31, 2025
858e468
fixup! also update ruler storage
ashwanthgoli Jan 31, 2025
a29f2e3
tidy
ashwanthgoli Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 61 additions & 34 deletions production/ksonnet/loki/config.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@
topology_spread_max_skew: 1,
},

// Use thanos object store clients
use_thanos_objstore: false,

// GCS variables
gcs_bucket_name: error 'must specify GCS bucket name',

Expand All @@ -91,6 +94,7 @@
s3_secret_access_key: '',
s3_address: error 'must specify s3_address',
s3_bucket_name: error 'must specify s3_bucket_name',
s3_bucket_region: '',
s3_path_style: false,

// Azure variables
Expand All @@ -101,28 +105,55 @@
// DNS Resolver
dns_resolver: 'kube-dns.kube-system.svc.cluster.local',

client_configs: {
s3: {
s3forcepathstyle: $._config.s3_path_style,
} + (
if $._config.s3_access_key != '' then {
s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name,
} else {
s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name,
}
),
gcs: {
bucket_name: $._config.gcs_bucket_name,
},
azure: {
container_name: $._config.azure_container_name,
account_name: $._config.azure_account_name,
} + (
if $._config.azure_account_key != '' then {
account_key: $._config.azure_account_key,
} else {}
),
},
object_store_config:
if $._config.storage_backend == 'gcs' then {
gcs: {
bucket_name: $._config.gcs_bucket_name,
},
} else if $._config.storage_backend == 's3' then {
aws: {
s3forcepathstyle: $._config.s3_path_style,
} + (
if $._config.s3_access_key != '' then {
s3: 's3://' + $._config.s3_access_key + ':' + $._config.s3_secret_access_key + '@' + $._config.s3_address + '/' + $._config.s3_bucket_name,
} else {
s3: 's3://' + $._config.s3_address + '/' + $._config.s3_bucket_name,
}
),
} else if $._config.storage_backend == 'azure' then {
azure: {
container_name: $._config.azure_container_name,
account_name: $._config.azure_account_name,
} + (
if $._config.azure_account_key != '' then {
account_key: $._config.azure_account_key,
} else {}
),
} else {},

// thanos object store config
thanos_object_store_config:
if $._config.storage_backend == 'gcs' then {
gcs: $._config.object_store_config.gcs,
} else if $._config.storage_backend == 's3' then {
s3: {
bucket_name: $._config.s3_bucket_name,
endpoint: $._config.s3_address,
} + (
if $._config.s3_access_key != '' && $._config.s3_secret_access_key != '' then {
access_key_id: $._config.s3_access_key,
secret_access_key: $._config.s3_secret_access_key,
}
else {}
) + (
if $._config.s3_bucket_region != '' then {
region: $._config.s3_bucket_region,
}
else {}
),
} else if $._config.storage_backend == 'azure' then {
azure: $._config.object_store_config.azure,
} else {},

// December 11 is when we first launched to the public.
// Assume we can ingest logs that are 5months old.
Expand Down Expand Up @@ -279,19 +310,11 @@
consistent_hash: true,
},
},
} + (
if $._config.storage_backend == 'gcs' then {
gcs: $._config.client_configs.gcs,
} else {}
) +
(
if $._config.storage_backend == 's3' then {
aws: $._config.client_configs.s3,
} else {}
) +
} + $._config.object_store_config +
(
if $._config.storage_backend == 'azure' then {
azure: $._config.client_configs.azure,
if $._config.use_thanos_objstore then {
use_thanos_objstore: true,
object_store: $._config.thanos_object_store_config,
} else {}
),

Expand Down Expand Up @@ -361,6 +384,10 @@
},
} else {},

ruler_storage: if $._config.ruler_enabled then {
backend: $._config.storage_backend,
} + $._config.thanos_object_store_config else {},

},
},

Expand Down