OTLPSpec defines which resource, scope and log attributes should be used as stream labels or -stored as structured metadata.
+OTLPSpec defines which resource, scope and log attributes should be used as stream labels or dropped before storing.
+Attributes need to be listed by their “OpenTelemetry name” and not the representation used by Loki. Please consult +the documentation of the collector or application emitting the OpenTelemetry data to find out which attributes +are emitted.
+Attributes not listed as stream labels or to be dropped are stored as structured metadata in Loki.
-structuredMetadata+ dropOTLPMetadataSpec @@ -2765,7 +2768,7 @@ OTLPMetadataSpec |
(Optional)
- StructuredMetadata configures which attributes are saved in structured metadata. +Drop configures which attributes are dropped from the log entry. |
(Optional)
- DisableRecommendedAttributes can be used to reduce the number of attributes used for stream labels and structured -metadata. +DisableRecommendedAttributes can be used to reduce the number of attributes used as stream labels. Enabling this setting removes the “recommended attributes” from the generated Loki configuration. This will cause -meta information to not be available as stream labels or structured metadata, potentially making queries more -expensive and less performant. +some stream labels to disappear from the index, potentially making queries more expensive and less performant.Note that there is a set of “required attributes”, needed for OpenShift Logging to work properly. Those will be added to the configuration, even if this field is set to true. -This option is supposed to be combined with a custom label configuration customizing the labels for the specific -usecase. +This option is supposed to be combined with a custom attribute configuration listing the stream labels that +should continue to exist. |
diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go
index 8530b7e63cfb6..04414ad3e48ff 100644
--- a/operator/internal/manifests/config_otlp.go
+++ b/operator/internal/manifests/config_otlp.go
@@ -6,7 +6,7 @@ import (
lokiv1 "github.com/grafana/loki/operator/api/loki/v1"
"github.com/grafana/loki/operator/internal/manifests/internal/config"
- "github.com/grafana/loki/operator/internal/manifests/openshift"
+ "github.com/grafana/loki/operator/internal/manifests/openshift/otlp"
)
func defaultOTLPAttributeConfig(ts *lokiv1.TenantsSpec) config.OTLPAttributeConfig {
@@ -19,7 +19,17 @@ func defaultOTLPAttributeConfig(ts *lokiv1.TenantsSpec) config.OTLPAttributeConf
disableRecommended = ts.Openshift.OTLP.DisableRecommendedAttributes
}
- return openshift.DefaultOTLPAttributes(disableRecommended)
+ return config.OTLPAttributeConfig{
+ RemoveDefaultLabels: true,
+ Global: &config.OTLPTenantAttributeConfig{
+ ResourceAttributes: []config.OTLPAttribute{
+ {
+ Action: config.OTLPAttributeActionStreamLabel,
+ Names: otlp.DefaultOTLPAttributes(disableRecommended),
+ },
+ },
+ },
+ }
}
func convertAttributeReferences(refs []lokiv1.OTLPAttributeReference, action config.OTLPAttributeAction) []config.OTLPAttribute {
@@ -92,20 +102,20 @@ func convertTenantAttributeReferences(otlpSpec *lokiv1.OTLPSpec, base *config.OT
convertAttributeReferences(streamLabels.ResourceAttributes, config.OTLPAttributeActionStreamLabel)...)
}
- if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil {
- if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 {
+ if dropLabels := otlpSpec.Drop; dropLabels != nil {
+ if resAttr := dropLabels.ResourceAttributes; len(resAttr) > 0 {
result.ResourceAttributes = append(result.ResourceAttributes,
- convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(resAttr, config.OTLPAttributeActionDrop)...)
}
- if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 {
+ if scopeAttr := dropLabels.ScopeAttributes; len(scopeAttr) > 0 {
result.ScopeAttributes = append(result.ScopeAttributes,
- convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(scopeAttr, config.OTLPAttributeActionDrop)...)
}
- if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 {
+ if logAttr := dropLabels.LogAttributes; len(logAttr) > 0 {
result.LogAttributes = append(result.LogAttributes,
- convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(logAttr, config.OTLPAttributeActionDrop)...)
}
}
@@ -233,24 +243,24 @@ func otlpAttributeConfig(ls *lokiv1.LokiStackSpec) config.OTLPAttributeConfig {
}
}
- if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil {
+ if dropLabels := globalOTLP.Drop; dropLabels != nil {
if result.Global == nil {
result.Global = &config.OTLPTenantAttributeConfig{}
}
- if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 {
+ if resAttr := dropLabels.ResourceAttributes; len(resAttr) > 0 {
result.Global.ResourceAttributes = append(result.Global.ResourceAttributes,
- convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(resAttr, config.OTLPAttributeActionDrop)...)
}
- if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 {
+ if scopeAttr := dropLabels.ScopeAttributes; len(scopeAttr) > 0 {
result.Global.ScopeAttributes = append(result.Global.ScopeAttributes,
- convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(scopeAttr, config.OTLPAttributeActionDrop)...)
}
- if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 {
+ if logAttr := dropLabels.LogAttributes; len(logAttr) > 0 {
result.Global.LogAttributes = append(result.Global.LogAttributes,
- convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...)
+ convertAttributeReferences(logAttr, config.OTLPAttributeActionDrop)...)
}
}
}
diff --git a/operator/internal/manifests/config_otlp_test.go b/operator/internal/manifests/config_otlp_test.go
index 73f4970c8dbf0..88f3e28ef3127 100644
--- a/operator/internal/manifests/config_otlp_test.go
+++ b/operator/internal/manifests/config_otlp_test.go
@@ -7,7 +7,7 @@ import (
lokiv1 "github.com/grafana/loki/operator/api/loki/v1"
"github.com/grafana/loki/operator/internal/manifests/internal/config"
- "github.com/grafana/loki/operator/internal/manifests/openshift"
+ "github.com/grafana/loki/operator/internal/manifests/openshift/otlp"
)
func TestOtlpAttributeConfig(t *testing.T) {
@@ -81,36 +81,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
},
{
- desc: "global metadata",
- spec: lokiv1.LokiStackSpec{
- Limits: &lokiv1.LimitsSpec{
- Global: &lokiv1.LimitsTemplateSpec{
- OTLP: &lokiv1.OTLPSpec{
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
- ResourceAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "metadata",
- },
- },
- },
- },
- },
- },
- },
- wantConfig: config.OTLPAttributeConfig{
- RemoveDefaultLabels: true,
- Global: &config.OTLPTenantAttributeConfig{
- ResourceAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"metadata"},
- },
- },
- },
- },
- },
- {
- desc: "global combined",
+ desc: "global with drop",
spec: lokiv1.LokiStackSpec{
Limits: &lokiv1.LimitsSpec{
Global: &lokiv1.LimitsTemplateSpec{
@@ -120,38 +91,12 @@ func TestOtlpAttributeConfig(t *testing.T) {
{
Name: "stream.label",
},
- {
- Name: "stream\\.label\\.regex\\..+",
- Regex: true,
- },
},
},
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
+ Drop: &lokiv1.OTLPMetadataSpec{
ResourceAttributes: []lokiv1.OTLPAttributeReference{
{
- Name: "resource.metadata",
- },
- {
- Name: "resource.metadata\\.other\\..+",
- Regex: true,
- },
- },
- ScopeAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "scope.metadata",
- },
- {
- Name: "scope.metadata\\.other\\..+",
- Regex: true,
- },
- },
- LogAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "log.metadata",
- },
- {
- Name: "log.metadata\\.other\\..+",
- Regex: true,
+ Name: "drop.attribute",
},
},
},
@@ -164,40 +109,16 @@ func TestOtlpAttributeConfig(t *testing.T) {
Global: &config.OTLPTenantAttributeConfig{
ResourceAttributes: []config.OTLPAttribute{
{
- Action: config.OTLPAttributeActionStreamLabel,
- Names: []string{"stream.label"},
+ Action: config.OTLPAttributeActionDrop,
+ Names: []string{
+ "drop.attribute",
+ },
},
{
Action: config.OTLPAttributeActionStreamLabel,
- Regex: "stream\\.label\\.regex\\..+",
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"resource.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "resource.metadata\\.other\\..+",
- },
- },
- ScopeAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"scope.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "scope.metadata\\.other\\..+",
- },
- },
- LogAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"log.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "log.metadata\\.other\\..+",
+ Names: []string{
+ "stream.label",
+ },
},
},
},
@@ -271,40 +192,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
},
{
- desc: "tenant metadata",
- spec: lokiv1.LokiStackSpec{
- Limits: &lokiv1.LimitsSpec{
- Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{
- "test-tenant": {
- OTLP: &lokiv1.OTLPSpec{
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
- ResourceAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "tenant.metadata",
- },
- },
- },
- },
- },
- },
- },
- },
- wantConfig: config.OTLPAttributeConfig{
- RemoveDefaultLabels: true,
- Tenants: map[string]*config.OTLPTenantAttributeConfig{
- "test-tenant": {
- ResourceAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"tenant.metadata"},
- },
- },
- },
- },
- },
- },
- {
- desc: "tenant combined",
+ desc: "tenant with drop",
spec: lokiv1.LokiStackSpec{
Limits: &lokiv1.LimitsSpec{
Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{
@@ -313,40 +201,14 @@ func TestOtlpAttributeConfig(t *testing.T) {
StreamLabels: &lokiv1.OTLPStreamLabelSpec{
ResourceAttributes: []lokiv1.OTLPAttributeReference{
{
- Name: "tenant.stream.label",
- },
- {
- Name: `tenant\.stream\.label\.regex\..+`,
- Regex: true,
+ Name: "stream.label",
},
},
},
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
+ Drop: &lokiv1.OTLPMetadataSpec{
ResourceAttributes: []lokiv1.OTLPAttributeReference{
{
- Name: "tenant.resource.metadata",
- },
- {
- Name: `tenant\.resource.metadata\.other\..+`,
- Regex: true,
- },
- },
- ScopeAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "tenant.scope.metadata",
- },
- {
- Name: `tenant\.scope\.metadata\.other\..+`,
- Regex: true,
- },
- },
- LogAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "tenant.log.metadata",
- },
- {
- Name: `tenant\.log\.metadata\.other\..+`,
- Regex: true,
+ Name: "drop.attribute",
},
},
},
@@ -361,40 +223,16 @@ func TestOtlpAttributeConfig(t *testing.T) {
"test-tenant": {
ResourceAttributes: []config.OTLPAttribute{
{
- Action: config.OTLPAttributeActionStreamLabel,
- Names: []string{"tenant.stream.label"},
+ Action: config.OTLPAttributeActionDrop,
+ Names: []string{
+ "drop.attribute",
+ },
},
{
Action: config.OTLPAttributeActionStreamLabel,
- Regex: "tenant\\.stream\\.label\\.regex\\..+",
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"tenant.resource.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `tenant\.resource.metadata\.other\..+`,
- },
- },
- ScopeAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"tenant.scope.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `tenant\.scope\.metadata\.other\..+`,
- },
- },
- LogAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"tenant.log.metadata"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `tenant\.log\.metadata\.other\..+`,
+ Names: []string{
+ "stream.label",
+ },
},
},
},
@@ -473,7 +311,17 @@ func TestOtlpAttributeConfig(t *testing.T) {
Mode: lokiv1.OpenshiftLogging,
},
},
- wantConfig: openshift.DefaultOTLPAttributes(false),
+ wantConfig: config.OTLPAttributeConfig{
+ RemoveDefaultLabels: true,
+ Global: &config.OTLPTenantAttributeConfig{
+ ResourceAttributes: []config.OTLPAttribute{
+ {
+ Action: config.OTLPAttributeActionStreamLabel,
+ Names: otlp.DefaultOTLPAttributes(false),
+ },
+ },
+ },
+ },
},
{
desc: "openshift-logging defaults without recommended",
@@ -487,10 +335,20 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
},
},
- wantConfig: openshift.DefaultOTLPAttributes(true),
+ wantConfig: config.OTLPAttributeConfig{
+ RemoveDefaultLabels: true,
+ Global: &config.OTLPTenantAttributeConfig{
+ ResourceAttributes: []config.OTLPAttribute{
+ {
+ Action: config.OTLPAttributeActionStreamLabel,
+ Names: otlp.DefaultOTLPAttributes(true),
+ },
+ },
+ },
+ },
},
{
- desc: "openshift-logging defaults with additional custom attributes",
+ desc: "openshift-logging defaults with drop",
spec: lokiv1.LokiStackSpec{
Limits: &lokiv1.LimitsSpec{
Global: &lokiv1.LimitsTemplateSpec{
@@ -502,7 +360,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
},
},
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
+ Drop: &lokiv1.OTLPMetadataSpec{
LogAttributes: []lokiv1.OTLPAttributeReference{
{
Name: "custom.log.metadata",
@@ -554,7 +412,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
LogAttributes: []config.OTLPAttribute{
{
- Action: config.OTLPAttributeActionMetadata,
+ Action: config.OTLPAttributeActionDrop,
Names: []string{"custom.log.metadata"},
},
},
@@ -579,7 +437,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
LogAttributes: []config.OTLPAttribute{
{
- Action: config.OTLPAttributeActionMetadata,
+ Action: config.OTLPAttributeActionDrop,
Names: []string{"custom.log.metadata"},
},
},
@@ -603,7 +461,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
},
},
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
+ Drop: &lokiv1.OTLPMetadataSpec{
LogAttributes: []lokiv1.OTLPAttributeReference{
{
Name: "custom.log.metadata",
@@ -642,7 +500,7 @@ func TestOtlpAttributeConfig(t *testing.T) {
},
LogAttributes: []config.OTLPAttribute{
{
- Action: config.OTLPAttributeActionMetadata,
+ Action: config.OTLPAttributeActionDrop,
Names: []string{"custom.log.metadata"},
},
},
@@ -671,22 +529,10 @@ func TestSortOTLPAttributes(t *testing.T) {
{
desc: "sort",
attrs: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"test.a"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.c",
- },
{
Action: config.OTLPAttributeActionStreamLabel,
Names: []string{"test.b"},
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.a",
- },
{
Action: config.OTLPAttributeActionStreamLabel,
Regex: "test.regex.b",
@@ -701,18 +547,6 @@ func TestSortOTLPAttributes(t *testing.T) {
Action: config.OTLPAttributeActionStreamLabel,
Regex: "test.regex.b",
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"test.a"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.a",
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.c",
- },
},
},
{
@@ -745,26 +579,14 @@ func TestSortOTLPAttributes(t *testing.T) {
Action: config.OTLPAttributeActionStreamLabel,
Names: []string{"test.c"},
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"test.d", "test.e"},
- },
{
Action: config.OTLPAttributeActionStreamLabel,
Names: []string{"test.b"},
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"test.f"},
- },
{
Action: config.OTLPAttributeActionStreamLabel,
Regex: "test.regex.b",
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.a",
- },
},
wantAttrs: []config.OTLPAttribute{
{
@@ -775,14 +597,6 @@ func TestSortOTLPAttributes(t *testing.T) {
Action: config.OTLPAttributeActionStreamLabel,
Regex: "test.regex.b",
},
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{"test.d", "test.e", "test.f"},
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: "test.regex.a",
- },
},
},
}
diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
index 1bf465b5e08c0..3d9c076656bbd 100644
--- a/operator/internal/manifests/internal/config/build_test.go
+++ b/operator/internal/manifests/internal/config/build_test.go
@@ -6164,16 +6164,16 @@ limits_config:
attributes:
- res.foo.bar
- res.bar.baz
- - action: structured_metadata
+ - action: drop
attributes:
- res.service.env
scope_attributes:
- - action: structured_metadata
+ - action: drop
attributes:
- scope.foo.bar
- scope.bar.baz
log_attributes:
- - action: structured_metadata
+ - action: drop
attributes:
- log.foo.bar
- log.bar.baz
@@ -6313,16 +6313,16 @@ overrides:
attributes:
- res.foo.bar
- res.bar.baz
- - action: structured_metadata
+ - action: drop
attributes:
- res.service.env
scope_attributes:
- - action: structured_metadata
+ - action: drop
attributes:
- scope.foo.bar
- scope.bar.baz
log_attributes:
- - action: structured_metadata
+ - action: drop
attributes:
- log.foo.bar
- log.bar.baz
@@ -6499,7 +6499,7 @@ overrides:
},
},
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"res.service.env",
},
@@ -6507,7 +6507,7 @@ overrides:
},
ScopeAttributes: []OTLPAttribute{
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"scope.foo.bar",
"scope.bar.baz",
@@ -6516,7 +6516,7 @@ overrides:
},
LogAttributes: []OTLPAttribute{
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"log.foo.bar",
"log.bar.baz",
@@ -6535,7 +6535,7 @@ overrides:
},
},
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"res.service.env",
},
@@ -6543,7 +6543,7 @@ overrides:
},
ScopeAttributes: []OTLPAttribute{
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"scope.foo.bar",
"scope.bar.baz",
@@ -6552,7 +6552,7 @@ overrides:
},
LogAttributes: []OTLPAttribute{
{
- Action: OTLPAttributeActionMetadata,
+ Action: OTLPAttributeActionDrop,
Names: []string{
"log.foo.bar",
"log.bar.baz",
diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go
index 365a768724eba..5f837cea72e7f 100644
--- a/operator/internal/manifests/internal/config/options.go
+++ b/operator/internal/manifests/internal/config/options.go
@@ -268,7 +268,7 @@ type OTLPAttributeAction string
const (
OTLPAttributeActionStreamLabel OTLPAttributeAction = "index_label"
- OTLPAttributeActionMetadata OTLPAttributeAction = "structured_metadata"
+ OTLPAttributeActionDrop OTLPAttributeAction = "drop"
)
type OTLPAttribute struct {
diff --git a/operator/internal/manifests/openshift/otlp.go b/operator/internal/manifests/openshift/otlp.go
deleted file mode 100644
index b74aa2e40aa7b..0000000000000
--- a/operator/internal/manifests/openshift/otlp.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package openshift
-
-import (
- "slices"
-
- "github.com/grafana/loki/operator/internal/manifests/internal/config"
-)
-
-// DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging.
-func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig {
- result := config.OTLPAttributeConfig{
- RemoveDefaultLabels: true,
- Global: &config.OTLPTenantAttributeConfig{
- ResourceAttributes: []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionStreamLabel,
- Names: []string{
- "k8s.namespace.name",
- "kubernetes.namespace_name",
- "log_source",
- "log_type",
- "openshift.cluster.uid",
- "openshift.log.source",
- "openshift.log.type",
- },
- },
- },
- },
- }
-
- if disableRecommended {
- return result
- }
-
- result.Global.ResourceAttributes[0].Names = append(result.Global.ResourceAttributes[0].Names,
- "k8s.container.name",
- "k8s.cronjob.name",
- "k8s.daemonset.name",
- "k8s.deployment.name",
- "k8s.job.name",
- "k8s.node.name",
- "k8s.pod.name",
- "k8s.statefulset.name",
- "kubernetes.container_name",
- "kubernetes.host",
- "kubernetes.pod_name",
- "service.name",
- )
- slices.Sort(result.Global.ResourceAttributes[0].Names)
-
- result.Global.ResourceAttributes = append(result.Global.ResourceAttributes,
- config.OTLPAttribute{
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{
- "k8s.node.uid",
- "k8s.pod.uid",
- "k8s.replicaset.name",
- "process.command_line",
- "process.executable.name",
- "process.executable.path",
- "process.pid",
- },
- },
- config.OTLPAttribute{
- Action: config.OTLPAttributeActionMetadata,
- Regex: `k8s\.pod\.labels\..+`,
- },
- config.OTLPAttribute{
- Action: config.OTLPAttributeActionMetadata,
- Regex: `openshift\.labels\..+`,
- },
- )
-
- result.Global.LogAttributes = []config.OTLPAttribute{
- {
- Action: config.OTLPAttributeActionMetadata,
- Names: []string{
- "k8s.event.level",
- "k8s.event.object_ref.api.group",
- "k8s.event.object_ref.api.version",
- "k8s.event.object_ref.name",
- "k8s.event.object_ref.resource",
- "k8s.event.request.uri",
- "k8s.event.response.code",
- "k8s.event.stage",
- "k8s.event.user_agent",
- "k8s.user.groups",
- "k8s.user.username",
- "level",
- "log.iostream",
- },
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `k8s\.event\.annotations\..+`,
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `systemd\.t\..+`,
- },
- {
- Action: config.OTLPAttributeActionMetadata,
- Regex: `systemd\.u\..+`,
- },
- }
-
- return result
-}
diff --git a/operator/internal/manifests/openshift/otlp/otlp.go b/operator/internal/manifests/openshift/otlp/otlp.go
new file mode 100644
index 0000000000000..96d9cab6d24c2
--- /dev/null
+++ b/operator/internal/manifests/openshift/otlp/otlp.go
@@ -0,0 +1,45 @@
+package otlp
+
+import (
+ "slices"
+)
+
+var (
+ requiredAttributes = []string{
+ "k8s.namespace.name",
+ "kubernetes.namespace_name",
+ "log_source",
+ "log_type",
+ "openshift.cluster.uid",
+ "openshift.log.source",
+ "openshift.log.type",
+ }
+
+ recommendedAttributes = []string{
+ "k8s.container.name",
+ "k8s.cronjob.name",
+ "k8s.daemonset.name",
+ "k8s.deployment.name",
+ "k8s.job.name",
+ "k8s.node.name",
+ "k8s.pod.name",
+ "k8s.statefulset.name",
+ "kubernetes.container_name",
+ "kubernetes.host",
+ "kubernetes.pod_name",
+ "service.name",
+ }
+)
+
+// DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging.
+func DefaultOTLPAttributes(disableRecommended bool) []string {
+ result := append([]string{}, requiredAttributes...)
+ if disableRecommended {
+ return result
+ }
+
+ result = append(result, recommendedAttributes...)
+ slices.Sort(result)
+
+ return result
+}
diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go
index c5eacd8d57299..8ec73456afec3 100644
--- a/operator/internal/validation/lokistack.go
+++ b/operator/internal/validation/lokistack.go
@@ -13,6 +13,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
lokiv1 "github.com/grafana/loki/operator/api/loki/v1"
+ "github.com/grafana/loki/operator/internal/validation/openshift"
)
// objectStorageSchemaMap defines the type for mapping a schema version with a date
@@ -108,7 +109,7 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe
if spec.Tenants.Mode == lokiv1.OpenshiftLogging {
// This tenancy mode always provides stream labels
- return nil
+ return openshift.ValidateOTLPInvalidDrop(spec)
}
if spec.Tenants.Mode == lokiv1.OpenshiftNetwork {
@@ -122,50 +123,59 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe
}
hasGlobalStreamLabels := false
+ errList := field.ErrorList{}
+ var globalOtlp *lokiv1.OTLPSpec
if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil {
- hasGlobalStreamLabels = v.hasOTLPStreamLabel(spec.Limits.Global.OTLP)
- }
+ globalOtlp = spec.Limits.Global.OTLP
- if hasGlobalStreamLabels {
- // When the global configuration has at least one stream label, then the configuration will be valid
- return nil
+ hasGlobalStreamLabels = v.hasOTLPStreamLabel(globalOtlp)
+ errList = append(errList, v.checkOTLPInvalidDrop(field.NewPath("spec", "limits", "global", "otlp"), globalOtlp, nil)...)
}
- if spec.Limits.Tenants == nil {
+ if !hasGlobalStreamLabels && spec.Limits.Tenants == nil {
// No tenant config and no global stream labels -> error
- return field.ErrorList{
- field.Invalid(
- field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"),
- nil,
- lokiv1.ErrOTLPGlobalNoStreamLabel.Error(),
- ),
- }
+ errList = append(errList, field.Invalid(
+ field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"),
+ nil,
+ lokiv1.ErrOTLPGlobalNoStreamLabel.Error(),
+ ))
}
- errList := field.ErrorList{}
+ errList = append(errList, v.validateOTLPTenantConfiguration(spec, globalOtlp, hasGlobalStreamLabels)...)
+ return errList
+}
+
+func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiStackSpec, globalOtlp *lokiv1.OTLPSpec, hasGlobalStreamLabel bool) (errList field.ErrorList) {
+ if spec.Limits == nil || spec.Limits.Tenants == nil {
+ return nil
+ }
+
+ errList = field.ErrorList{}
for _, tenant := range spec.Tenants.Authentication {
tenantName := tenant.TenantName
tenantLimits, ok := spec.Limits.Tenants[tenantName]
if !ok || tenantLimits.OTLP == nil {
- // No tenant limits defined and no global stream labels -> error
- errList = append(errList, field.Invalid(
- field.NewPath("spec", "limits", "tenants", tenantName, "otlp"),
- nil,
- lokiv1.ErrOTLPTenantMissing.Error(),
- ))
+ if !hasGlobalStreamLabel {
+ // No tenant limits defined and no global stream labels -> error
+ errList = append(errList, field.Invalid(
+ field.NewPath("spec", "limits", "tenants", tenantName, "otlp"),
+ nil,
+ lokiv1.ErrOTLPTenantMissing.Error(),
+ ))
+ }
continue
}
- if v.hasOTLPStreamLabel(tenantLimits.OTLP) {
- continue
+ if !hasGlobalStreamLabel && !v.hasOTLPStreamLabel(tenantLimits.OTLP) {
+ errList = append(errList, field.Invalid(
+ field.NewPath("spec", "limits", "tenants", tenantName, "otlp", "streamLabels", "resourceAttributes"),
+ nil,
+ lokiv1.ErrOTLPTenantNoStreamLabel.Error(),
+ ))
}
- errList = append(errList, field.Invalid(
- field.NewPath("spec", "limits", "tenants", tenantName, "otlp", "streamLabels", "resourceAttributes"),
- nil,
- lokiv1.ErrOTLPTenantNoStreamLabel.Error(),
- ))
+ errList = append(errList, v.checkOTLPInvalidDrop(field.NewPath("spec", "limits", "tenants", tenantName, "otlp"), tenantLimits.OTLP, globalOtlp)...)
}
return errList
@@ -183,6 +193,69 @@ func (v *LokiStackValidator) hasOTLPStreamLabel(otlp *lokiv1.OTLPSpec) bool {
return len(otlp.StreamLabels.ResourceAttributes) > 0
}
+func (v *LokiStackValidator) checkOTLPInvalidDrop(basePath *field.Path, otlp, inheritedOtlp *lokiv1.OTLPSpec) field.ErrorList {
+ if otlp.Drop == nil {
+ return nil
+ }
+
+ errList := field.ErrorList{}
+ streamAttributes := [][]lokiv1.OTLPAttributeReference{}
+ if streamLabels := otlp.StreamLabels; streamLabels != nil {
+ streamAttributes = append(streamAttributes, streamLabels.ResourceAttributes)
+ }
+ if inheritedOtlp != nil && inheritedOtlp.StreamLabels != nil && len(inheritedOtlp.StreamLabels.ResourceAttributes) > 0 {
+ streamAttributes = append(streamAttributes, inheritedOtlp.StreamLabels.ResourceAttributes)
+ }
+ errList = append(errList, v.checkOTLPInvalidDropReference(
+ basePath.Child("drop", "resourceAttributes"),
+ otlp.Drop.ResourceAttributes,
+ streamAttributes,
+ )...)
+
+ return errList
+}
+
+func (v *LokiStackValidator) checkOTLPInvalidDropReference(basePath *field.Path, dropList []lokiv1.OTLPAttributeReference, keepLists [][]lokiv1.OTLPAttributeReference) field.ErrorList {
+ if len(dropList) == 0 {
+ return nil
+ }
+
+ if len(keepLists) == 0 {
+ return nil
+ }
+
+ attributeNames := map[string]bool{}
+ errList := field.ErrorList{}
+
+ for _, keeps := range keepLists {
+ for _, attr := range keeps {
+ if attr.Regex {
+ // skip regular expressions for this check
+ continue
+ }
+ attributeNames[attr.Name] = true
+ }
+ }
+
+ for i, attr := range dropList {
+ if attr.Regex {
+ continue
+ }
+
+ if !attributeNames[attr.Name] {
+ continue
+ }
+
+ errList = append(errList, field.Invalid(
+ basePath.Index(i),
+ attr.Name,
+ lokiv1.ErrOTLPInvalidDrop.Error(),
+ ))
+ }
+
+ return errList
+}
+
func (v *LokiStackValidator) validateHashRingSpec(s lokiv1.LokiStackSpec) field.ErrorList {
if s.HashRing == nil {
return nil
diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go
index 18434c5fa20b2..b51098e586045 100644
--- a/operator/internal/validation/lokistack_test.go
+++ b/operator/internal/validation/lokistack_test.go
@@ -440,15 +440,7 @@ var ltt = []struct {
},
Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{
"test-tenant": {
- OTLP: &lokiv1.OTLPSpec{
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
- ResourceAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "custom.resource.attribute",
- },
- },
- },
- },
+ OTLP: &lokiv1.OTLPSpec{},
},
},
},
@@ -517,15 +509,7 @@ var ltt = []struct {
Spec: lokiv1.LokiStackSpec{
Limits: &lokiv1.LimitsSpec{
Global: &lokiv1.LimitsTemplateSpec{
- OTLP: &lokiv1.OTLPSpec{
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
- ResourceAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "custom.resource.attribute",
- },
- },
- },
- },
+ OTLP: &lokiv1.OTLPSpec{},
},
},
Storage: lokiv1.ObjectStorageSpec{
@@ -612,15 +596,7 @@ var ltt = []struct {
Limits: &lokiv1.LimitsSpec{
Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{
"test-tenant": {
- OTLP: &lokiv1.OTLPSpec{
- StructuredMetadata: &lokiv1.OTLPMetadataSpec{
- ResourceAttributes: []lokiv1.OTLPAttributeReference{
- {
- Name: "tenant.resource.attribute",
- },
- },
- },
- },
+ OTLP: &lokiv1.OTLPSpec{},
},
},
},
@@ -654,6 +630,55 @@ var ltt = []struct {
},
),
},
+ {
+ desc: "lokistack with custom OTLP configuration listing an attribute as both stream-label and drop",
+ spec: lokiv1.LokiStack{
+ Spec: lokiv1.LokiStackSpec{
+ Limits: &lokiv1.LimitsSpec{
+ Global: &lokiv1.LimitsTemplateSpec{
+ OTLP: &lokiv1.OTLPSpec{
+ StreamLabels: &lokiv1.OTLPStreamLabelSpec{
+ ResourceAttributes: []lokiv1.OTLPAttributeReference{
+ {
+ Name: "global.stream.label",
+ },
+ },
+ },
+ Drop: &lokiv1.OTLPMetadataSpec{
+ ResourceAttributes: []lokiv1.OTLPAttributeReference{
+ {
+ Name: "global.stream.label",
+ },
+ },
+ },
+ },
+ },
+ },
+ Storage: lokiv1.ObjectStorageSpec{
+ Schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2024-10-22",
+ },
+ },
+ },
+ Tenants: &lokiv1.TenantsSpec{
+ Mode: lokiv1.Static,
+ },
+ },
+ },
+ err: apierrors.NewInvalid(
+ schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"},
+ "testing-stack",
+ field.ErrorList{
+ field.Invalid(
+ field.NewPath("spec", "limits", "global", "otlp", "drop", "resourceAttributes").Index(0),
+ "global.stream.label",
+ lokiv1.ErrOTLPInvalidDrop.Error(),
+ ),
+ },
+ ),
+ },
}
func TestLokiStackValidationWebhook_ValidateCreate(t *testing.T) {
diff --git a/operator/internal/validation/openshift/otlp.go b/operator/internal/validation/openshift/otlp.go
new file mode 100644
index 0000000000000..9ddf86cd79da1
--- /dev/null
+++ b/operator/internal/validation/openshift/otlp.go
@@ -0,0 +1,67 @@
+package openshift
+
+import (
+ "k8s.io/apimachinery/pkg/util/validation/field"
+
+ lokiv1 "github.com/grafana/loki/operator/api/loki/v1"
+ "github.com/grafana/loki/operator/internal/manifests/openshift/otlp"
+)
+
+// ValidateOTLPInvalidDrop validates that a spec does not drop required OTLP attributes in the openshift-logging tenancy mode.
+func ValidateOTLPInvalidDrop(spec *lokiv1.LokiStackSpec) field.ErrorList {
+ if spec.Limits == nil {
+ return nil
+ }
+
+ disableRecommendedAttributes := false
+ if spec.Tenants != nil &&
+ spec.Tenants.Openshift != nil &&
+ spec.Tenants.Openshift.OTLP != nil {
+ disableRecommendedAttributes = spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes
+ }
+
+ requiredAttributes := map[string]bool{}
+ for _, label := range otlp.DefaultOTLPAttributes(disableRecommendedAttributes) {
+ requiredAttributes[label] = true
+ }
+
+ errList := field.ErrorList{}
+ if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil {
+ errList = append(errList, validateOTLPSpec(requiredAttributes, spec.Limits.Global.OTLP, field.NewPath("spec", "limits", "global", "otlp"))...)
+ }
+
+ if len(spec.Limits.Tenants) > 0 {
+ for name, tenant := range spec.Limits.Tenants {
+ if tenant.OTLP != nil {
+ errList = append(errList, validateOTLPSpec(requiredAttributes, tenant.OTLP, field.NewPath("spec", "limits", "tenants", name, "otlp"))...)
+ }
+ }
+ }
+
+ return errList
+}
+
+func validateOTLPSpec(requiredAttributes map[string]bool, otlp *lokiv1.OTLPSpec, basePath *field.Path) field.ErrorList {
+ if otlp.Drop == nil {
+ return nil
+ }
+
+ errList := field.ErrorList{}
+ for i, attr := range otlp.Drop.ResourceAttributes {
+ if attr.Regex {
+ continue
+ }
+
+ if !requiredAttributes[attr.Name] {
+ continue
+ }
+
+ errList = append(errList, field.Invalid(
+ basePath.Child("drop", "resourceAttributes").Index(i).Child("name"),
+ attr.Name,
+ lokiv1.ErrOTLPInvalidDrop.Error(),
+ ))
+ }
+
+ return errList
+}
diff --git a/operator/internal/validation/openshift/otlp_test.go b/operator/internal/validation/openshift/otlp_test.go
new file mode 100644
index 0000000000000..23047621f6afb
--- /dev/null
+++ b/operator/internal/validation/openshift/otlp_test.go
@@ -0,0 +1,81 @@
+package openshift
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apimachinery/pkg/util/validation/field"
+
+ lokiv1 "github.com/grafana/loki/operator/api/loki/v1"
+)
+
+func TestValidateOTLPInvalidDrop(t *testing.T) {
+ tt := []struct {
+ desc string
+ spec lokiv1.LokiStack
+ err *apierrors.StatusError
+ }{
+ {
+ desc: "lokistack using openshift-logging tenancy mode trying to remove a required stream label",
+ spec: lokiv1.LokiStack{
+ Spec: lokiv1.LokiStackSpec{
+ Limits: &lokiv1.LimitsSpec{
+ Global: &lokiv1.LimitsTemplateSpec{
+ OTLP: &lokiv1.OTLPSpec{
+ Drop: &lokiv1.OTLPMetadataSpec{
+ ResourceAttributes: []lokiv1.OTLPAttributeReference{
+ {
+ Name: "kubernetes.namespace_name",
+ },
+ },
+ },
+ },
+ },
+ },
+ Storage: lokiv1.ObjectStorageSpec{
+ Schemas: []lokiv1.ObjectStorageSchema{
+ {
+ Version: lokiv1.ObjectStorageSchemaV13,
+ EffectiveDate: "2024-10-22",
+ },
+ },
+ },
+ Tenants: &lokiv1.TenantsSpec{
+ Mode: lokiv1.OpenshiftLogging,
+ },
+ },
+ },
+ err: apierrors.NewInvalid(
+ schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"},
+ "testing-stack",
+ field.ErrorList{
+ field.Invalid(
+ field.NewPath("spec", "limits", "global", "otlp", "drop", "resourceAttributes").Index(0).Child("name"),
+ "kubernetes.namespace_name",
+ lokiv1.ErrOTLPInvalidDrop.Error(),
+ ),
+ },
+ ),
+ },
+ }
+
+ for _, tc := range tt {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ errList := ValidateOTLPInvalidDrop(&tc.spec.Spec)
+ if tc.err != nil {
+ testErr := apierrors.NewInvalid(
+ schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"},
+ "testing-stack",
+ errList,
+ )
+ require.Equal(t, tc.err, testErr)
+ } else {
+ require.Nil(t, errList)
+ }
+ })
+ }
+}