add config to drop certain labels from target info (#2510)

* testing

* undo logs

* add config to drop certain labels from target info

* comment and refactor

* documentation

* hopefully improve perf

* rebase

* lint

* oops

* lint

docs/sources/tempo/configuration/_index.md

@@ -323,9 +323,11 @@ metrics_generator:
             # "name" appears in the metrics, "source_labels" are the actual
             # attributes that will make up the value of the label and "join" is the
             # separator if multiple source_labels are provided
-            [MetricsGeneratorProcessorSpanMetricsDimensionMappings: <list of map>]
-            # Enable target_info metrics
-            [MetricsGeneratorProcessorSpanMetricsEnableTargetInfo: <bool>]
+            [dimension_mappings: <list of map>]
+            # Enable traces_target_info metrics
+            [enable_target_info: <bool>]
+            # Drop specific labels from traces_target_info metrics
+            [target_info_excluded_dimensions: <list of string>]
             # Attribute Key to multiply span metrics
             [span_multiplier_key: <string> | default = ""]
 
@@ -1309,9 +1311,11 @@ overrides:
     # Allowed keys for intrinsic dimensions are: service, span_name, span_kind, status_code, and status_message.
     [metrics_generator_processor_span_metrics_intrinsic_dimensions: <map string to bool>]
     [metrics_generator_processor_span_metrics_dimensions: <list of string>]
-    [MetricsGeneratorProcessorSpanMetricsDimensionMappings: <list of map>]
+    [metrics_generator_processor_span_metrics_dimension_mapings: <list of map>]
     # Enable target_info metrics
-    [MetricsGeneratorProcessorSpanMetricsEnableTargetInfo: <bool>]
+    [metrics_generator_processor_span_metrics_enable_target_info: <bool>]
+    # Drop specific resource labels from traces_target_info
+    [metrics_generator_processor_span_metrics_target_info_excluded_dimensions: <list of string>]
 
     # Maximum number of active series in the registry, per instance of the metrics-generator. A
     # value of 0 disables this check.

modules/generator/overrides.go

@@ -29,6 +29,7 @@ type metricsGeneratorOverrides interface {
 	MetricsGeneratorProcessorSpanMetricsDimensionMappings(userID string) []sharedconfig.DimensionMappings
 	MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(userID string) bool
 	MetricsGeneratorProcessorServiceGraphsEnableClientServerPrefix(userID string) bool
+	MetricsGeneratorProcessorSpanMetricsTargetInfoExcludedDimensions(userID string) []string
 }
 
 var _ metricsGeneratorOverrides = (overrides.Interface)(nil)

modules/generator/overrides_test.go

@@ -8,23 +8,24 @@ import (
 )
 
 type mockOverrides struct {
-	processors                            map[string]struct{}
-	serviceGraphsHistogramBuckets         []float64
-	serviceGraphsDimensions               []string
-	serviceGraphsPeerAttributes           []string
-	serviceGraphsEnableClientServerPrefix bool
-	spanMetricsHistogramBuckets           []float64
-	spanMetricsDimensions                 []string
-	spanMetricsIntrinsicDimensions        map[string]bool
-	spanMetricsFilterPolicies             []filterconfig.FilterPolicy
-	spanMetricsDimensionMappings          []sharedconfig.DimensionMappings
-	spanMetricsEnableTargetInfo           bool
-	localBlocksMaxLiveTraces              uint64
-	localBlocksMaxBlockDuration           time.Duration
-	localBlocksMaxBlockBytes              uint64
-	localBlocksFlushCheckPeriod           time.Duration
-	localBlocksTraceIdlePeriod            time.Duration
-	localBlocksCompleteBlockTimeout       time.Duration
+	processors                              map[string]struct{}
+	serviceGraphsHistogramBuckets           []float64
+	serviceGraphsDimensions                 []string
+	serviceGraphsPeerAttributes             []string
+	serviceGraphsEnableClientServerPrefix   bool
+	spanMetricsHistogramBuckets             []float64
+	spanMetricsDimensions                   []string
+	spanMetricsIntrinsicDimensions          map[string]bool
+	spanMetricsFilterPolicies               []filterconfig.FilterPolicy
+	spanMetricsDimensionMappings            []sharedconfig.DimensionMappings
+	spanMetricsEnableTargetInfo             bool
+	spanMetricsTargetInfoExcludedDimensions []string
+	localBlocksMaxLiveTraces                uint64
+	localBlocksMaxBlockDuration             time.Duration
+	localBlocksMaxBlockBytes                uint64
+	localBlocksFlushCheckPeriod             time.Duration
+	localBlocksTraceIdlePeriod              time.Duration
+	localBlocksCompleteBlockTimeout         time.Duration
 }
 
 var _ metricsGeneratorOverrides = (*mockOverrides)(nil)
@@ -110,3 +111,7 @@ func (m *mockOverrides) MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(str
 func (m *mockOverrides) MetricsGeneratorProcessorServiceGraphsEnableClientServerPrefix(string) bool {
 	return m.serviceGraphsEnableClientServerPrefix
 }
+
+func (m *mockOverrides) MetricsGeneratorProcessorSpanMetricsTargetInfoExcludedDimensions(string) []string {
+	return m.spanMetricsTargetInfoExcludedDimensions
+}

modules/generator/processor/spanmetrics/config.go

@@ -45,6 +45,9 @@ type Config struct {
 
 	// FilterPolicies is a list of policies that will be applied to spans for inclusion or exlusion.
 	FilterPolicies []filterconfig.FilterPolicy `yaml:"filter_policies"`
+
+	// Allow user to specify labels they want to drop from target_info
+	TargetInfoExcludedDimensions []string `yaml:"target_info_excluded_dimensions"`
 }
 
 func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) {

modules/generator/processor/spanmetrics/spanmetrics.go

@@ -123,7 +123,7 @@ func (p *Processor) aggregateMetrics(resourceSpans []*v1_trace.ResourceSpans) {
 		resourceValues := make([]string, 0)
 
 		if p.Cfg.EnableTargetInfo {
-			resourceLabels, resourceValues = processor_util.GetTargetInfoAttributesValues(rs.Resource.Attributes)
+			resourceLabels, resourceValues = processor_util.GetTargetInfoAttributesValues(rs.Resource.Attributes, p.Cfg.TargetInfoExcludedDimensions)
 		}
 		for _, ils := range rs.ScopeSpans {
 			for _, span := range ils.Spans {
@@ -234,7 +234,7 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, jobName string, inst
 
 		// only register target info if at least (job or instance) AND one other attribute are present
 		if resourceAttributesCount > 0 && len(targetInfoLabels) > resourceAttributesCount {
-			p.spanMetricsTargetInfo.Set(targetInfoRegistryLabelValues, 1)
+			p.spanMetricsTargetInfo.SetForTargetInfo(targetInfoRegistryLabelValues, 1)
 		}
 	}
 }
@@ -250,10 +250,5 @@ func sanitizeLabelNameWithCollisions(name string) string {
 }
 
 func isIntrinsicDimension(name string) bool {
-	return name == dimJob ||
-		name == dimSpanName ||
-		name == dimSpanKind ||
-		name == dimStatusCode ||
-		name == dimStatusMessage ||
-		name == dimInstance
+	return processor_util.Contains(name, []string{dimJob, dimSpanName, dimSpanKind, dimStatusCode, dimStatusMessage, dimInstance})
 }

modules/generator/processor/spanmetrics/spanmetrics_test.go

@@ -623,6 +623,67 @@ func TestTargetInfoDisabled(t *testing.T) {
 	assert.Equal(t, false, targetInfoExist)
 }
 
+func TestTargetInfoWithExclusion(t *testing.T) {
+	// no service.name = no job label/dimension
+	// if the only labels are job and instance then target_info should not exist
+	testRegistry := registry.NewTestRegistry()
+	filteredSpansCounter := metricSpansDiscarded.WithLabelValues("test-tenant", "filtered")
+
+	cfg := Config{}
+	cfg.RegisterFlagsAndApplyDefaults("", nil)
+	cfg.EnableTargetInfo = true
+	cfg.TargetInfoExcludedDimensions = []string{"container", "container.id"}
+	cfg.HistogramBuckets = []float64{0.5, 1}
+
+	p, err := New(cfg, testRegistry, filteredSpansCounter)
+	require.NoError(t, err)
+	defer p.Shutdown(context.Background())
+
+	// TODO give these spans some duration so we can verify latencies are recorded correctly, in fact we should also test with various span names etc.
+	batch := test.MakeBatch(10, nil)
+
+	// add instance
+	batch.Resource.Attributes = append(batch.Resource.Attributes, &common_v1.KeyValue{
+		Key:   "service.instance.id",
+		Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "abc-instance-id-test-def"}},
+	})
+
+	// add additional source attributes
+	batch.Resource.Attributes = append(batch.Resource.Attributes, &common_v1.KeyValue{
+		Key:   "cluster",
+		Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "eu-west-0"}},
+	})
+
+	batch.Resource.Attributes = append(batch.Resource.Attributes, &common_v1.KeyValue{
+		Key:   "ip",
+		Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "1.1.1.1"}},
+	})
+
+	// add attribute for labels that we want to drop
+	batch.Resource.Attributes = append(batch.Resource.Attributes, &common_v1.KeyValue{
+		Key:   "container",
+		Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "test-service"}},
+	})
+
+	batch.Resource.Attributes = append(batch.Resource.Attributes, &common_v1.KeyValue{
+		Key:   "container.id",
+		Value: &common_v1.AnyValue{Value: &common_v1.AnyValue_StringValue{StringValue: "xyz123"}},
+	})
+
+	p.PushSpans(context.Background(), &tempopb.PushSpansRequest{Batches: []*trace_v1.ResourceSpans{batch}})
+
+	fmt.Println(testRegistry)
+
+	lbls := labels.FromMap(map[string]string{
+		"job":      "test-service",
+		"instance": "abc-instance-id-test-def",
+		"cluster":  "eu-west-0",
+		"ip":       "1.1.1.1",
+	})
+
+	assert.Equal(t, 1.0, testRegistry.Query("traces_target_info", lbls))
+}
+
 func TestTargetInfoSanitizeLabelName(t *testing.T) {
 	// no service.name = no job label/dimension
 	// if the only labels are job and instance then target_info should not exist

modules/generator/processor/util/util.go

@@ -59,18 +59,27 @@ func GetJobValue(attributes []*v1_common.KeyValue) string {
 	return namespace + svName
 }
 
-func GetTargetInfoAttributesValues(attributes []*v1_common.KeyValue) ([]string, []string) {
+func GetTargetInfoAttributesValues(attributes []*v1_common.KeyValue, exclude []string) ([]string, []string) {
 	keys := make([]string, 0)
 	values := make([]string, 0)
 	for _, attrs := range attributes {
 		// ignoring job and instance
 		key := attrs.Key
 		value := tempo_util.StringifyAnyValue(attrs.Value)
-		if key != "service.name" && key != "service.namespace" && key != "service.instance.id" {
+		if key != "service.name" && key != "service.namespace" && key != "service.instance.id" && !Contains(key, exclude) {
 			keys = append(keys, key)
 			values = append(values, value)
 		}
 	}
 
 	return keys, values
 }
+
+func Contains(key string, list []string) bool {
+	for _, exclude := range list {
+		if key == exclude {
+			return true
+		}
+	}
+	return false
+}

modules/generator/registry/gauge.go

@@ -58,21 +58,29 @@ func newGauge(name string, onAddSeries func(uint32) bool, onRemoveSeries func(co
 }
 
 func (g *gauge) Set(labelValueCombo *LabelValueCombo, value float64) {
-	g.updateSeries(labelValueCombo, value, set)
+	g.updateSeries(labelValueCombo, value, set, true)
 }
 
 func (g *gauge) Inc(labelValueCombo *LabelValueCombo, value float64) {
-	g.updateSeries(labelValueCombo, value, add)
+	g.updateSeries(labelValueCombo, value, add, true)
 }
 
-func (g *gauge) updateSeries(labelValueCombo *LabelValueCombo, value float64, operation string) {
+func (g *gauge) SetForTargetInfo(labelValueCombo *LabelValueCombo, value float64) {
+	g.updateSeries(labelValueCombo, value, set, false)
+}
+
+func (g *gauge) updateSeries(labelValueCombo *LabelValueCombo, value float64, operation string, updateIfAlreadyExist bool) {
 	hash := labelValueCombo.getHash()
 
 	g.seriesMtx.RLock()
 	s, ok := g.series[hash]
 	g.seriesMtx.RUnlock()
 
 	if ok {
+		// target_info will always be 1 so if the series exists, we don't need to go through this loop
+		if !updateIfAlreadyExist {
+			return
+		}
 		g.updateSeriesValue(s, value, operation)
 		return
 	}

modules/generator/registry/interface.go

@@ -28,6 +28,7 @@ type Gauge interface {
 	// Set sets the Gauge to an arbitrary value.
 	Set(labelValueCombo *LabelValueCombo, value float64)
 	Inc(labelValueCombo *LabelValueCombo, value float64)
+	SetForTargetInfo(labelValueCombo *LabelValueCombo, value float64)
 }
 
 // LabelValueCombo is a wrapper around a slice of label values. It has the ability to cache the hash of

modules/generator/registry/test.go

@@ -134,6 +134,10 @@ func (t *testGauge) Set(labelValueCombo *LabelValueCombo, value float64) {
 	t.registry.setMetric(t.name, lbls, value)
 }
 
+func (t *testGauge) SetForTargetInfo(labelValueCombo *LabelValueCombo, value float64) {
+	t.Set(labelValueCombo, value)
+}
+
 type testHistogram struct {
 	nameSum    string
 	nameCount  string

modules/overrides/interface.go

@@ -54,6 +54,7 @@ type Interface interface {
 	MetricsGeneratorProcessorSpanMetricsDimensionMappings(userID string) []sharedconfig.DimensionMappings
 	MetricsGeneratorProcessorSpanMetricsEnableTargetInfo(userID string) bool
 	MetricsGeneratorProcessorServiceGraphsEnableClientServerPrefix(userID string) bool
+	MetricsGeneratorProcessorSpanMetricsTargetInfoExcludedDimensions(userID string) []string
 	BlockRetention(userID string) time.Duration
 	MaxSearchDuration(userID string) time.Duration
 	DedicatedColumns(userID string) backend.DedicatedColumns