From ac9537f2774cefd9e2b60ba202ee468d14151096 Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Wed, 22 Mar 2023 17:08:37 +0000 Subject: [PATCH 1/6] add new flags and exporter opts for defaulting the exemplar project_id label --- pkg/export/export.go | 6 ++++++ pkg/export/setup/setup.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/export/export.go b/pkg/export/export.go index 193771ee36..7b3946b9a9 100644 --- a/pkg/export/export.go +++ b/pkg/export/export.go @@ -170,6 +170,12 @@ type ExporterOpts struct { Location string Cluster string + // If true, automatically populate the project id in an exemplar labelset. + PopulateExemplarProjectID bool + // Automatically populate exemplar labelset with this project id + // if PopulateExemplarProjectID is true. + ExemplarProjectID string + // A list of metric matchers. Only Prometheus time series satisfying at // least one of the matchers are exported. // This option matches the semantics of the Prometheus federation match[] diff --git a/pkg/export/setup/setup.go b/pkg/export/setup/setup.go index 54435497ac..1d7dce529b 100644 --- a/pkg/export/setup/setup.go +++ b/pkg/export/setup/setup.go @@ -133,6 +133,12 @@ func FromFlags(a *kingpin.Application, userAgentProduct string) func(log.Logger, a.Flag("export.label.project-id", fmt.Sprintf("Default project ID set for all exported data. Prefer setting the external label %q in the Prometheus configuration if not using the auto-discovered default.", export.KeyProjectID)). Default(opts.ProjectID).StringVar(&opts.ProjectID) + a.Flag("export.exemplars.populate-exemplar-project-id", "If true, automatically populate the 'project_id' label in an exemplar labelset."). + Default("true").BoolVar(&opts.PopulateExemplarProjectID) + + a.Flag("export.exemplars.project-id", "If non-empty and export.exemplars.populate-exemplar-project-id is true, use this project_id as the project_id for exemplar labelsets."). + Default(opts.ProjectID).StringVar(&opts.ExemplarProjectID) + a.Flag("export.user-agent-mode", fmt.Sprintf("Mode for user agent used for requests against the GCM API. Valid values are %q, %q, %q, %q or %q.", UAModeGKE, UAModeKubectl, UAModeAVMW, UAModeABM, UAModeUnspecified)). Default("unspecified").EnumVar(&opts.UserAgentMode, UAModeUnspecified, UAModeGKE, UAModeKubectl, UAModeAVMW, UAModeABM) From 27d44cd4781785a2dbac6f2c24dda87e4f110fef Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Thu, 23 Mar 2023 12:11:02 +0000 Subject: [PATCH 2/6] add exemplarOpts to autopopulate the project_id label for exemplars if requested. --- pkg/export/export.go | 6 +++++- pkg/export/transform.go | 28 +++++++++++++++++++++------- pkg/export/transform_test.go | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/export/export.go b/pkg/export/export.go index 7b3946b9a9..fac53291ef 100644 --- a/pkg/export/export.go +++ b/pkg/export/export.go @@ -426,7 +426,11 @@ func (e *Exporter) Export(metadata MetadataFunc, batch []record.RefSample, exemp samplesDropped.WithLabelValues("no-ha-range").Add(float64(batchSize)) return } - builder := newSampleBuilder(e.seriesCache) + exOpts := &exemplarOpts{ + autoPopulateProjectID: e.opts.PopulateExemplarProjectID, + exemplarProjectID: e.opts.ExemplarProjectID, + } + builder := newSampleBuilder(e.seriesCache, exOpts) defer builder.close() exemplarsExported.Add(float64(len(exemplarMap))) diff --git a/pkg/export/transform.go b/pkg/export/transform.go index 2fe8989637..876599ae46 100644 --- a/pkg/export/transform.go +++ b/pkg/export/transform.go @@ -69,15 +69,24 @@ func discardExemplarIncIfExists(series storage.SeriesRef, exemplars map[storage. } } +type exemplarOpts struct { + autoPopulateProjectID bool + exemplarProjectID string +} type sampleBuilder struct { series *seriesCache dists map[uint64]*distribution + exOpts *exemplarOpts } -func newSampleBuilder(c *seriesCache) *sampleBuilder { +func newSampleBuilder(c *seriesCache, exOpts *exemplarOpts) *sampleBuilder { + if exOpts == nil { + exOpts = &exemplarOpts{} + } return &sampleBuilder{ series: c, dists: make(map[uint64]*distribution, 128), + exOpts: exOpts, } } @@ -270,7 +279,7 @@ func (d *distribution) Swap(i, j int) { d.values[i], d.values[j] = d.values[j], d.values[i] } -func (d *distribution) build(lset labels.Labels) (*distribution_pb.Distribution, error) { +func (d *distribution) build(lset labels.Labels, exOpts *exemplarOpts) (*distribution_pb.Distribution, error) { // The exposition format in general requires buckets to be in-order but we observed // some cases in the wild where this was not the case. // Ensure sorting here to gracefully handle those cases sometimes. This cannot handle @@ -354,7 +363,7 @@ func (d *distribution) build(lset labels.Labels) (*distribution_pb.Distribution, }, }, BucketCounts: values, - Exemplars: buildExemplars(d.exemplars), + Exemplars: buildExemplars(d.exemplars, exOpts), } return dp, nil } @@ -467,7 +476,7 @@ Loop: if !dist.complete() { continue } - dp, err := dist.build(e.lset) + dp, err := dist.build(e.lset, b.exOpts) if err != nil { return nil, 0, samples[consumed:], err } @@ -482,7 +491,7 @@ Loop: return nil, 0, samples[consumed:], nil } -func buildExemplars(exemplars []record.RefExemplar) []*distribution_pb.Distribution_Exemplar { +func buildExemplars(exemplars []record.RefExemplar, exOpts *exemplarOpts) []*distribution_pb.Distribution_Exemplar { // The exemplars field of a distribution value field must be in increasing order of value // (https://cloud.google.com/monitoring/api/ref_v3/rpc/google.api#distribution) -- let's sort them. sort.Slice(exemplars, func(i, j int) bool { @@ -490,7 +499,7 @@ func buildExemplars(exemplars []record.RefExemplar) []*distribution_pb.Distribut }) var result []*distribution_pb.Distribution_Exemplar for _, pex := range exemplars { - attachments := buildExemplarAttachments(pex.Labels) + attachments := buildExemplarAttachments(pex.Labels, exOpts) result = append(result, &distribution_pb.Distribution_Exemplar{ Value: pex.V, Timestamp: getTimestamp(pex.T), @@ -511,7 +520,7 @@ func buildExemplars(exemplars []record.RefExemplar) []*distribution_pb.Distribut // This is to maintain comptability with CloudTrace. // Note that the project_id needs to be the project_id where the span was written. // This may not necessarily be the same project_id where the metric was written. -func buildExemplarAttachments(lset labels.Labels) []*anypb.Any { +func buildExemplarAttachments(lset labels.Labels, exOpts *exemplarOpts) []*anypb.Any { var projectID, spanID, traceID string var attachments []*anypb.Any droppedLabels := make(map[string]string) @@ -526,6 +535,11 @@ func buildExemplarAttachments(lset labels.Labels) []*anypb.Any { droppedLabels[label.Name] = label.Value } } + // If the project_id is not in the labelset, and autoPopulateProjectID is true, + // autopopulate projectID, but only if spanID and traceID are set. + if projectID == "" && spanID != "" && traceID != "" && exOpts.autoPopulateProjectID { + projectID = exOpts.exemplarProjectID + } if projectID != "" && spanID != "" && traceID != "" { spanCtx, err := anypb.New(&monitoring_pb.SpanContext{ SpanName: fmt.Sprintf(spanContextFormat, projectID, traceID, spanID), diff --git a/pkg/export/transform_test.go b/pkg/export/transform_test.go index dfbb351661..6756ad1ccf 100644 --- a/pkg/export/transform_test.go +++ b/pkg/export/transform_test.go @@ -1489,7 +1489,7 @@ func TestSampleBuilder(t *testing.T) { var result []*monitoring_pb.TimeSeries for i, batch := range c.samples { - b := newSampleBuilder(cache) + b := newSampleBuilder(cache, nil) for k := 0; len(batch) > 0; k++ { var exemplars map[storage.SeriesRef]record.RefExemplar From fa7f1b7744e99276928eab3395acd73a99acc32c Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Thu, 23 Mar 2023 13:52:45 +0000 Subject: [PATCH 3/6] add exemplarOpts to unit tests for transform_test.go --- pkg/export/transform_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/export/transform_test.go b/pkg/export/transform_test.go index 6756ad1ccf..1570424b2a 100644 --- a/pkg/export/transform_test.go +++ b/pkg/export/transform_test.go @@ -72,6 +72,7 @@ func TestSampleBuilder(t *testing.T) { matchers Matchers wantSeries []*monitoring_pb.TimeSeries wantFail bool + exOpts *exemplarOpts }{ { doc: "convert gauge", @@ -1489,7 +1490,7 @@ func TestSampleBuilder(t *testing.T) { var result []*monitoring_pb.TimeSeries for i, batch := range c.samples { - b := newSampleBuilder(cache, nil) + b := newSampleBuilder(cache, c.exOpts) for k := 0; len(batch) > 0; k++ { var exemplars map[storage.SeriesRef]record.RefExemplar From 4cd8efa4d443c5c5549fbca52ae0abf61e4f0f64 Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Fri, 24 Mar 2023 17:12:24 +0000 Subject: [PATCH 4/6] add unit test --- pkg/export/transform_test.go | 143 +++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/pkg/export/transform_test.go b/pkg/export/transform_test.go index 1570424b2a..49f270211e 100644 --- a/pkg/export/transform_test.go +++ b/pkg/export/transform_test.go @@ -1476,6 +1476,149 @@ func TestSampleBuilder(t *testing.T) { }, }, }, + { + doc: "convert histogram with exemplars project_id default", + metadata: testMetadataFunc(metricMetadataMap{ + "metric1": {Type: textparse.MetricTypeHistogram, Help: "metric1 help text"}, + "metric1_a_count": {Type: textparse.MetricTypeGauge, Help: "metric1_a_count help text"}, + }), + series: seriesMap{ + 1: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_sum"), + 2: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_count"), + 3: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_bucket", "le", "0.1"), + 4: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_bucket", "le", "0.5"), + 5: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_bucket", "le", "1"), + 6: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_bucket", "le", "2.5"), + 7: labels.FromStrings("job", "job1", "instance", "instance1", "__name__", "metric1_bucket", "le", "+Inf"), + }, + samples: [][]record.RefSample{ + // First sample set, should be skipped by reset handling. + // The buckets must be in ascending order for an individual histogram but otherwise + // no order or grouping constraints apply for series of a given histogram metric. + { + {Ref: 1, T: 1000, V: 55.1}, // hist1, sum + {Ref: 3, T: 1000, V: 2}, // hist1, 0.1 + {Ref: 4, T: 1000, V: 5}, // hist1, 0.5 + {Ref: 5, T: 1000, V: 6}, // hist1, 1 + {Ref: 6, T: 1000, V: 8}, // hist1, 2.5 + {Ref: 7, T: 1000, V: 10}, // hist1, inf + {Ref: 2, T: 1000, V: 10}, // hist1, count + }, + // Second sample set should actually be emitted. + { + // Second samples for histograms should produce a distribution. + {Ref: 3, T: 2000, V: 4}, // hist1, 0.1 + {Ref: 2, T: 2000, V: 21}, // hist1, count + {Ref: 1, T: 2000, V: 123.4}, // hist1, sum + {Ref: 4, T: 2000, V: 9}, // hist1, 0.5 + {Ref: 5, T: 2000, V: 11}, // hist1, 1 + {Ref: 6, T: 2000, V: 15}, // hist1, 2.5 + {Ref: 7, T: 2000, V: 21}, // hist1, inf + }, + }, + exemplars: []map[storage.SeriesRef]record.RefExemplar{ + // First sample set is skipped by reset handling. + {}, + { + 3: {Ref: 3, T: 1500, V: .099, Labels: labels.New( + labels.Label{Name: "trace_id", Value: "2"}, + labels.Label{Name: "span_id", Value: "3"}, + )}, + 4: {Ref: 4, T: 1500, V: .4, Labels: labels.New( + labels.Label{Name: "project_id", Value: "explicit-project-id"}, + labels.Label{Name: "trace_id", Value: "2"}, + labels.Label{Name: "span_id", Value: "3"}, + )}, + 5: {Ref: 5, T: 1500, V: .99}, + 6: {Ref: 6, T: 1500, V: 2}, + 7: {Ref: 7, T: 1500, V: 11}, + }, + }, + exOpts: &exemplarOpts{ + autoPopulateProjectID: true, + exemplarProjectID: "exemplar-project-id", + }, + wantSeries: []*monitoring_pb.TimeSeries{ + // 0: skipped by reset handling. + { // 1 + Resource: &monitoredres_pb.MonitoredResource{ + Type: "prometheus_target", + Labels: map[string]string{ + "project_id": "example-project", + "location": "europe", + "cluster": "foo-cluster", + "namespace": "", + "job": "job1", + "instance": "instance1", + }, + }, + Metric: &metric_pb.Metric{ + Type: "prometheus.googleapis.com/metric1/histogram", + Labels: map[string]string{}, + }, + MetricKind: metric_pb.MetricDescriptor_CUMULATIVE, + ValueType: metric_pb.MetricDescriptor_DISTRIBUTION, + Points: []*monitoring_pb.Point{{ + Interval: &monitoring_pb.TimeInterval{ + StartTime: ×tamp_pb.Timestamp{Seconds: 1}, + EndTime: ×tamp_pb.Timestamp{Seconds: 2}, + }, + Value: &monitoring_pb.TypedValue{ + Value: &monitoring_pb.TypedValue_DistributionValue{ + DistributionValue: &distribution_pb.Distribution{ + Count: 11, + Mean: 6.20909090909091, + SumOfSquaredDeviation: 270.301590909091, + BucketOptions: &distribution_pb.Distribution_BucketOptions{ + Options: &distribution_pb.Distribution_BucketOptions_ExplicitBuckets{ + ExplicitBuckets: &distribution_pb.Distribution_BucketOptions_Explicit{ + Bounds: []float64{0.1, 0.5, 1, 2.5}, + }, + }, + }, + BucketCounts: []int64{2, 2, 1, 2, 4}, + Exemplars: []*distribution_pb.Distribution_Exemplar{ + { + Value: .099, + Timestamp: ×tamp_pb.Timestamp{Seconds: 1, Nanos: 500000000}, + Attachments: []*anypb.Any{ + wrapAsAny(&monitoring_pb.SpanContext{ + SpanName: "projects/exemplar-project-id/traces/2/spans/3", + }), + wrapAsAny(&monitoring_pb.DroppedLabels{ + Label: map[string]string{"random": "4"}, + }), + }, + }, + { + Value: .4, + Timestamp: ×tamp_pb.Timestamp{Seconds: 1, Nanos: 500000000}, + Attachments: []*anypb.Any{ + wrapAsAny(&monitoring_pb.SpanContext{ + SpanName: "projects/explicit-project-id/traces/2/spans/3", + }), + }, + }, + { + Value: .99, + Timestamp: ×tamp_pb.Timestamp{Seconds: 1, Nanos: 500000000}, + }, + { + Value: 2, + Timestamp: ×tamp_pb.Timestamp{Seconds: 1, Nanos: 500000000}, + }, + { + Value: 11, + Timestamp: ×tamp_pb.Timestamp{Seconds: 1, Nanos: 500000000}, + }, + }, + }, + }, + }, + }}, + }, + }, + }, } for i, c := range cases { From af70e767fe3a6c4d79ae874c588f428551b96ffe Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Fri, 24 Mar 2023 17:17:10 +0000 Subject: [PATCH 5/6] fix unit test --- pkg/export/transform_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/export/transform_test.go b/pkg/export/transform_test.go index 49f270211e..1ddd7fddfb 100644 --- a/pkg/export/transform_test.go +++ b/pkg/export/transform_test.go @@ -1585,9 +1585,6 @@ func TestSampleBuilder(t *testing.T) { wrapAsAny(&monitoring_pb.SpanContext{ SpanName: "projects/exemplar-project-id/traces/2/spans/3", }), - wrapAsAny(&monitoring_pb.DroppedLabels{ - Label: map[string]string{"random": "4"}, - }), }, }, { From 1542c50839fa20b5419eaddffddb562823de7b7b Mon Sep 17 00:00:00 2001 From: Joshua Pollock Date: Tue, 28 Mar 2023 14:38:43 +0000 Subject: [PATCH 6/6] address flag comment --- pkg/export/setup/setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/export/setup/setup.go b/pkg/export/setup/setup.go index 1d7dce529b..ff99252d99 100644 --- a/pkg/export/setup/setup.go +++ b/pkg/export/setup/setup.go @@ -133,7 +133,7 @@ func FromFlags(a *kingpin.Application, userAgentProduct string) func(log.Logger, a.Flag("export.label.project-id", fmt.Sprintf("Default project ID set for all exported data. Prefer setting the external label %q in the Prometheus configuration if not using the auto-discovered default.", export.KeyProjectID)). Default(opts.ProjectID).StringVar(&opts.ProjectID) - a.Flag("export.exemplars.populate-exemplar-project-id", "If true, automatically populate the 'project_id' label in an exemplar labelset."). + a.Flag("export.exemplars.populate-exemplar-project-id", "If true, automatically populate the 'project_id' label in an exemplar labelset if a 'span_id' and 'trace_id' are also present."). Default("true").BoolVar(&opts.PopulateExemplarProjectID) a.Flag("export.exemplars.project-id", "If non-empty and export.exemplars.populate-exemplar-project-id is true, use this project_id as the project_id for exemplar labelsets.").