Implement configuration option for the querier to only hit the ingest…

…ers based on the trace id hashes instead of all healthy ingesters (grafana#1484) * metrics-generator: don't inject X-Scope-OrgID header for single-tenant setups (grafana#1417) * metrics-generator: don't inject X-Scope-OrgID header for single-tenant setups * Update CHANGELOG.md * Fix compactor missing compaction_objects_combined_total metric and not honoring max_bytes_per_trace (grafana#1420) * Fix compactor to record compaction_objects_combined_total and honor max_bytes_per_trace again * changelog * distributor: prevent panics when concurrently calling to forwarder's… (grafana#1422) * distributor: prevent panics when concurrently calling to forwarder's queueManager * Update PR number * Pin k6 version to v0.37.0 * update querier to get ingester instance based on trace id * rebase with main * added change log for the config * update config name as per PR comments * Doc tweak Signed-off-by: Joe Elliott <[email protected]> Co-authored-by: Koenraad Verheyden <[email protected]> Co-authored-by: Martin Disibio <[email protected]> Co-authored-by: Mario <[email protected]> Co-authored-by: Joe Elliott <[email protected]>
anovateam · Jun 13, 2022 · cd21a9a · cd21a9a
1 parent b59263c
commit cd21a9a
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,7 @@ Additionally, default label `span_status` is renamed to `status_code`.
 * [BUGFIX] Fix nil pointer panic when the trace by id path errors. [#1441](https://github.com/grafana/tempo/pull/1441) (@joe-elliott)
 * [BUGFIX] Update tempo microservices Helm values example which missed the 'enabled' key for thriftHttp. [#1472](https://github.com/grafana/tempo/pull/1472) (@hajowieland)
 * [BUGFIX] Fix race condition in forwarder overrides loop. [1468](https://github.com/grafana/tempo/pull/1468) (@mapno)
+* [ENHANCEMENT] Add a config to query single ingester instance based on trace id hash for Trace By ID API. (1484)[https://github.com/grafana/tempo/pull/1484] (@sagarwala, @bikashmishra100, @ashwinidulams)
 
 ## v1.4.1 / 2022-05-05
 

diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md
@@ -349,6 +349,13 @@ querier:
     # not distinguish between the types of queries.
     [max_concurrent_queries: <int> | default = 5]
 
+    # The query frontend sents sharded requests to ingesters and querier (/api/traces/<id>)
+    # By default, all healthy ingesters are queried for the trace id.
+    # When true the querier will hash the trace id in the same way that distributors do and then 
+    # only query those ingesters who own the trace id hash as determined by the ring.
+    # If this parameter is set, the number of 404s could increase during rollout or scaling of ingesters.
+    [query_relevant_ingesters: <bool> | default = false]
+
     search:
         # Timeout for search requests    
         [query_timeout: <duration> | default = 30s]

diff --git a/modules/querier/config.go b/modules/querier/config.go
@@ -17,6 +17,7 @@ type Config struct {
 	ExtraQueryDelay         time.Duration `yaml:"extra_query_delay,omitempty"`
 	MaxConcurrentQueries    int           `yaml:"max_concurrent_queries"`
 	Worker                  worker.Config `yaml:"frontend_worker"`
+	QueryRelevantIngesters  bool          `yaml:"query_relevant_ingesters"`
 }
 
 type SearchConfig struct {
@@ -30,6 +31,7 @@ type SearchConfig struct {
 // RegisterFlagsAndApplyDefaults register flags.
 func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	cfg.TraceLookupQueryTimeout = 10 * time.Second
+	cfg.QueryRelevantIngesters = false
 	cfg.ExtraQueryDelay = 0
 	cfg.MaxConcurrentQueries = 5
 	cfg.Search.PreferSelf = 2

diff --git a/modules/querier/querier.go b/modules/querier/querier.go
@@ -184,7 +184,14 @@ func (q *Querier) FindTraceByID(ctx context.Context, req *tempopb.TraceByIDReque
 	combiner := trace.NewCombiner()
 	var spanCount, spanCountTotal, traceCountTotal int
 	if req.QueryMode == QueryModeIngesters || req.QueryMode == QueryModeAll {
-		replicationSet, err := q.ring.GetReplicationSetForOperation(ring.Read)
+		var replicationSet ring.ReplicationSet
+		var err error
+		if q.cfg.QueryRelevantIngesters {
+			traceKey := util.TokenFor(userID, req.TraceID)
+			replicationSet, err = q.ring.Get(traceKey, ring.Read, nil, nil, nil)
+		} else {
+			replicationSet, err = q.ring.GetReplicationSetForOperation(ring.Read)
+		}
 		if err != nil {
 			return nil, errors.Wrap(err, "error finding ingesters in Querier.FindTraceByID")
 		}