client: make TSO client request duration include failed requests (#8410)

ref #8281 Make TSO client request duration include failed requests. Signed-off-by: JmPotato <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
tikv · Jul 18, 2024 · c65577c · c65577c
1 parent 3330a44
commit c65577c
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/client/metrics.go b/client/metrics.go
@@ -159,11 +159,13 @@ var (
 	cmdFailedDurationUpdateServiceGCSafePoint prometheus.Observer
 	cmdFailedDurationLoadKeyspace             prometheus.Observer
 	cmdFailedDurationUpdateKeyspaceState      prometheus.Observer
-	requestDurationTSO                        prometheus.Observer
 	cmdFailedDurationGet                      prometheus.Observer
 	cmdFailedDurationPut                      prometheus.Observer
 	cmdFailedDurationUpdateGCSafePointV2      prometheus.Observer
 	cmdFailedDurationUpdateServiceSafePointV2 prometheus.Observer
+
+	requestDurationTSO       prometheus.Observer
+	requestFailedDurationTSO prometheus.Observer
 )
 
 func initCmdDurations() {
@@ -207,11 +209,13 @@ func initCmdDurations() {
 	cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point")
 	cmdFailedDurationLoadKeyspace = cmdFailedDuration.WithLabelValues("load_keyspace")
 	cmdFailedDurationUpdateKeyspaceState = cmdFailedDuration.WithLabelValues("update_keyspace_state")
-	requestDurationTSO = requestDuration.WithLabelValues("tso")
 	cmdFailedDurationGet = cmdFailedDuration.WithLabelValues("get")
 	cmdFailedDurationPut = cmdFailedDuration.WithLabelValues("put")
 	cmdFailedDurationUpdateGCSafePointV2 = cmdFailedDuration.WithLabelValues("update_gc_safe_point_v2")
 	cmdFailedDurationUpdateServiceSafePointV2 = cmdFailedDuration.WithLabelValues("update_service_safe_point_v2")
+
+	requestDurationTSO = requestDuration.WithLabelValues("tso")
+	requestFailedDurationTSO = requestDuration.WithLabelValues("tso-failed")
 }
 
 func registerMetrics() {

diff --git a/client/tso_stream.go b/client/tso_stream.go
@@ -141,15 +141,17 @@ func (s *pdTSOStream) processRequests(
 	}
 	tsoBatchSendLatency.Observe(time.Since(batchStartTime).Seconds())
 	resp, err := s.stream.Recv()
+	duration := time.Since(start).Seconds()
 	if err != nil {
+		requestFailedDurationTSO.Observe(duration)
 		if err == io.EOF {
 			err = errs.ErrClientTSOStreamClosed
 		} else {
 			err = errors.WithStack(err)
 		}
 		return
 	}
-	requestDurationTSO.Observe(time.Since(start).Seconds())
+	requestDurationTSO.Observe(duration)
 	tsoBatchSize.Observe(float64(count))
 
 	if resp.GetCount() != uint32(count) {
@@ -197,15 +199,17 @@ func (s *tsoTSOStream) processRequests(
 	}
 	tsoBatchSendLatency.Observe(time.Since(batchStartTime).Seconds())
 	resp, err := s.stream.Recv()
+	duration := time.Since(start).Seconds()
 	if err != nil {
+		requestFailedDurationTSO.Observe(duration)
 		if err == io.EOF {
 			err = errs.ErrClientTSOStreamClosed
 		} else {
 			err = errors.WithStack(err)
 		}
 		return
 	}
-	requestDurationTSO.Observe(time.Since(start).Seconds())
+	requestDurationTSO.Observe(duration)
 	tsoBatchSize.Observe(float64(count))
 
 	if resp.GetCount() != uint32(count) {