diff --git a/client/metrics.go b/client/metrics.go
index f3c47d7e787..a11362669b3 100644
--- a/client/metrics.go
+++ b/client/metrics.go
@@ -159,11 +159,13 @@ var (
 	cmdFailedDurationUpdateServiceGCSafePoint prometheus.Observer
 	cmdFailedDurationLoadKeyspace             prometheus.Observer
 	cmdFailedDurationUpdateKeyspaceState      prometheus.Observer
-	requestDurationTSO                        prometheus.Observer
 	cmdFailedDurationGet                      prometheus.Observer
 	cmdFailedDurationPut                      prometheus.Observer
 	cmdFailedDurationUpdateGCSafePointV2      prometheus.Observer
 	cmdFailedDurationUpdateServiceSafePointV2 prometheus.Observer
+
+	requestDurationTSO       prometheus.Observer
+	requestFailedDurationTSO prometheus.Observer
 )
 
 func initCmdDurations() {
@@ -207,11 +209,13 @@ func initCmdDurations() {
 	cmdFailedDurationUpdateServiceGCSafePoint = cmdFailedDuration.WithLabelValues("update_service_gc_safe_point")
 	cmdFailedDurationLoadKeyspace = cmdFailedDuration.WithLabelValues("load_keyspace")
 	cmdFailedDurationUpdateKeyspaceState = cmdFailedDuration.WithLabelValues("update_keyspace_state")
-	requestDurationTSO = requestDuration.WithLabelValues("tso")
 	cmdFailedDurationGet = cmdFailedDuration.WithLabelValues("get")
 	cmdFailedDurationPut = cmdFailedDuration.WithLabelValues("put")
 	cmdFailedDurationUpdateGCSafePointV2 = cmdFailedDuration.WithLabelValues("update_gc_safe_point_v2")
 	cmdFailedDurationUpdateServiceSafePointV2 = cmdFailedDuration.WithLabelValues("update_service_safe_point_v2")
+
+	requestDurationTSO = requestDuration.WithLabelValues("tso")
+	requestFailedDurationTSO = requestDuration.WithLabelValues("tso-failed")
 }
 
 func registerMetrics() {
diff --git a/client/tso_stream.go b/client/tso_stream.go
index dd5b9422aae..9c4d78dfe18 100644
--- a/client/tso_stream.go
+++ b/client/tso_stream.go
@@ -141,7 +141,9 @@ func (s *pdTSOStream) processRequests(
 	}
 	tsoBatchSendLatency.Observe(time.Since(batchStartTime).Seconds())
 	resp, err := s.stream.Recv()
+	duration := time.Since(start).Seconds()
 	if err != nil {
+		requestFailedDurationTSO.Observe(duration)
 		if err == io.EOF {
 			err = errs.ErrClientTSOStreamClosed
 		} else {
@@ -149,7 +151,7 @@ func (s *pdTSOStream) processRequests(
 		}
 		return
 	}
-	requestDurationTSO.Observe(time.Since(start).Seconds())
+	requestDurationTSO.Observe(duration)
 	tsoBatchSize.Observe(float64(count))
 
 	if resp.GetCount() != uint32(count) {
@@ -197,7 +199,9 @@ func (s *tsoTSOStream) processRequests(
 	}
 	tsoBatchSendLatency.Observe(time.Since(batchStartTime).Seconds())
 	resp, err := s.stream.Recv()
+	duration := time.Since(start).Seconds()
 	if err != nil {
+		requestFailedDurationTSO.Observe(duration)
 		if err == io.EOF {
 			err = errs.ErrClientTSOStreamClosed
 		} else {
@@ -205,7 +209,7 @@ func (s *tsoTSOStream) processRequests(
 		}
 		return
 	}
-	requestDurationTSO.Observe(time.Since(start).Seconds())
+	requestDurationTSO.Observe(duration)
 	tsoBatchSize.Observe(float64(count))
 
 	if resp.GetCount() != uint32(count) {