diff --git a/go/vt/discovery/healthcheck.go b/go/vt/discovery/healthcheck.go index d57d568db40..7bb98fb63cd 100644 --- a/go/vt/discovery/healthcheck.go +++ b/go/vt/discovery/healthcheck.go @@ -39,6 +39,7 @@ import ( "fmt" "hash/crc32" "net/http" + "runtime" "sort" "strings" "sync" @@ -647,6 +648,12 @@ func (hc *HealthCheckImpl) Unsubscribe(c chan *TabletHealth) { delete(hc.subscribers, c) } +var printStack = sync.OnceFunc(func() { + buf := make([]byte, 10240) // Allocate buffer large enough + n := runtime.Stack(buf, true) + fmt.Printf("All Goroutines Stack Trace:\n%s\n", buf[:n]) +}) + func (hc *HealthCheckImpl) broadcast(th *TabletHealth) { hc.subMu.Lock() defer hc.subMu.Unlock() @@ -657,6 +664,8 @@ func (hc *HealthCheckImpl) broadcast(th *TabletHealth) { // If the channel is full, we drop the message. hcChannelFullCounter.Add(1) log.Warningf("HealthCheck broadcast channel is full for %v, dropping message for %s", subscriber, topotools.TabletIdent(th.Tablet)) + // Print the stack trace only once. + printStack() } } }