diff --git a/cmd/litefs/mount_linux.go b/cmd/litefs/mount_linux.go index dee0c74..5662a6a 100644 --- a/cmd/litefs/mount_linux.go +++ b/cmd/litefs/mount_linux.go @@ -359,6 +359,10 @@ func (c *MountCommand) initStore(ctx context.Context) error { if err := c.initStoreBackupClient(ctx); err != nil { return err } + + // Initialize as a singleton so we can automatically collect metrics. + litefs.GlobalStore.Store(c.Store) + return nil } diff --git a/db.go b/db.go index ffe8c7f..c6072af 100644 --- a/db.go +++ b/db.go @@ -3602,7 +3602,7 @@ var ( }, []string{"db"}) dbLatencySecondsMetricVec = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "litefs_db_latency_seconds", + Name: "litefs_db_lag_seconds", Help: "Latency between generating an LTX file and consuming it.", }, []string{"db"}) ) diff --git a/store.go b/store.go index fbf6339..b041cdc 100644 --- a/store.go +++ b/store.go @@ -45,8 +45,15 @@ const ( DefaultBackupFullSyncInterval = 10 * time.Second ) +const ( + MetricsMonitorInterval = 1 * time.Second +) + var ErrStoreClosed = fmt.Errorf("store closed") +// GlobalStore represents a single store used for metrics collection. +var GlobalStore atomic.Value + // Store represents a collection of databases. type Store struct { mu sync.Mutex @@ -1595,6 +1602,18 @@ func (s *Store) setPrimaryTimestamp(ts int64) { } } +// Lag returns the number of seconds that the local instance is lagging +// behind the primary node. Returns 0 if the node is the primary or if the +// node is not marked as ready yet. +func (s *Store) Lag() time.Duration { + switch ts := s.PrimaryTimestamp(); ts { + case 0, -1: + return 0 // primary or not ready + default: + return time.Duration(time.Now().UnixMilli()-ts) * time.Millisecond + } +} + // Expvar returns a variable for debugging output. func (s *Store) Expvar() expvar.Var { return (*StoreVar)(s) } @@ -1821,4 +1840,14 @@ var ( Name: "litefs_subscriber_count", Help: "Number of connected subscribers", }) + + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "litefs_lag_seconds", + Help: "Lag behind the primary node, in seconds", + }, func() float64 { + if s := GlobalStore.Load(); s != nil { + return s.(*Store).Lag().Seconds() + } + return 0 + }) )