db: add metrics for num backing sstables and size

Pr just adds backing size/count into the Metrics struct and
grabs them from the version set. Needs loglock to grab these
as the fields are protected by the loglock.

db.go

@@ -1963,6 +1963,17 @@ func (d *DB) Metrics() *Metrics {
 
 	d.mu.versions.logLock()
 	metrics.private.manifestFileSize = uint64(d.mu.versions.manifest.Size())
+	metrics.Table.BackingTableCount = uint64(len(d.mu.versions.backingState.fileBackingMap))
+	metrics.Table.BackingTableSize = d.mu.versions.backingState.fileBackingSize
+	if invariants.Enabled {
+		var totalSize uint64
+		for _, backing := range d.mu.versions.backingState.fileBackingMap {
+			totalSize += backing.Size
+		}
+		if totalSize != metrics.Table.BackingTableSize {
+			panic("pebble: invalid backing table size accounting")
+		}
+	}
 	d.mu.versions.logUnlock()
 
 	metrics.LogWriter.FsyncLatency = d.mu.log.metrics.fsyncLatency

metrics.go

@@ -253,6 +253,10 @@ type Metrics struct {
 		ZombieSize uint64
 		// The count of zombie tables.
 		ZombieCount int64
+		// The count of the backing sstables.
+		BackingTableCount uint64
+		// The sum of the sizes of the all of the backing sstables.
+		BackingTableSize uint64
 	}
 
 	TableCache CacheMetrics
@@ -528,6 +532,10 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.Table.ZombieCount),
 		humanize.Bytes.Uint64(m.Table.ZombieSize))
 
+	w.Printf("Backing tables: %d (%s)\n",
+		redact.Safe(m.Table.BackingTableCount),
+		humanize.Bytes.Uint64(m.Table.BackingTableSize))
+
 	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
 		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
 			name,

metrics_test.go

@@ -5,6 +5,7 @@
 package pebble
 
 import (
+	"bytes"
 	"fmt"
 	"strconv"
 	"strings"
@@ -48,6 +49,8 @@ func exampleMetrics() Metrics {
 	m.Snapshots.Count = 4
 	m.Snapshots.EarliestSeqNum = 1024
 	m.Table.ZombieSize = 15
+	m.Table.BackingTableCount = 1
+	m.Table.BackingTableSize = 2 << 20
 	m.Table.ZombieCount = 16
 	m.TableCache.Size = 17
 	m.TableCache.Count = 18
@@ -183,6 +186,18 @@ func TestMetrics(t *testing.T) {
 			}
 			return ""
 
+		case "lsm":
+			d.mu.Lock()
+			s := d.mu.versions.currentVersion().String()
+			d.mu.Unlock()
+			return s
+
+		case "ingest-and-excise":
+			if err := runIngestAndExciseCmd(td, d, d.opts.FS); err != nil {
+				return err.Error()
+			}
+			return ""
+
 		case "iter-close":
 			if len(td.CmdArgs) != 1 {
 				return "iter-close <name>"
@@ -229,6 +244,31 @@ func TestMetrics(t *testing.T) {
 
 			return d.Metrics().StringForTests()
 
+		case "metrics-value":
+			// metrics-value confirms the value of a given metric. Note that there
+			// are some metrics which aren't deterministic and behave differently
+			// for invariant/non-invariant builds. An example of this is cache
+			// hit rates. Under invariant builds, the excising code will try
+			// to create iterators and confirm that the virtual sstable bounds
+			// are accurate. Reads on these iterators will change the cache hit
+			// rates.
+			lines := strings.Split(td.Input, "\n")
+			m := d.Metrics()
+			// TODO(bananabrick): Use reflection to pull the values associated
+			// with the metrics fields.
+			var buf bytes.Buffer
+			for i := range lines {
+				line := lines[i]
+				if line == "num-backing" {
+					buf.WriteString(fmt.Sprintf("%d\n", m.Table.BackingTableCount))
+				} else if line == "backing-size" {
+					buf.WriteString(fmt.Sprintf("%s\n", humanize.Bytes.Uint64(m.Table.BackingTableSize)))
+				} else {
+					panic(fmt.Sprintf("invalid field: %s", line))
+				}
+			}
+			return buf.String()
+
 		case "disk-usage":
 			return humanize.Bytes.Uint64(d.Metrics().DiskSpaceUsage()).String()
 

testdata/event_listener

@@ -286,6 +286,7 @@ Compactions: 1  estimated debt: 2.0KB  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 6 entries (1.1KB)  hit rate: 11.1%
 Table cache: 1 entries (800B)  hit rate: 40.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -383,6 +384,7 @@ Compactions: 1  estimated debt: 4.0KB  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (512KB)  zombie: 1 (512KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 12 entries (2.3KB)  hit rate: 14.3%
 Table cache: 1 entries (800B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%

testdata/ingest

@@ -49,6 +49,7 @@ Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
              default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 6 entries (1.2KB)  hit rate: 35.7%
 Table cache: 1 entries (800B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%

testdata/metrics

@@ -18,6 +18,7 @@ Compactions: 5  estimated debt: 6B  in progress: 2 (7B)
              default: 27  delete: 28  elision: 29  move: 30  read: 31  rewrite: 32  multi-level: 33
 MemTables: 12 (11B)  zombie: 14 (13B)
 Zombie tables: 16 (15B)
+Backing tables: 1 (2.0MB)
 Block cache: 2 entries (1B)  hit rate: 42.9%
 Table cache: 18 entries (17B)  hit rate: 48.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -63,6 +64,7 @@ Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
              default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 3 entries (556B)  hit rate: 0.0%
 Table cache: 1 entries (800B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -115,6 +117,7 @@ Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.3KB)
+Backing tables: 0 (0B)
 Block cache: 5 entries (1.1KB)  hit rate: 42.9%
 Table cache: 2 entries (1.6KB)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -152,6 +155,7 @@ Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 2 (1.3KB)
+Backing tables: 0 (0B)
 Block cache: 5 entries (1.1KB)  hit rate: 42.9%
 Table cache: 2 entries (1.6KB)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -186,6 +190,7 @@ Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 2 (512KB)
 Zombie tables: 1 (661B)
+Backing tables: 0 (0B)
 Block cache: 3 entries (556B)  hit rate: 42.9%
 Table cache: 1 entries (800B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -223,6 +228,7 @@ Compactions: 1  estimated debt: 0B  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 0 entries (0B)  hit rate: 42.9%
 Table cache: 0 entries (0B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -286,6 +292,7 @@ Compactions: 1  estimated debt: 2.9KB  in progress: 0 (0B)
              default: 1  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 0 entries (0B)  hit rate: 42.9%
 Table cache: 0 entries (0B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -333,6 +340,7 @@ Compactions: 2  estimated debt: 0B  in progress: 0 (0B)
              default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 1 (256KB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 0 entries (0B)  hit rate: 27.3%
 Table cache: 0 entries (0B)  hit rate: 58.3%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -426,10 +434,105 @@ Compactions: 2  estimated debt: 5.0KB  in progress: 0 (0B)
              default: 2  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (1.0MB)  zombie: 1 (1.0MB)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 12 entries (2.4KB)  hit rate: 31.1%
 Table cache: 3 entries (2.3KB)  hit rate: 57.9%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
 Snapshots: 0  earliest seq num: 0
 Table iters: 0
 Filter utility: 0.0%
 Ingestions: 0  as flushable: 2 (2.1KB in 3 tables)
+
+batch
+set g g
+set h h
+set i i
+set j j
+set k k
+set l l
+set m m
+----
+
+flush
+----
+0.1:
+  000015:[d#22,SET-d#22,SET]
+  000016:[e#23,SET-e#23,SET]
+  000019:[f#24,SET-f#24,SET]
+0.0:
+  000023:[d#19,SET-f#21,SET]
+  000025:[g#25,SET-i#27,SET]
+  000026:[j#28,SET-l#30,SET]
+  000027:[m#31,SET-m#31,SET]
+6:
+  000008:[a#0,SET-b#0,SET]
+  000013:[c@20#0,SET-c@16#0,SET]
+  000014:[c@15#0,SET-c@14#0,SET]
+
+build ext1
+set z z
+----
+
+ingest-and-excise ext1 excise=i-k
+----
+
+# sstable 29, 30 were created as virtual when i-k was excised.
+lsm
+----
+0.1:
+  000015:[d#22,SET-d#22,SET]
+  000016:[e#23,SET-e#23,SET]
+  000019:[f#24,SET-f#24,SET]
+0.0:
+  000023:[d#19,SET-f#21,SET]
+  000029:[g#25,SET-h#26,SET]
+  000030:[k#29,SET-l#30,SET]
+  000027:[m#31,SET-m#31,SET]
+6:
+  000008:[a#0,SET-b#0,SET]
+  000013:[c@20#0,SET-c@16#0,SET]
+  000014:[c@15#0,SET-c@14#0,SET]
+  000028:[z#32,SET-z#32,SET]
+
+# There should be 2 backing tables.
+metrics-value
+num-backing
+backing-size
+----
+2
+1.3KB
+
+# Virtualize a virtual sstable.
+build ext1
+set zz zz
+----
+
+ingest-and-excise ext1 excise=k-l
+----
+
+# sstable 32 created when k-l was excised, but no new backing file should be
+# created.
+lsm
+----
+0.1:
+  000015:[d#22,SET-d#22,SET]
+  000016:[e#23,SET-e#23,SET]
+  000019:[f#24,SET-f#24,SET]
+0.0:
+  000023:[d#19,SET-f#21,SET]
+  000029:[g#25,SET-h#26,SET]
+  000032:[l#30,SET-l#30,SET]
+  000027:[m#31,SET-m#31,SET]
+6:
+  000008:[a#0,SET-b#0,SET]
+  000013:[c@20#0,SET-c@16#0,SET]
+  000014:[c@15#0,SET-c@14#0,SET]
+  000028:[z#32,SET-z#32,SET]
+  000031:[zz#33,SET-zz#33,SET]
+
+metrics-value
+num-backing
+backing-size
+----
+2
+1.3KB

tool/testdata/db_lsm

@@ -28,6 +28,7 @@ Compactions: 0  estimated debt: 0B  in progress: 0 (0B)
              default: 0  delete: 0  elision: 0  move: 0  read: 0  rewrite: 0  multi-level: 0
 MemTables: 1 (256KB)  zombie: 0 (0B)
 Zombie tables: 0 (0B)
+Backing tables: 0 (0B)
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%