Merge pull request #1086 from dolthub/taylor/metrics

Implement metrics, `describe` system table tests

go.mod

@@ -6,13 +6,13 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/cockroachdb/apd/v2 v2.0.3-0.20200518165714-d020e156310a
 	github.com/cockroachdb/errors v1.7.5
-	github.com/dolthub/dolt/go v0.40.5-0.20241218181729-d62130f67059
+	github.com/dolthub/dolt/go v0.40.5-0.20241227214751-bfb33fe9c9e1
 	github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
 	github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90
-	github.com/dolthub/go-mysql-server v0.18.2-0.20241217205639-85adcd5e580f
+	github.com/dolthub/go-mysql-server v0.19.1-0.20241227200914-69b2934b5468
 	github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216
-	github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54
+	github.com/dolthub/vitess v0.0.0-20241220202600-b18f18d0cde7
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.10.2
 	github.com/gogo/protobuf v1.3.2

go.sum

@@ -214,8 +214,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/dolthub/dolt/go v0.40.5-0.20241218181729-d62130f67059 h1:ughDGwhqfIsfCF8LpGhTXqqVu/Cx2UsbJoMuvzIX8ME=
-github.com/dolthub/dolt/go v0.40.5-0.20241218181729-d62130f67059/go.mod h1:AQKcVKL5Dbv14cz1E0rEhhSKiSpQs358Gyiso19RCFk=
+github.com/dolthub/dolt/go v0.40.5-0.20241227214751-bfb33fe9c9e1 h1:WOyXDfnn/z3q9yzkx+vuXyeZkVYb5NHRJLrMbdXiDNo=
+github.com/dolthub/dolt/go v0.40.5-0.20241227214751-bfb33fe9c9e1/go.mod h1:kJKbaXFI6g8fpv8WJip6wxGfxwXmAcx8fsmF5aItNJI=
 github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d h1:gO9+wrmNHXukPNCO1tpfCcXIdMlW/qppbUStfLvqz/U=
 github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20241119094239-f4e529af734d/go.mod h1:L5RDYZbC9BBWmoU2+TjTekeqqhFXX5EqH9ln00O0stY=
 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww=
@@ -224,8 +224,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
-github.com/dolthub/go-mysql-server v0.18.2-0.20241217205639-85adcd5e580f h1:TTUk8P8Rt4YUrQ9A6yp91v5/kIlvWl30Mt3inK+V1Z4=
-github.com/dolthub/go-mysql-server v0.18.2-0.20241217205639-85adcd5e580f/go.mod h1:elfIatfq2fkU5lqTBrTcpL0RcHZOgYPE8EzBD7yQFiY=
+github.com/dolthub/go-mysql-server v0.19.1-0.20241227200914-69b2934b5468 h1:b0yy2NKmg6P0sqrvxWw229uVchHjpgAcg6jtsKjSJSQ=
+github.com/dolthub/go-mysql-server v0.19.1-0.20241227200914-69b2934b5468/go.mod h1:ToNOAVZAJ6iQBpigxYZo3q8JZDRxpI2/VRrtUoZeehI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=
@@ -238,8 +238,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216 h1:JWkKRE4
 github.com/dolthub/sqllogictest/go v0.0.0-20240618184124-ca47f9354216/go.mod h1:e/FIZVvT2IR53HBCAo41NjqgtEnjMJGKca3Y/dAmZaA=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54 h1:nzBnC0Rt1gFtscJEz4veYd/mazZEdbdmed+tujdaKOo=
-github.com/dolthub/vitess v0.0.0-20241211024425-b00987f7ba54/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
+github.com/dolthub/vitess v0.0.0-20241220202600-b18f18d0cde7 h1:w130WLeARGGNYWmhGPugsHXzJEelKKimt3kTWg6/Puk=
+github.com/dolthub/vitess v0.0.0-20241220202600-b18f18d0cde7/go.mod h1:1gQZs/byeHLMSul3Lvl3MzioMtOW1je79QYGyi2fd70=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

server/connection_handler.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqlserver"
+	"github.com/dolthub/go-mysql-server/server"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/vitess/go/mysql"
 	"github.com/dolthub/vitess/go/vt/sqlparser"
@@ -87,7 +88,7 @@ func init() {
 }
 
 // NewConnectionHandler returns a new ConnectionHandler for the connection provided
-func NewConnectionHandler(conn net.Conn, handler mysql.Handler) *ConnectionHandler {
+func NewConnectionHandler(conn net.Conn, handler mysql.Handler, sel server.ServerEventListener) *ConnectionHandler {
 	mysqlConn := &mysql.Conn{
 		Conn:        conn,
 		PrepareData: make(map[uint32]*mysql.PrepareData),
@@ -111,6 +112,9 @@ func NewConnectionHandler(conn net.Conn, handler mysql.Handler) *ConnectionHandl
 		encodeLoggedQuery: false, // cfg.EncodeLoggedQuery,
 		pgTypeMap:         pgtype.NewMap(),
 	}
+	if sel != nil {
+		doltgresHandler.sel = sel
+	}
 
 	return &ConnectionHandler{
 		mysqlConn:          mysqlConn,

server/doltgres_handler.go

@@ -84,6 +84,7 @@ type DoltgresHandler struct {
 	readTimeout       time.Duration
 	encodeLoggedQuery bool
 	pgTypeMap         *pgtype.Map
+	sel               server.ServerEventListener
 }
 
 var _ Handler = &DoltgresHandler{}
@@ -120,11 +121,21 @@ func (h *DoltgresHandler) ComExecuteBound(ctx context.Context, conn *mysql.Conn,
 		return fmt.Errorf("boundQuery must be a sql.Node, but got %T", boundQuery)
 	}
 
+	// TODO: This technically isn't query start and underestimates query execution time
+	start := time.Now()
+	if h.sel != nil {
+		h.sel.QueryStarted()
+	}
+
 	err := h.doQuery(ctx, conn, query, nil, analyzedPlan, h.executeBoundPlan, callback)
 	if err != nil {
 		err = sql.CastSQLError(err)
 	}
 
+	if h.sel != nil {
+		h.sel.QueryCompleted(err == nil, time.Since(start))
+	}
+
 	return err
 }
 
@@ -163,10 +174,21 @@ func (h *DoltgresHandler) ComPrepareParsed(ctx context.Context, c *mysql.Conn, q
 
 // ComQuery implements the Handler interface.
 func (h *DoltgresHandler) ComQuery(ctx context.Context, c *mysql.Conn, query string, parsed sqlparser.Statement, callback func(*Result) error) error {
+	// TODO: This technically isn't query start and underestimates query execution time
+	start := time.Now()
+	if h.sel != nil {
+		h.sel.QueryStarted()
+	}
+
 	err := h.doQuery(ctx, c, query, parsed, nil, h.executeQuery, callback)
 	if err != nil {
 		err = sql.CastSQLError(err)
 	}
+
+	if h.sel != nil {
+		h.sel.QueryCompleted(err == nil, time.Since(start))
+	}
+
 	return err
 }
 
@@ -191,6 +213,12 @@ func (h *DoltgresHandler) ComResetConnection(c *mysql.Conn) error {
 
 // ConnectionClosed implements the Handler interface.
 func (h *DoltgresHandler) ConnectionClosed(c *mysql.Conn) {
+	defer func() {
+		if h.sel != nil {
+			h.sel.ClientDisconnected()
+		}
+	}()
+
 	defer h.sm.RemoveConn(c)
 	defer h.e.CloseSession(c.ConnectionID)
 
@@ -201,6 +229,10 @@ func (h *DoltgresHandler) ConnectionClosed(c *mysql.Conn) {
 
 // NewConnection implements the Handler interface.
 func (h *DoltgresHandler) NewConnection(c *mysql.Conn) {
+	if h.sel != nil {
+		h.sel.ClientConnected()
+	}
+
 	h.sm.AddConn(c)
 	sql.StatusVariables.IncrementGlobal("Connections", 1)
 

server/listener.go

@@ -35,6 +35,7 @@ var (
 type Listener struct {
 	listener net.Listener
 	cfg      mysql.ListenerConfig
+	sel      server.ServerEventListener
 }
 
 var _ server.ProtocolListener = (*Listener)(nil)
@@ -48,14 +49,15 @@ func WithCertificate(cert tls.Certificate) ListenerOpt {
 }
 
 // NewListener creates a new Listener.
-func NewListener(listenerCfg mysql.ListenerConfig) (server.ProtocolListener, error) {
-	return NewListenerWithOpts(listenerCfg)
+func NewListener(listenerCfg mysql.ListenerConfig, sel server.ServerEventListener) (server.ProtocolListener, error) {
+	return NewListenerWithOpts(listenerCfg, sel)
 }
 
-func NewListenerWithOpts(listenerCfg mysql.ListenerConfig, opts ...ListenerOpt) (server.ProtocolListener, error) {
+func NewListenerWithOpts(listenerCfg mysql.ListenerConfig, sel server.ServerEventListener, opts ...ListenerOpt) (server.ProtocolListener, error) {
 	l := &Listener{
 		listener: listenerCfg.Listener,
 		cfg:      listenerCfg,
+		sel:      sel,
 	}
 
 	for _, opt := range opts {
@@ -83,7 +85,7 @@ func (l *Listener) Accept() {
 			conn = netutil.NewConnWithTimeouts(conn, l.cfg.ConnReadTimeout, l.cfg.ConnWriteTimeout)
 		}
 
-		connectionHandler := NewConnectionHandler(conn, l.cfg.Handler)
+		connectionHandler := NewConnectionHandler(conn, l.cfg.Handler, l.sel)
 		go connectionHandler.HandleConnection()
 	}
 }

testing/go/dolt_tables_test.go

@@ -1885,6 +1885,31 @@ func TestUserSpaceDoltTables(t *testing.T) {
 					Query:    `SELECT * FROM dolt_status`,
 					Expected: []sql.Row{{"public.t", "f", "new table"}},
 				},
+				{
+					Query: `DESCRIBE dolt."status"`,
+					Expected: []sql.Row{
+						{"table_name", "text", "NO", "PRI", nil, ""},
+						{"staged", "boolean", "NO", "PRI", nil, ""},
+						{"status", "text", "NO", "PRI", nil, ""},
+					},
+				},
+				{
+					Skip:  true, // TODO: ERROR: at or near "status": syntax error
+					Query: `DESCRIBE dolt.status`,
+					Expected: []sql.Row{
+						{"table_name", "text", "NO", "PRI", nil, ""},
+						{"staged", "boolean", "NO", "PRI", nil, ""},
+						{"status", "text", "NO", "PRI", nil, ""},
+					},
+				},
+				{
+					Query: `DESCRIBE dolt_status`,
+					Expected: []sql.Row{
+						{"table_name", "text", "NO", "PRI", nil, ""},
+						{"staged", "boolean", "NO", "PRI", nil, ""},
+						{"status", "text", "NO", "PRI", nil, ""},
+					},
+				},
 				{
 					Query:    `SELECT * FROM dolt.status WHERE staged=true`,
 					Expected: []sql.Row{},

testing/go/enginetest/doltgres_engine_test.go

@@ -1735,12 +1735,12 @@ func TestStatsHistograms(t *testing.T) {
 	denginetest.RunStatsHistogramTests(t, h)
 }
 
-// TestStatsIO force a provider reload in-between setup and assertions that
+// TestStatsStorage force a provider reload in-between setup and assertions that
 // forces a round trip of the statistics table before inspecting values.
-func TestStatsIO(t *testing.T) {
+func TestStatsStorage(t *testing.T) {
 	t.Skip()
 	h := newDoltgresServerHarness(t)
-	denginetest.RunStatsIOTests(t, h)
+	denginetest.RunStatsStorageTests(t, h)
 }
 
 func TestJoinStats(t *testing.T) {

testing/go/ssl_test.go

@@ -23,7 +23,7 @@ type SSLListener struct {
 	*dserver.Listener
 }
 
-func NewSslListener(listenerCfg mysql.ListenerConfig) (server.ProtocolListener, error) {
+func NewSslListener(listenerCfg mysql.ListenerConfig, sel server.ServerEventListener) (server.ProtocolListener, error) {
 	// Since this is intended for testing, we'll configure a test certificate so that we can test for SSL support
 	cert, key, err := testcerts.GenerateCerts()
 	if err != nil {
@@ -35,7 +35,7 @@ func NewSslListener(listenerCfg mysql.ListenerConfig) (server.ProtocolListener,
 		panic(err)
 	}
 
-	listener, err := dserver.NewListenerWithOpts(listenerCfg, dserver.WithCertificate(certificate))
+	listener, err := dserver.NewListenerWithOpts(listenerCfg, sel, dserver.WithCertificate(certificate))
 	if err != nil {
 		return nil, err
 	}