[no-release-notes] fix stats errors (#7399)

* [no-release-notes] fix stats errors

* bump

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* bump

---------

Co-authored-by: max-hoffman <[email protected]>

go/cmd/dolt/commands/engine/sqlengine.go

@@ -34,7 +34,6 @@ import (
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/libraries/doltcore/branch_control"
 	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
-	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	dblr "github.com/dolthub/dolt/go/libraries/doltcore/sqle/binlogreplication"
@@ -104,16 +103,6 @@ func NewSqlEngine(
 		return nil, err
 	}
 
-	var doltdbs []*doltdb.DoltDB
-	var dbNames []string
-	for _, db := range dbs {
-		ddbs := db.DoltDatabases()
-		if len(ddbs) > 0 {
-			dbNames = append(dbNames, strings.ToLower(db.Name()))
-			doltdbs = append(doltdbs, ddbs[0])
-		}
-	}
-
 	config.ClusterController.ManageSystemVariables(sql.SystemVariables)
 
 	err = config.ClusterController.ApplyStandbyReplicationConfig(ctx, bThreads, mrEnv, dbs...)
@@ -197,7 +186,7 @@ func NewSqlEngine(
 	sqlEngine.dsessFactory = sessFactory
 
 	engine.Analyzer.Catalog.StatsProvider = stats.NewProvider()
-	engine.Analyzer.Catalog.StatsProvider.(*stats.Provider).Load(sql.NewContext(ctx), dbNames, doltdbs)
+	engine.Analyzer.Catalog.StatsProvider.(*stats.Provider).Load(sql.NewContext(ctx), dbs)
 
 	// Load MySQL Db information
 	if err = engine.Analyzer.Catalog.MySQLDb.LoadData(sql.NewEmptyContext(), data); err != nil {

go/go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c
+	github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2
 	github.com/google/go-github/v57 v57.0.0

go/go.sum

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c h1:fny2lObIxkMCusLhYG5WcE5ZxIoRlSmLpglkPib7pmA=
-github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c/go.mod h1:4hIbiUsJmVHyqdeuAe+zuy7OnXQltSwkZ3Nx6xAjwJQ=
+github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28 h1:gVRoMPxdlkMhLa3JjJQz5IfObEVTiU4xNUoELjcNs/A=
+github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28/go.mod h1:4hIbiUsJmVHyqdeuAe+zuy7OnXQltSwkZ3Nx6xAjwJQ=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
 github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ=

go/libraries/doltcore/doltdb/doltdb.go

@@ -1761,6 +1761,7 @@ func (ddb *DoltDB) GetStatistics(ctx context.Context) (prolly.Map, error) {
 	if err != nil {
 		return prolly.Map{}, err
 	}
+
 	return stats.Map(), nil
 
 }

go/libraries/doltcore/sqle/enginetest/stats_queries.go

@@ -26,7 +26,6 @@ import (
 	"github.com/dolthub/go-mysql-server/sql/types"
 	"github.com/stretchr/testify/require"
 
-	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/stats"
@@ -423,18 +422,9 @@ func TestProviderReloadScriptWithEngine(t *testing.T, e enginetest.QueryEngine,
 			if !ok {
 				t.Errorf("expected *sqle.DoltDatabaseProvider but found: %T", eng.Analyzer.Catalog.DbProvider)
 			}
-			var doltdbs []*doltdb.DoltDB
-			var dbNames []string
-			for _, db := range dbProv.DoltDatabases() {
-				ddbs := db.DoltDatabases()
-				if len(ddbs) > 0 {
-					dbNames = append(dbNames, strings.ToLower(db.Name()))
-					doltdbs = append(doltdbs, ddbs[0])
-				}
-			}
 
 			newProv := stats.NewProvider()
-			err := newProv.Load(ctx, dbNames, doltdbs)
+			err := newProv.Load(ctx, dbProv.DoltDatabases())
 			require.NoError(t, err)
 
 			eng.Analyzer.Catalog.StatsProvider = newProv

go/libraries/doltcore/sqle/stats/read.go

@@ -16,6 +16,7 @@ package stats
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -34,8 +35,8 @@ import (
 	"github.com/dolthub/dolt/go/store/val"
 )
 
-func loadStats(ctx *sql.Context, dbName string, m prolly.Map) (*dbStats, error) {
-	dbStat := &dbStats{db: dbName, active: make(map[hash.Hash]int), stats: make(map[sql.StatQualifier]*DoltStats)}
+func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (*dbStats, error) {
+	dbStat := &dbStats{db: db.Name(), active: make(map[hash.Hash]int), stats: make(map[sql.StatQualifier]*DoltStats)}
 
 	iter, err := dtables.NewStatsIter(ctx, m)
 	if err != nil {
@@ -111,11 +112,17 @@ func loadStats(ctx *sql.Context, dbName string, m prolly.Map) (*dbStats, error)
 
 		qual := sql.NewStatQualifier(dbName, tableName, indexName)
 		if currentStat.Qual.String() != qual.String() {
-			if currentStat.Qual.String() != "" {
+			if !currentStat.Qual.Empty() {
 				currentStat.LowerBound, err = loadLowerBound(ctx, currentStat.Qual)
 				if err != nil {
 					return nil, err
 				}
+				fds, colSet, err := loadFuncDeps(ctx, db, currentStat.Qual)
+				if err != nil {
+					return nil, err
+				}
+				currentStat.fds = fds
+				currentStat.colSet = colSet
 				dbStat.stats[currentStat.Qual] = currentStat
 			}
 			currentStat = &DoltStats{Qual: qual, Columns: columns, LowerBound: lowerBound}
@@ -199,3 +206,36 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) {
 	}
 	return firstRow, nil
 }
+
+func loadFuncDeps(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier) (*sql.FuncDepSet, sql.ColSet, error) {
+	tab, ok, err := db.GetTableInsensitive(ctx, qual.Table())
+	if err != nil {
+		return nil, sql.ColSet{}, err
+	} else if !ok {
+		return nil, sql.ColSet{}, fmt.Errorf("%w: table not found: '%s'", ErrFailedToLoad, qual.Table())
+	}
+
+	iat, ok := tab.(sql.IndexAddressable)
+	if !ok {
+		return nil, sql.ColSet{}, fmt.Errorf("%w: table does not have indexes: '%s'", ErrFailedToLoad, qual.Table())
+	}
+
+	indexes, err := iat.GetIndexes(ctx)
+	if err != nil {
+		return nil, sql.ColSet{}, err
+	}
+
+	var idx sql.Index
+	for _, i := range indexes {
+		if strings.EqualFold(i.ID(), qual.Index()) {
+			idx = i
+			break
+		}
+	}
+
+	if idx == nil {
+		return nil, sql.ColSet{}, fmt.Errorf("%w: index not found: '%s'", ErrFailedToLoad, qual.Index())
+	}
+
+	return stats.IndexFds(qual.Table(), tab.Schema(), idx)
+}

go/libraries/doltcore/sqle/stats/refresh.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/dolthub/go-mysql-server/sql"
+	"github.com/dolthub/go-mysql-server/sql/stats"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
@@ -56,6 +57,11 @@ func refreshStats(ctx *sql.Context, indexes []sql.Index, idxMetas []indexMeta) (
 		return nil, fmt.Errorf("error creating statistics for table: %s; table not found", idxMetas[0].table)
 	}
 
+	nameToIdx := make(map[string]sql.Index)
+	for _, idx := range indexes {
+		nameToIdx[strings.ToLower(idx.ID())] = idx
+	}
+
 	var dTab *doltdb.Table
 	switch t := tab.(type) {
 	case *sqle.AlterableDoltTable:
@@ -178,9 +184,18 @@ func refreshStats(ctx *sql.Context, indexes []sql.Index, idxMetas []indexMeta) (
 			bucket.Chunk = addrs[i]
 			ret[updater.qual].Histogram = append(ret[updater.qual].Histogram, bucket)
 		}
+
+		sqlIdx := nameToIdx[strings.ToLower(qual.Index())]
+		fds, colSet, err := stats.IndexFds(qual.Table(), tab.Schema(), sqlIdx)
+		if err != nil {
+			return nil, err
+		}
+
 		ret[updater.qual].DistinctCount = uint64(updater.globalDistinct)
 		ret[updater.qual].RowCount = uint64(updater.globalCount)
 		ret[updater.qual].LowerBound = firstRow
+		ret[updater.qual].fds = fds
+		ret[updater.qual].colSet = colSet
 	}
 	return ret, nil
 }

go/libraries/doltcore/sqle/stats/stats_provider.go

@@ -177,18 +177,18 @@ var _ sql.StatsProvider = (*Provider)(nil)
 
 // Init scans the statistics tables, populating the |stats| attribute.
 // Statistics are not available for reading until we've finished loading.
-func (p *Provider) Load(ctx *sql.Context, dbNames []string, dbs []*doltdb.DoltDB) error {
+func (p *Provider) Load(ctx *sql.Context, dbs []dsess.SqlDatabase) error {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	for _, name := range dbNames {
+	for _, db := range dbs {
 		// set map keys so concurrent orthogonal writes are OK
-		p.dbStats[name] = &dbStats{db: strings.ToLower(name), stats: make(map[sql.StatQualifier]*DoltStats)}
+		p.dbStats[strings.ToLower(db.Name())] = &dbStats{db: strings.ToLower(db.Name()), stats: make(map[sql.StatQualifier]*DoltStats)}
 	}
 	eg, ctx := ctx.NewErrgroup()
-	for i, db := range dbs {
+	for _, db := range dbs {
 		// copy closure variables
-		dbName := dbNames[i]
+		dbName := strings.ToLower(db.Name())
 		db := db
 		eg.Go(func() (err error) {
 			defer func() {
@@ -203,13 +203,13 @@ func (p *Provider) Load(ctx *sql.Context, dbNames []string, dbs []*doltdb.DoltDB
 				}
 			}()
 
-			m, err := db.GetStatistics(ctx)
+			m, err := db.DbData().Ddb.GetStatistics(ctx)
 			if errors.Is(err, doltdb.ErrNoStatistics) {
 				return nil
 			} else if err != nil {
 				return err
 			}
-			stats, err := loadStats(ctx, dbName, m)
+			stats, err := loadStats(ctx, db, m)
 			if errors.Is(err, dtables.ErrIncompatibleVersion) {
 				ctx.Warn(0, err.Error())
 				return nil