diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 7909fe055a2..9c17fd111d2 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -34,7 +34,6 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" "github.com/dolthub/dolt/go/libraries/doltcore/dconfig" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle" dblr "github.com/dolthub/dolt/go/libraries/doltcore/sqle/binlogreplication" @@ -104,16 +103,6 @@ func NewSqlEngine( return nil, err } - var doltdbs []*doltdb.DoltDB - var dbNames []string - for _, db := range dbs { - ddbs := db.DoltDatabases() - if len(ddbs) > 0 { - dbNames = append(dbNames, strings.ToLower(db.Name())) - doltdbs = append(doltdbs, ddbs[0]) - } - } - config.ClusterController.ManageSystemVariables(sql.SystemVariables) err = config.ClusterController.ApplyStandbyReplicationConfig(ctx, bThreads, mrEnv, dbs...) @@ -197,7 +186,7 @@ func NewSqlEngine( sqlEngine.dsessFactory = sessFactory engine.Analyzer.Catalog.StatsProvider = stats.NewProvider() - engine.Analyzer.Catalog.StatsProvider.(*stats.Provider).Load(sql.NewContext(ctx), dbNames, doltdbs) + engine.Analyzer.Catalog.StatsProvider.(*stats.Provider).Load(sql.NewContext(ctx), dbs) // Load MySQL Db information if err = engine.Analyzer.Catalog.MySQLDb.LoadData(sql.NewEmptyContext(), data); err != nil { diff --git a/go/go.mod b/go/go.mod index cd6b141019b..e37bc6c4ec3 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c + github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/go-github/v57 v57.0.0 diff --git a/go/go.sum b/go/go.sum index 4bb41df3afa..30034d9a953 100644 --- a/go/go.sum +++ b/go/go.sum @@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c h1:fny2lObIxkMCusLhYG5WcE5ZxIoRlSmLpglkPib7pmA= -github.com/dolthub/go-mysql-server v0.17.1-0.20240124232230-3821405a573c/go.mod h1:4hIbiUsJmVHyqdeuAe+zuy7OnXQltSwkZ3Nx6xAjwJQ= +github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28 h1:gVRoMPxdlkMhLa3JjJQz5IfObEVTiU4xNUoELjcNs/A= +github.com/dolthub/go-mysql-server v0.17.1-0.20240125221447-e42adc9dbc28/go.mod h1:4hIbiUsJmVHyqdeuAe+zuy7OnXQltSwkZ3Nx6xAjwJQ= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 8952eee6889..7246d270b56 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -1761,6 +1761,7 @@ func (ddb *DoltDB) GetStatistics(ctx context.Context) (prolly.Map, error) { if err != nil { return prolly.Map{}, err } + return stats.Map(), nil } diff --git a/go/libraries/doltcore/sqle/enginetest/stats_queries.go b/go/libraries/doltcore/sqle/enginetest/stats_queries.go index b9dd23fc004..a7d9cd36838 100644 --- a/go/libraries/doltcore/sqle/enginetest/stats_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/stats_queries.go @@ -26,7 +26,6 @@ import ( "github.com/dolthub/go-mysql-server/sql/types" "github.com/stretchr/testify/require" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/stats" @@ -423,18 +422,9 @@ func TestProviderReloadScriptWithEngine(t *testing.T, e enginetest.QueryEngine, if !ok { t.Errorf("expected *sqle.DoltDatabaseProvider but found: %T", eng.Analyzer.Catalog.DbProvider) } - var doltdbs []*doltdb.DoltDB - var dbNames []string - for _, db := range dbProv.DoltDatabases() { - ddbs := db.DoltDatabases() - if len(ddbs) > 0 { - dbNames = append(dbNames, strings.ToLower(db.Name())) - doltdbs = append(doltdbs, ddbs[0]) - } - } newProv := stats.NewProvider() - err := newProv.Load(ctx, dbNames, doltdbs) + err := newProv.Load(ctx, dbProv.DoltDatabases()) require.NoError(t, err) eng.Analyzer.Catalog.StatsProvider = newProv diff --git a/go/libraries/doltcore/sqle/stats/read.go b/go/libraries/doltcore/sqle/stats/read.go index 7f32c264ae3..3efe708d125 100644 --- a/go/libraries/doltcore/sqle/stats/read.go +++ b/go/libraries/doltcore/sqle/stats/read.go @@ -16,6 +16,7 @@ package stats import ( "errors" + "fmt" "io" "strconv" "strings" @@ -34,8 +35,8 @@ import ( "github.com/dolthub/dolt/go/store/val" ) -func loadStats(ctx *sql.Context, dbName string, m prolly.Map) (*dbStats, error) { - dbStat := &dbStats{db: dbName, active: make(map[hash.Hash]int), stats: make(map[sql.StatQualifier]*DoltStats)} +func loadStats(ctx *sql.Context, db dsess.SqlDatabase, m prolly.Map) (*dbStats, error) { + dbStat := &dbStats{db: db.Name(), active: make(map[hash.Hash]int), stats: make(map[sql.StatQualifier]*DoltStats)} iter, err := dtables.NewStatsIter(ctx, m) if err != nil { @@ -111,11 +112,17 @@ func loadStats(ctx *sql.Context, dbName string, m prolly.Map) (*dbStats, error) qual := sql.NewStatQualifier(dbName, tableName, indexName) if currentStat.Qual.String() != qual.String() { - if currentStat.Qual.String() != "" { + if !currentStat.Qual.Empty() { currentStat.LowerBound, err = loadLowerBound(ctx, currentStat.Qual) if err != nil { return nil, err } + fds, colSet, err := loadFuncDeps(ctx, db, currentStat.Qual) + if err != nil { + return nil, err + } + currentStat.fds = fds + currentStat.colSet = colSet dbStat.stats[currentStat.Qual] = currentStat } currentStat = &DoltStats{Qual: qual, Columns: columns, LowerBound: lowerBound} @@ -199,3 +206,36 @@ func loadLowerBound(ctx *sql.Context, qual sql.StatQualifier) (sql.Row, error) { } return firstRow, nil } + +func loadFuncDeps(ctx *sql.Context, db dsess.SqlDatabase, qual sql.StatQualifier) (*sql.FuncDepSet, sql.ColSet, error) { + tab, ok, err := db.GetTableInsensitive(ctx, qual.Table()) + if err != nil { + return nil, sql.ColSet{}, err + } else if !ok { + return nil, sql.ColSet{}, fmt.Errorf("%w: table not found: '%s'", ErrFailedToLoad, qual.Table()) + } + + iat, ok := tab.(sql.IndexAddressable) + if !ok { + return nil, sql.ColSet{}, fmt.Errorf("%w: table does not have indexes: '%s'", ErrFailedToLoad, qual.Table()) + } + + indexes, err := iat.GetIndexes(ctx) + if err != nil { + return nil, sql.ColSet{}, err + } + + var idx sql.Index + for _, i := range indexes { + if strings.EqualFold(i.ID(), qual.Index()) { + idx = i + break + } + } + + if idx == nil { + return nil, sql.ColSet{}, fmt.Errorf("%w: index not found: '%s'", ErrFailedToLoad, qual.Index()) + } + + return stats.IndexFds(qual.Table(), tab.Schema(), idx) +} diff --git a/go/libraries/doltcore/sqle/stats/refresh.go b/go/libraries/doltcore/sqle/stats/refresh.go index 12005cb08fd..d7b179f6ea5 100644 --- a/go/libraries/doltcore/sqle/stats/refresh.go +++ b/go/libraries/doltcore/sqle/stats/refresh.go @@ -24,6 +24,7 @@ import ( "time" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/stats" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" @@ -56,6 +57,11 @@ func refreshStats(ctx *sql.Context, indexes []sql.Index, idxMetas []indexMeta) ( return nil, fmt.Errorf("error creating statistics for table: %s; table not found", idxMetas[0].table) } + nameToIdx := make(map[string]sql.Index) + for _, idx := range indexes { + nameToIdx[strings.ToLower(idx.ID())] = idx + } + var dTab *doltdb.Table switch t := tab.(type) { case *sqle.AlterableDoltTable: @@ -178,9 +184,18 @@ func refreshStats(ctx *sql.Context, indexes []sql.Index, idxMetas []indexMeta) ( bucket.Chunk = addrs[i] ret[updater.qual].Histogram = append(ret[updater.qual].Histogram, bucket) } + + sqlIdx := nameToIdx[strings.ToLower(qual.Index())] + fds, colSet, err := stats.IndexFds(qual.Table(), tab.Schema(), sqlIdx) + if err != nil { + return nil, err + } + ret[updater.qual].DistinctCount = uint64(updater.globalDistinct) ret[updater.qual].RowCount = uint64(updater.globalCount) ret[updater.qual].LowerBound = firstRow + ret[updater.qual].fds = fds + ret[updater.qual].colSet = colSet } return ret, nil } diff --git a/go/libraries/doltcore/sqle/stats/stats_provider.go b/go/libraries/doltcore/sqle/stats/stats_provider.go index 6dd069e304c..775132dabc4 100644 --- a/go/libraries/doltcore/sqle/stats/stats_provider.go +++ b/go/libraries/doltcore/sqle/stats/stats_provider.go @@ -177,18 +177,18 @@ var _ sql.StatsProvider = (*Provider)(nil) // Init scans the statistics tables, populating the |stats| attribute. // Statistics are not available for reading until we've finished loading. -func (p *Provider) Load(ctx *sql.Context, dbNames []string, dbs []*doltdb.DoltDB) error { +func (p *Provider) Load(ctx *sql.Context, dbs []dsess.SqlDatabase) error { p.mu.Lock() defer p.mu.Unlock() - for _, name := range dbNames { + for _, db := range dbs { // set map keys so concurrent orthogonal writes are OK - p.dbStats[name] = &dbStats{db: strings.ToLower(name), stats: make(map[sql.StatQualifier]*DoltStats)} + p.dbStats[strings.ToLower(db.Name())] = &dbStats{db: strings.ToLower(db.Name()), stats: make(map[sql.StatQualifier]*DoltStats)} } eg, ctx := ctx.NewErrgroup() - for i, db := range dbs { + for _, db := range dbs { // copy closure variables - dbName := dbNames[i] + dbName := strings.ToLower(db.Name()) db := db eg.Go(func() (err error) { defer func() { @@ -203,13 +203,13 @@ func (p *Provider) Load(ctx *sql.Context, dbNames []string, dbs []*doltdb.DoltDB } }() - m, err := db.GetStatistics(ctx) + m, err := db.DbData().Ddb.GetStatistics(ctx) if errors.Is(err, doltdb.ErrNoStatistics) { return nil } else if err != nil { return err } - stats, err := loadStats(ctx, dbName, m) + stats, err := loadStats(ctx, db, m) if errors.Is(err, dtables.ErrIncompatibleVersion) { ctx.Warn(0, err.Error()) return nil