[rowexec] dolt-side lookup execution operator (#8072)

* [rowexec] custom rowexec

* fix build

* testing progress

* save progress

* fix more tests

* skip optimization for system table indexes

* fix headers

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* progressive refactors

* more refactoring

* del comment

* fix more tests

* fix nullSafeEq

* validate lookup join type safety

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* fix more tests

* remove prints

* reformat

* delete unused file

* undo int->string conversion that isn't always valid

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* formatting

* sometimes dstSchema nil, which is not used

* add tests

* more tests and cleanup

* nick comments

* fmt

* more comments

* simplify according to extra testing

* better comments

---------

Co-authored-by: max-hoffman <[email protected]>

go/cmd/dolt/commands/engine/sqlengine.go

@@ -43,6 +43,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/mysql_file_handler"
+	drowexec "github.com/dolthub/dolt/go/libraries/doltcore/sqle/rowexec"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statsnoms"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
@@ -194,7 +195,7 @@ func NewSqlEngine(
 	statsPro := statspro.NewProvider(pro, statsnoms.NewNomsStatsFactory(mrEnv.RemoteDialProvider()))
 	engine.Analyzer.Catalog.StatsProvider = statsPro
 
-	engine.Analyzer.ExecBuilder = rowexec.DefaultBuilder
+	engine.Analyzer.ExecBuilder = rowexec.NewOverrideBuilder(drowexec.Builder{})
 	sessFactory := doltSessionFactory(pro, statsPro, mrEnv.Config(), bcController, config.Autocommit)
 	sqlEngine.provider = pro
 	sqlEngine.contextFactory = sqlContextFactory()

go/go.mod

@@ -57,7 +57,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.18.2-0.20240717222343-23b065691b11
+	github.com/dolthub/go-mysql-server v0.18.2-0.20240718204205-22783f1f7266
 	github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2

go/go.sum

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.18.2-0.20240717222343-23b065691b11 h1:hBmki6phLRQcIzALub15QlZWnaxgTW+FHqYtWjSoq0A=
-github.com/dolthub/go-mysql-server v0.18.2-0.20240717222343-23b065691b11/go.mod h1:P6bG0p+3mH4LS4DLo3BySh10ZJTDqgWyfWBu8gGE3eU=
+github.com/dolthub/go-mysql-server v0.18.2-0.20240718204205-22783f1f7266 h1:89gBiQU+gwbYkV0Q7cGqJGAnM5ngzTB87XVjM/RgxaA=
+github.com/dolthub/go-mysql-server v0.18.2-0.20240718204205-22783f1f7266/go.mod h1:P6bG0p+3mH4LS4DLo3BySh10ZJTDqgWyfWBu8gGE3eU=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=

go/libraries/doltcore/sqle/enginetest/dolt_harness.go

@@ -35,6 +35,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
+	drowexec "github.com/dolthub/dolt/go/libraries/doltcore/sqle/rowexec"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statsnoms"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
@@ -251,7 +252,7 @@ func (d *DoltHarness) NewEngine(t *testing.T) (enginetest.QueryEngine, error) {
 		if err != nil {
 			return nil, err
 		}
-		e.Analyzer.ExecBuilder = rowexec.DefaultBuilder
+		e.Analyzer.ExecBuilder = rowexec.NewOverrideBuilder(drowexec.Builder{})
 		d.engine = e
 
 		ctx := enginetest.NewContext(d)