Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create dolt_help system table #8739

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
3 changes: 3 additions & 0 deletions go/cmd/dolt/dolt.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables"
"github.com/dolthub/dolt/go/libraries/events"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
"github.com/dolthub/dolt/go/libraries/utils/config"
Expand Down Expand Up @@ -226,6 +227,8 @@ func init() {
if _, ok := os.LookupEnv(disableEventFlushEnvVar); ok {
eventFlushDisabled = true
}

dtables.DoltCommand = doltCommand
}

const pprofServerFlag = "--pprof-server"
Expand Down
10 changes: 10 additions & 0 deletions go/libraries/doltcore/doltdb/system_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ var getGeneratedSystemTables = func() []string {
GetCommitAncestorsTableName(),
GetStatusTableName(),
GetRemotesTableName(),
GetHelpTableName(),
}
}

Expand Down Expand Up @@ -380,6 +381,11 @@ var GetTagsTableName = func() string {
return TagsTableName
}

// GetHelpTableName returns the help table name
var GetHelpTableName = func() string {
return HelpTableName
}

const (
// LogTableName is the log system table name
LogTableName = "dolt_log"
Expand Down Expand Up @@ -585,3 +591,7 @@ const (
// was originally defined. Mode settings, such as ANSI_QUOTES, are needed to correctly parse the fragment.
ProceduresTableSqlModeCol = "sql_mode"
)

const (
HelpTableName = "dolt_help"
)
8 changes: 8 additions & 0 deletions go/libraries/doltcore/sqle/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,14 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
return nil, false, err
}
dt = NewSchemaTable(backingTable)
case doltdb.GetHelpTableName(), doltdb.HelpTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
dt, found = dtables.NewHelpTable(ctx, db.Name(), lwrName), true
}
}

if found {
Expand Down
210 changes: 210 additions & 0 deletions go/libraries/doltcore/sqle/dtables/help_table.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
// Copyright 2025 Dolthub, Inc.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dtables

import (
"encoding/json"
"fmt"
"io"
"strings"

"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/go-mysql-server/sql"
sqlTypes "github.com/dolthub/go-mysql-server/sql/types"
)

type HelpTable struct {
dbName string
tableName string
}

var HelpTableTypes = []string{
"system_table",
"procedure",
"function",
"variable",
}

// NewHelpTable creates a HelpTable
func NewHelpTable(_ *sql.Context, dbName, tableName string) sql.Table {
return &HelpTable{dbName: dbName, tableName: tableName}
}

// Name is a sql.Table interface function which returns the name of the table.
func (ht *HelpTable) Name() string {
return ht.tableName
}

// String is a sql.Table interface function which returns the name of the table.
func (ht *HelpTable) String() string {
return ht.tableName
}

// Schema is a sql.Table interface function that gets the sql.Schema of the help system table.
func (ht *HelpTable) Schema() sql.Schema {
return []*sql.Column{
{
Name: "target",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would call this name instead

Type: sqlTypes.TinyText,
Source: ht.tableName,
PrimaryKey: true,
DatabaseSource: ht.dbName,
},
{
Name: "type",
Type: sqlTypes.MustCreateEnumType(HelpTableTypes, sql.Collation_Default),
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
zachmu marked this conversation as resolved.
Show resolved Hide resolved
Name: "short_description",
Type: sqlTypes.LongText,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
Name: "long_description",
Type: sqlTypes.LongText,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
Name: "arguments",
Type: sqlTypes.JSON,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
}
}

// Collation implements the sql.Table interface.
func (ht *HelpTable) Collation() sql.CollationID {
return sql.Collation_Default
}

// Partitions is a sql.Table interface function that returns a partition
// of the data. Currently the data is unpartitioned.
func (ht *HelpTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
return index.SinglePartitionIterFromNomsMap(nil), nil
}

// PartitionRows is a sql.Table interface function that gets a row iterator for a partition.
func (ht *HelpTable) PartitionRows(_ *sql.Context, _ sql.Partition) (sql.RowIter, error) {
return NewHelpRowIter(), nil
}

type HelpRowIter struct {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Best practice here is to use a pointer for the receiver type and not the fields

idx *int
rows *[]sql.Row
}

func NewHelpRowIter() HelpRowIter {
idx := 0
var nilRows []sql.Row
return HelpRowIter{idx: &idx, rows: &nilRows}
}

// DoltCommand is set in cmd/dolt/dolt.go to avoid circular dependency.
var DoltCommand cli.SubCommandHandler

func (itr HelpRowIter) Next(_ *sql.Context) (sql.Row, error) {
if *itr.rows == nil {
var err error
*itr.rows, err = generateProcedureHelpRows(DoltCommand.Name(), DoltCommand.Subcommands)
if err != nil {
return nil, err
}
}

helpRows := *itr.rows

if *itr.idx >= len(helpRows) {
return nil, io.EOF
}

row := helpRows[*itr.idx]
(*itr.idx)++
return row, nil
}

func (itr HelpRowIter) Close(_ *sql.Context) error {
return nil
}

// generateProcedureHelpRows generates a sql row for each procedure that has an equivalent CLI command.
func generateProcedureHelpRows(cmdStr string, subCommands []cli.Command) ([]sql.Row, error) {
rows := []sql.Row{}

for _, curr := range subCommands {
if hidCmd, ok := curr.(cli.HiddenCommand); ok && hidCmd.Hidden() {
continue
}

if subCmdHandler, ok := curr.(cli.SubCommandHandler); ok {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This loop can be simplified, because there are no procedures that correspond to sub commands. E.g. there is no dolt_table_ls command.

if subCmdHandler.Unspecified != nil {
newRows, err := generateProcedureHelpRows(cmdStr, []cli.Command{subCmdHandler.Unspecified})
if err != nil {
return nil, err
}
rows = append(rows, newRows...)
}
newRows, err := generateProcedureHelpRows(cmdStr+"_"+subCmdHandler.Name(), subCmdHandler.Subcommands)
if err != nil {
return nil, err
}
rows = append(rows, newRows...)
} else {
nameFormatted := fmt.Sprintf("%s_%s", cmdStr, strings.ReplaceAll(curr.Name(), "-", "_"))

hasProcedure := false
for _, procedure := range dprocedures.DoltProcedures {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would pull out a boolean procedureExists method for this, and check for this condition before beginning the loop

if procedure.Name == nameFormatted {
hasProcedure = true
break
}
}

docs := curr.Docs()

if hasProcedure && docs != nil {
argsMap := map[string]string{}
for _, argHelp := range curr.Docs().ArgParser.ArgListHelp {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is missing all the options (flags)

argsMap[argHelp[0]] = argHelp[1]
}

argsJson, err := json.Marshal(argsMap)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why, but something about this seems to be producing invalid json in a subtle and weird way. Check this out:

db1/main> select json_pretty(arguments) from dolt_help where name = 'dolt_revert';
invalid data type for JSON data in argument 1 to function json_pretty; a JSON string or JSON type is required

db1/main> create table json_test (j json);
db1/main*> insert into json_test (select arguments from dolt_help where name = 'dolt_revert');
db1/main*> select json_pretty(j) from json_test;
+-----------------------------------------------------------------------------------------------------------------------------------+
| json_pretty(j)
  |
+-----------------------------------------------------------------------------------------------------------------------------------+
| {
  |
|   "--author=\u003cauthor\u003e": "Specify an explicit author using the standard A U Thor \[email protected]\u003e format.", |
|   "\u003crevision\u003e": "The commit revisions. If multiple revisions are given, they're applied in the order given."
  |
| }
  |
+-----------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

So whatever it is, it's getting handled by our round-trip into storage, but can't get processed by as returned by this table. This seems to affect all our json functions, so it needs to be fixed. Another example:

db1/main*> select arguments->>"$.<revision>" from dolt_help where name = 'dolt_revert';
invalid data type for JSON data in argument 1 to function json_extract; a JSON string or JSON type is required

db1/main*> select j->>"$.<revision>" from json_test;
+--------------------------------------------------------------------------------------------+
| j->>"$.<revision>"                                                                         |
+--------------------------------------------------------------------------------------------+
| The commit revisions. If multiple revisions are given, they're applied in the order given. |
+--------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it has something to do with the < and > characters. I would expect json.Marshall to do something reasonable there but it apparently is not.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Above examples work great now, but there's a related problem where ascii command characters (used for bolding on the terminal) are making it into the SQL output like this:

 "-f, --force": "Reset \u003cbranchname\u003e to \u003cstartpoint\u003e, even if \u003cbranchname\u003e exists already

I think what we need here is the OptionsUsageList() method to accept a Formatter object (needs to be defined) that we can use to change out the formatting behavior. For the SQL use case, we want to pass a formatter that just deletes the template options, rather than replacing them with command chars like the CLI implementation does.

if err != nil {
return nil, err
}

rows = append(rows, sql.NewRow(
nameFormatted,
"procedure",
curr.Docs().ShortDesc,
curr.Docs().LongDesc,
argsJson,
))
}
}
}

return rows, nil
}
21 changes: 21 additions & 0 deletions integration-tests/bats/system-tables.bats
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ teardown() {
[[ "$output" =~ "dolt_remote_branches" ]] || false
[[ "$output" =~ "dolt_remotes" ]] || false
[[ "$output" =~ "dolt_status" ]] || false
[[ "$output" =~ "dolt_help" ]] || false
[[ "$output" =~ "test" ]] || false

dolt add test
Expand Down Expand Up @@ -620,3 +621,23 @@ SQL
[ "$status" -eq 0 ]
[ "$output" = "" ]
}

@test "system-tables: query dolt_help system table" {
run dolt sql -q "select type from dolt_help where target='dolt_rebase'"
[ "$status" -eq 0 ]
[[ "$output" =~ "procedure" ]] || false

run dolt sql -q "select short_description from dolt_help where target='dolt_commit'"
[ "$status" -eq 0 ]
[[ "$output" =~ "Record changes to the database" ]] || false

run dolt sql -q "select long_description from dolt_help where target='dolt_add'"
[ "$status" -eq 0 ]
[[ "$output" =~ "This command updates the list of tables using the current content found in the working root" ]] || false
[[ "$output" =~ "This command can be performed multiple times before a commit." ]] || false

run dolt sql -q "select arguments from dolt_help where target='dolt_pull'"
[ "$status" -eq 0 ]
[[ "$output" =~ "remote".*"The name of the remote to pull from." ]] || false
[[ "$output" =~ "remoteBranch".*"The name of a branch on the specified remote to be merged into the current working set." ]] || false
}
Loading