Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create dolt_help system table #8739

Merged
merged 22 commits into from
Jan 21, 2025
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 32 additions & 18 deletions go/cmd/dolt/cli/help.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,21 +166,35 @@ func terminalSize() (width, height int) {
func OptionsUsage(ap *argparser.ArgParser, indent string, lineLen int) string {
var lines []string

for _, kvTuple := range ap.ArgListHelp {
k, v := kvTuple[0], kvTuple[1]
lines = append(lines, "<"+k+">")
l, err := templateDocStringHelper(v, CliFormat)
if err != nil {
panic(err)
}
l = embolden(l)
descLines := toParagraphLines(l, lineLen)
for _, usage := range OptionsUsageList(ap) {
name, description := usage[0], usage[1]

lines = append(lines, name)

descLines := toParagraphLines(description, lineLen)
descLines = indentLines(descLines, " ")
descLines = append(descLines, "")

lines = append(lines, descLines...)
}

lines = indentLines(lines, indent)
return strings.Join(lines, "\n")
}

// OptionsUsageList returns a pair of strings for each option/argument in |ap|, where the first string
// is the name of the option/argument and the second string is the description of the option/argument.
func OptionsUsageList(ap *argparser.ArgParser) [][2]string {
res := [][2]string{}

for _, help := range ap.ArgListHelp {
name, description := help[0], help[1]

nameFormatted := "<" + name + ">"

res = append(res, [2]string{nameFormatted, description})
}

for _, supOpt := range ap.Supported {
argHelpFmt := "--%[2]s"

Expand All @@ -192,22 +206,22 @@ func OptionsUsage(ap *argparser.ArgParser, indent string, lineLen int) string {
argHelpFmt = "--%[2]s=<%[3]s>"
}

lines = append(lines, fmt.Sprintf(argHelpFmt, supOpt.Abbrev, supOpt.Name, supOpt.ValDesc))
nameFormatted := fmt.Sprintf(argHelpFmt, supOpt.Abbrev, supOpt.Name, supOpt.ValDesc)

l, err := templateDocStringHelper(supOpt.Desc, CliFormat)
res = append(res, [2]string{nameFormatted, supOpt.Desc})
}

for i := range res {
descriptionFormatted, err := templateDocStringHelper(res[i][1], CliFormat)
if err != nil {
panic(err)
}
l = embolden(l)
descLines := toParagraphLines(l, lineLen)
descLines = indentLines(descLines, " ")
descLines = append(descLines, "")
descriptionFormatted = embolden(descriptionFormatted)

lines = append(lines, descLines...)
res[i][1] = descriptionFormatted
}

lines = indentLines(lines, indent)
return strings.Join(lines, "\n")
return res
}

func ToIndentedParagraph(inStr, indent string, lineLen int) string {
Expand Down
3 changes: 3 additions & 0 deletions go/cmd/dolt/dolt.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables"
"github.com/dolthub/dolt/go/libraries/events"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
"github.com/dolthub/dolt/go/libraries/utils/config"
Expand Down Expand Up @@ -226,6 +227,8 @@ func init() {
if _, ok := os.LookupEnv(disableEventFlushEnvVar); ok {
eventFlushDisabled = true
}

dtables.DoltCommand = doltCommand
}

const pprofServerFlag = "--pprof-server"
Expand Down
10 changes: 10 additions & 0 deletions go/libraries/doltcore/doltdb/system_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ var getGeneratedSystemTables = func() []string {
GetCommitAncestorsTableName(),
GetStatusTableName(),
GetRemotesTableName(),
GetHelpTableName(),
}
}

Expand Down Expand Up @@ -380,6 +381,11 @@ var GetTagsTableName = func() string {
return TagsTableName
}

// GetHelpTableName returns the help table name
var GetHelpTableName = func() string {
return HelpTableName
}

const (
// LogTableName is the log system table name
LogTableName = "dolt_log"
Expand Down Expand Up @@ -585,3 +591,7 @@ const (
// was originally defined. Mode settings, such as ANSI_QUOTES, are needed to correctly parse the fragment.
ProceduresTableSqlModeCol = "sql_mode"
)

const (
HelpTableName = "dolt_help"
)
8 changes: 8 additions & 0 deletions go/libraries/doltcore/sqle/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,14 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds
return nil, false, err
}
dt = NewSchemaTable(backingTable)
case doltdb.GetHelpTableName(), doltdb.HelpTableName:
isDoltgresSystemTable, err := resolve.IsDoltgresSystemTable(ctx, tname, root)
if err != nil {
return nil, false, err
}
if !resolve.UseSearchPath || isDoltgresSystemTable {
dt, found = dtables.NewHelpTable(ctx, db.Name(), lwrName), true
}
}

if found {
Expand Down
232 changes: 232 additions & 0 deletions go/libraries/doltcore/sqle/dtables/help_table.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,232 @@
// Copyright 2025 Dolthub, Inc.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dtables

import (
"encoding/json"
"io"
"strings"

"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/go-mysql-server/sql"
sqlTypes "github.com/dolthub/go-mysql-server/sql/types"
)

type HelpTable struct {
dbName string
tableName string
}

var HelpTableTypes = []string{
"system_table",
"procedure",
"function",
"variable",
}

// NewHelpTable creates a HelpTable
func NewHelpTable(_ *sql.Context, dbName, tableName string) sql.Table {
return &HelpTable{dbName: dbName, tableName: tableName}
}

// Name is a sql.Table interface function which returns the name of the table.
func (ht *HelpTable) Name() string {
return ht.tableName
}

// String is a sql.Table interface function which returns the name of the table.
func (ht *HelpTable) String() string {
return ht.tableName
}

// Schema is a sql.Table interface function that gets the sql.Schema of the help system table.
func (ht *HelpTable) Schema() sql.Schema {
return []*sql.Column{
{
Name: "name",
Type: sqlTypes.TinyText,
Source: ht.tableName,
PrimaryKey: true,
DatabaseSource: ht.dbName,
},
{
Name: "type",
Type: sqlTypes.MustCreateEnumType(HelpTableTypes, sql.Collation_Default),
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
zachmu marked this conversation as resolved.
Show resolved Hide resolved
Name: "synopsis",
Type: sqlTypes.LongText,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
Name: "short_description",
Type: sqlTypes.LongText,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
Name: "long_description",
Type: sqlTypes.LongText,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
{
Name: "arguments",
Type: sqlTypes.JSON,
Source: ht.tableName,
PrimaryKey: false,
DatabaseSource: ht.dbName,
},
}
}

// Collation implements the sql.Table interface.
func (ht *HelpTable) Collation() sql.CollationID {
return sql.Collation_Default
}

// Partitions is a sql.Table interface function that returns a partition
// of the data. Currently the data is unpartitioned.
func (ht *HelpTable) Partitions(*sql.Context) (sql.PartitionIter, error) {
return index.SinglePartitionIterFromNomsMap(nil), nil
}

// PartitionRows is a sql.Table interface function that gets a row iterator for a partition.
func (ht *HelpTable) PartitionRows(_ *sql.Context, _ sql.Partition) (sql.RowIter, error) {
return NewHelpRowIter(), nil
}

type HelpRowIter []sql.Row
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prefer to define an actual struct here, with []sql.Row field and a counter int to keep track of the place in it.


func NewHelpRowIter() *HelpRowIter {
var nilIter HelpRowIter
return &nilIter
}

// DoltCommand is set in cmd/dolt/dolt.go to avoid circular dependency.
var DoltCommand cli.SubCommandHandler

func (itr *HelpRowIter) Next(_ *sql.Context) (sql.Row, error) {
if *itr == nil {
var err error
*itr, err = generateProcedureHelpRows(DoltCommand.Name(), DoltCommand.Subcommands)
if err != nil {
return nil, err
}
}

helpRows := *itr

if len(helpRows) == 0 {
return nil, io.EOF
}

row := helpRows[0]
*itr = helpRows[1:]

return row, nil
}

func (itr *HelpRowIter) Close(_ *sql.Context) error {
return nil
}

// generateProcedureHelpRows generates a sql row for each procedure that has an equivalent CLI command.
func generateProcedureHelpRows(cmdStr string, subCommands []cli.Command) ([]sql.Row, error) {
rows := []sql.Row{}

for _, curr := range subCommands {
if hidCmd, ok := curr.(cli.HiddenCommand); ok && hidCmd.Hidden() {
continue
}

if subCmdHandler, ok := curr.(cli.SubCommandHandler); ok {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This loop can be simplified, because there are no procedures that correspond to sub commands. E.g. there is no dolt_table_ls command.

if subCmdHandler.Unspecified != nil {
newRows, err := generateProcedureHelpRows(cmdStr, []cli.Command{subCmdHandler.Unspecified})
if err != nil {
return nil, err
}
rows = append(rows, newRows...)
}
newRows, err := generateProcedureHelpRows(cmdStr+"_"+subCmdHandler.Name(), subCmdHandler.Subcommands)
if err != nil {
return nil, err
}
rows = append(rows, newRows...)
} else {
fullName := cmdStr + "_" + curr.Name()
procedureName := strings.ReplaceAll(fullName, "-", "_")

hasProcedure := false
for _, procedure := range dprocedures.DoltProcedures {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would pull out a boolean procedureExists method for this, and check for this condition before beginning the loop

if procedure.Name == procedureName {
hasProcedure = true
break
}
}

docs := curr.Docs()

if hasProcedure && docs != nil {
argsMap := map[string]string{}
for _, usage := range cli.OptionsUsageList(docs.ArgParser) {
argsMap[usage[0]] = usage[1]
}

argsJson, err := json.Marshal(argsMap)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why, but something about this seems to be producing invalid json in a subtle and weird way. Check this out:

db1/main> select json_pretty(arguments) from dolt_help where name = 'dolt_revert';
invalid data type for JSON data in argument 1 to function json_pretty; a JSON string or JSON type is required

db1/main> create table json_test (j json);
db1/main*> insert into json_test (select arguments from dolt_help where name = 'dolt_revert');
db1/main*> select json_pretty(j) from json_test;
+-----------------------------------------------------------------------------------------------------------------------------------+
| json_pretty(j)
  |
+-----------------------------------------------------------------------------------------------------------------------------------+
| {
  |
|   "--author=\u003cauthor\u003e": "Specify an explicit author using the standard A U Thor \[email protected]\u003e format.", |
|   "\u003crevision\u003e": "The commit revisions. If multiple revisions are given, they're applied in the order given."
  |
| }
  |
+-----------------------------------------------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

So whatever it is, it's getting handled by our round-trip into storage, but can't get processed by as returned by this table. This seems to affect all our json functions, so it needs to be fixed. Another example:

db1/main*> select arguments->>"$.<revision>" from dolt_help where name = 'dolt_revert';
invalid data type for JSON data in argument 1 to function json_extract; a JSON string or JSON type is required

db1/main*> select j->>"$.<revision>" from json_test;
+--------------------------------------------------------------------------------------------+
| j->>"$.<revision>"                                                                         |
+--------------------------------------------------------------------------------------------+
| The commit revisions. If multiple revisions are given, they're applied in the order given. |
+--------------------------------------------------------------------------------------------+
1 row in set (0.00 sec)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it has something to do with the < and > characters. I would expect json.Marshall to do something reasonable there but it apparently is not.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Above examples work great now, but there's a related problem where ascii command characters (used for bolding on the terminal) are making it into the SQL output like this:

 "-f, --force": "Reset \u003cbranchname\u003e to \u003cstartpoint\u003e, even if \u003cbranchname\u003e exists already

I think what we need here is the OptionsUsageList() method to accept a Formatter object (needs to be defined) that we can use to change out the formatting behavior. For the SQL use case, we want to pass a formatter that just deletes the template options, rather than replacing them with command chars like the CLI implementation does.

if err != nil {
return nil, err
}

synopsis, err := docs.GetSynopsis(cli.CliFormat)
zachmu marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, err
}

cliName := strings.ReplaceAll(fullName, "_", " ")
for i := range synopsis {
synopsis[i] = cliName + " " + synopsis[i]
}

shortDesc := docs.GetShortDesc()

longDesc, err := docs.GetLongDesc(cli.CliFormat)
if err != nil {
return nil, err
}

rows = append(rows, sql.NewRow(
procedureName,
"procedure",
strings.Join(synopsis, "\n"),
shortDesc,
longDesc,
argsJson,
))
}
}
}

return rows, nil
}
1 change: 1 addition & 0 deletions go/libraries/doltcore/sqle/enginetest/dolt_queries.go
Original file line number Diff line number Diff line change
Expand Up @@ -7565,6 +7565,7 @@ var DoltSystemVariables = []queries.ScriptTest{
{"dolt_constraint_violations"},
{"dolt_constraint_violations_test"},
{"dolt_diff_test"},
{"dolt_help"},
{"dolt_history_test"},
{"dolt_log"},
{"dolt_remote_branches"},
Expand Down
3 changes: 2 additions & 1 deletion integration-tests/bats/ls.bats
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ teardown() {
@test "ls: --system shows system tables" {
run dolt ls --system
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 22 ]
[ "${#lines[@]}" -eq 23 ]
[[ "$output" =~ "System tables:" ]] || false
[[ "$output" =~ "dolt_status" ]] || false
[[ "$output" =~ "dolt_commits" ]] || false
Expand All @@ -71,6 +71,7 @@ teardown() {
[[ "$output" =~ "dolt_remotes" ]] || false
[[ "$output" =~ "dolt_branches" ]] || false
[[ "$output" =~ "dolt_remote_branches" ]] || false
[[ "$output" =~ "dolt_help" ]] || false
[[ "$output" =~ "dolt_constraint_violations_table_one" ]] || false
[[ "$output" =~ "dolt_history_table_one" ]] || false
[[ "$output" =~ "dolt_conflicts_table_one" ]] || false
Expand Down
Loading
Loading