Skip to content

Commit

Permalink
Implement EXPLAIN ANALYZE (#66)
Browse files Browse the repository at this point in the history 
* Implement prototype of EXPLAIN ANALYZE

* Fix format

* Refactor getStringValueFromPath

* Refactor

* Rename func

* Use protojson to embed stats

* Use tab to separate text and stats

* Embed display_name and link_type into protojson

* Some cleanup

* Add some newlines

* Rename some variable

* Set ForceVerbose when EXPLAIN ANALYZE

* Do go fmt

* Add key name

* Add key name

* Unwrap struct

* Simplify the usage of protojson

* Split branch and protojson by \t

* Add unit of latency total

* Rename fields name

* Use more descriptive name

* Some cleanup

* Do go fmt

* Add TestRenderTreeWithStats

* Refactor TestRenderTreeWithStats

* Add description of EXPLAIN ANALYZE into README.md

* Update README.md

* Fix test

* Fix to use SplitN

* Narrow variable scope

* Update comment
  • Loading branch information
@apstndb
apstndb authored Jun 16, 2020
1 parent 3897e3a commit 8706473
Show file tree
Hide file tree
Showing 5 changed files with 316 additions and 53 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ The syntax is case-insensitive.
| DML | `INSERT / UPDATE / DELETE ...;` | |
| Partitioned DML | | Not supported yet |
| Show Query Execution Plan | `EXPLAIN SELECT ...;` | |
| Show Query Execution Plan with Stats | `EXPLAIN ANALYZE SELECT ...;` | EXPERIMENTAL |
| Start Read-Write Transaction | `BEGIN (RW);` | |
| Commit Read-Write Transaction | `COMMIT;` | |
| Rollback Read-Write Transaction | `ROLLBACK;` | |
Expand Down
4 changes: 3 additions & 1 deletion cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,9 @@ func printResult(out io.Writer, result *Result, mode DisplayMode, interactive, v
}
}

if interactive {
if result.ForceVerbose {
fmt.Fprint(out, resultLine(result, true))
} else if interactive {
fmt.Fprint(out, resultLine(result, verbose))
}
}
Expand Down
90 changes: 89 additions & 1 deletion query_plan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,15 @@
package main

import (
"encoding/json"
"fmt"
"sort"
"strings"

"github.com/xlab/treeprint"
pb "google.golang.org/genproto/googleapis/spanner/v1"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)

func init() {
Expand Down Expand Up @@ -79,6 +82,58 @@ func (n *Node) Render() string {
return "\n" + tree.String()
}

type RenderedTreeWithStats struct {
Text string
RowsTotal string
Execution string
LatencyTotal string
}

func (n *Node) RenderTreeWithStats() []RenderedTreeWithStats {
tree := treeprint.New()
renderTreeWithStats(tree, "", n)
var result []RenderedTreeWithStats
for _, line := range strings.Split(tree.String(), "\n") {
if line == "" {
continue
}

split := strings.SplitN(line, "\t", 2)
// Handle the case of the root node of treeprint
if len(split) != 2 {
result = append(result, RenderedTreeWithStats{Text: line})
continue
}
branchText, protojsonText := split[0], split[1]

var value structpb.Value
if err := protojson.Unmarshal([]byte(protojsonText), &value); err != nil {
result = append(result, RenderedTreeWithStats{Text: line})
continue
}

displayName := getStringValueByPath(value.GetStructValue(), "display_name")
linkType := getStringValueByPath(value.GetStructValue(), "link_type")

var text string
if linkType != "" {
text = fmt.Sprintf("[%s] %s", linkType, displayName)
} else {
text = displayName
}

result = append(result, RenderedTreeWithStats{
Text: branchText + text,
RowsTotal: getStringValueByPath(value.GetStructValue(), "execution_stats", "rows", "total"),
Execution: getStringValueByPath(value.GetStructValue(), "execution_stats", "execution_summary", "num_executions"),
LatencyTotal: fmt.Sprintf("%s %s",
getStringValueByPath(value.GetStructValue(), "execution_stats", "latency", "total"),
getStringValueByPath(value.GetStructValue(), "execution_stats", "latency", "unit")),
})
}
return result
}

func (n *Node) IsVisible() bool {
operator := n.PlanNode.DisplayName
if operator == "Function" || operator == "Reference" || operator == "Constant" {
Expand Down Expand Up @@ -107,7 +162,6 @@ func (n *Node) String() string {
operator = strings.Join(components, " ")
}


var metadata string
{
fields := make([]string, 0)
Expand Down Expand Up @@ -166,3 +220,37 @@ func renderTree(tree treeprint.Tree, linkType string, node *Node) {
}
}
}

func getStringValueByPath(s *structpb.Struct, first string, path ...string) string {
current := s.GetFields()[first]
for _, p := range path {
current = current.GetStructValue().GetFields()[p]
}
return current.GetStringValue()
}

func renderTreeWithStats(tree treeprint.Tree, linkType string, node *Node) {
if !node.IsVisible() {
return
}

statsJson, _ := protojson.Marshal(node.PlanNode.GetExecutionStats())
b, _ := json.Marshal(
map[string]interface{}{
"execution_stats": json.RawMessage(statsJson),
"display_name": node.String(),
"link_type": linkType,
},
)
// Prefixed by tab to ease to split
str := "\t" + string(b)

if len(node.Children) > 0 {
branch := tree.AddBranch(str)
for _, child := range node.Children {
renderTreeWithStats(branch, child.Type, child.Dest)
}
} else {
tree.AddNode(str)
}
}
215 changes: 164 additions & 51 deletions query_plan_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,85 +3,198 @@ package main
import (
"testing"

"github.com/google/go-cmp/cmp"
"google.golang.org/genproto/googleapis/spanner/v1"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)

func protojsonAsStruct(t *testing.T, j string) *structpb.Struct {
t.Helper()
var result structpb.Struct
if err := protojson.Unmarshal([]byte(j), &result); err != nil {
t.Fatal("protojsonAsStruct fails, invalid test case", err)
}
return &result
}

func TestRenderTreeWithStats(t *testing.T) {
for _, test := range []struct {
title string
plan *spanner.QueryPlan
want []RenderedTreeWithStats
}{
{title: "Simple Query",
plan: &spanner.QueryPlan{
PlanNodes: []*spanner.PlanNode{
{
ChildLinks: []*spanner.PlanNode_ChildLink{
{
ChildIndex: 1,
},
},
DisplayName: "Distributed Union",
Kind: spanner.PlanNode_RELATIONAL,
ExecutionStats: protojsonAsStruct(t, `
{
"latency": {"total": "1", "unit": "msec"},
"rows": {"total": "9"},
"execution_summary": {"num_executions": "1"}
}`),
},
{
ChildLinks: []*spanner.PlanNode_ChildLink{
{
ChildIndex: 2,
},
},
DisplayName: "Distributed Union",
Kind: spanner.PlanNode_RELATIONAL,
Metadata: protojsonAsStruct(t, `{"call_type": "Local"}`),
ExecutionStats: protojsonAsStruct(t, `
{
"latency": {"total": "1", "unit": "msec"},
"rows": {"total": "9"},
"execution_summary": {"num_executions": "1"}
}`),
},
{
ChildLinks: []*spanner.PlanNode_ChildLink{
{
ChildIndex: 3,
},
},
DisplayName: "Serialize Result",
Kind: spanner.PlanNode_RELATIONAL,
ExecutionStats: protojsonAsStruct(t, `
{
"latency": {"total": "1", "unit": "msec"},
"rows": {"total": "9"},
"execution_summary": {"num_executions": "1"}
}`),
},
{
DisplayName: "Scan",
Kind: spanner.PlanNode_RELATIONAL,
Metadata: protojsonAsStruct(t, `{"scan_type": "IndexScan", "scan_target": "SongsBySingerAlbumSongNameDesc", "Full scan": "true"}`),
ExecutionStats: protojsonAsStruct(t, `
{
"latency": {"total": "1", "unit": "msec"},
"rows": {"total": "9"},
"execution_summary": {"num_executions": "1"}
}`),
},
},
},
want: []RenderedTreeWithStats{
{Text: "."},
{
Text: "+- Distributed Union",
RowsTotal: "9",
Execution: "1",
LatencyTotal: "1 msec",
},
{
Text: " +- Local Distributed Union",
RowsTotal: "9",
Execution: "1",
LatencyTotal: "1 msec",
},
{
Text: " +- Serialize Result",
RowsTotal: "9",
Execution: "1",
LatencyTotal: "1 msec",
},
{
Text: " +- Index Scan (Full scan: true, Index: SongsBySingerAlbumSongNameDesc)",
RowsTotal: "9",
Execution: "1",
LatencyTotal: "1 msec",
},
}},
} {
tree := BuildQueryPlanTree(test.plan, 0)
if got := tree.RenderTreeWithStats(); !cmp.Equal(test.want, got) {
t.Errorf("%s: node.RenderTreeWithStats() differ: %s", test.title, cmp.Diff(test.want, got))
}
}
}
func TestNodeString(t *testing.T) {
for _, test := range []struct {
title string
node *Node
want string
title string
node *Node
want string
}{
{"Distributed Union with call_type=Local",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Distributed Union",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Local"}},
"subquery_cluster_node": {Kind: &structpb.Value_StringValue{StringValue: "4"}},
DisplayName: "Distributed Union",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Local"}},
"subquery_cluster_node": {Kind: &structpb.Value_StringValue{StringValue: "4"}},
},
},
},
}}, "Local Distributed Union",
}}, "Local Distributed Union",
},
{"Scan with scan_type=IndexScan and Full scan=true",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "IndexScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "SongsBySongName"}},
"Full scan": {Kind: &structpb.Value_StringValue{StringValue: "true"}},
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "IndexScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "SongsBySongName"}},
"Full scan": {Kind: &structpb.Value_StringValue{StringValue: "true"}},
},
},
},
}}, "Index Scan (Full scan: true, Index: SongsBySongName)"},
{ "Scan with scan_type=TableScan",
}}, "Index Scan (Full scan: true, Index: SongsBySongName)"},
{"Scan with scan_type=TableScan",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "TableScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "Songs"}},
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "TableScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "Songs"}},
},
},
},
}}, "Table Scan (Table: Songs)"},
}}, "Table Scan (Table: Songs)"},
{"Scan with scan_type=BatchScan",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "BatchScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "$v2"}},
DisplayName: "Scan",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"scan_type": {Kind: &structpb.Value_StringValue{StringValue: "BatchScan"}},
"scan_target": {Kind: &structpb.Value_StringValue{StringValue: "$v2"}},
},
},
},
}}, "Batch Scan (Batch: $v2)"},
}}, "Batch Scan (Batch: $v2)"},
{"Sort Limit with call_type=Local",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Sort Limit",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Local"}},
DisplayName: "Sort Limit",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Local"}},
},
},
},
}}, "Local Sort Limit"},
}}, "Local Sort Limit"},
{"Sort Limit with call_type=Global",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Sort Limit",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Global"}},
DisplayName: "Sort Limit",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"call_type": {Kind: &structpb.Value_StringValue{StringValue: "Global"}},
},
},
},
}}, "Global Sort Limit"},
}}, "Global Sort Limit"},
{"Aggregate with iterator_type=Stream",
&Node{PlanNode: &spanner.PlanNode{
DisplayName: "Aggregate",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"iterator_type": {Kind: &structpb.Value_StringValue{StringValue: "Stream"}},
DisplayName: "Aggregate",
Metadata: &structpb.Struct{
Fields: map[string]*structpb.Value{
"iterator_type": {Kind: &structpb.Value_StringValue{StringValue: "Stream"}},
},
},
},
}}, "Stream Aggregate"},
}}, "Stream Aggregate"},
} {
if got := test.node.String(); got != test.want {
t.Errorf("%s: node.String() = %q but want %q", test.title, got, test.want)
Expand Down
Loading

0 comments on commit 8706473

Please sign in to comment.