diff --git a/lib/coverage.go b/lib/coverage.go new file mode 100644 index 0000000..0ecdbe0 --- /dev/null +++ b/lib/coverage.go @@ -0,0 +1,244 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package lib + +import ( + "bytes" + "errors" + "fmt" + "sort" + "strings" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/interpreter" +) + +// NewCoverage return an execution coverage statistics collector for the +// provided AST. +func NewCoverage(ast *cel.Ast) *Coverage { + return &Coverage{ + ast: ast, + decorator: coverage{ + all: make(map[int64]bool), + cov: make(map[int64]bool), + }, + } +} + +// Coverage is a CEL program execution coverage statistics collector. +type Coverage struct { + ast *cel.Ast + decorator coverage +} + +// ProgramOption return a cel.ProgramOption that can be used in a call to +// cel.Env.Program to collect coverage information from the program's execution. +func (c *Coverage) ProgramOption() cel.ProgramOption { + return cel.CustomDecorator(func(i interpreter.Interpretable) (interpreter.Interpretable, error) { + c.decorator.all[i.ID()] = true + switch i := i.(type) { + case interpreter.InterpretableAttribute: + return coverageAttribute{InterpretableAttribute: i, cov: c.decorator.cov}, nil + default: + return coverage{Interpretable: i, cov: c.decorator.cov}, nil + } + }) +} + +func (c *Coverage) String() string { + var buf bytes.Buffer + for i, d := range c.Details() { + if i != 0 { + fmt.Fprintln(&buf) + } + fmt.Fprintf(&buf, "%s", d) + } + return buf.String() +} + +// Merge adds node coverage from o into c. If c was constructed with NewCoverage +// o and c must have been constructed with the AST from the same source. If o is +// nil, Merge is a no-op. +func (c *Coverage) Merge(o *Coverage) error { + if o == nil { + return nil + } + if c.decorator.all == nil { + *c = *o + return nil + } + if !equalNodes(c.decorator.all, o.decorator.all) { + return errors.New("cannot merge unrelated coverage: mismatched nodes") + } + if c.ast.Source().Content() != o.ast.Source().Content() { + return errors.New("cannot merge unrelated coverage: mismatched source") + } + for id := range o.decorator.cov { + c.decorator.cov[id] = true + } + return nil +} + +func equalNodes(a, b map[int64]bool) bool { + if len(a) != len(b) { + return false + } + for k, v1 := range a { + if v2, ok := b[k]; !ok || v1 != v2 { + return false + } + } + return true +} + +// Details returns the coverage details from running the target CEL program. +func (c *Coverage) Details() []LineCoverage { + nodes := make(map[int][]int64) + for id := range c.decorator.all { + line := c.ast.NativeRep().SourceInfo().GetStartLocation(id).Line() + nodes[line] = append(nodes[line], id) + } + hits := make(map[int][]int64) + for id := range c.decorator.cov { + line := c.ast.NativeRep().SourceInfo().GetStartLocation(id).Line() + hits[line] = append(hits[line], id) + } + stats := make(map[int]float64) + var lines []int + for l := range nodes { + sort.Slice(nodes[l], func(i, j int) bool { return nodes[l][i] < nodes[l][j] }) + sort.Slice(hits[l], func(i, j int) bool { return hits[l][i] < hits[l][j] }) + stats[l] = float64(len(hits[l])) / float64(len(nodes[l])) + lines = append(lines, l) + } + sort.Ints(lines) + cov := make([]LineCoverage, 0, len(lines)) + src := c.ast.Source() + for _, l := range lines { + var missed []int64 + i, j := 0, 0 + for i < len(nodes[l]) && j < len(hits[l]) { + if nodes[l][i] == hits[l][j] { + i++ + j++ + continue + } + missed = append(missed, nodes[l][i]) + i++ + } + missed = append(missed, nodes[l][i:]...) + cov = append(cov, LineCoverage{ + Line: l, + Coverage: stats[l], + Nodes: nodes[l], + Covered: hits[l], + Missed: missed, + Annotation: srcAnnot(c.ast, src, missed, "!"), + }) + } + return cov +} + +func srcAnnot(ast *cel.Ast, src common.Source, nodes []int64, mark string) string { + if len(nodes) == 0 { + return "" + } + var buf bytes.Buffer + columns := make(map[int]bool) + var snippet string + for _, id := range nodes { + loc := ast.NativeRep().SourceInfo().GetStopLocation(id) + if columns[loc.Column()] { + continue + } + columns[loc.Column()] = true + if snippet == "" { + var ok bool + snippet, ok = src.Snippet(loc.Line()) + if !ok { + continue + } + } + } + missed := make([]int, 0, len(columns)) + for col := range columns { + missed = append(missed, col) + } + sort.Ints(missed) + fmt.Fprintln(&buf, " | "+strings.Replace(snippet, "\t", " ", -1)) + fmt.Fprint(&buf, " | ") + var last int + for _, col := range missed { + fmt.Fprint(&buf, strings.Repeat(" ", minInt(col, len(snippet))-last)+mark) + last = col + 1 + } + return buf.String() +} + +type coverage struct { + interpreter.Interpretable + all map[int64]bool + cov map[int64]bool +} + +func (c coverage) Eval(a interpreter.Activation) ref.Val { + c.cov[c.ID()] = true + return c.Interpretable.Eval(a) +} + +type coverageAttribute struct { + interpreter.InterpretableAttribute + all map[int64]bool + cov map[int64]bool +} + +func (c coverageAttribute) Eval(a interpreter.Activation) ref.Val { + c.cov[c.ID()] = true + return c.InterpretableAttribute.Eval(a) +} + +// LineCoverage is the execution coverage data for a single line of a CEL +// program. +type LineCoverage struct { + // Line is the line number of the program. + Line int `json:"line"` + // Coverage is the fraction of CEL expression nodes + // executed on the line. + Coverage float64 `json:"coverage"` + // Nodes is the full set of expression nodes on + // the line. + Nodes []int64 `json:"nodes"` + // Nodes is the set of expression nodes that were + // executed. + Covered []int64 `json:"covered"` + // Nodes is the set of expression nodes that were + // not executed. + Missed []int64 `json:"missed"` + // Annotation is a textual representation of the + // line, marking positions that were not executed. + Annotation string `json:"annotation"` +} + +func (c LineCoverage) String() string { + if c.Annotation == "" { + return fmt.Sprintf("%d: %0.2f (%d/%d)", c.Line, c.Coverage, len(c.Covered), len(c.Nodes)) + } + return fmt.Sprintf("%d: %0.2f (%d/%d) %v\n%s", c.Line, c.Coverage, len(c.Covered), len(c.Nodes), c.Missed, c.Annotation) +} diff --git a/mito.go b/mito.go index 77cc65e..f58ac5b 100644 --- a/mito.go +++ b/mito.go @@ -75,6 +75,7 @@ func Main() int { maxTraceBody := flag.Int("max_log_body", 1000, "maximum length of body logged in request traces (go1.21+)") fold := flag.Bool("fold", false, "apply constant folding optimisation") dumpState := flag.String("dump", "", "dump eval state ('always' or 'error')") + coverage := flag.String("coverage", "", "file to write an execution coverage report to (prefix if multiple executions are run)") version := flag.Bool("version", false, "print version and exit") flag.Parse() if *version { @@ -195,8 +196,13 @@ func Main() int { input = map[string]interface{}{root: input} } + var cov lib.Coverage for n := int(0); *maxExecutions < 0 || n < *maxExecutions; n++ { - res, val, dump, err := eval(string(b), root, input, *fold, *dumpState != "", libs...) + res, val, dump, c, err := eval(string(b), root, input, *fold, *dumpState != "", *coverage != "", libs...) + if err := cov.Merge(c); err != nil { + fmt.Fprintf(os.Stderr, "internal error merging coverage: %v\n", err) + return 2 + } if *dumpState == "always" { fmt.Fprint(os.Stderr, dump) } @@ -220,6 +226,22 @@ func Main() int { } input = map[string]any{"state": val} } + if *coverage != "" { + f, err := os.Create(*coverage) + if err != nil { + fmt.Fprintf(os.Stderr, "internal error opening coverage file: %v\n", err) + return 2 + } + defer func() { + f.Sync() + f.Close() + }() + _, err = f.WriteString(cov.String() + "\n") + if err != nil { + fmt.Fprintf(os.Stderr, "internal error writing coverage file: %v\n", err) + return 2 + } + } return 0 } @@ -332,53 +354,58 @@ func debug(tag string, value any) { fmt.Fprintf(os.Stderr, "%s: logging %q: %v\n", level, tag, value) } -func eval(src, root string, input interface{}, fold, details bool, libs ...cel.EnvOption) (string, any, *lib.Dump, error) { - prg, ast, err := compile(src, root, fold, details, libs...) +func eval(src, root string, input interface{}, fold, details, coverage bool, libs ...cel.EnvOption) (string, any, *lib.Dump, *lib.Coverage, error) { + prg, ast, cov, err := compile(src, root, fold, details, coverage, libs...) if err != nil { - return "", nil, nil, fmt.Errorf("failed program instantiation: %v", err) + return "", nil, nil, nil, fmt.Errorf("failed program instantiation: %v", err) } res, val, det, err := run(prg, ast, false, input) var dump *lib.Dump if details { dump = lib.NewDump(ast, det) } - return res, val, dump, err + return res, val, dump, cov, err } -func compile(src, root string, fold, details bool, libs ...cel.EnvOption) (cel.Program, *cel.Ast, error) { +func compile(src, root string, fold, details, coverage bool, libs ...cel.EnvOption) (cel.Program, *cel.Ast, *lib.Coverage, error) { opts := append([]cel.EnvOption{ cel.Declarations(decls.NewVar(root, decls.Dyn)), }, libs...) env, err := cel.NewEnv(opts...) if err != nil { - return nil, nil, fmt.Errorf("failed to create env: %v", err) + return nil, nil, nil, fmt.Errorf("failed to create env: %v", err) } ast, iss := env.Compile(src) if iss.Err() != nil { - return nil, nil, fmt.Errorf("failed compilation: %v", iss.Err()) + return nil, nil, nil, fmt.Errorf("failed compilation: %v", iss.Err()) } if fold { folder, err := cel.NewConstantFoldingOptimizer() if err != nil { - return nil, nil, fmt.Errorf("failed folding optimization: %v", err) + return nil, nil, nil, fmt.Errorf("failed folding optimization: %v", err) } ast, iss = cel.NewStaticOptimizer(folder).Optimize(env, ast) if iss.Err() != nil { - return nil, nil, fmt.Errorf("failed optimization: %v", iss.Err()) + return nil, nil, nil, fmt.Errorf("failed optimization: %v", iss.Err()) } } + var cov *lib.Coverage var progOpts []cel.ProgramOption + if coverage { + cov = lib.NewCoverage(ast) + progOpts = []cel.ProgramOption{cov.ProgramOption()} + } if details { - progOpts = []cel.ProgramOption{cel.EvalOptions(cel.OptTrackState)} + progOpts = append(progOpts, cel.EvalOptions(cel.OptTrackState)) } prg, err := env.Program(ast, progOpts...) if err != nil { - return nil, nil, fmt.Errorf("failed program instantiation: %v", err) + return nil, nil, nil, fmt.Errorf("failed program instantiation: %v", err) } - return prg, ast, nil + return prg, ast, cov, nil } func run(prg cel.Program, ast *cel.Ast, fast bool, input interface{}) (string, any, *cel.EvalDetails, error) { diff --git a/mito_bench_test.go b/mito_bench_test.go index 9770eaa..ef0c4f4 100644 --- a/mito_bench_test.go +++ b/mito_bench_test.go @@ -42,11 +42,12 @@ var benchmarks = []struct { { name: "hello_world_static", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `"hello world"`, root, fold, false, + false, ) return prg, ast, nil, err }, @@ -54,11 +55,12 @@ var benchmarks = []struct { { name: "hello_world_object_static", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `{"greeting":"hello world"}`, root, fold, false, + false, ) return prg, ast, nil, err }, @@ -66,11 +68,12 @@ var benchmarks = []struct { { name: "nested_static", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `{"a":{"b":{"c":{"d":{"e":"f"}}}}}`, root, fold, false, + false, ) return prg, ast, nil, err }, @@ -78,11 +81,12 @@ var benchmarks = []struct { { name: "encode_json_static", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `{"a":{"b":{"c":{"d":{"e":"f"}}}}}.encode_json()`, root, fold, false, + false, lib.JSON(nil), ) return prg, ast, nil, err @@ -91,11 +95,12 @@ var benchmarks = []struct { { name: "nested_collate_static", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `{"a":{"b":{"c":{"d":{"e":"f"}}}}}.collate("a.b.c.d.e")`, root, fold, false, + false, lib.Collections(), ) return prg, ast, nil, err @@ -106,7 +111,7 @@ var benchmarks = []struct { { name: "hello_world_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile(root, root, fold, false) + prg, ast, _, err := compile(root, root, fold, false, false) state := map[string]any{root: "hello world"} return prg, ast, state, err }, @@ -114,11 +119,12 @@ var benchmarks = []struct { { name: "hello_world_object_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile( + prg, ast, _, err := compile( `{"greeting":state.greeting}`, root, fold, false, + false, ) state := map[string]any{root: mustParseJSON(`{"greeting": "hello world}"}`)} return prg, ast, state, err @@ -127,7 +133,7 @@ var benchmarks = []struct { { name: "nested_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile(root, root, fold, false) + prg, ast, _, err := compile(root, root, fold, false, false) state := map[string]any{root: mustParseJSON(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)} return prg, ast, state, err }, @@ -135,10 +141,11 @@ var benchmarks = []struct { { name: "encode_json_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile(`state.encode_json()`, + prg, ast, _, err := compile(`state.encode_json()`, root, fold, false, + false, lib.JSON(nil), ) state := map[string]any{root: mustParseJSON(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)} @@ -155,10 +162,11 @@ var benchmarks = []struct { { name: "nested_collate_list_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile(`[state].collate("a.b.c.d.e")`, + prg, ast, _, err := compile(`[state].collate("a.b.c.d.e")`, root, fold, false, + false, lib.Collections(), ) state := map[string]any{root: mustParseJSON(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)} @@ -168,10 +176,11 @@ var benchmarks = []struct { { name: "nested_collate_map_state", setup: func(b *testing.B, fold bool) (cel.Program, *cel.Ast, any, error) { - prg, ast, err := compile(`{"state": state}.collate("state.a.b.c.d.e")`, + prg, ast, _, err := compile(`{"state": state}.collate("state.a.b.c.d.e")`, root, fold, false, + false, lib.Collections(), ) state := map[string]any{root: mustParseJSON(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)} @@ -190,11 +199,12 @@ var benchmarks = []struct { w.WriteHeader(http.StatusOK) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`get(%q).size()`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), ) return prg, ast, nil, err @@ -207,11 +217,12 @@ var benchmarks = []struct { w.Write([]byte("hello world")) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`string(get(%q).Body)`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), ) return prg, ast, nil, err @@ -224,11 +235,12 @@ var benchmarks = []struct { w.Write([]byte(`{"greeting":"hello world"}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`{"greeting":bytes(get(%q).Body).decode_json().greeting}`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), ) @@ -242,11 +254,12 @@ var benchmarks = []struct { w.Write([]byte(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`bytes(get(%q).Body).decode_json()`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), ) @@ -260,11 +273,12 @@ var benchmarks = []struct { w.Write([]byte(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`get(%q).Body`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), ) @@ -280,11 +294,12 @@ var benchmarks = []struct { w.Write([]byte(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`bytes(get(%q).Body).decode_json().encode_json()`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), ) @@ -298,11 +313,12 @@ var benchmarks = []struct { w.Write([]byte(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`[bytes(get(%q).Body).decode_json()].collate("a.b.c.d.e")`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), lib.Collections(), @@ -317,11 +333,12 @@ var benchmarks = []struct { w.Write([]byte(`{"a":{"b":{"c":{"d":{"e":"f"}}}}}`)) })) b.Cleanup(func() { srv.Close() }) - prg, ast, err := compile( + prg, ast, _, err := compile( fmt.Sprintf(`{"body": bytes(get(%q).Body).decode_json()}.collate("body.a.b.c.d.e")`, srv.URL), root, fold, false, + false, lib.HTTP(srv.Client(), nil, nil), lib.JSON(nil), lib.Collections(), diff --git a/mito_test.go b/mito_test.go index d39dd11..c3f1a9d 100644 --- a/mito_test.go +++ b/mito_test.go @@ -150,7 +150,7 @@ func TestSend(t *testing.T) { got = <-chans["ch"] }() - res, _, _, err := eval(`42.send_to("ch").close("ch")`, "", nil, fold, false, send) + res, _, _, _, err := eval(`42.send_to("ch").close("ch")`, "", nil, fold, false, false, send) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -257,7 +257,7 @@ func TestVars(t *testing.T) { name = "folded" } t.Run(name, func(t *testing.T) { - got, _, _, err := eval(src, "", interpreter.EmptyActivation(), fold, false, vars) + got, _, _, _, err := eval(src, "", interpreter.EmptyActivation(), fold, false, false, vars) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -381,7 +381,7 @@ func TestRegaxp(t *testing.T) { name = "folded" } t.Run(name, func(t *testing.T) { - got, _, _, err := eval(test.src, "", interpreter.EmptyActivation(), fold, false, lib.Regexp(test.regexps)) + got, _, _, _, err := eval(test.src, "", interpreter.EmptyActivation(), fold, false, false, lib.Regexp(test.regexps)) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/testdata/want_more_coverage.txt b/testdata/want_more_coverage.txt new file mode 100644 index 0000000..f559b5b --- /dev/null +++ b/testdata/want_more_coverage.txt @@ -0,0 +1,66 @@ +mito -coverage cov.txt -data state.json src.cel +! stderr . +cmp stdout want.txt + +cmp cov.txt want_cov.txt + +-- state.json -- +{"n": 0} +-- src.cel -- +int(state.n).as(n, { + "n": n+1, + "want_more": n+1 < 5, + "probe": n < 2 ? + "little" + : + "big", + "fail_probe": n < 0 ? + "negative" + : + "non-negative", +}) +-- want.txt -- +{ + "fail_probe": "non-negative", + "n": 1, + "probe": "little", + "want_more": true +} +{ + "fail_probe": "non-negative", + "n": 2, + "probe": "little", + "want_more": true +} +{ + "fail_probe": "non-negative", + "n": 3, + "probe": "big", + "want_more": true +} +{ + "fail_probe": "non-negative", + "n": 4, + "probe": "big", + "want_more": true +} +{ + "fail_probe": "non-negative", + "n": 5, + "probe": "big", + "want_more": false +} +-- want_cov.txt -- +1: 0.62 (5/8) [2 36 37] + | int(state.n).as(n, { + | ! ! +2: 1.00 (4/4) +3: 1.00 (6/6) +4: 1.00 (5/5) +5: 1.00 (1/1) +7: 1.00 (1/1) +8: 1.00 (5/5) +9: 0.00 (0/1) [33] + | "negative" + | ! +11: 1.00 (1/1)