From 1ce1c9f23c07755e0be18702c7d0c8f7748e0b81 Mon Sep 17 00:00:00 2001 From: Alex Pana <8968914+acpana@users.noreply.github.com> Date: Fri, 7 Apr 2023 01:45:30 +0000 Subject: [PATCH] feat: stats in gator Signed-off-by: Alex Pana <8968914+acpana@users.noreply.github.com> --- cmd/gator/test/gatortest_test.go | 2 +- cmd/gator/test/test.go | 123 ++++++++++++++++++++++++++----- pkg/gator/test/test.go | 45 +++++++++-- pkg/gator/test/test_test.go | 83 ++++++++++++++++++++- pkg/gator/test/types.go | 6 +- 5 files changed, 227 insertions(+), 32 deletions(-) diff --git a/cmd/gator/test/gatortest_test.go b/cmd/gator/test/gatortest_test.go index d56b4531236..be711905e6e 100644 --- a/cmd/gator/test/gatortest_test.go +++ b/cmd/gator/test/gatortest_test.go @@ -90,7 +90,7 @@ func Test_formatOutput(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - output := formatOutput(tc.inputFormat, tc.input) + output := formatOutput(tc.inputFormat, tc.input, nil) if diff := cmp.Diff(tc.expectedOutput, output); diff != "" { t.Fatal(diff) } diff --git a/cmd/gator/test/test.go b/cmd/gator/test/test.go index 87dc4f769e4..504aa9ae2cd 100644 --- a/cmd/gator/test/test.go +++ b/cmd/gator/test/test.go @@ -7,6 +7,8 @@ import ( "os" "strings" + "github.com/open-policy-agent/frameworks/constraint/pkg/instrumentation" + "github.com/open-policy-agent/gatekeeper/cmd/gator/commons" "github.com/open-policy-agent/gatekeeper/pkg/gator/reader" "github.com/open-policy-agent/gatekeeper/pkg/gator/test" "github.com/open-policy-agent/gatekeeper/pkg/util" @@ -44,18 +46,21 @@ var Cmd = &cobra.Command{ } var ( - flagFilenames []string - flagOutput string - flagIncludeTrace bool - flagImages []string - flagTempDir string + flagFilenames []string + flagOutput string + flagIncludeTrace bool + flagGatherStats bool + flagImages []string + flagTempDir string + flagStatsOutputFile string ) const ( - flagNameFilename = "filename" - flagNameOutput = "output" - flagNameImage = "image" - flagNameTempDir = "tempdir" + flagNameFilename = "filename" + flagNameOutput = "output" + flagNameImage = "image" + flagNameTempDir = "tempdir" + flagStatsOutputFileName = "stats-ofile" stringJSON = "json" stringYAML = "yaml" @@ -65,7 +70,10 @@ const ( func init() { Cmd.Flags().StringArrayVarP(&flagFilenames, flagNameFilename, "f", []string{}, "a file or directory containing Kubernetes resources. Can be specified multiple times.") Cmd.Flags().StringVarP(&flagOutput, flagNameOutput, "o", "", fmt.Sprintf("Output format. One of: %s|%s.", stringJSON, stringYAML)) - Cmd.Flags().BoolVarP(&flagIncludeTrace, "trace", "t", false, `include a trace for the underlying constraint framework evaluation`) + Cmd.Flags().BoolVarP(&flagIncludeTrace, "trace", "t", false, "include a trace for the underlying Constraint Framework evaluation.") + Cmd.Flags().BoolVarP(&flagGatherStats, "stats", "", false, "include performance stats returned from the Constraint Framework.") + Cmd.Flags().StringVarP(&flagStatsOutputFile, flagStatsOutputFileName, "", "", "the output file for the stats from the Constraint Framework."+ + "If specified, all stats are going in this file and not the main response from the command. The outout from the-o flag for json/ yaml is respected, otherwise it defaults to JSON.") Cmd.Flags().StringArrayVarP(&flagImages, flagNameImage, "i", []string{}, "a URL to an OCI image containing policies. Can be specified multiple times.") Cmd.Flags().StringVarP(&flagTempDir, flagNameTempDir, "d", "", fmt.Sprintf("Specifies the temporary directory to download and unpack images to, if using the --%s flag. Optional.", flagNameImage)) } @@ -79,13 +87,27 @@ func run(cmd *cobra.Command, args []string) { errFatalf("no input data identified") } - responses, err := test.Test(unstrucs, flagIncludeTrace) + responses, err := test.Test(unstrucs, test.TestOpts{IncludeTrace: flagIncludeTrace, GatherStats: flagGatherStats}) if err != nil { errFatalf("auditing objects: %v\n", err) } results := responses.Results() - fmt.Print(formatOutput(flagOutput, results)) + if flagStatsOutputFile != "" { + var statsString string + switch strings.ToLower(flagOutput) { + case stringYAML: + statsString = statsToYAMLString(responses.StatsEntries) + default: // default is string json + statsString = statsToJSONString(responses.StatsEntries) + } + + commons.StringToFile(statsString, flagStatsOutputFile) + + fmt.Print(formatOutput(flagOutput, results, nil)) + } else { + fmt.Print(formatOutput(flagOutput, results, responses.StatsEntries)) + } // Whether or not we return non-zero depends on whether we have a `deny` // enforcementAction on one of the violated constraints @@ -96,14 +118,25 @@ func run(cmd *cobra.Command, args []string) { os.Exit(exitCode) } -func formatOutput(flagOutput string, results []*test.GatorResult) string { +func formatOutput(flagOutput string, results []*test.GatorResult, stats []*instrumentation.StatsEntry) string { switch strings.ToLower(flagOutput) { case stringJSON: - b, err := json.MarshalIndent(results, "", " ") - if err != nil { - errFatalf("marshaling validation json results: %v", err) + var jsonB []byte + var err error + if stats != nil { + statsAndResuluts := map[string]interface{}{"results": results, "stats": stats} + jsonB, err = json.MarshalIndent(statsAndResuluts, "", " ") + if err != nil { + errFatalf("marshaling validation json results and stats: %v", err) + } + } else { + jsonB, err = json.MarshalIndent(results, "", " ") + if err != nil { + errFatalf("marshaling validation json results: %v", err) + } } - return string(b) + + return string(jsonB) case stringYAML: yamlResults := test.GetYamlFriendlyResults(results) jsonb, err := json.Marshal(yamlResults) @@ -117,10 +150,32 @@ func formatOutput(flagOutput string, results []*test.GatorResult) string { errFatalf("pre-unmarshaling results from json: %v", err) } - yamlb, err := yaml.Marshal(unmarshalled) - if err != nil { - errFatalf("marshaling validation yaml results: %v", err) + var yamlb []byte + if stats != nil { + statsAndResuluts := map[string]interface{}{"results": results, "stats": stats} + + statsJsonB, err := json.Marshal(stats) + if err != nil { + errFatalf("pre-marshaling stats to json: %v", err) + } + + unmarshalledStats := []*instrumentation.StatsEntry{} + err = json.Unmarshal(statsJsonB, &unmarshalledStats) + if err != nil { + errFatalf("pre-unmarshaling stats from json: %v", err) + } + + yamlb, err = yaml.Marshal(statsAndResuluts) + if err != nil { + errFatalf("marshaling validation yaml results and stats: %v", err) + } + } else { + yamlb, err = yaml.Marshal(unmarshalled) + if err != nil { + errFatalf("marshaling validation yaml results: %v", err) + } } + return string(yamlb) case stringHumanFriendly: default: @@ -154,3 +209,31 @@ func errFatalf(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, format, a...) os.Exit(1) } + +func statsToYAMLString(stats []*instrumentation.StatsEntry) string { + jsonb, err := json.Marshal(stats) + if err != nil { + commons.ErrFatalf("pre-marshaling stats to json: %v", err) + } + + unmarshalled := []*instrumentation.StatsEntry{} + err = json.Unmarshal(jsonb, &unmarshalled) + if err != nil { + commons.ErrFatalf("pre-unmarshaling stats from json: %v", err) + } + + var b bytes.Buffer + yamlEncoder := yaml.NewEncoder(&b) + if err := yamlEncoder.Encode(unmarshalled); err != nil { + commons.ErrFatalf("marshaling validation yaml stats: %v", err) + } + return b.String() +} + +func statsToJSONString(stats []*instrumentation.StatsEntry) string { + b, err := json.MarshalIndent(stats, "", " ") + if err != nil { + commons.ErrFatalf("marshaling validation stats resource: %v", err) + } + return string(b) +} diff --git a/pkg/gator/test/test.go b/pkg/gator/test/test.go index 06aada6c98f..93b3be534b1 100644 --- a/pkg/gator/test/test.go +++ b/pkg/gator/test/test.go @@ -27,10 +27,16 @@ func init() { } } -func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses, error) { - // create the client +// options for the Test func +type TestOpts struct { + // Driver specific options + IncludeTrace bool + GatherStats bool +} - driver, err := rego.New(rego.Tracing(includeTrace)) +func Test(objs []*unstructured.Unstructured, tOpts TestOpts) (*GatorResponses, error) { + // create the client + driver, err := makeRegoDriver(tOpts) if err != nil { return nil, err } @@ -40,9 +46,12 @@ func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses return nil, fmt.Errorf("creating OPA client: %w", err) } + // mark off which indices hold objs that are templates or constraints + templatesOrConstraints := make([]bool, len(objs), len(objs)) + // search for templates, add them if they exist ctx := context.Background() - for _, obj := range objs { + for idx, obj := range objs { if !isTemplate(obj) { continue } @@ -56,11 +65,13 @@ func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses if err != nil { return nil, fmt.Errorf("adding template %q: %w", templ.GetName(), err) } + + templatesOrConstraints[idx] = true } // add all constraints. A constraint must be added after its associated // template or OPA will return an error - for _, obj := range objs { + for idx, obj := range objs { if !isConstraint(obj) { continue } @@ -69,6 +80,8 @@ func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses if err != nil { return nil, fmt.Errorf("adding constraint %q: %w", obj.GetName(), err) } + + templatesOrConstraints[idx] = true } // finally, add all the data. @@ -89,7 +102,12 @@ func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses responses := &GatorResponses{ ByTarget: make(map[string]*GatorResponse), } - for _, obj := range objs { + for idx, obj := range objs { + if templatesOrConstraints[idx] { + // skip review on anything that is a constraint or a template + continue + } + // Try to attach the namespace if it was supplied (ns will be nil otherwise) ns, _ := er.NamespaceForResource(obj) au := target.AugmentedUnstructured{ @@ -148,9 +166,10 @@ func Test(objs []*unstructured.Unstructured, includeTrace bool) (*GatorResponses trace = trace + "\n\n" + *r.Trace targetResponse.Trace = &trace } - responses.ByTarget[targetName] = targetResponse } + + responses.StatsEntries = append(responses.StatsEntries, review.StatsEntries...) } return responses, nil @@ -165,3 +184,15 @@ func isConstraint(u *unstructured.Unstructured) bool { gvk := u.GroupVersionKind() return gvk.Group == "constraints.gatekeeper.sh" } + +func makeRegoDriver(tOpts TestOpts) (*rego.Driver, error) { + var args []rego.Arg + if tOpts.GatherStats { + args = append(args, rego.GatherStats()) + } + if tOpts.IncludeTrace { + args = append(args, rego.Tracing(tOpts.IncludeTrace)) + } + + return rego.New(args...) +} diff --git a/pkg/gator/test/test_test.go b/pkg/gator/test/test_test.go index 57c37f11056..b1aa9544d5e 100644 --- a/pkg/gator/test/test_test.go +++ b/pkg/gator/test/test_test.go @@ -6,9 +6,11 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" constraintclient "github.com/open-policy-agent/frameworks/constraint/pkg/client" + "github.com/open-policy-agent/frameworks/constraint/pkg/instrumentation" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "github.com/open-policy-agent/gatekeeper/pkg/gator/fixtures" "github.com/open-policy-agent/gatekeeper/pkg/target" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" @@ -190,7 +192,7 @@ func TestTest(t *testing.T) { objs = append(objs, u) } - resps, err := Test(objs, false) + resps, err := Test(objs, TestOpts{}) if tc.err != nil { require.ErrorIs(t, err, tc.err) } else if err != nil { @@ -230,7 +232,7 @@ func Test_Test_withTrace(t *testing.T) { objs = append(objs, u) } - resps, err := Test(objs, true) + resps, err := Test(objs, TestOpts{IncludeTrace: true}) if err != nil { t.Errorf("got err '%v', want nil", err) } @@ -276,6 +278,83 @@ func Test_Test_withTrace(t *testing.T) { } } +// Test_Test_withStats proves that we can get a stats populated when we ask for it. +func Test_Test_withStats(t *testing.T) { + inputs := []string{ + fixtures.TemplateNeverValidate, + fixtures.ConstraintNeverValidate, + fixtures.Object, + } + + var objs []*unstructured.Unstructured + for _, input := range inputs { + u, err := readUnstructured([]byte(input)) + assert.NoErrorf(t, err, "readUnstructured for input %q: %v", input, err) + objs = append(objs, u) + } + + resps, err := Test(objs, TestOpts{GatherStats: true}) + assert.NoError(t, err) + + actualStats := resps.StatsEntries + expectedStats := []*instrumentation.StatsEntry{ + { + Scope: "template", + StatsFor: "NeverValidate", + Stats: []*instrumentation.Stat{ + { + Name: "templateRunTimeNS", + // Value: 0, // will be checled later + Source: instrumentation.Source{ + Type: "engine", + Value: "Rego", + }, + }, + { + Name: "constraintCount", + Value: 1, + Source: instrumentation.Source{ + Type: "engine", + Value: "Rego", + }, + }, + }, + Labels: []*instrumentation.Label{ + { + Name: "TracingEnabled", + Value: false, + }, + { + Name: "PrintEnabled", + Value: false, + }, + { + Name: "target", + Value: "admission.k8s.gatekeeper.sh", + }, + }, + }, + } + + diff := cmp.Diff(actualStats, expectedStats, cmpopts.IgnoreFields( + instrumentation.Stat{}, "Value", + )) + if diff != "" { + t.Errorf("diff in StatsEntries (-want +got):\n%s", diff) + } + + // there should be one stats entry with two stats + assert.Len(t, actualStats[0].Stats, 2) + for _, stat := range actualStats[0].Stats { + if stat.Name == "templateRunTimeNS" { + require.NotZero(t, stat.Value) + } + if stat.Name == "constraintCount" { + require.Equal(t, stat.Value, 1) + } + } +} + func readUnstructured(bytes []byte) (*unstructured.Unstructured, error) { u := &unstructured.Unstructured{ Object: make(map[string]interface{}), diff --git a/pkg/gator/test/types.go b/pkg/gator/test/types.go index 36f98ab90cf..33db9b4451b 100644 --- a/pkg/gator/test/types.go +++ b/pkg/gator/test/types.go @@ -3,6 +3,7 @@ package test import ( "sort" + "github.com/open-policy-agent/frameworks/constraint/pkg/instrumentation" "github.com/open-policy-agent/frameworks/constraint/pkg/types" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -41,8 +42,9 @@ type GatorResponse struct { } type GatorResponses struct { - ByTarget map[string]*GatorResponse - Handled map[string]bool + ByTarget map[string]*GatorResponse + Handled map[string]bool + StatsEntries []*instrumentation.StatsEntry } func (r *GatorResponses) Results() []*GatorResult {