From d3a583a910ac6a613192d9778781a6e330f8f048 Mon Sep 17 00:00:00 2001 From: Ville Vesilehto Date: Sat, 18 Jan 2025 22:09:28 +0200 Subject: [PATCH] feat: add SARIF output format support Add Static Analysis Results Interchange Format (SARIF) v2.1.0 output support to conftest. SARIF is a standard JSON format for static analysis tools. - SARIF v2.1.0 schema compliance - Includes file locations and rule metadata - Tracks execution timing and status - Test coverage - Documentation Signed-off-by: Ville Vesilehto --- docs/options.md | 66 ++ output/output.go | 4 + output/output_test.go | 4 + output/sarif.go | 496 ++++++++++++++ output/sarif_test.go | 1448 +++++++++++++++++++++++++++++++++++++++++ policy/engine.go | 11 +- 6 files changed, 2027 insertions(+), 2 deletions(-) create mode 100644 output/sarif.go create mode 100644 output/sarif_test.go diff --git a/docs/options.md b/docs/options.md index ae7a77ac83..f06987842d 100644 --- a/docs/options.md +++ b/docs/options.md @@ -173,6 +173,7 @@ As of today Conftest supports the following output types: - JUnit `--output=junit` - GitHub `--output=github` - AzureDevOps `--output=azuredevops` +- SARIF `--output=sarif` ### Plaintext @@ -322,6 +323,71 @@ success file=examples/kubernetes/deployment.yaml 1 5 tests, 1 passed, 0 warnings, 4 failures, 0 exceptions ``` +### SARIF + +```console +$ conftest test --proto-file-dirs examples/textproto/protos -p examples/textproto/policy examples/textproto/fail.textproto -o sarif +{ + "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "conftest", + "informationUri": "https://github.com/open-policy-agent/conftest", + "rules": [ + { + "id": "conftest-failure-main-deny", + "shortDescription": { + "text": "fail: Power level must be over 9000" + }, + "properties": { + "namespace": "main", + "query": "data.main.deny" + } + } + ] + } + }, + "results": [ + { + "ruleId": "conftest-failure-main-deny", + "ruleIndex": 0, + "kind": "fail", + "level": "error", + "message": { + "text": "fail: Power level must be over 9000" + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "examples/textproto/fail.textproto" + } + } + } + ], + "properties": { + "namespace": "main", + "query": "data.main.deny" + } + } + ], + "invocations": [ + { + "executionSuccessful": true, + "exitCode": 1, + "exitCodeDescription": "Policy violations found", + "startTimeUtc": "2025-01-19T13:14:11Z", + "endTimeUtc": "2025-01-19T13:14:11Z" + } + ] + } + ] +} +``` + ## `--parser` Conftest normally detects which parser to used based on the file extension of the file, even when multiple input files are passed in. However, it is possible force a specific parser to be used with the `--parser` flag. diff --git a/output/output.go b/output/output.go index 9b1ec6a66d..981026bf9f 100644 --- a/output/output.go +++ b/output/output.go @@ -34,6 +34,7 @@ const ( OutputJUnit = "junit" OutputGitHub = "github" OutputAzureDevOps = "azuredevops" + OutputSARIF = "sarif" ) // Get returns a type that can render output in the given format. @@ -57,6 +58,8 @@ func Get(format string, options Options) Outputter { return NewGitHub(options.File) case OutputAzureDevOps: return NewAzureDevOps(options.File) + case OutputSARIF: + return NewSARIF(options.File) default: return NewStandard(options.File) } @@ -72,5 +75,6 @@ func Outputs() []string { OutputJUnit, OutputGitHub, OutputAzureDevOps, + OutputSARIF, } } diff --git a/output/output_test.go b/output/output_test.go index 4a4669a908..97ad8b1e61 100644 --- a/output/output_test.go +++ b/output/output_test.go @@ -39,6 +39,10 @@ func TestGetOutputter(t *testing.T) { input: OutputAzureDevOps, expected: NewAzureDevOps(os.Stdout), }, + { + input: OutputSARIF, + expected: NewSARIF(os.Stdout), + }, { input: "unknown_format", expected: NewStandard(os.Stdout), diff --git a/output/sarif.go b/output/sarif.go new file mode 100644 index 0000000000..8ecad93e57 --- /dev/null +++ b/output/sarif.go @@ -0,0 +1,496 @@ +package output + +import ( + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/open-policy-agent/opa/tester" +) + +const ( + // SARIF schema and version + sarifSchemaURI = "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json" + sarifVersion = "2.1.0" + + // Tool information + toolName = "conftest" + toolURI = "https://github.com/open-policy-agent/conftest" + + // SARIF levels + levelError = "error" + levelWarning = "warning" + levelNote = "note" + levelNone = "none" + + // SARIF kinds + kindFail = "fail" + kindReview = "review" + kindInfo = "informational" + kindPass = "pass" + kindSkipped = "notApplicable" + + // Rule ID prefixes + ruleFailureBase = "conftest-failure" + ruleWarningBase = "conftest-warning" + ruleExceptionBase = "conftest-exception" + rulePassBase = "conftest-pass" + ruleSkippedBase = "conftest-skipped" + + // Result descriptions + successDesc = "Policy was satisfied successfully" + skippedDesc = "Policy check was skipped" + failureDesc = "Policy violation" + warningDesc = "Policy warning" + exceptionDesc = "Policy exception" + + // Exit code descriptions + exitNoViolations = "No policy violations found" + exitViolations = "Policy violations found" + exitWarnings = "Policy warnings found" + exitExceptions = "Policy exceptions found" +) + +// SARIF represents an Outputter that outputs results in SARIF format. +// https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html +type SARIF struct { + writer io.Writer +} + +// NewSARIF creates a new SARIF with the given writer. +func NewSARIF(w io.Writer) *SARIF { + return &SARIF{ + writer: w, + } +} + +// sarifReport represents the root object of a SARIF log file +type sarifReport struct { + Schema string `json:"$schema"` + Version string `json:"version"` + Runs []sarifRun `json:"runs"` +} + +// sarifRun represents a single run of a tool +type sarifRun struct { + Tool sarifTool `json:"tool"` + Results []sarifResult `json:"results"` + Invocations []sarifInvocation `json:"invocations"` +} + +// sarifTool represents the analysis tool that was run +type sarifTool struct { + Driver sarifDriver `json:"driver"` +} + +// sarifDriver represents the analysis tool component that contains rule metadata +type sarifDriver struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + InformationURI string `json:"informationUri"` + Rules []sarifRule `json:"rules"` +} + +// sarifRule represents a rule that was evaluated during the scan +type sarifRule struct { + ID string `json:"id"` + ShortDescription sarifMessage `json:"shortDescription"` + FullDescription *sarifMessage `json:"fullDescription,omitempty"` + Help *sarifMessage `json:"help,omitempty"` + HelpURI string `json:"helpUri,omitempty"` + Properties map[string]interface{} `json:"properties,omitempty"` +} + +// sarifResult represents a single analysis result +type sarifResult struct { + RuleID string `json:"ruleId"` + RuleIndex int `json:"ruleIndex"` + Kind string `json:"kind"` + Level string `json:"level"` + Message sarifMessage `json:"message"` + Locations []sarifLocation `json:"locations"` + Properties map[string]interface{} `json:"properties,omitempty"` +} + +// sarifLocation represents a location within a programming artifact +type sarifLocation struct { + PhysicalLocation sarifPhysicalLocation `json:"physicalLocation"` +} + +// sarifPhysicalLocation represents the physical location where the result was detected +type sarifPhysicalLocation struct { + ArtifactLocation sarifArtifactLocation `json:"artifactLocation"` +} + +// sarifArtifactLocation represents the location of an artifact +type sarifArtifactLocation struct { + URI string `json:"uri"` +} + +// sarifMessage represents a message string or message with arguments +type sarifMessage struct { + Text string `json:"text"` +} + +// sarifInvocation represents the runtime environment of the analysis tool run +type sarifInvocation struct { + ExecutionSuccessful bool `json:"executionSuccessful"` + ExitCode int `json:"exitCode"` + ExitCodeDescription string `json:"exitCodeDescription"` + StartTimeUtc string `json:"startTimeUtc"` + EndTimeUtc string `json:"endTimeUtc"` +} + +// resultKind represents the type of result being processed +type resultKind int + +const ( + resultKindSuccess resultKind = iota + resultKindSkipped + resultKindException + resultKindFailure + resultKindWarning +) + +// resultType represents the type of result being processed +type resultType struct { + kind resultKind + ruleIDPrefix string + kindStr string + level string + description string +} + +var ( + failureResultType = resultType{ + kind: resultKindFailure, + ruleIDPrefix: ruleFailureBase, + kindStr: kindFail, + level: levelError, + description: failureDesc, + } + warningResultType = resultType{ + kind: resultKindWarning, + ruleIDPrefix: ruleWarningBase, + kindStr: kindReview, + level: levelWarning, + description: warningDesc, + } + exceptionResultType = resultType{ + kind: resultKindException, + ruleIDPrefix: ruleExceptionBase, + kindStr: kindInfo, + level: levelNote, + description: exceptionDesc, + } + successResultType = resultType{ + kind: resultKindSuccess, + ruleIDPrefix: rulePassBase, + kindStr: kindPass, + level: levelNone, + description: successDesc, + } + skippedResultType = resultType{ + kind: resultKindSkipped, + ruleIDPrefix: ruleSkippedBase, + kindStr: kindSkipped, + level: levelNone, + description: skippedDesc, + } +) + +// getRuleID generates a unique rule ID based on metadata and result type +func getRuleID(result Result, rType resultType, namespace string) string { + // Always use base ID for success, skipped, and exception results + switch rType.kind { + case resultKindSuccess, resultKindSkipped, resultKindException: + return rType.ruleIDPrefix + } + + // Use package and rule from metadata when available + if pkg, ok := result.Metadata["package"].(string); ok { + if rule, ok := result.Metadata["rule"].(string); ok { + return fmt.Sprintf("%s/%s/%s", namespace, pkg, rule) + } + } + + // Use query path if available + if query, ok := result.Metadata["query"].(string); ok { + // Remove "data." prefix and convert dots to dashes + query = strings.TrimPrefix(query, "data.") + query = strings.ReplaceAll(query, ".", "-") + return fmt.Sprintf("%s-%s", rType.ruleIDPrefix, query) + } + + // Use description if available + if desc, ok := result.Metadata["description"].(string); ok { + return fmt.Sprintf("%s/%s", rType.ruleIDPrefix, strings.ToLower(strings.ReplaceAll(desc, " ", "-"))) + } + + // Fallback to base ID if no identifying information is available + return rType.ruleIDPrefix +} + +// createRule creates a new SARIF rule from a result +func createRule(result Result, rType resultType, namespace string) sarifRule { + ruleID := getRuleID(result, rType, namespace) + + rule := sarifRule{ + ID: ruleID, + ShortDescription: sarifMessage{ + Text: result.Message, + }, + Properties: make(map[string]interface{}), + } + + // Add policy metadata to rule properties + if pkg, ok := result.Metadata["package"].(string); ok { + rule.Properties["package"] = pkg + } + if ruleName, ok := result.Metadata["rule"].(string); ok { + rule.Properties["rule"] = ruleName + } + if query, ok := result.Metadata["query"].(string); ok { + rule.Properties["query"] = query + } + rule.Properties["namespace"] = namespace + + // Add additional rule metadata if available + if desc, ok := result.Metadata["description"].(string); ok { + rule.FullDescription = &sarifMessage{Text: desc} + } + if url, ok := result.Metadata["url"].(string); ok { + rule.HelpURI = url + } + if help, ok := result.Metadata["help"].(string); ok { + rule.Help = &sarifMessage{Text: help} + } + + // Add any remaining metadata to properties + for k, v := range result.Metadata { + switch k { + case "package", "rule", "description", "url", "help", "query": + // Skip already processed fields + continue + default: + rule.Properties[k] = v + } + } + + return rule +} + +// createLocation creates a new SARIF location from a file path +func createLocation(filePath string) sarifLocation { + return sarifLocation{ + PhysicalLocation: sarifPhysicalLocation{ + ArtifactLocation: sarifArtifactLocation{ + URI: filepath.ToSlash(filePath), + }, + }, + } +} + +// createProperties creates a new properties map with namespace information +func createProperties(metadata map[string]interface{}, namespace string) map[string]interface{} { + properties := make(map[string]interface{}) + for k, v := range metadata { + switch k { + case "package", "rule", "description", "url", "help": + // Skip rule-level metadata + continue + case "query", "traces", "outputs": + // Include query-related information + properties[k] = v + default: + properties[k] = v + } + } + + // Always include namespace + properties["namespace"] = namespace + + return properties +} + +// processResults processes a slice of results and adds them to the SARIF run +func processResults(run *sarifRun, results []Result, rType resultType, fileName, namespace string, ruleMap map[string]bool) error { + for _, result := range results { + // Create or get rule + rule := createRule(result, rType, namespace) + if !ruleMap[rule.ID] { + run.Tool.Driver.Rules = append(run.Tool.Driver.Rules, rule) + ruleMap[rule.ID] = true + } + + // Find rule index + ruleIndex := -1 + for i, r := range run.Tool.Driver.Rules { + if r.ID == rule.ID { + ruleIndex = i + break + } + } + + if ruleIndex == -1 { + return fmt.Errorf("rule %s not found in rules array after being added", rule.ID) + } + + // Create result + run.Results = append(run.Results, sarifResult{ + RuleID: rule.ID, + RuleIndex: ruleIndex, + Kind: rType.kindStr, + Level: rType.level, + Message: sarifMessage{Text: result.Message}, + Locations: []sarifLocation{createLocation(fileName)}, + Properties: createProperties(result.Metadata, namespace), + }) + } + return nil +} + +// createSuccessResult creates a success result for a given file and namespace +func createSuccessResult(run *sarifRun, fileName, namespace string, ruleMap map[string]bool) error { + result := Result{ + Message: successResultType.description, + } + return processResults(run, []Result{result}, successResultType, fileName, namespace, ruleMap) +} + +// createSkippedResult creates a skipped result for a given file and namespace +func createSkippedResult(run *sarifRun, fileName, namespace string, ruleMap map[string]bool) error { + result := Result{ + Message: skippedResultType.description, + } + return processResults(run, []Result{result}, skippedResultType, fileName, namespace, ruleMap) +} + +// Output outputs the results in SARIF format. +func (s *SARIF) Output(results []CheckResult) error { + startTime := time.Now().UTC() + + // Create SARIF report structure + report := sarifReport{ + Schema: sarifSchemaURI, + Version: sarifVersion, + Runs: []sarifRun{ + { + Tool: sarifTool{ + Driver: sarifDriver{ + Name: toolName, + InformationURI: toolURI, + Rules: []sarifRule{}, + }, + }, + Results: []sarifResult{}, + Invocations: []sarifInvocation{}, + }, + }, + } + + run := &report.Runs[0] + ruleMap := make(map[string]bool) + + // Process all results + for _, result := range results { + err := processResults(run, result.Failures, failureResultType, result.FileName, result.Namespace, ruleMap) + if err != nil { + return fmt.Errorf("process failures: %w", err) + } + err = processResults(run, result.Warnings, warningResultType, result.FileName, result.Namespace, ruleMap) + if err != nil { + return fmt.Errorf("process warnings: %w", err) + } + err = processResults(run, result.Exceptions, exceptionResultType, result.FileName, result.Namespace, ruleMap) + if err != nil { + return fmt.Errorf("process exceptions: %w", err) + } + + // Add success result if no failures/warnings/exceptions + if len(result.Failures) == 0 && len(result.Warnings) == 0 && len(result.Exceptions) == 0 { + if result.Successes > 0 { + err = createSuccessResult(run, result.FileName, result.Namespace, ruleMap) + if err != nil { + return fmt.Errorf("create success result: %w", err) + } + } else { + err = createSkippedResult(run, result.FileName, result.Namespace, ruleMap) + if err != nil { + return fmt.Errorf("create skipped result: %w", err) + } + } + } + } + + // Add invocation information + exitCode := 0 + exitDesc := exitNoViolations + if hasFailures(results) { + exitCode = 1 + exitDesc = exitViolations + } else if hasWarnings(results) { + exitCode = 0 + exitDesc = exitWarnings + } else if hasExceptions(results) { + exitCode = 0 + exitDesc = exitExceptions + } + + run.Invocations = []sarifInvocation{ + { + ExecutionSuccessful: true, + ExitCode: exitCode, + ExitCodeDescription: exitDesc, + StartTimeUtc: startTime.Format(time.RFC3339), + EndTimeUtc: time.Now().UTC().Format(time.RFC3339), + }, + } + + // Marshal to JSON + encoder := json.NewEncoder(s.writer) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return fmt.Errorf("encode sarif: %w", err) + } + + return nil +} + +// Report is not supported in SARIF output +func (s *SARIF) Report(_ []*tester.Result, _ string) error { + return fmt.Errorf("report is not supported in SARIF output") +} + +// hasFailures returns true if any of the results contain failures +func hasFailures(results []CheckResult) bool { + for _, result := range results { + if len(result.Failures) > 0 { + return true + } + } + return false +} + +// hasWarnings returns true if any of the results contain warnings +func hasWarnings(results []CheckResult) bool { + for _, result := range results { + if len(result.Warnings) > 0 { + return true + } + } + return false +} + +// hasExceptions returns true if any of the results contain exceptions +func hasExceptions(results []CheckResult) bool { + for _, result := range results { + if len(result.Exceptions) > 0 { + return true + } + } + return false +} diff --git a/output/sarif_test.go b/output/sarif_test.go new file mode 100644 index 0000000000..4db205a47b --- /dev/null +++ b/output/sarif_test.go @@ -0,0 +1,1448 @@ +package output + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +const ( + // Test file and namespace + testFileName = "examples/kubernetes/service.yaml" + testNamespace = "namespace" + + // Test messages + testFailureMsg = "first failure" + testWarningMsg = "first warning" + testSecondWarn = "second warning" + + // Test metadata + testFailureDesc = "A detailed description of the failure" + testFailureURL = "https://example.com/docs" + testFailureHelp = "How to fix this failure" + testWarnDesc = "A detailed description of the warning" + testWarnURL = "https://example.com/warnings" + testWarnHelp = "How to fix this warning" + + // Test policy metadata + testPackage = "security.container" + testRuleName = "no_root_user" +) + +func TestSARIF(t *testing.T) { + tests := []struct { + name string + input []CheckResult + validate func(t *testing.T, output string) + }{ + { + name: "success path - no violations", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result (skipped), got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelNone, + kind: kindSkipped, + message: skippedDesc, + ruleID: ruleSkippedBase, + namespace: testNamespace, + }) + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 0, + exitDesc: exitNoViolations, + }) + + validateTimestamps(t, run.Invocations[0]) + }, + }, + { + name: "single failure with basic result structure", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{Message: testFailureMsg}}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: ruleFailureBase, + namespace: testNamespace, + }) + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "multiple warnings and failures", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Warnings: []Result{{Message: testWarningMsg}, {Message: testSecondWarn}}, + Failures: []Result{{Message: testFailureMsg}}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 3 { + t.Errorf("expected 3 results, got %d", len(run.Results)) + } + + // Count warnings and failures while validating each result + warningCount := 0 + failureCount := 0 + for _, result := range run.Results { + switch result.Level { + case levelWarning: + warningCount++ + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelWarning, + kind: kindReview, + message: result.Message.Text, + ruleID: ruleWarningBase, + namespace: testNamespace, + }) + case levelError: + failureCount++ + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: ruleFailureBase, + namespace: testNamespace, + }) + } + } + + if warningCount != 2 { + t.Errorf("expected 2 warnings, got %d", warningCount) + } + if failureCount != 1 { + t.Errorf("expected 1 failure, got %d", failureCount) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "handles stdin input", + input: []CheckResult{ + { + FileName: "-", + Namespace: testNamespace, + Failures: []Result{{Message: testFailureMsg}}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + result := run.Results[0] + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: ruleFailureBase, + namespace: testNamespace, + }) + + // Verify location URI for stdin + if result.Locations[0].PhysicalLocation.ArtifactLocation.URI != "-" { + t.Errorf("expected URI '-' for stdin, got '%s'", result.Locations[0].PhysicalLocation.ArtifactLocation.URI) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "includes metadata in rules and results", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{ + Message: testFailureMsg, + Metadata: map[string]interface{}{ + "description": testFailureDesc, + "url": testFailureURL, + "help": testFailureHelp, + }, + }}, + Warnings: []Result{{ + Message: testWarningMsg, + Metadata: map[string]interface{}{ + "description": testWarnDesc, + "url": testWarnURL, + "help": testWarnHelp, + }, + }}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 2 { + t.Errorf("expected 2 results, got %d", len(run.Results)) + } + + // Find and validate failure rule/result + expectedFailureRuleID := fmt.Sprintf("%s/%s", ruleFailureBase, strings.ToLower(strings.ReplaceAll(testFailureDesc, " ", "-"))) + failureRule, failureResult, err := findRule(run, expectedFailureRuleID) + if err != nil { + t.Fatal(err) + } + + validateRuleMetadata(t, failureRule, struct { + description string + helpURI string + helpText string + namespace string + }{ + description: testFailureDesc, + helpURI: testFailureURL, + helpText: testFailureHelp, + namespace: testNamespace, + }) + + validateResult(t, *failureResult, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: expectedFailureRuleID, + namespace: testNamespace, + }) + + // Find and validate warning rule/result + expectedWarningRuleID := fmt.Sprintf("%s/%s", ruleWarningBase, strings.ToLower(strings.ReplaceAll(testWarnDesc, " ", "-"))) + warningRule, warningResult, err := findRule(run, expectedWarningRuleID) + if err != nil { + t.Fatal(err) + } + + validateRuleMetadata(t, warningRule, struct { + description string + helpURI string + helpText string + namespace string + }{ + description: testWarnDesc, + helpURI: testWarnURL, + helpText: testWarnHelp, + namespace: testNamespace, + }) + + validateResult(t, *warningResult, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelWarning, + kind: kindReview, + message: testWarningMsg, + ruleID: expectedWarningRuleID, + namespace: testNamespace, + }) + }, + }, + { + name: "handles exceptions", + input: []CheckResult{ + { + FileName: "examples/exceptions/deployments.yaml", + Namespace: "main", + Failures: []Result{{ + Message: "Containers must not run as root", + }}, + Exceptions: []Result{{ + Message: "data.main.exception[_][_] == \"run_as_root\"", + Metadata: map[string]interface{}{ + "description": "Exception for running as root", + "error": "run_as_root", + "code": "policy_exception", + }, + }}, + }, + { + FileName: "examples/exceptions/other.yaml", + Namespace: "main", + Exceptions: []Result{{ + Message: "data.main.exception[_][_] == \"other\"", + }}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 3 { + t.Errorf("expected 3 results (1 failure, 2 exceptions), got %d", len(run.Results)) + } + + exceptionCount := 0 + failureCount := 0 + for _, result := range run.Results { + switch result.Kind { + case kindInfo: + exceptionCount++ + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelNote, + kind: kindInfo, + message: result.Message.Text, + ruleID: ruleExceptionBase, + namespace: "main", + }) + + // Verify error details if present + if result.Message.Text == "data.main.exception[_][_] == \"run_as_root\"" { + if errType, ok := result.Properties["error"].(string); !ok || errType != "run_as_root" { + t.Error("expected error type in properties") + } + if code, ok := result.Properties["code"].(string); !ok || code != "policy_exception" { + t.Error("expected error code in properties") + } + } + + case kindFail: + failureCount++ + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: "Containers must not run as root", + ruleID: ruleFailureBase, + namespace: "main", + }) + } + } + + if exceptionCount != 2 { + t.Errorf("expected 2 exceptions, got %d", exceptionCount) + } + if failureCount != 1 { + t.Errorf("expected 1 failure, got %d", failureCount) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "success result when checks pass", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Successes: 1, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelNone, + kind: kindPass, + message: successDesc, + ruleID: rulePassBase, + namespace: testNamespace, + }) + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 0, + exitDesc: exitNoViolations, + }) + }, + }, + { + name: "skipped result when no checks run", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Successes: 0, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelNone, + kind: kindSkipped, + message: skippedDesc, + ruleID: ruleSkippedBase, + namespace: testNamespace, + }) + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 0, + exitDesc: exitNoViolations, + }) + }, + }, + { + name: "no success/skipped result when failures exist", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Successes: 5, + Failures: []Result{{Message: testFailureMsg}}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result (failure only), got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: ruleFailureBase, + namespace: testNamespace, + }) + }, + }, + { + name: "rule ID generation with policy metadata", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{ + Message: testFailureMsg, + Metadata: map[string]interface{}{ + "package": testPackage, + "rule": testRuleName, + "description": testFailureDesc, + }, + }}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + result := run.Results[0] + expectedRuleID := fmt.Sprintf("%s/%s/%s", testNamespace, testPackage, testRuleName) + if result.RuleID != expectedRuleID { + t.Errorf("expected rule ID '%s', got '%s'", expectedRuleID, result.RuleID) + } + + // Find and validate the rule + rule, foundResult, err := findRule(run, expectedRuleID) + if err != nil { + t.Fatal(err) + } + if foundResult == nil { + t.Fatal("no result found for rule") + } + result = *foundResult + + // Verify package and rule in rule properties + if pkg, ok := rule.Properties["package"].(string); !ok || pkg != testPackage { + t.Errorf("expected package '%s' in rule properties, got '%v'", testPackage, rule.Properties["package"]) + } + if ruleName, ok := rule.Properties["rule"].(string); !ok || ruleName != testRuleName { + t.Errorf("expected rule name '%s' in rule properties, got '%v'", testRuleName, rule.Properties["rule"]) + } + + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: expectedRuleID, + namespace: testNamespace, + }) + }, + }, + { + name: "rule ID generation with description fallback", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{ + Message: testFailureMsg, + Metadata: map[string]interface{}{ + "description": testFailureDesc, + }, + }}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + result := run.Results[0] + expectedRuleID := fmt.Sprintf("%s/%s", ruleFailureBase, strings.ToLower(strings.ReplaceAll(testFailureDesc, " ", "-"))) + if result.RuleID != expectedRuleID { + t.Errorf("expected rule ID '%s', got '%s'", expectedRuleID, result.RuleID) + } + + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: expectedRuleID, + namespace: testNamespace, + }) + }, + }, + { + name: "rule ID generation with no metadata", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{ + Message: testFailureMsg, + Metadata: map[string]interface{}{}, + }}, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: testFailureMsg, + ruleID: ruleFailureBase, + namespace: testNamespace, + }) + }, + }, + { + name: "rule ID generation with message hash", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{{ + Message: testFailureMsg, + Metadata: map[string]interface{}{ + "severity": "HIGH", + }, + }}, + }, + }, + validate: func(t *testing.T, output string) { + var report sarifReport + if err := json.NewDecoder(strings.NewReader(output)).Decode(&report); err != nil { + t.Fatalf("failed to decode SARIF output: %v", err) + } + + run := report.Runs[0] + result := run.Results[0] + expectedRuleID := ruleFailureBase + if result.RuleID != expectedRuleID { + t.Errorf("expected rule ID '%s', got '%s'", expectedRuleID, result.RuleID) + } + }, + }, + { + name: "multiple violations from same rule", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Failures: []Result{ + { + Message: "Container 'app1' runs as privileged", + Metadata: map[string]interface{}{ + "rule": "no_privileged", + "container": "app1", + "query": "data.kubernetes.deny_privileged_container", + }, + }, + { + Message: "Container 'app2' runs as privileged", + Metadata: map[string]interface{}{ + "rule": "no_privileged", + "container": "app2", + "query": "data.kubernetes.deny_privileged_container", + }, + }, + }, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 2 { + t.Errorf("expected 2 results, got %d", len(run.Results)) + } + + // Both results should have the same ruleId and ruleIndex + firstResult := run.Results[0] + secondResult := run.Results[1] + if firstResult.RuleID != secondResult.RuleID { + t.Errorf("expected same rule ID, got '%s' and '%s'", firstResult.RuleID, secondResult.RuleID) + } + if firstResult.RuleIndex != secondResult.RuleIndex { + t.Errorf("expected same rule index, got %d and %d", firstResult.RuleIndex, secondResult.RuleIndex) + } + + // Validate each result + for _, result := range run.Results { + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: result.Message.Text, + ruleID: fmt.Sprintf("%s-kubernetes-deny_privileged_container", ruleFailureBase), + namespace: testNamespace, + }) + + // Verify query path in properties + if query, ok := result.Properties["query"].(string); !ok || query != "data.kubernetes.deny_privileged_container" { + t.Errorf("unexpected query: %v", result.Properties["query"]) + } + + // Verify container in properties + if container, ok := result.Properties["container"].(string); !ok || (container != "app1" && container != "app2") { + t.Errorf("unexpected container: %v", result.Properties["container"]) + } + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "cross package rules", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: "kubernetes", + Failures: []Result{ + { + Message: "Container runs as privileged", + Metadata: map[string]interface{}{ + "query": "data.kubernetes.deny_privileged_container", + }, + }, + }, + }, + { + FileName: testFileName, + Namespace: "custom", + Failures: []Result{ + { + Message: "Custom rule violation", + Metadata: map[string]interface{}{ + "query": "data.custom.deny_custom", + }, + }, + }, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 2 { + t.Errorf("expected 2 results, got %d", len(run.Results)) + } + + // Validate kubernetes rule result + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: "Container runs as privileged", + ruleID: fmt.Sprintf("%s-kubernetes-deny_privileged_container", ruleFailureBase), + namespace: "kubernetes", + }) + + // Validate custom rule result + validateResult(t, run.Results[1], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: "Custom rule violation", + ruleID: fmt.Sprintf("%s-custom-deny_custom", ruleFailureBase), + namespace: "custom", + }) + + // Verify query paths in properties + if query, ok := run.Results[0].Properties["query"].(string); !ok || query != "data.kubernetes.deny_privileged_container" { + t.Errorf("unexpected query in first result: %v", run.Results[0].Properties["query"]) + } + if query, ok := run.Results[1].Properties["query"].(string); !ok || query != "data.custom.deny_custom" { + t.Errorf("unexpected query in second result: %v", run.Results[1].Properties["query"]) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) + }, + }, + { + name: "exception handling", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: testNamespace, + Exceptions: []Result{ + { + Message: "Division by zero in policy evaluation", + Metadata: map[string]interface{}{ + "error": "divide by zero", + "code": "rego_type_error", + }, + }, + }, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + result := run.Results[0] + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelNote, + kind: kindInfo, + message: "Division by zero in policy evaluation", + ruleID: ruleExceptionBase, + namespace: testNamespace, + }) + + // Verify error details in properties + if errMsg, ok := result.Properties["error"].(string); !ok || errMsg != "divide by zero" { + t.Errorf("expected error 'divide by zero', got '%v'", result.Properties["error"]) + } + if code, ok := result.Properties["code"].(string); !ok || code != "rego_type_error" { + t.Errorf("expected code 'rego_type_error', got '%v'", result.Properties["code"]) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 0, + exitDesc: exitExceptions, + }) + }, + }, + { + name: "query path inclusion", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: "kubernetes.security", + Failures: []Result{ + { + Message: "Security violation", + Metadata: map[string]interface{}{ + "query": "data.kubernetes.security.deny_privileged", + "package": "kubernetes.security", + "rule": "deny_privileged", + }, + }, + }, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + result := run.Results[0] + expectedRuleID := fmt.Sprintf("%s/%s/%s", "kubernetes.security", "kubernetes.security", "deny_privileged") + + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: "Security violation", + ruleID: expectedRuleID, + namespace: "kubernetes.security", + }) + + // Verify query path in properties + if query, ok := result.Properties["query"].(string); !ok || query != "data.kubernetes.security.deny_privileged" { + t.Errorf("unexpected query path: %v", result.Properties["query"]) + } + + // Find and validate the rule + rule, _, err := findRule(run, expectedRuleID) + if err != nil { + t.Fatal(err) + } + + // Verify package and rule in rule properties + if pkg, ok := rule.Properties["package"].(string); !ok || pkg != "kubernetes.security" { + t.Errorf("unexpected package: %v", rule.Properties["package"]) + } + if ruleName, ok := rule.Properties["rule"].(string); !ok || ruleName != "deny_privileged" { + t.Errorf("unexpected rule: %v", rule.Properties["rule"]) + } + }, + }, + { + name: "policy warnings", + input: []CheckResult{ + { + FileName: testFileName, + Namespace: "kubernetes", + Warnings: []Result{ + { + Message: "Memory limits not set", + Metadata: map[string]interface{}{ + "query": "data.kubernetes.warn_no_memory_limits", + "package": "kubernetes", + "rule": "warn_memory_limits", + "container": "app", + }, + }, + { + Message: "CPU limits not set", + Metadata: map[string]interface{}{ + "query": "data.kubernetes.warn_no_cpu_limits", + "package": "kubernetes", + "rule": "warn_cpu_limits", + "container": "app", + }, + }, + }, + }, + }, + validate: func(t *testing.T, output string) { + report := decodeSARIFReport(t, output) + run := report.Runs[0] + + if len(run.Results) != 2 { + t.Errorf("expected 2 warning results, got %d", len(run.Results)) + } + + // Expected rule IDs based on package and rule names + expectedRuleIDs := map[string]bool{ + "kubernetes/kubernetes/warn_memory_limits": false, + "kubernetes/kubernetes/warn_cpu_limits": false, + } + + for _, result := range run.Results { + // Mark rule ID as found + expectedRuleIDs[result.RuleID] = true + + validateResult(t, result, struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelWarning, + kind: kindReview, + message: result.Message.Text, + ruleID: result.RuleID, + namespace: "kubernetes", + }) + + // Find and validate the rule + rule, _, err := findRule(run, result.RuleID) + if err != nil { + t.Fatalf("rule not found: %s", result.RuleID) + } + + // Verify rule properties + if pkg, ok := rule.Properties["package"].(string); !ok || pkg != "kubernetes" { + t.Errorf("unexpected package in rule: %v", rule.Properties["package"]) + } + if ruleName, ok := rule.Properties["rule"].(string); !ok || !strings.HasPrefix(ruleName, "warn_") { + t.Errorf("unexpected rule name: %v", rule.Properties["rule"]) + } + + // Verify query path and container in properties + if query, ok := result.Properties["query"].(string); !ok || !strings.HasPrefix(query, "data.kubernetes.warn_") { + t.Errorf("unexpected query: %v", result.Properties["query"]) + } + if container, ok := result.Properties["container"].(string); !ok || container != "app" { + t.Errorf("unexpected container: %v", result.Properties["container"]) + } + } + + // Verify all expected rule IDs were found + for ruleID, found := range expectedRuleIDs { + if !found { + t.Errorf("expected rule ID not found: %s", ruleID) + } + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 0, + exitDesc: exitWarnings, + }) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + s := NewSARIF(&buf) + + if err := s.Output(tt.input); err != nil { + t.Fatalf("failed to output SARIF: %v", err) + } + + tt.validate(t, buf.String()) + }) + } +} + +func TestSARIFReport(t *testing.T) { + var buf bytes.Buffer + sarif := NewSARIF(&buf) + + err := sarif.Report(nil, "") + if err == nil { + t.Error("expected error for Report call") + } + + wantErrMsg := "report is not supported in SARIF output" + if !strings.Contains(err.Error(), wantErrMsg) { + t.Errorf("expected error containing '%s', got: %v", wantErrMsg, err) + } +} + +func TestGetRuleID(t *testing.T) { + tests := []struct { + name string + result Result + rType resultType + namespace string + want string + }{ + { + name: "success result uses base ID", + result: Result{ + Message: "success", + Metadata: map[string]interface{}{ + "query": "data.main.deny", + }, + }, + rType: successResultType, + namespace: "main", + want: "conftest-pass", + }, + { + name: "skipped result uses base ID", + result: Result{ + Message: "skipped", + Metadata: map[string]interface{}{ + "query": "data.main.deny", + }, + }, + rType: skippedResultType, + namespace: "main", + want: "conftest-skipped", + }, + { + name: "uses package and rule from metadata when available", + result: Result{ + Message: "violation", + Metadata: map[string]interface{}{ + "package": "kubernetes", + "rule": "deny_privileged", + "query": "data.kubernetes.deny", + }, + }, + rType: failureResultType, + namespace: "main", + want: "main/kubernetes/deny_privileged", + }, + { + name: "uses query path when package/rule not available", + result: Result{ + Message: "violation", + Metadata: map[string]interface{}{ + "query": "data.kubernetes.deny", + }, + }, + rType: failureResultType, + namespace: "main", + want: "conftest-failure-kubernetes-deny", + }, + { + name: "uses description when no query or package/rule", + result: Result{ + Message: "violation", + Metadata: map[string]interface{}{ + "description": "No privileged containers allowed", + }, + }, + rType: failureResultType, + namespace: "main", + want: "conftest-failure/no-privileged-containers-allowed", + }, + { + name: "falls back to base ID when no identifying information", + result: Result{ + Message: "violation", + Metadata: map[string]interface{}{}, + }, + rType: failureResultType, + namespace: "main", + want: "conftest-failure", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getRuleID(tt.result, tt.rType, tt.namespace) + if got != tt.want { + t.Errorf("getRuleID() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestQueryInformation(t *testing.T) { + var buf bytes.Buffer + s := NewSARIF(&buf) + + results := []CheckResult{ + { + FileName: "test.yaml", + Namespace: "main", + Failures: []Result{ + { + Message: "violation found", + Metadata: map[string]interface{}{ + "query": "data.main.deny", + "traces": []string{"trace1", "trace2"}, + "outputs": []string{"output1", "output2"}, + "package": "main", + "rule": "deny", + }, + }, + }, + }, + } + + if err := s.Output(results); err != nil { + t.Fatal(err) + } + + report := decodeSARIFReport(t, buf.String()) + run := report.Runs[0] + + if len(run.Results) != 1 { + t.Errorf("expected 1 result, got %d", len(run.Results)) + } + + expectedRuleID := "main/main/deny" + validateResult(t, run.Results[0], struct { + level string + kind string + message string + ruleID string + namespace string + }{ + level: levelError, + kind: kindFail, + message: "violation found", + ruleID: expectedRuleID, + namespace: "main", + }) + + // Find and validate the rule + _, foundResult, err := findRule(run, expectedRuleID) + if err != nil { + t.Fatal(err) + } + if foundResult == nil { + t.Fatal("no result found for rule") + } + result := *foundResult + + // Verify query information in properties + if query, ok := result.Properties["query"].(string); !ok || query != "data.main.deny" { + t.Errorf("unexpected query: %v", result.Properties["query"]) + } + + // Verify traces and outputs are preserved + if traces, ok := result.Properties["traces"].([]interface{}); !ok || len(traces) != 2 { + t.Errorf("unexpected traces: %v", result.Properties["traces"]) + } + if outputs, ok := result.Properties["outputs"].([]interface{}); !ok || len(outputs) != 2 { + t.Errorf("unexpected outputs: %v", result.Properties["outputs"]) + } + + validateInvocation(t, run.Invocations[0], struct { + successful bool + exitCode int + exitDesc string + }{ + successful: true, + exitCode: 1, + exitDesc: exitViolations, + }) +} + +// Helper functions for testing +func decodeSARIFReport(t *testing.T, output string) sarifReport { + t.Helper() + var report sarifReport + if err := json.NewDecoder(strings.NewReader(output)).Decode(&report); err != nil { + t.Fatalf("failed to decode SARIF output: %v", err) + } + return report +} + +// validateResult validates that a SARIF result matches the expected values for level, kind, +// message, rule ID and namespace +func validateResult(t *testing.T, result sarifResult, expected struct { + level string + kind string + message string + ruleID string + namespace string +}) { + t.Helper() + if result.Level != expected.level { + t.Errorf("expected level '%s', got '%s'", expected.level, result.Level) + } + if result.Kind != expected.kind { + t.Errorf("expected kind '%s', got '%s'", expected.kind, result.Kind) + } + if result.Message.Text != expected.message { + t.Errorf("expected message '%s', got '%s'", expected.message, result.Message.Text) + } + if result.RuleID != expected.ruleID { + t.Errorf("expected rule ID '%s', got '%s'", expected.ruleID, result.RuleID) + } + if ns, ok := result.Properties["namespace"].(string); !ok || ns != expected.namespace { + t.Errorf("expected namespace '%s' in properties, got '%v'", expected.namespace, result.Properties["namespace"]) + } +} + +// validateInvocation validates that a SARIF invocation matches the expected values for +// execution success, exit code and exit code description. +func validateInvocation(t *testing.T, invocation sarifInvocation, expected struct { + successful bool + exitCode int + exitDesc string +}) { + t.Helper() + if invocation.ExecutionSuccessful != expected.successful { + t.Error("unexpected execution success state") + } + if invocation.ExitCode != expected.exitCode { + t.Errorf("expected exit code %d, got %d", expected.exitCode, invocation.ExitCode) + } + if invocation.ExitCodeDescription != expected.exitDesc { + t.Errorf("unexpected exit code description: %s", invocation.ExitCodeDescription) + } +} + +// validateTimestamps validates that the start and end times in a SARIF invocation +// are valid RFC3339 timestamps and chronologically ordered. +func validateTimestamps(t *testing.T, invocation sarifInvocation) { + t.Helper() + startTime, err := time.Parse(time.RFC3339, invocation.StartTimeUtc) + if err != nil { + t.Errorf("invalid start time format: %v", err) + } + endTime, err := time.Parse(time.RFC3339, invocation.EndTimeUtc) + if err != nil { + t.Errorf("invalid end time format: %v", err) + } + if endTime.Before(startTime) { + t.Error("end time should not be before start time") + } +} + +// findRule returns the rule and its associated result for a given rule ID. +// If the rule is not found, it returns an error. +func findRule(run sarifRun, ruleID string) (rule *sarifRule, result *sarifResult, err error) { + for i := range run.Tool.Driver.Rules { + r := run.Tool.Driver.Rules[i] // Take reference to array element, not loop variable + if r.ID == ruleID { + rule = &r + // Find the first result that references this rule + for j := range run.Results { + res := run.Results[j] // Take reference to array element, not loop variable + if res.RuleIndex == i { + result = &res + return rule, result, nil + } + } + // Rule found but no results reference it + return rule, nil, nil + } + } + return nil, nil, fmt.Errorf("rule not found: %s", ruleID) +} + +// validateRuleMetadata validates that a SARIF rule matches the expected values for description, help URI, +// help text and namespace. +func validateRuleMetadata(t *testing.T, rule *sarifRule, expected struct { + description string + helpURI string + helpText string + namespace string +}) { + t.Helper() + if rule == nil { + t.Fatal("rule is nil") + } + if rule.FullDescription == nil || rule.FullDescription.Text != expected.description { + t.Error("invalid rule description") + } + if rule.HelpURI != expected.helpURI { + t.Error("invalid help URI") + } + if rule.Help == nil || rule.Help.Text != expected.helpText { + t.Error("invalid help text") + } + if ns, ok := rule.Properties["namespace"].(string); !ok || ns != expected.namespace { + t.Errorf("expected namespace '%s' in rule properties, got '%v'", expected.namespace, ns) + } +} diff --git a/policy/engine.go b/policy/engine.go index 2fcf16aabf..5934221d2d 100644 --- a/policy/engine.go +++ b/policy/engine.go @@ -478,7 +478,6 @@ func (e *Engine) query(ctx context.Context, input interface{}, query string) (ou var results []output.Result for _, result := range resultSet { for _, expression := range result.Expressions { - // Rego rules that are intended for evaluation should return a slice of values. // For example, deny[msg] or violation[{"msg": msg}]. // @@ -495,11 +494,13 @@ func (e *Engine) query(ctx context.Context, input interface{}, query string) (ou for _, v := range expressionValues { switch val := v.(type) { - // Policies that only return a single string (e.g. deny[msg]) case string: result := output.Result{ Message: val, + Metadata: map[string]interface{}{ + "query": query, + }, } results = append(results, result) @@ -510,6 +511,12 @@ func (e *Engine) query(ctx context.Context, input interface{}, query string) (ou return output.QueryResult{}, fmt.Errorf("new result: %w", err) } + // Add query to metadata + if result.Metadata == nil { + result.Metadata = make(map[string]interface{}) + } + result.Metadata["query"] = query + results = append(results, result) } }