🌱 use ValidateTestReturn for SAST tests (ossf#3872)

* check logger counts for SAST tests previously, we only checked the result score. test failures with this method dont produce as actionable feedback. Signed-off-by: Spencer Schrock <[email protected]> * clarify test names and score constants used Signed-off-by: Spencer Schrock <[email protected]> --------- Signed-off-by: Spencer Schrock <[email protected]>
diogoteles08 · Feb 13, 2024 · 68d8d93 · 68d8d93
1 parent 9b65bde
commit 68d8d93
Showing 1 changed file with 45 additions and 24 deletions.
diff --git a/checks/sast_test.go b/checks/sast_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	mockrepo "github.com/ossf/scorecard/v4/clients/mockclients"
+	sce "github.com/ossf/scorecard/v4/errors"
 	scut "github.com/ossf/scorecard/v4/utests"
 )
 
@@ -39,24 +40,31 @@ func Test_SAST(t *testing.T) {
 		searchRequest clients.SearchRequest
 		name          string
 		path          string
-		expected      checker.CheckResult
 		commits       []clients.Commit
 		checkRuns     []clients.CheckRun
 		searchresult  clients.SearchResponse
+		expected      scut.TestReturn
 	}{
 		{
-			name:         "SAST checker should return failed status when no PRs are found",
+			name:         "SAST checker should return min score when no PRs are found",
 			commits:      []clients.Commit{},
 			searchresult: clients.SearchResponse{},
 			checkRuns:    []clients.CheckRun{},
+			expected: scut.TestReturn{
+				Score:        checker.MinResultScore,
+				NumberOfWarn: 1,
+			},
 		},
 		{
-			name:         "SAST checker should return failed status when no PRs are found",
+			name:         "SAST checker should return failed status when an error occurs",
 			err:          errors.New("error"),
 			commits:      []clients.Commit{},
 			searchresult: clients.SearchResponse{},
 			checkRuns:    []clients.CheckRun{},
-			expected:     checker.CheckResult{Score: -1},
+			expected: scut.TestReturn{
+				Score: checker.InconclusiveResultScore,
+				Error: sce.ErrScorecardInternal,
+			},
 		},
 		{
 			name: "Successful SAST checker should return success status for github-advanced-security",
@@ -77,8 +85,10 @@ func Test_SAST(t *testing.T) {
 					},
 				},
 			},
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  1,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -100,8 +110,10 @@ func Test_SAST(t *testing.T) {
 					},
 				},
 			},
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  1,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -124,8 +136,10 @@ func Test_SAST(t *testing.T) {
 				},
 			},
 			path: "",
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  1,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -147,8 +161,10 @@ func Test_SAST(t *testing.T) {
 					},
 				},
 			},
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  1,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -163,8 +179,10 @@ func Test_SAST(t *testing.T) {
 			},
 			searchresult: clients.SearchResponse{},
 			path:         ".github/workflows/airflow-codeql-workflow.yaml",
-			expected: checker.CheckResult{
-				Score: 7,
+			expected: scut.TestReturn{
+				Score:        7,
+				NumberOfWarn: 1,
+				NumberOfInfo: 1,
 			},
 		},
 		{
@@ -195,8 +213,10 @@ func Test_SAST(t *testing.T) {
 				},
 			},
 			path: ".github/workflows/airflow-codeql-workflow.yaml",
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  2,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -228,8 +248,10 @@ func Test_SAST(t *testing.T) {
 				},
 			},
 			path: ".github/workflows/airflow-codeql-workflow.yaml",
-			expected: checker.CheckResult{
-				Score: 10,
+			expected: scut.TestReturn{
+				Score:         checker.MaxResultScore,
+				NumberOfInfo:  2,
+				NumberOfDebug: 1,
 			},
 		},
 		{
@@ -266,8 +288,10 @@ func Test_SAST(t *testing.T) {
 				},
 			},
 			path: ".github/workflows/airflow-codeql-workflow.yaml",
-			expected: checker.CheckResult{
-				Score: 7,
+			expected: scut.TestReturn{
+				Score:        7,
+				NumberOfWarn: 1,
+				NumberOfInfo: 1,
 			},
 		},
 	}
@@ -315,10 +339,7 @@ func Test_SAST(t *testing.T) {
 			}
 			res := SAST(&req)
 
-			if res.Score != tt.expected.Score {
-				t.Errorf("Expected score %d, got %d for %v", tt.expected.Score, res.Score, tt.name)
-			}
-			ctrl.Finish()
+			scut.ValidateTestReturn(t, tt.name, &tt.expected, &res, &dl)
 		})
 	}
 }