diff --git a/check/check.go b/check/check.go index a6ebdb03b..b4c2b5a74 100644 --- a/check/check.go +++ b/check/check.go @@ -38,9 +38,12 @@ const ( WARN State = "WARN" // INFO informational message INFO State = "INFO" - // SKIP for when a check should be skipped. - SKIP = "skip" + SKIP State = "SKIP" + // MANU for when a check is manual. + MANU State = "MANU" + // ERRO for errors in tests. + ERRO State = "ERRO" // MASTER a master node MASTER NodeType = "master" @@ -58,8 +61,11 @@ const ( // MANAGEDSERVICES a node to run managedservices from MANAGEDSERVICES = "managedservices" - // MANUAL Check Type - MANUAL string = "manual" + // TypeSkip is skip check type. + TypeSkip = "skip" + + // TypeManual is manual check type. + TypeManual = "manual" ) // Check contains information about a recommendation in the @@ -118,18 +124,18 @@ func (c *Check) run() State { return c.State } - // If check type is skip, force result to INFO - if c.Type == SKIP { + // If check type is skip, force result to SKIP + if c.Type == TypeSkip { c.Reason = "Test marked as skip" - c.State = INFO + c.State = SKIP glog.V(3).Info(c.Reason) return c.State } - // If check type is manual force result to WARN - if c.Type == MANUAL { + // If check type is manual, force result to MANU + if c.Type == TypeManual { c.Reason = "Test marked as a manual test" - c.State = WARN + c.State = MANU glog.V(3).Info(c.Reason) return c.State } @@ -172,11 +178,7 @@ func (c *Check) run() State { if err != nil { c.Reason = err.Error() - if c.Scored { - c.State = FAIL - } else { - c.State = WARN - } + c.State = ERRO glog.V(3).Info(c.Reason) } diff --git a/check/check_test.go b/check/check_test.go index fd18927bf..9cd9a717a 100644 --- a/check/check_test.go +++ b/check/check_test.go @@ -27,9 +27,27 @@ func TestCheck_Run(t *testing.T) { } testCases := []TestCase{ - {name: "Manual check should WARN", check: Check{Type: MANUAL}, Expected: WARN}, - {name: "Skip check should INFO", check: Check{Type: "skip"}, Expected: INFO}, - {name: "Unscored check (with no type) should WARN on failure", check: Check{Scored: false}, Expected: WARN}, + { + name: "Manual check should MANU", + check: Check{ + Type: TypeManual, + }, + Expected: MANU, + }, + { + name: "Skip check should SKIP", + check: Check{ + Type: TypeSkip, + }, + Expected: SKIP, + }, + { + name: "Unscored check (with no type) should WARN on failure", + check: Check{ + Scored: false, + }, + Expected: WARN, + }, { name: "Unscored check that pass should PASS", check: Check{ @@ -42,9 +60,21 @@ func TestCheck_Run(t *testing.T) { }, Expected: PASS, }, - - {name: "Check with no tests should WARN", check: Check{Scored: true}, Expected: WARN}, - {name: "Scored check with empty tests should FAIL", check: Check{Scored: true, Tests: &tests{}}, Expected: FAIL}, + { + name: "Check with no tests should WARN", + check: Check{ + Scored: true, + }, + Expected: WARN, + }, + { + name: "Scored check with empty tests should FAIL", + check: Check{ + Scored: true, + Tests: &tests{}, + }, + Expected: FAIL, + }, { name: "Scored check that doesn't pass should FAIL", check: Check{ diff --git a/check/controls.go b/check/controls.go index f84acf7b6..7d8fa210c 100644 --- a/check/controls.go +++ b/check/controls.go @@ -64,6 +64,9 @@ type Group struct { Fail int `json:"fail"` Warn int `json:"warn"` Info int `json:"info"` + Skip int `json:"skip"` + Manu int `json:"manu"` + Erro int `json:"erro"` Text string `json:"desc"` Checks []*Check `json:"results"` } @@ -74,6 +77,9 @@ type Summary struct { Fail int `json:"total_fail"` Warn int `json:"total_warn"` Info int `json:"total_info"` + Skip int `json:"total_skip"` + Manu int `json:"total_manu"` + Erro int `json:"total_erro"` } // Predicate a predicate on the given Group and Check arguments. @@ -99,7 +105,7 @@ func NewControls(t NodeType, in []byte, detectedVersion string) (*Controls, erro func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap map[string]bool) Summary { var g []*Group m := make(map[string]*Group) - controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn, controls.Info = 0, 0, 0, 0 + controls.Summary = Summary{} for _, group := range controls.Groups { for _, check := range group.Checks { @@ -111,8 +117,8 @@ func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap m _, groupSkippedViaCmd := skipIDMap[group.ID] _, checkSkippedViaCmd := skipIDMap[check.ID] - if group.Type == SKIP || groupSkippedViaCmd || checkSkippedViaCmd { - check.Type = SKIP + if group.Type == TypeSkip || groupSkippedViaCmd || checkSkippedViaCmd { + check.Type = TypeSkip } state := runner.Run(check) @@ -158,8 +164,14 @@ func (controls *Controls) JUnit() ([]byte, error) { suite := reporters.JUnitTestSuite{ Name: controls.Text, TestCases: []reporters.JUnitTestCase{}, - Tests: controls.Summary.Pass + controls.Summary.Fail + controls.Summary.Info + controls.Summary.Warn, - Failures: controls.Summary.Fail, + Tests: controls.Summary.Pass + + controls.Summary.Fail + + controls.Summary.Info + + controls.Summary.Warn + + controls.Summary.Skip + + controls.Summary.Manu + + controls.Summary.Erro, + Failures: controls.Summary.Fail, } for _, g := range controls.Groups { for _, check := range g.Checks { @@ -179,11 +191,10 @@ func (controls *Controls) JUnit() ([]byte, error) { } switch check.State { - case FAIL: + case FAIL, ERRO: tc.FailureMessage = &reporters.JUnitFailureMessage{Message: check.Remediation} - case WARN, INFO: - // WARN and INFO are two different versions of skipped tests. Either way it would be a false positive/negative to report - // it any other way. + case WARN, INFO, SKIP, MANU: + // Different versions of skipped tests. It would be a false positive/negative to report it any other way. tc.Skipped = &reporters.JUnitSkipped{} case PASS: default: @@ -226,7 +237,7 @@ func (controls *Controls) ASFF() ([]*securityhub.AwsSecurityFinding, error) { tf := ti.Format(time.RFC3339) for _, g := range controls.Groups { for _, check := range g.Checks { - if check.State == FAIL || check.State == WARN { + if check.State == FAIL || check.State == WARN || check.State == MANU || check.State == ERRO { // ASFF ProductFields['Actual result'] can't be longer than 1024 characters actualValue := check.ActualValue remediation := check.Remediation @@ -292,6 +303,7 @@ func getConfig(name string) (string, error) { } return r, nil } + func summarize(controls *Controls, state State) { switch state { case PASS: @@ -302,6 +314,12 @@ func summarize(controls *Controls, state State) { controls.Summary.Warn++ case INFO: controls.Summary.Info++ + case SKIP: + controls.Summary.Skip++ + case MANU: + controls.Summary.Manu++ + case ERRO: + controls.Summary.Erro++ default: glog.Warningf("Unrecognized state %s", state) } @@ -317,6 +335,12 @@ func summarizeGroup(group *Group, state State) { group.Warn++ case INFO: group.Info++ + case SKIP: + group.Skip++ + case MANU: + group.Manu++ + case ERRO: + group.Erro++ default: glog.Warningf("Unrecognized state %s", state) } diff --git a/check/controls_test.go b/check/controls_test.go index a0015facb..17fdaf416 100644 --- a/check/controls_test.go +++ b/check/controls_test.go @@ -132,10 +132,10 @@ groups: controls.RunChecks(normalRunner, allChecks, skipMap) G1 := controls.Groups[0] - assertEqualGroupSummary(t, 0, 0, 3, 0, G1) + assertEqualGroupSummary(t, 0, 0, 0, 0, 3, 0, 0, G1) G2 := controls.Groups[1] - assertEqualGroupSummary(t, 0, 0, 2, 0, G2) + assertEqualGroupSummary(t, 0, 0, 0, 0, 2, 0, 0, G2) }) } @@ -163,7 +163,7 @@ groups: controls.RunChecks(normalRunner, allChecks, emptySkipList) G1 := controls.Groups[0] - assertEqualGroupSummary(t, 0, 0, 1, 0, G1) + assertEqualGroupSummary(t, 0, 0, 0, 0, 1, 0, 0, G1) }) } @@ -214,7 +214,7 @@ groups: G1 := controls.Groups[0] assert.Equal(t, "G1", G1.ID) assert.Equal(t, "G1/C1", G1.Checks[0].ID) - assertEqualGroupSummary(t, 1, 0, 0, 0, G1) + assertEqualGroupSummary(t, 1, 0, 0, 0, 0, 0, 0, G1) // and G2 := controls.Groups[1] assert.Equal(t, "G2", G2.ID) @@ -225,12 +225,15 @@ groups: assert.Equal(t, "SomeSampleFlag=true", G2.Checks[0].Tests.TestItems[0].Flag) assert.Equal(t, "Edit the config file /this/is/a/file/path and set SomeSampleFlag to true.\n", G2.Checks[0].Remediation) assert.Equal(t, true, G2.Checks[0].Scored) - assertEqualGroupSummary(t, 0, 1, 0, 0, G2) + assertEqualGroupSummary(t, 0, 1, 0, 0, 0, 0, 0, G2) // and assert.Equal(t, 1, controls.Summary.Pass) assert.Equal(t, 1, controls.Summary.Fail) assert.Equal(t, 0, controls.Summary.Info) assert.Equal(t, 0, controls.Summary.Warn) + assert.Equal(t, 0, controls.Summary.Skip) + assert.Equal(t, 0, controls.Summary.Manu) + assert.Equal(t, 0, controls.Summary.Erro) // and runner.AssertExpectations(t) }) @@ -267,6 +270,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) { Pass: 100, Warn: 101, Info: 102, + Skip: 0, + Manu: 0, + Erro: 0, }, Groups: []*Group{ { @@ -283,7 +289,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) { `), }, { - desc: "Warn and Info are considered skips and failed tests properly reported", + desc: "WARN, INFO, SKIP, MANU are considered skips and failed tests properly reported", input: &Controls{ Groups: []*Group{ { @@ -293,6 +299,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) { {ID: "check2id", Text: "check2text", State: INFO}, {ID: "check3id", Text: "check3text", State: WARN}, {ID: "check4id", Text: "check4text", State: FAIL}, + {ID: "check5id", Text: "check5text", State: SKIP}, + {ID: "check6id", Text: "check6text", State: MANU}, + {ID: "check7id", Text: "check7text", State: ERRO}, }, }, }, @@ -313,6 +322,18 @@ func TestControls_JUnitIncludesJSON(t *testing.T) { {"test_number":"check4id","test_desc":"check4text","audit":"","AuditEnv":"","AuditConfig":"","type":"","remediation":"","test_info":null,"status":"FAIL","actual_value":"","scored":false,"IsMultiple":false,"expected_result":""} + + + {"test_number":"check5id","test_desc":"check5text","audit":"","AuditEnv":"","AuditConfig":"","type":"","remediation":"","test_info":null,"status":"SKIP","actual_value":"","scored":false,"IsMultiple":false,"expected_result":""} + + + + {"test_number":"check6id","test_desc":"check6text","audit":"","AuditEnv":"","AuditConfig":"","type":"","remediation":"","test_info":null,"status":"MANU","actual_value":"","scored":false,"IsMultiple":false,"expected_result":""} + + + + {"test_number":"check7id","test_desc":"check7text","audit":"","AuditEnv":"","AuditConfig":"","type":"","remediation":"","test_info":null,"status":"ERRO","actual_value":"","scored":false,"IsMultiple":false,"expected_result":""} + `), }, } @@ -355,12 +376,15 @@ func TestControls_JUnitIncludesJSON(t *testing.T) { } } -func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn int, actual *Group) { +func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn, skip, manu, erro int, actual *Group) { t.Helper() assert.Equal(t, pass, actual.Pass) assert.Equal(t, fail, actual.Fail) assert.Equal(t, info, actual.Info) assert.Equal(t, warn, actual.Warn) + assert.Equal(t, skip, actual.Skip) + assert.Equal(t, manu, actual.Manu) + assert.Equal(t, erro, actual.Erro) } func TestControls_ASFF(t *testing.T) { @@ -388,6 +412,9 @@ func TestControls_ASFF(t *testing.T) { Pass: 100, Warn: 101, Info: 102, + Skip: 0, + Manu: 0, + Erro: 0, }, Groups: []*Group{ { diff --git a/cmd/common.go b/cmd/common.go index 60f291332..a9f09f99e 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -173,7 +173,9 @@ func prettyPrint(r *check.Controls, summary check.Summary) { for _, c := range g.Checks { colorPrint(c.State, fmt.Sprintf("%s %s\n", c.ID, c.Text)) - if includeTestOutput && c.State == check.FAIL && len(c.ActualValue) > 0 { + if includeTestOutput && + (c.State == check.FAIL || c.State == check.WARN || c.State == check.ERRO) && + len(c.ActualValue) > 0 { printRawOutput(c.ActualValue) } } @@ -184,20 +186,22 @@ func prettyPrint(r *check.Controls, summary check.Summary) { // Print remediations. if !noRemediations { - if summary.Fail > 0 || summary.Warn > 0 { + if summary.Fail > 0 || summary.Warn > 0 || summary.Manu > 0 || summary.Erro > 0 { colors[check.WARN].Printf("== Remediations %s ==\n", r.Type) for _, g := range r.Groups { for _, c := range g.Checks { - if c.State == check.FAIL { + if c.State == check.FAIL || c.State == check.WARN { fmt.Printf("%s %s\n", c.ID, c.Remediation) } - if c.State == check.WARN { - // Print the error if test failed due to problem with the audit command - if c.Reason != "" && c.Type != "manual" { - fmt.Printf("%s audit test did not run: %s\n", c.ID, c.Reason) - } else { - fmt.Printf("%s %s\n", c.ID, c.Remediation) + if c.State == check.ERRO { + fmt.Printf("%s audit test error: %s\n", c.ID, c.Reason) + } + if c.State == check.MANU { + fmt.Printf("%s audit test is a manual test", c.ID) + if c.Remediation != "" { + fmt.Printf(": %s", c.Remediation) } + fmt.Println() } } } @@ -213,7 +217,9 @@ func prettyPrint(r *check.Controls, summary check.Summary) { func printSummary(summary check.Summary, sectionName string) { var res check.State - if summary.Fail > 0 { + if summary.Erro > 0 { + res = check.ERRO + } else if summary.Fail > 0 { res = check.FAIL } else if summary.Warn > 0 { res = check.WARN @@ -222,8 +228,14 @@ func printSummary(summary check.Summary, sectionName string) { } colors[res].Printf("== Summary %s ==\n", sectionName) - fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n%d checks INFO\n\n", - summary.Pass, summary.Fail, summary.Warn, summary.Info, + fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n%d checks INFO\n%d checks SKIP\n%d checks MANU\n%d checks ERRO\n\n", + summary.Pass, + summary.Fail, + summary.Warn, + summary.Info, + summary.Skip, + summary.Manu, + summary.Erro, ) } @@ -386,7 +398,7 @@ func isThisNodeRunning(nodeType check.NodeType) bool { func exitCodeSelection(controlsCollection []*check.Controls) int { for _, control := range controlsCollection { - if control.Fail > 0 { + if control.Fail > 0 || control.Erro > 0 { return exitCode } } @@ -492,6 +504,9 @@ func getSummaryTotals(controlsCollection []*check.Controls) check.Summary { totalSummary.Warn = totalSummary.Warn + summary.Warn totalSummary.Pass = totalSummary.Pass + summary.Pass totalSummary.Info = totalSummary.Info + summary.Info + totalSummary.Skip = totalSummary.Skip + summary.Skip + totalSummary.Manu = totalSummary.Manu + summary.Manu + totalSummary.Erro = totalSummary.Erro + summary.Erro } return totalSummary } diff --git a/cmd/common_test.go b/cmd/common_test.go index e0f38d049..7e91221a2 100644 --- a/cmd/common_test.go +++ b/cmd/common_test.go @@ -653,6 +653,9 @@ func TestGetSummaryTotals(t *testing.T) { assert.Equal(t, 14, resultTotals.Warn) assert.Equal(t, 0, resultTotals.Info) assert.Equal(t, 49, resultTotals.Pass) + assert.Equal(t, 0, resultTotals.Skip) + assert.Equal(t, 0, resultTotals.Manu) + assert.Equal(t, 0, resultTotals.Erro) } func TestPrintSummary(t *testing.T) { @@ -670,7 +673,7 @@ func TestPrintSummary(t *testing.T) { out, _ := ioutil.ReadAll(r) os.Stdout = rescueStdout - assert.Contains(t, string(out), "49 checks PASS\n12 checks FAIL\n14 checks WARN\n0 checks INFO\n\n") + assert.Contains(t, string(out), "49 checks PASS\n12 checks FAIL\n14 checks WARN\n0 checks INFO\n0 checks SKIP\n0 checks MANU\n0 checks ERRO\n") } func TestPrettyPrintNoSummary(t *testing.T) { diff --git a/cmd/testdata/controlsCollection.json b/cmd/testdata/controlsCollection.json index db71728ca..d7d7c4b31 100644 --- a/cmd/testdata/controlsCollection.json +++ b/cmd/testdata/controlsCollection.json @@ -11,6 +11,9 @@ "fail": 0, "warn": 0, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Etcd Node Configuration Files", "results": [ { @@ -34,7 +37,10 @@ "total_pass": 7, "total_fail": 0, "total_warn": 0, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "3", @@ -48,6 +54,9 @@ "fail": 0, "warn": 1, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Authentication and Authorization", "results": [ { @@ -72,7 +81,10 @@ "total_pass": 0, "total_fail": 0, "total_warn": 3, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "1", @@ -86,6 +98,9 @@ "fail": 1, "warn": 5, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Master Node Configuration Files", "results": [ { @@ -109,6 +124,9 @@ "total_pass": 42, "total_fail": 12, "total_warn": 11, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 } -] \ No newline at end of file +] diff --git a/cmd/testdata/passedControlsCollection.json b/cmd/testdata/passedControlsCollection.json index 5235dbb52..cf6d86692 100644 --- a/cmd/testdata/passedControlsCollection.json +++ b/cmd/testdata/passedControlsCollection.json @@ -11,6 +11,9 @@ "fail": 0, "warn": 0, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Etcd Node Configuration Files", "results": [ { @@ -34,7 +37,10 @@ "total_pass": 7, "total_fail": 0, "total_warn": 0, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "3", @@ -48,6 +54,9 @@ "fail": 0, "warn": 1, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Authentication and Authorization", "results": [ { @@ -72,6 +81,9 @@ "total_pass": 0, "total_fail": 0, "total_warn": 3, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 } ] diff --git a/cmd/testdata/result.json b/cmd/testdata/result.json index 871483da2..8ad0d9376 100644 --- a/cmd/testdata/result.json +++ b/cmd/testdata/result.json @@ -12,6 +12,9 @@ "fail": 1, "warn": 5, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Master Node Configuration Files", "results": [ { @@ -35,7 +38,10 @@ "total_pass": 42, "total_fail": 12, "total_warn": 11, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "2", @@ -49,6 +55,9 @@ "fail": 0, "warn": 0, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Etcd Node Configuration Files", "results": [ { @@ -72,7 +81,10 @@ "total_pass": 7, "total_fail": 0, "total_warn": 0, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "3", @@ -86,6 +98,9 @@ "fail": 0, "warn": 1, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Authentication and Authorization", "results": [ { @@ -110,13 +125,19 @@ "total_pass": 0, "total_fail": 0, "total_warn": 3, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 } ], "Totals": { "total_pass": 49, "total_fail": 12, "total_warn": 14, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 } } diff --git a/cmd/testdata/result_no_totals.json b/cmd/testdata/result_no_totals.json index 6589a507c..53a283765 100644 --- a/cmd/testdata/result_no_totals.json +++ b/cmd/testdata/result_no_totals.json @@ -11,6 +11,9 @@ "fail": 1, "warn": 5, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Master Node Configuration Files", "results": [ { @@ -34,7 +37,10 @@ "total_pass": 42, "total_fail": 12, "total_warn": 11, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "2", @@ -48,6 +54,9 @@ "fail": 0, "warn": 0, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Etcd Node Configuration Files", "results": [ { @@ -71,7 +80,10 @@ "total_pass": 7, "total_fail": 0, "total_warn": 0, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 }, { "id": "3", @@ -85,6 +97,9 @@ "fail": 0, "warn": 1, "info": 0, + "skip": 0, + "manu": 0, + "erro": 0, "desc": "Authentication and Authorization", "results": [ { @@ -109,6 +124,9 @@ "total_pass": 0, "total_fail": 0, "total_warn": 3, - "total_info": 0 + "total_info": 0, + "total_skip": 0, + "total_manu": 0, + "total_erro": 0 } -] \ No newline at end of file +] diff --git a/cmd/util.go b/cmd/util.go index a60ca8618..cf87cdcf3 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -22,7 +22,10 @@ var ( check.PASS: color.New(color.FgGreen), check.FAIL: color.New(color.FgRed), check.WARN: color.New(color.FgYellow), - check.INFO: color.New(color.FgBlue), + check.INFO: color.New(color.FgCyan), + check.SKIP: color.New(color.FgBlue), + check.MANU: color.New(color.FgWhite), + check.ERRO: color.New(color.FgMagenta), } )