Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce new statuses: SKIP, MANU, ERRO #1035

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Introduce new statuses: SKIP, MANU, ERRO #1035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 17 additions & 15 deletions check/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,12 @@ const (
WARN State = "WARN"
// INFO informational message
INFO State = "INFO"

// SKIP for when a check should be skipped.
SKIP = "skip"
SKIP State = "SKIP"
// MANU for when a check is manual.
MANU State = "MANU"
// ERRO for errors in tests.
ERRO State = "ERRO"

// MASTER a master node
MASTER NodeType = "master"
Expand All @@ -58,8 +61,11 @@ const (
// MANAGEDSERVICES a node to run managedservices from
MANAGEDSERVICES = "managedservices"

// MANUAL Check Type
MANUAL string = "manual"
// TypeSkip is skip check type.
TypeSkip = "skip"

// TypeManual is manual check type.
TypeManual = "manual"
)

// Check contains information about a recommendation in the
Expand Down Expand Up @@ -118,18 +124,18 @@ func (c *Check) run() State {
return c.State
}

// If check type is skip, force result to INFO
if c.Type == SKIP {
// If check type is skip, force result to SKIP
if c.Type == TypeSkip {
c.Reason = "Test marked as skip"
c.State = INFO
c.State = SKIP
glog.V(3).Info(c.Reason)
return c.State
}

// If check type is manual force result to WARN
if c.Type == MANUAL {
// If check type is manual, force result to MANU
if c.Type == TypeManual {
c.Reason = "Test marked as a manual test"
c.State = WARN
c.State = MANU
glog.V(3).Info(c.Reason)
return c.State
}
Expand Down Expand Up @@ -172,11 +178,7 @@ func (c *Check) run() State {

if err != nil {
c.Reason = err.Error()
if c.Scored {
c.State = FAIL
} else {
c.State = WARN
}
c.State = ERRO
glog.V(3).Info(c.Reason)
}

Expand Down
42 changes: 36 additions & 6 deletions check/check_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,27 @@ func TestCheck_Run(t *testing.T) {
}

testCases := []TestCase{
{name: "Manual check should WARN", check: Check{Type: MANUAL}, Expected: WARN},
{name: "Skip check should INFO", check: Check{Type: "skip"}, Expected: INFO},
{name: "Unscored check (with no type) should WARN on failure", check: Check{Scored: false}, Expected: WARN},
{
name: "Manual check should MANU",
check: Check{
Type: TypeManual,
},
Expected: MANU,
},
{
name: "Skip check should SKIP",
check: Check{
Type: TypeSkip,
},
Expected: SKIP,
},
{
name: "Unscored check (with no type) should WARN on failure",
check: Check{
Scored: false,
},
Expected: WARN,
},
{
name: "Unscored check that pass should PASS",
check: Check{
Expand All @@ -42,9 +60,21 @@ func TestCheck_Run(t *testing.T) {
},
Expected: PASS,
},

{name: "Check with no tests should WARN", check: Check{Scored: true}, Expected: WARN},
{name: "Scored check with empty tests should FAIL", check: Check{Scored: true, Tests: &tests{}}, Expected: FAIL},
{
name: "Check with no tests should WARN",
check: Check{
Scored: true,
},
Expected: WARN,
},
{
name: "Scored check with empty tests should FAIL",
check: Check{
Scored: true,
Tests: &tests{},
},
Expected: FAIL,
},
{
name: "Scored check that doesn't pass should FAIL",
check: Check{
Expand Down
44 changes: 34 additions & 10 deletions check/controls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ type Group struct {
Fail int `json:"fail"`
Warn int `json:"warn"`
Info int `json:"info"`
Skip int `json:"skip"`
Manu int `json:"manu"`
Erro int `json:"erro"`
Text string `json:"desc"`
Checks []*Check `json:"results"`
}
Expand All @@ -74,6 +77,9 @@ type Summary struct {
Fail int `json:"total_fail"`
Warn int `json:"total_warn"`
Info int `json:"total_info"`
Skip int `json:"total_skip"`
Manu int `json:"total_manu"`
Erro int `json:"total_erro"`
}

// Predicate a predicate on the given Group and Check arguments.
Expand All @@ -99,7 +105,7 @@ func NewControls(t NodeType, in []byte, detectedVersion string) (*Controls, erro
func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap map[string]bool) Summary {
var g []*Group
m := make(map[string]*Group)
controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn, controls.Info = 0, 0, 0, 0
controls.Summary = Summary{}

for _, group := range controls.Groups {
for _, check := range group.Checks {
Expand All @@ -111,8 +117,8 @@ func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap m
_, groupSkippedViaCmd := skipIDMap[group.ID]
_, checkSkippedViaCmd := skipIDMap[check.ID]

if group.Type == SKIP || groupSkippedViaCmd || checkSkippedViaCmd {
check.Type = SKIP
if group.Type == TypeSkip || groupSkippedViaCmd || checkSkippedViaCmd {
check.Type = TypeSkip
}

state := runner.Run(check)
Expand Down Expand Up @@ -158,8 +164,14 @@ func (controls *Controls) JUnit() ([]byte, error) {
suite := reporters.JUnitTestSuite{
Name: controls.Text,
TestCases: []reporters.JUnitTestCase{},
Tests: controls.Summary.Pass + controls.Summary.Fail + controls.Summary.Info + controls.Summary.Warn,
Failures: controls.Summary.Fail,
Tests: controls.Summary.Pass +
controls.Summary.Fail +
controls.Summary.Info +
controls.Summary.Warn +
controls.Summary.Skip +
controls.Summary.Manu +
controls.Summary.Erro,
Failures: controls.Summary.Fail,
}
for _, g := range controls.Groups {
for _, check := range g.Checks {
Expand All @@ -179,11 +191,10 @@ func (controls *Controls) JUnit() ([]byte, error) {
}

switch check.State {
case FAIL:
case FAIL, ERRO:
tc.FailureMessage = &reporters.JUnitFailureMessage{Message: check.Remediation}
case WARN, INFO:
// WARN and INFO are two different versions of skipped tests. Either way it would be a false positive/negative to report
// it any other way.
case WARN, INFO, SKIP, MANU:
// Different versions of skipped tests. It would be a false positive/negative to report it any other way.
tc.Skipped = &reporters.JUnitSkipped{}
case PASS:
default:
Expand Down Expand Up @@ -226,7 +237,7 @@ func (controls *Controls) ASFF() ([]*securityhub.AwsSecurityFinding, error) {
tf := ti.Format(time.RFC3339)
for _, g := range controls.Groups {
for _, check := range g.Checks {
if check.State == FAIL || check.State == WARN {
if check.State == FAIL || check.State == WARN || check.State == MANU || check.State == ERRO {
// ASFF ProductFields['Actual result'] can't be longer than 1024 characters
actualValue := check.ActualValue
remediation := check.Remediation
Expand Down Expand Up @@ -292,6 +303,7 @@ func getConfig(name string) (string, error) {
}
return r, nil
}

func summarize(controls *Controls, state State) {
switch state {
case PASS:
Expand All @@ -302,6 +314,12 @@ func summarize(controls *Controls, state State) {
controls.Summary.Warn++
case INFO:
controls.Summary.Info++
case SKIP:
controls.Summary.Skip++
case MANU:
controls.Summary.Manu++
case ERRO:
controls.Summary.Erro++
default:
glog.Warningf("Unrecognized state %s", state)
}
Expand All @@ -317,6 +335,12 @@ func summarizeGroup(group *Group, state State) {
group.Warn++
case INFO:
group.Info++
case SKIP:
group.Skip++
case MANU:
group.Manu++
case ERRO:
group.Erro++
default:
glog.Warningf("Unrecognized state %s", state)
}
Expand Down
41 changes: 34 additions & 7 deletions check/controls_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,10 @@ groups:
controls.RunChecks(normalRunner, allChecks, skipMap)

G1 := controls.Groups[0]
assertEqualGroupSummary(t, 0, 0, 3, 0, G1)
assertEqualGroupSummary(t, 0, 0, 0, 0, 3, 0, 0, G1)

G2 := controls.Groups[1]
assertEqualGroupSummary(t, 0, 0, 2, 0, G2)
assertEqualGroupSummary(t, 0, 0, 0, 0, 2, 0, 0, G2)
})
}

Expand Down Expand Up @@ -163,7 +163,7 @@ groups:
controls.RunChecks(normalRunner, allChecks, emptySkipList)

G1 := controls.Groups[0]
assertEqualGroupSummary(t, 0, 0, 1, 0, G1)
assertEqualGroupSummary(t, 0, 0, 0, 0, 1, 0, 0, G1)
})
}

Expand Down Expand Up @@ -214,7 +214,7 @@ groups:
G1 := controls.Groups[0]
assert.Equal(t, "G1", G1.ID)
assert.Equal(t, "G1/C1", G1.Checks[0].ID)
assertEqualGroupSummary(t, 1, 0, 0, 0, G1)
assertEqualGroupSummary(t, 1, 0, 0, 0, 0, 0, 0, G1)
// and
G2 := controls.Groups[1]
assert.Equal(t, "G2", G2.ID)
Expand All @@ -225,12 +225,15 @@ groups:
assert.Equal(t, "SomeSampleFlag=true", G2.Checks[0].Tests.TestItems[0].Flag)
assert.Equal(t, "Edit the config file /this/is/a/file/path and set SomeSampleFlag to true.\n", G2.Checks[0].Remediation)
assert.Equal(t, true, G2.Checks[0].Scored)
assertEqualGroupSummary(t, 0, 1, 0, 0, G2)
assertEqualGroupSummary(t, 0, 1, 0, 0, 0, 0, 0, G2)
// and
assert.Equal(t, 1, controls.Summary.Pass)
assert.Equal(t, 1, controls.Summary.Fail)
assert.Equal(t, 0, controls.Summary.Info)
assert.Equal(t, 0, controls.Summary.Warn)
assert.Equal(t, 0, controls.Summary.Skip)
assert.Equal(t, 0, controls.Summary.Manu)
assert.Equal(t, 0, controls.Summary.Erro)
// and
runner.AssertExpectations(t)
})
Expand Down Expand Up @@ -267,6 +270,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
Pass: 100,
Warn: 101,
Info: 102,
Skip: 0,
Manu: 0,
Erro: 0,
},
Groups: []*Group{
{
Expand All @@ -283,7 +289,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
</testcase>
</testsuite>`),
}, {
desc: "Warn and Info are considered skips and failed tests properly reported",
desc: "WARN, INFO, SKIP, MANU are considered skips and failed tests properly reported",
input: &Controls{
Groups: []*Group{
{
Expand All @@ -293,6 +299,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
{ID: "check2id", Text: "check2text", State: INFO},
{ID: "check3id", Text: "check3text", State: WARN},
{ID: "check4id", Text: "check4text", State: FAIL},
{ID: "check5id", Text: "check5text", State: SKIP},
{ID: "check6id", Text: "check6text", State: MANU},
{ID: "check7id", Text: "check7text", State: ERRO},
},
},
},
Expand All @@ -313,6 +322,18 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
<failure type=""></failure>
<system-out>{&#34;test_number&#34;:&#34;check4id&#34;,&#34;test_desc&#34;:&#34;check4text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;FAIL&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check5id check5text" classname="" time="0">
<skipped></skipped>
<system-out>{&#34;test_number&#34;:&#34;check5id&#34;,&#34;test_desc&#34;:&#34;check5text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;SKIP&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check6id check6text" classname="" time="0">
<skipped></skipped>
<system-out>{&#34;test_number&#34;:&#34;check6id&#34;,&#34;test_desc&#34;:&#34;check6text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;MANU&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check7id check7text" classname="" time="0">
<failure type=""></failure>
<system-out>{&#34;test_number&#34;:&#34;check7id&#34;,&#34;test_desc&#34;:&#34;check7text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;ERRO&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
</testsuite>`),
},
}
Expand Down Expand Up @@ -355,12 +376,15 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
}
}

func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn int, actual *Group) {
func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn, skip, manu, erro int, actual *Group) {
t.Helper()
assert.Equal(t, pass, actual.Pass)
assert.Equal(t, fail, actual.Fail)
assert.Equal(t, info, actual.Info)
assert.Equal(t, warn, actual.Warn)
assert.Equal(t, skip, actual.Skip)
assert.Equal(t, manu, actual.Manu)
assert.Equal(t, erro, actual.Erro)
}

func TestControls_ASFF(t *testing.T) {
Expand Down Expand Up @@ -388,6 +412,9 @@ func TestControls_ASFF(t *testing.T) {
Pass: 100,
Warn: 101,
Info: 102,
Skip: 0,
Manu: 0,
Erro: 0,
},
Groups: []*Group{
{
Expand Down
Loading