Skip to content

Commit

Permalink
pkg/hubtest: extract methods + consistent error handling (#2756)
Browse files Browse the repository at this point in the history 
* pkg/hubtest: extract methods + consistent error handling
* lint
* rename variables for further refactor
  • Loading branch information
@mmetc
mmetc authored Feb 14, 2024
1 parent 2bbf0b4 commit 8de8bf0
Show file tree
Hide file tree
Showing 4 changed files with 185 additions and 170 deletions.
80 changes: 43 additions & 37 deletions pkg/hubtest/appsecrule.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,75 +11,81 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)

func (t *HubTestItem) installAppsecRuleItem(hubAppsecRule *cwhub.Item) error {
appsecRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubAppsecRule.RemotePath))
func (t *HubTestItem) installAppsecRuleItem(item *cwhub.Item) error {
sourcePath, err := filepath.Abs(filepath.Join(t.HubPath, item.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", appsecRuleSource, err)
return fmt.Errorf("can't get absolute path of '%s': %w", sourcePath, err)
}

appsecRuleFilename := filepath.Base(appsecRuleSource)
sourceFilename := filepath.Base(sourcePath)

// runtime/hub/appsec-rules/author/appsec-rule
hubDirAppsecRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubAppsecRule.RemotePath))
hubDirAppsecRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(item.RemotePath))

// runtime/appsec-rules/
appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)
itemTypeDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)

if err := os.MkdirAll(hubDirAppsecRuleDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirAppsecRuleDest, err)
return fmt.Errorf("unable to create folder '%s': %w", hubDirAppsecRuleDest, err)
}

if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err)
if err := os.MkdirAll(itemTypeDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %w", itemTypeDirDest, err)
}

// runtime/hub/appsec-rules/crowdsecurity/rule.yaml
hubDirAppsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename)
if err := Copy(appsecRuleSource, hubDirAppsecRulePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %s", appsecRuleSource, hubDirAppsecRulePath, err)
hubDirAppsecRulePath := filepath.Join(itemTypeDirDest, sourceFilename)
if err := Copy(sourcePath, hubDirAppsecRulePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %w", sourcePath, hubDirAppsecRulePath, err)
}

// runtime/appsec-rules/rule.yaml
appsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename)
appsecRulePath := filepath.Join(itemTypeDirDest, sourceFilename)
if err := os.Symlink(hubDirAppsecRulePath, appsecRulePath); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("unable to symlink appsec-rule '%s' to '%s': %s", hubDirAppsecRulePath, appsecRulePath, err)
return fmt.Errorf("unable to symlink appsec-rule '%s' to '%s': %w", hubDirAppsecRulePath, appsecRulePath, err)
}
}

return nil
}

func (t *HubTestItem) installAppsecRuleCustomFrom(appsecrule string, customPath string) (bool, error) {
// we check if its a custom appsec-rule
customAppsecRulePath := filepath.Join(customPath, appsecrule)
if _, err := os.Stat(customAppsecRulePath); os.IsNotExist(err) {
return false, nil
}

customAppsecRulePathSplit := strings.Split(customAppsecRulePath, "/")
customAppsecRuleName := customAppsecRulePathSplit[len(customAppsecRulePathSplit)-1]

itemTypeDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)
if err := os.MkdirAll(itemTypeDirDest, os.ModePerm); err != nil {
return false, fmt.Errorf("unable to create folder '%s': %w", itemTypeDirDest, err)
}

customAppsecRuleDest := fmt.Sprintf("%s/appsec-rules/%s", t.RuntimePath, customAppsecRuleName)
if err := Copy(customAppsecRulePath, customAppsecRuleDest); err != nil {
return false, fmt.Errorf("unable to copy appsec-rule from '%s' to '%s': %w", customAppsecRulePath, customAppsecRuleDest, err)
}

return true, nil
}

func (t *HubTestItem) installAppsecRuleCustom(appsecrule string) error {
customAppsecRuleExist := false
for _, customPath := range t.CustomItemsLocation {
// we check if its a custom appsec-rule
customAppsecRulePath := filepath.Join(customPath, appsecrule)
if _, err := os.Stat(customAppsecRulePath); os.IsNotExist(err) {
continue
}
customAppsecRulePathSplit := strings.Split(customAppsecRulePath, "/")
customAppsecRuleName := customAppsecRulePathSplit[len(customAppsecRulePathSplit)-1]

appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)
if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err)
found, err := t.installAppsecRuleCustomFrom(appsecrule, customPath)
if err != nil {
return err
}

// runtime/appsec-rules/
customAppsecRuleDest := fmt.Sprintf("%s/appsec-rules/%s", t.RuntimePath, customAppsecRuleName)
// if path to postoverflow exist, copy it
if err := Copy(customAppsecRulePath, customAppsecRuleDest); err != nil {
continue
if found {
return nil
}
customAppsecRuleExist = true
break
}
if !customAppsecRuleExist {
return fmt.Errorf("couldn't find custom appsec-rule '%s' in the following location: %+v", appsecrule, t.CustomItemsLocation)
}

return nil
return fmt.Errorf("couldn't find custom appsec-rule '%s' in the following location: %+v", appsecrule, t.CustomItemsLocation)
}

func (t *HubTestItem) installAppsecRule(name string) error {
Expand Down
99 changes: 50 additions & 49 deletions pkg/hubtest/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,89 +9,90 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)

func (t *HubTestItem) installParserItem(hubParser *cwhub.Item) error {
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
func (t *HubTestItem) installParserItem(item *cwhub.Item) error {
sourcePath, err := filepath.Abs(filepath.Join(t.HubPath, item.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
return fmt.Errorf("can't get absolute path of '%s': %w", sourcePath, err)
}

parserFileName := filepath.Base(parserSource)
sourceFilename := filepath.Base(sourcePath)

// runtime/hub/parsers/s00-raw/crowdsecurity/
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(item.RemotePath))

// runtime/parsers/s00-raw/
parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
itemTypeDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, item.Stage)

if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
return fmt.Errorf("unable to create folder '%s': %w", hubDirParserDest, err)
}

if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
if err := os.MkdirAll(itemTypeDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %w", itemTypeDirDest, err)
}

// runtime/hub/parsers/s00-raw/crowdsecurity/syslog-logs.yaml
hubDirParserPath := filepath.Join(hubDirParserDest, parserFileName)
if err := Copy(parserSource, hubDirParserPath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %s", parserSource, hubDirParserPath, err)
hubDirParserPath := filepath.Join(hubDirParserDest, sourceFilename)
if err := Copy(sourcePath, hubDirParserPath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %w", sourcePath, hubDirParserPath, err)
}

// runtime/parsers/s00-raw/syslog-logs.yaml
parserDirParserPath := filepath.Join(parserDirDest, parserFileName)
parserDirParserPath := filepath.Join(itemTypeDirDest, sourceFilename)
if err := os.Symlink(hubDirParserPath, parserDirParserPath); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("unable to symlink parser '%s' to '%s': %s", hubDirParserPath, parserDirParserPath, err)
return fmt.Errorf("unable to symlink parser '%s' to '%s': %w", hubDirParserPath, parserDirParserPath, err)
}
}

return nil
}

func (t *HubTestItem) installParserCustom(parser string) error {
customParserExist := false
for _, customPath := range t.CustomItemsLocation {
// we check if its a custom parser
customParserPath := filepath.Join(customPath, parser)
if _, err := os.Stat(customParserPath); os.IsNotExist(err) {
continue
//return fmt.Errorf("parser '%s' doesn't exist in the hub and doesn't appear to be a custom one.", parser)
}
func (t *HubTestItem) installParserCustomFrom(parser string, customPath string) (bool, error) {
// we check if its a custom parser
customParserPath := filepath.Join(customPath, parser)
if _, err := os.Stat(customParserPath); os.IsNotExist(err) {
return false, nil
}

customParserPathSplit, customParserName := filepath.Split(customParserPath)
// because path is parsers/<stage>/<author>/parser.yaml and we wan't the stage
splittedPath := strings.Split(customParserPathSplit, string(os.PathSeparator))
customParserStage := splittedPath[len(splittedPath)-3]
customParserPathSplit, customParserName := filepath.Split(customParserPath)
// because path is parsers/<stage>/<author>/parser.yaml and we wan't the stage
splitPath := strings.Split(customParserPathSplit, string(os.PathSeparator))
customParserStage := splitPath[len(splitPath)-3]

// check if stage exist
hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("parsers/%s", customParserStage))
// check if stage exist
hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("parsers/%s", customParserStage))
if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
return false, fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
}

if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
continue
//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
}
stageDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
if err := os.MkdirAll(stageDirDest, os.ModePerm); err != nil {
return false, fmt.Errorf("unable to create folder '%s': %w", stageDirDest, err)
}

parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
continue
//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
}
customParserDest := filepath.Join(stageDirDest, customParserName)
// if path to parser exist, copy it
if err := Copy(customParserPath, customParserDest); err != nil {
return false, fmt.Errorf("unable to copy custom parser '%s' to '%s': %w", customParserPath, customParserDest, err)
}

return true, nil
}

customParserDest := filepath.Join(parserDirDest, customParserName)
// if path to parser exist, copy it
if err := Copy(customParserPath, customParserDest); err != nil {
continue
//return fmt.Errorf("unable to copy custom parser '%s' to '%s': %s", customParserPath, customParserDest, err)
func (t *HubTestItem) installParserCustom(parser string) error {
for _, customPath := range t.CustomItemsLocation {
found, err := t.installParserCustomFrom(parser, customPath)
if err != nil {
return err
}

customParserExist = true
break
}
if !customParserExist {
return fmt.Errorf("couldn't find custom parser '%s' in the following location: %+v", parser, t.CustomItemsLocation)
if found {
return nil
}
}

return nil
return fmt.Errorf("couldn't find custom parser '%s' in the following locations: %+v", parser, t.CustomItemsLocation)
}

func (t *HubTestItem) installParser(name string) error {
Expand Down
98 changes: 50 additions & 48 deletions pkg/hubtest/postoverflow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,88 +9,90 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)

func (t *HubTestItem) installPostoverflowItem(hubPostOverflow *cwhub.Item) error {
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
func (t *HubTestItem) installPostoverflowItem(item *cwhub.Item) error {
sourcePath, err := filepath.Abs(filepath.Join(t.HubPath, item.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
return fmt.Errorf("can't get absolute path of '%s': %w", sourcePath, err)
}

postoverflowFileName := filepath.Base(postoverflowSource)
sourceFilename := filepath.Base(sourcePath)

// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(item.RemotePath))

// runtime/postoverflows/s00-enrich
postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
itemTypeDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, item.Stage)

if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
return fmt.Errorf("unable to create folder '%s': %w", hubDirPostoverflowDest, err)
}

if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
if err := os.MkdirAll(itemTypeDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %w", itemTypeDirDest, err)
}

// runtime/hub/postoverflows/s00-enrich/crowdsecurity/rdns.yaml
hubDirPostoverflowPath := filepath.Join(hubDirPostoverflowDest, postoverflowFileName)
if err := Copy(postoverflowSource, hubDirPostoverflowPath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %s", postoverflowSource, hubDirPostoverflowPath, err)
hubDirPostoverflowPath := filepath.Join(hubDirPostoverflowDest, sourceFilename)
if err := Copy(sourcePath, hubDirPostoverflowPath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %w", sourcePath, hubDirPostoverflowPath, err)
}

// runtime/postoverflows/s00-enrich/rdns.yaml
postoverflowDirParserPath := filepath.Join(postoverflowDirDest, postoverflowFileName)
postoverflowDirParserPath := filepath.Join(itemTypeDirDest, sourceFilename)
if err := os.Symlink(hubDirPostoverflowPath, postoverflowDirParserPath); err != nil {
if !os.IsExist(err) {
return fmt.Errorf("unable to symlink postoverflow '%s' to '%s': %s", hubDirPostoverflowPath, postoverflowDirParserPath, err)
return fmt.Errorf("unable to symlink postoverflow '%s' to '%s': %w", hubDirPostoverflowPath, postoverflowDirParserPath, err)
}
}

return nil
}

func (t *HubTestItem) installPostoverflowCustom(postoverflow string) error {
customPostoverflowExist := false
for _, customPath := range t.CustomItemsLocation {
// we check if its a custom postoverflow
customPostOverflowPath := filepath.Join(customPath, postoverflow)
if _, err := os.Stat(customPostOverflowPath); os.IsNotExist(err) {
continue
//return fmt.Errorf("postoverflow '%s' doesn't exist in the hub and doesn't appear to be a custom one.", postoverflow)
}
func (t *HubTestItem) installPostoverflowCustomFrom(postoverflow string, customPath string) (bool, error) {
// we check if its a custom postoverflow
customPostOverflowPath := filepath.Join(customPath, postoverflow)
if _, err := os.Stat(customPostOverflowPath); os.IsNotExist(err) {
return false, nil
}

customPostOverflowPathSplit := strings.Split(customPostOverflowPath, "/")
customPostoverflowName := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-1]
// because path is postoverflows/<stage>/<author>/parser.yaml and we wan't the stage
customPostoverflowStage := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-3]
customPostOverflowPathSplit := strings.Split(customPostOverflowPath, "/")
customPostoverflowName := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-1]
// because path is postoverflows/<stage>/<author>/parser.yaml and we wan't the stage
customPostoverflowStage := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-3]

// check if stage exist
hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("postoverflows/%s", customPostoverflowStage))
// check if stage exist
hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("postoverflows/%s", customPostoverflowStage))
if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
return false, fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
}

if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
continue
//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
}
stageDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
if err := os.MkdirAll(stageDirDest, os.ModePerm); err != nil {
return false, fmt.Errorf("unable to create folder '%s': %w", stageDirDest, err)
}

postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
continue
//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
customPostoverflowDest := filepath.Join(stageDirDest, customPostoverflowName)
// if path to postoverflow exist, copy it
if err := Copy(customPostOverflowPath, customPostoverflowDest); err != nil {
return false, fmt.Errorf("unable to copy custom parser '%s' to '%s': %w", customPostOverflowPath, customPostoverflowDest, err)
}

return true, nil
}

func (t *HubTestItem) installPostoverflowCustom(postoverflow string) error {
for _, customPath := range t.CustomItemsLocation {
found, err := t.installPostoverflowCustomFrom(postoverflow, customPath)
if err != nil {
return err
}

customPostoverflowDest := filepath.Join(postoverflowDirDest, customPostoverflowName)
// if path to postoverflow exist, copy it
if err := Copy(customPostOverflowPath, customPostoverflowDest); err != nil {
continue
//return fmt.Errorf("unable to copy custom parser '%s' to '%s': %s", customPostOverflowPath, customPostoverflowDest, err)
if found {
return nil
}
customPostoverflowExist = true
break
}
if !customPostoverflowExist {
return fmt.Errorf("couldn't find custom postoverflow '%s' in the following location: %+v", postoverflow, t.CustomItemsLocation)
}

return nil
return fmt.Errorf("couldn't find custom postoverflow '%s' in the following location: %+v", postoverflow, t.CustomItemsLocation)
}

func (t *HubTestItem) installPostoverflow(name string) error {
Expand Down
Loading

0 comments on commit 8de8bf0

Please sign in to comment.