From 7f3188d2e32b352ae9fcfdf6cb56af31288551fb Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Sun, 15 Sep 2024 14:38:34 +0300 Subject: [PATCH] fix: folder check without glob --- checks/folder.go | 51 +++++++++++---------------- checks/folder_gcs.go | 2 +- checks/folder_s3.go | 2 +- checks/folder_sftp.go | 2 +- checks/folder_smb.go | 2 +- fixtures/datasources/folder_fail.yaml | 11 +++++- fixtures/datasources/folder_pass.yaml | 6 ++++ go.mod | 11 ++---- go.sum | 18 ++-------- 9 files changed, 44 insertions(+), 61 deletions(-) diff --git a/checks/folder.go b/checks/folder.go index 324c95e05..08d71d936 100644 --- a/checks/folder.go +++ b/checks/folder.go @@ -94,12 +94,14 @@ func checkLocalFolder(ctx *context.Context, check v1.FolderCheck) pkg.Results { results = append(results, result) // Form a dummy connection to get a local filesystem - localFS, err := artifacts.GetFSForConnection(ctx.Context, models.Connection{Type: models.ConnectionTypeFolder}) + localFS, err := artifacts.GetFSForConnection(ctx.Context, models.Connection{ + Type: models.ConnectionTypeFolder, + }) if err != nil { return results.ErrorMessage(err) } - folders, err := genericFolderCheck(localFS, check.Path, check.Recursive, check.Filter) + folders, err := genericFolderCheck(ctx, localFS, check.Path, check.Recursive, check.Filter) result.AddDetails(folders) if err != nil { @@ -112,18 +114,18 @@ func checkLocalFolder(ctx *context.Context, check v1.FolderCheck) pkg.Results { return results } -func genericFolderCheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { - return _genericFolderCheck(true, dirFS, path, recursive, filter) +func genericFolderCheck(ctx *context.Context, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { + return _genericFolderCheck(ctx, true, dirFS, path, recursive, filter) } // genericFolderCheckWithoutPrecheck is used for those filesystems that do not support fetching the stat of a directory. // Eg: s3, gcs. // It will not pre check whether the given path is a directory. -func genericFolderCheckWithoutPrecheck(dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { - return _genericFolderCheck(false, dirFS, path, recursive, filter) +func genericFolderCheckWithoutPrecheck(ctx *context.Context, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { + return _genericFolderCheck(ctx, false, dirFS, path, recursive, filter) } -func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { +func _genericFolderCheck(ctx *context.Context, supportsDirStat bool, dirFS artifactFS.Filesystem, path string, recursive bool, filter v1.FolderFilter) (FolderCheck, error) { result := FolderCheck{} _filter, err := filter.New() if err != nil { @@ -131,30 +133,24 @@ func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path } _filter.AllowDir = recursive - var fileInfo os.FileInfo - if supportsDirStat { + files, err := getFolderContents(ctx, dirFS, path, _filter) + if os.IsNotExist(err) { + return result, nil + } else if err != nil { + return result, err + } + + if len(files) == 0 { fileInfo, err := dirFS.Stat(path) if err != nil { - if os.IsNotExist(err) { - return result, nil - } return result, err - } else if !fileInfo.IsDir() { - return result, fmt.Errorf("%s is not a directory", path) } - } - - files, err := getFolderContents(dirFS, path, _filter) - if err != nil { - return result, err - } - if len(files) == 0 { if fileInfo == nil { return FolderCheck{}, nil } - // directory is empty. returning duration of directory + // listing is empty, returning duration of directory return FolderCheck{ Oldest: newFile(fileInfo), Newest: newFile(fileInfo), @@ -171,7 +167,7 @@ func _genericFolderCheck(supportsDirStat bool, dirFS artifactFS.Filesystem, path // getFolderContents walks the folder and returns all files. // Also supports recursively fetching contents -func getFolderContents(dirFs artifactFS.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) { +func getFolderContents(ctx *context.Context, dirFs artifactFS.Filesystem, path string, filter *v1.FolderFilterContext) ([]fs.FileInfo, error) { files, err := dirFs.ReadDir(path) if err != nil { return nil, err @@ -184,18 +180,11 @@ func getFolderContents(dirFs artifactFS.Filesystem, path string, filter *v1.Fold var result []fs.FileInfo for _, info := range files { if !filter.Filter(info) { + ctx.Logger.V(3).Infof("skipping %s, does not match filter", info.Name()) continue } result = append(result, info) - if info.IsDir() { // This excludes even directory symlinks - subFiles, err := getFolderContents(dirFs, path+"/"+info.Name(), filter) - if err != nil { - return nil, err - } - - result = append(result, subFiles...) - } } return result, err diff --git a/checks/folder_gcs.go b/checks/folder_gcs.go index 7ec668885..5e5e32a18 100644 --- a/checks/folder_gcs.go +++ b/checks/folder_gcs.go @@ -49,7 +49,7 @@ func CheckGCSBucket(ctx *context.Context, check v1.FolderCheck) pkg.Results { return results.ErrorMessage(err) } - folders, err := genericFolderCheckWithoutPrecheck(fs, check.Path, check.Recursive, check.Filter) + folders, err := genericFolderCheckWithoutPrecheck(ctx, fs, check.Path, check.Recursive, check.Filter) if err != nil { return results.ErrorMessage(err) } diff --git a/checks/folder_s3.go b/checks/folder_s3.go index 9cc540bc4..c15f60de6 100644 --- a/checks/folder_s3.go +++ b/checks/folder_s3.go @@ -47,7 +47,7 @@ func CheckS3Bucket(ctx *context.Context, check v1.FolderCheck) pkg.Results { limitFS.SetMaxListItems(ctx.Properties().Int("s3.list.max-objects", 50_000)) } - folders, err := genericFolderCheckWithoutPrecheck(fs, check.Path, check.Recursive, check.Filter) + folders, err := genericFolderCheckWithoutPrecheck(ctx, fs, check.Path, check.Recursive, check.Filter) if err != nil { return results.ErrorMessage(err) } diff --git a/checks/folder_sftp.go b/checks/folder_sftp.go index ffb8f05e7..4104ba565 100644 --- a/checks/folder_sftp.go +++ b/checks/folder_sftp.go @@ -21,7 +21,7 @@ func CheckSFTP(ctx *context.Context, check v1.FolderCheck) pkg.Results { return results.ErrorMessage(err) } - folders, err := genericFolderCheck(fs, check.Path, check.Recursive, check.Filter) + folders, err := genericFolderCheck(ctx, fs, check.Path, check.Recursive, check.Filter) if err != nil { return results.ErrorMessage(err) } diff --git a/checks/folder_smb.go b/checks/folder_smb.go index 5941b3d50..8c94a32f4 100644 --- a/checks/folder_smb.go +++ b/checks/folder_smb.go @@ -38,7 +38,7 @@ func CheckSmb(ctx *context.Context, check v1.FolderCheck) pkg.Results { return results.ErrorMessage(err) } - folders, err := genericFolderCheck(fs, path, check.Recursive, check.Filter) + folders, err := genericFolderCheck(ctx, fs, path, check.Recursive, check.Filter) if err != nil { return results.ErrorMessage(err) } diff --git a/fixtures/datasources/folder_fail.yaml b/fixtures/datasources/folder_fail.yaml index c9106d13f..359e2bdfa 100644 --- a/fixtures/datasources/folder_fail.yaml +++ b/fixtures/datasources/folder_fail.yaml @@ -8,4 +8,13 @@ spec: - path: /etc/ name: min count fail minCount: 100000 - maxAge: 4m \ No newline at end of file + maxAge: 4m + - path: /etc/ + recursive: true + name: min count recursive + minCount: 100000 + maxAge: 4m + - path: /etc/**/* + name: min count glob + minCount: 100000 + maxAge: 4m diff --git a/fixtures/datasources/folder_pass.yaml b/fixtures/datasources/folder_pass.yaml index ef824518d..5ca12ffa0 100644 --- a/fixtures/datasources/folder_pass.yaml +++ b/fixtures/datasources/folder_pass.yaml @@ -5,6 +5,12 @@ metadata: spec: interval: 300 folder: + - path: /etc/* + minCount: 1 + name: min count glob + - path: /etc/**/* + minCount: 1 + name: min count doublestar - path: /etc/ name: Check for updated /etc files filter: diff --git a/go.mod b/go.mod index 2f2523071..f56edb109 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/eko/gocache/lib/v4 v4.1.6 github.com/eko/gocache/store/bigcache/v4 v4.2.1 github.com/elastic/go-elasticsearch/v8 v8.13.1 - github.com/flanksource/artifacts v1.0.14 + github.com/flanksource/artifacts v1.0.15 github.com/flanksource/commons v1.29.10 github.com/flanksource/duty v1.0.652 github.com/flanksource/gomplate/v3 v3.24.30 @@ -119,11 +119,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect github.com/aws/smithy-go v1.20.4 // indirect - github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect - github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -198,7 +196,6 @@ require ( github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/jsonschema v0.12.0 // indirect github.com/itchyny/gojq v0.12.16 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect @@ -273,14 +270,10 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xlab/treeprint v1.2.0 // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect @@ -332,7 +325,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) -// replace github.com/flanksource/duty => ../duty +// replace github.com/flanksource/duty => /Users/moshe/go/src/github.com/flanksource/duty // replace github.com/flanksource/artifacts => ../artifacts diff --git a/go.sum b/go.sum index 9c3744d30..b7aa648e4 100644 --- a/go.sum +++ b/go.sum @@ -748,8 +748,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3Co github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= -github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -761,8 +759,6 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= @@ -859,8 +855,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fergusstrange/embedded-postgres v1.25.0 h1:sa+k2Ycrtz40eCRPOzI7Ry7TtkWXXJ+YRsxpKMDhxK0= github.com/fergusstrange/embedded-postgres v1.25.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw= -github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z5JYCTJU= -github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70= +github.com/flanksource/artifacts v1.0.15 h1:3ImJr2y0ZCXw/QrMhfJJktAT7pYD3sMZR5ixcvqDGbs= +github.com/flanksource/artifacts v1.0.15/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70= github.com/flanksource/commons v1.29.10 h1:T/S95Pl8kASEFvQjQ7fJjTUqeVdhxQXg1vfkULTYFJQ= github.com/flanksource/commons v1.29.10/go.mod h1:iTbrXOSp3Spv570Nly97D/U9cQjLZoVlmWCXqWzsvRU= github.com/flanksource/duty v1.0.652 h1:u5u2apQGJbF5qEpXZ75PMKuUnua0+WaH89OpO+RkQdY= @@ -1172,8 +1168,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= -github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4= github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g= github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM= @@ -1575,8 +1569,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= -github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -1585,12 +1577,6 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=