From 265aac9d5f4a16775a223e87909b7676de3bc6c9 Mon Sep 17 00:00:00 2001 From: Maik Date: Wed, 6 Apr 2022 10:30:23 +0200 Subject: [PATCH] AWS-API Throttling feature implemented. --- deploy/helm/Chart.yaml | 2 +- deploy/helm/templates/config.yaml | 3 +++ deploy/helm/values.yaml | 6 +++++ go.mod | 1 + go.sum | 4 ++++ pkg/plugin/trivy/plugin.go | 38 +++++++++++++++++++++++++------ pkg/utils/durationutil.go | 22 ++++++++++++++++++ 7 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 pkg/utils/durationutil.go diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index da67215d3..07b5d9015 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -6,7 +6,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.10.1 +version: 0.10.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/helm/templates/config.yaml b/deploy/helm/templates/config.yaml index ae412da76..61b2daf4c 100644 --- a/deploy/helm/templates/config.yaml +++ b/deploy/helm/templates/config.yaml @@ -66,6 +66,9 @@ data: {{- if .useEcrRoleCreds }} trivy.useEcrRoleCreds: {{ .useEcrRoleCreds | quote }} {{- end }} + {{- if .useEcrRoleCreds }} + trivy.ecrTokenRefreshTTL: {{ .ecrTokenRefreshTTL | quote }} + {{- end }} {{- if .timeout }} trivy.timeout: {{ .timeout | quote }} {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index cf9748f77..edbafa4af 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -124,6 +124,12 @@ trivy: # # useEcrRoleCreds: false + # Set the TokenExpiry for the next AWS-Tokengeneration + # AWS authorization token is valid for 12 hours by default + # Unit is hours(h) + # + # ecrTokenRefreshTTL = 11h + # Registries without SSL. There can be multiple registries with different keys. nonSslRegistries: {} # pocRegistry: poc.myregistry.harbor.com.pl diff --git a/go.mod b/go.mod index fa9cce01f..9b0a7a0cd 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xhit/go-str2duration/v2 v2.0.0 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect diff --git a/go.sum b/go.sum index 0fecd8b46..0a57c9d2f 100644 --- a/go.sum +++ b/go.sum @@ -615,6 +615,7 @@ github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -963,6 +964,8 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xhit/go-str2duration/v2 v2.0.0 h1:uFtk6FWB375bP7ewQl+/1wBcn840GPhnySOdcz/okPE= +github.com/xhit/go-str2duration/v2 v2.0.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= @@ -1167,6 +1170,7 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/pkg/plugin/trivy/plugin.go b/pkg/plugin/trivy/plugin.go index 36cc13ae8..e55a38ab3 100644 --- a/pkg/plugin/trivy/plugin.go +++ b/pkg/plugin/trivy/plugin.go @@ -9,12 +9,14 @@ import ( "io" "regexp" "strings" + "time" "github.com/aquasecurity/starboard/pkg/apis/aquasecurity/v1alpha1" "github.com/aquasecurity/starboard/pkg/docker" "github.com/aquasecurity/starboard/pkg/ext" "github.com/aquasecurity/starboard/pkg/kube" "github.com/aquasecurity/starboard/pkg/starboard" + "github.com/aquasecurity/starboard/pkg/utils" "github.com/aquasecurity/starboard/pkg/vulnerabilityreport" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -55,6 +57,7 @@ const ( keyTrivySkipFiles = "trivy.skipFiles" keyTrivySkipDirs = "trivy.skipDirs" keyTrivyUseECRRoleCreds = "trivy.useEcrRoleCreds" + keyTrivyECRTokenRefreshTTL = "trivy.ecrTokenRefreshTTL" keyTrivyServerURL = "trivy.serverURL" keyTrivyServerTokenHeader = "trivy.serverTokenHeader" @@ -71,6 +74,10 @@ const ( // Mode in which Trivy client operates. type Mode string +var aws_creds [][]string +var EcrTokenTTL time.Duration = 0 +var EcrTokenGen time.Time + const ( Standalone Mode = "Standalone" ClientServer Mode = "ClientServer" @@ -146,6 +153,10 @@ func (c Config) UseECRCredentials() bool { } } +func (c Config) GetECRRefreshTTL() (string, error) { + return c.GetRequiredData(keyTrivyECRTokenRefreshTTL) +} + func (c Config) GetServerInsecure() bool { _, ok := c.Data[keyTrivyServerInsecure] return ok @@ -599,10 +610,16 @@ func (p *plugin) getPodSpecForStandaloneMode(ctx starboard.PluginContext, config }) } + TTLresult, err := config.GetECRRefreshTTL() + if config.UseECRCredentials() && CheckAwsEcrPrivateRegistry(c.Image) != "" { - var aws_creds, err = GetAuthorizationToken(CheckAwsEcrPrivateRegistry(c.Image)) - if err != nil { - return corev1.PodSpec{}, nil, err + + if utils.TokenTTLValidation(EcrTokenGen, TTLresult) { + EcrTokenGen = time.Now() + aws_creds, err = GetAuthorizationToken(CheckAwsEcrPrivateRegistry(c.Image)) + if err != nil { + return corev1.PodSpec{}, nil, err + } } var creds (ecr_credentials) = ecr_credentials{aws_creds[0][1], aws_creds[0][2]} @@ -882,11 +899,18 @@ func (p *plugin) getPodSpecForClientServerMode(ctx starboard.PluginContext, conf }, } + TTLresult, err := config.GetECRRefreshTTL() + if config.UseECRCredentials() && CheckAwsEcrPrivateRegistry(container.Image) != "" { - var aws_creds, err = GetAuthorizationToken(CheckAwsEcrPrivateRegistry(container.Image)) - if err != nil { - return corev1.PodSpec{}, nil, err + + if utils.TokenTTLValidation(EcrTokenGen, TTLresult) { + EcrTokenGen = time.Now() + aws_creds, err = GetAuthorizationToken(CheckAwsEcrPrivateRegistry(container.Image)) + if err != nil { + return corev1.PodSpec{}, nil, err + } } + var creds (ecr_credentials) = ecr_credentials{aws_creds[0][1], aws_creds[0][2]} env = append(env, corev1.EnvVar{ @@ -1450,7 +1474,7 @@ func GetAuthorizationToken(AwsEcrRegion string) ([][]string, error) { errormsg = "GetAuthorizationToken (AWS-API): " + aerr.Error() } } else { - fmt.Println(err.Error()) + errormsg = err.Error() } return nil, errors.New((errormsg)) } diff --git a/pkg/utils/durationutil.go b/pkg/utils/durationutil.go new file mode 100644 index 000000000..d666e9944 --- /dev/null +++ b/pkg/utils/durationutil.go @@ -0,0 +1,22 @@ +package utils + +import ( + "time" + + "github.com/xhit/go-str2duration/v2" +) + +func TokenTTLValidation(TokenGen time.Time, TokenTTL string) bool { + duration, err := str2duration.ParseDuration(TokenTTL) + + if err == nil { + if time.Now().Sub(TokenGen) >= duration { + return true + } else { + return false + } + } else { + return false + } + +}