From 74e80885905ab503145831054b1496290305da3b Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Fri, 11 Oct 2024 14:13:58 +0200 Subject: [PATCH 01/25] First commit --- scanner/k8s-assets/client/generated.go | 99 +++++++++++++++---- .../client/query/component_create.graphql | 13 +++ .../query/componentversion_create.graphql | 5 +- scanner/k8s-assets/go.mod | 5 + scanner/k8s-assets/go.sum | 13 +++ scanner/k8s-assets/processor/processor.go | 27 +++++ 6 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 scanner/k8s-assets/client/query/component_create.graphql diff --git a/scanner/k8s-assets/client/generated.go b/scanner/k8s-assets/client/generated.go index 6e16a4d5..587c5698 100644 --- a/scanner/k8s-assets/client/generated.go +++ b/scanner/k8s-assets/client/generated.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors -// SPDX-License-Identifier: Apache-2.0 - // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client @@ -64,6 +61,17 @@ type ComponentFilter struct { // GetComponentName returns ComponentFilter.ComponentName, and is useful for accessing the field via an interface. func (v *ComponentFilter) GetComponentName() []string { return v.ComponentName } +type ComponentInput struct { + Name string `json:"name"` + Type ComponentTypeValues `json:"type"` +} + +// GetName returns ComponentInput.Name, and is useful for accessing the field via an interface. +func (v *ComponentInput) GetName() string { return v.Name } + +// GetType returns ComponentInput.Type, and is useful for accessing the field via an interface. +func (v *ComponentInput) GetType() ComponentTypeValues { return v.Type } + // ComponentInstance includes the requested fields of the GraphQL type ComponentInstance. type ComponentInstance struct { Id string `json:"id"` @@ -89,11 +97,23 @@ func (v *ComponentInstance) GetComponentVersionId() string { return v.ComponentV func (v *ComponentInstance) GetServiceId() string { return v.ServiceId } type ComponentInstanceFilter struct { - IssueMatchId []string `json:"issueMatchId"` + ServiceName []string `json:"serviceName"` + Ccrn []string `json:"ccrn"` + SupportGroup []string `json:"supportGroup"` + Search []string `json:"search"` } -// GetIssueMatchId returns ComponentInstanceFilter.IssueMatchId, and is useful for accessing the field via an interface. -func (v *ComponentInstanceFilter) GetIssueMatchId() []string { return v.IssueMatchId } +// GetServiceName returns ComponentInstanceFilter.ServiceName, and is useful for accessing the field via an interface. +func (v *ComponentInstanceFilter) GetServiceName() []string { return v.ServiceName } + +// GetCcrn returns ComponentInstanceFilter.Ccrn, and is useful for accessing the field via an interface. +func (v *ComponentInstanceFilter) GetCcrn() []string { return v.Ccrn } + +// GetSupportGroup returns ComponentInstanceFilter.SupportGroup, and is useful for accessing the field via an interface. +func (v *ComponentInstanceFilter) GetSupportGroup() []string { return v.SupportGroup } + +// GetSearch returns ComponentInstanceFilter.Search, and is useful for accessing the field via an interface. +func (v *ComponentInstanceFilter) GetSearch() []string { return v.Search } type ComponentInstanceInput struct { Ccrn string `json:"ccrn"` @@ -202,29 +222,21 @@ func (v *CreateComponentInstanceResponse) GetCreateComponentInstance() *Componen return v.CreateComponentInstance } -// CreateComponentVersionCreateComponentVersion includes the requested fields of the GraphQL type ComponentVersion. -type CreateComponentVersionCreateComponentVersion struct { - Id string `json:"id"` - Version string `json:"version"` - ComponentId string `json:"componentId"` +// CreateComponentResponse is returned by CreateComponent on success. +type CreateComponentResponse struct { + CreateComponent *Component `json:"createComponent"` } -// GetId returns CreateComponentVersionCreateComponentVersion.Id, and is useful for accessing the field via an interface. -func (v *CreateComponentVersionCreateComponentVersion) GetId() string { return v.Id } - -// GetVersion returns CreateComponentVersionCreateComponentVersion.Version, and is useful for accessing the field via an interface. -func (v *CreateComponentVersionCreateComponentVersion) GetVersion() string { return v.Version } - -// GetComponentId returns CreateComponentVersionCreateComponentVersion.ComponentId, and is useful for accessing the field via an interface. -func (v *CreateComponentVersionCreateComponentVersion) GetComponentId() string { return v.ComponentId } +// GetCreateComponent returns CreateComponentResponse.CreateComponent, and is useful for accessing the field via an interface. +func (v *CreateComponentResponse) GetCreateComponent() *Component { return v.CreateComponent } // CreateComponentVersionResponse is returned by CreateComponentVersion on success. type CreateComponentVersionResponse struct { - CreateComponentVersion *CreateComponentVersionCreateComponentVersion `json:"createComponentVersion"` + CreateComponentVersion *ComponentVersion `json:"createComponentVersion"` } // GetCreateComponentVersion returns CreateComponentVersionResponse.CreateComponentVersion, and is useful for accessing the field via an interface. -func (v *CreateComponentVersionResponse) GetCreateComponentVersion() *CreateComponentVersionCreateComponentVersion { +func (v *CreateComponentVersionResponse) GetCreateComponentVersion() *ComponentVersion { return v.CreateComponentVersion } @@ -488,6 +500,14 @@ func (v *__AddServiceToSupportGroupInput) GetSupportGroupId() string { return v. // GetServiceId returns __AddServiceToSupportGroupInput.ServiceId, and is useful for accessing the field via an interface. func (v *__AddServiceToSupportGroupInput) GetServiceId() string { return v.ServiceId } +// __CreateComponentInput is used internally by genqlient +type __CreateComponentInput struct { + Input *ComponentInput `json:"input,omitempty"` +} + +// GetInput returns __CreateComponentInput.Input, and is useful for accessing the field via an interface. +func (v *__CreateComponentInput) GetInput() *ComponentInput { return v.Input } + // __CreateComponentInstanceInput is used internally by genqlient type __CreateComponentInstanceInput struct { Input *ComponentInstanceInput `json:"input,omitempty"` @@ -598,6 +618,43 @@ func AddServiceToSupportGroup( return &data_, err_ } +// The query or mutation executed by CreateComponent. +const CreateComponent_Operation = ` +mutation CreateComponent ($input: ComponentInput!) { + createComponent(input: $input) { + id + name + type + } +} +` + +func CreateComponent( + ctx_ context.Context, + client_ graphql.Client, + input *ComponentInput, +) (*CreateComponentResponse, error) { + req_ := &graphql.Request{ + OpName: "CreateComponent", + Query: CreateComponent_Operation, + Variables: &__CreateComponentInput{ + Input: input, + }, + } + var err_ error + + var data_ CreateComponentResponse + resp_ := &graphql.Response{Data: &data_} + + err_ = client_.MakeRequest( + ctx_, + req_, + resp_, + ) + + return &data_, err_ +} + // The query or mutation executed by CreateComponentInstance. const CreateComponentInstance_Operation = ` mutation CreateComponentInstance ($input: ComponentInstanceInput!) { diff --git a/scanner/k8s-assets/client/query/component_create.graphql b/scanner/k8s-assets/client/query/component_create.graphql new file mode 100644 index 00000000..33849abf --- /dev/null +++ b/scanner/k8s-assets/client/query/component_create.graphql @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +# SPDX-License-Identifier: Apache-2.0 + +mutation CreateComponent($input: ComponentInput!) { + # @genqlient(typename: "Component") + createComponent ( + input: $input + ) { + id + name + type + } +} \ No newline at end of file diff --git a/scanner/k8s-assets/client/query/componentversion_create.graphql b/scanner/k8s-assets/client/query/componentversion_create.graphql index 3f5bb725..215cd0ae 100644 --- a/scanner/k8s-assets/client/query/componentversion_create.graphql +++ b/scanner/k8s-assets/client/query/componentversion_create.graphql @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors # SPDX-License-Identifier: Apache-2.0 -mutation CreateComponentVersion ($input: ComponentVersionInput!) { +mutation CreateComponentVersion($input: ComponentVersionInput!) { + # @genqlient(typename: "ComponentVersion") createComponentVersion ( input: $input ) { @@ -9,4 +10,4 @@ mutation CreateComponentVersion ($input: ComponentVersionInput!) { version componentId } -} +} \ No newline at end of file diff --git a/scanner/k8s-assets/go.mod b/scanner/k8s-assets/go.mod index e4088208..5b3dbe1d 100644 --- a/scanner/k8s-assets/go.mod +++ b/scanner/k8s-assets/go.mod @@ -10,9 +10,14 @@ require ( ) require ( + github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/alexflint/go-arg v1.4.2 // indirect + github.com/alexflint/go-scalar v1.0.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect github.com/vektah/gqlparser/v2 v2.5.15 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect ) diff --git a/scanner/k8s-assets/go.sum b/scanner/k8s-assets/go.sum index dc7df577..641a12d2 100644 --- a/scanner/k8s-assets/go.sum +++ b/scanner/k8s-assets/go.sum @@ -1,12 +1,20 @@ github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= +github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0= +github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= +github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70= +github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -82,6 +90,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -100,6 +109,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -111,6 +122,8 @@ golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index 94a7c78e..1ba98792 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -315,6 +315,33 @@ func (p *Processor) ProcessContainer( Container: containerInfo.Name, } + // Create new Component + componentInput := &client.ComponentInput{ + // TODO: Put name from container info + Name: fmt.Sprintf("%s/%s/%s", "registry", "account.Name", "repository.Name"), + Type: client.ComponentTypeValuesContainerimage, + } + createComponentResp, err := client.CreateComponent(ctx, *p.Client, componentInput) + if err != nil { + return fmt.Errorf("failed to create Component: %w", err) + } + log.WithFields(log.Fields{ + "componentId": createComponentResp.CreateComponent.Id, + }).Info("Component created") + + // Create new ComponentVersion + componentVersionInput := &client.ComponentVersionInput{ + Version: containerInfo.ImageHash, + ComponentId: createComponentResp.CreateComponent.Id, + } + createCompVersionResp, err := client.CreateComponentVersion(ctx, *p.Client, componentVersionInput) + if err != nil { + return fmt.Errorf("failed to create ComponentVersion: %w", err) + } + log.WithFields(log.Fields{ + "componentId": createCompVersionResp.CreateComponentVersion.Id, + }).Info("ComponentVersion created") + // Create new ComponentInstance componentInstanceInput := &client.ComponentInstanceInput{ Ccrn: ccrn.String(), From 29fbaabdc4f4292aebd1d9008fb2f1bdc0bdd0fa Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Fri, 11 Oct 2024 15:27:59 +0200 Subject: [PATCH 02/25] Wip --- scanner/k8s-assets/processor/processor.go | 1 + 1 file changed, 1 insertion(+) diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index 1ba98792..a029fd8f 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -300,6 +300,7 @@ func (p *Processor) ProcessContainer( // Find component version by container image hash componentVersionId, err := p.getComponentVersion(ctx, containerInfo.ImageHash) if err != nil { + // TODO: Create componentVersion if API call failed return fmt.Errorf("Couldn't find ComponentVersion (imageHash: %s): %w", containerInfo.ImageHash, err) } From 8e13dbd3751b6cf3b1d528b8b3c531e908e2ebe1 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 14 Oct 2024 09:02:56 +0200 Subject: [PATCH 03/25] Revert https://github.com/cloudoperators/heureka/commit/f04feabc6b81521f81efa368c0356b613e9fc763 --- scanner/k8s-assets/processor/processor.go | 33 ----------------------- 1 file changed, 33 deletions(-) diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index a029fd8f..57394be6 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -240,39 +240,6 @@ func (p *Processor) getComponentVersion(ctx context.Context, versionHash string) image := imageAndVersion[0] version := imageAndVersion[1] - //@todo Temporary Start - // AS we do not scan all the individual registries, we replace the registry string - // this is a temporary "hack" we need to move this to the heureka core with a registry configuration - //so that the respective versions are created correctly during version creation - - var myMap map[string]string = make(map[string]string) - myMap["keppel.global.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.qa-de-1.cloud.sap/ccloud-mirror"] = "keppel.eu-de-1.cloud.sap/ccloud" - myMap["keppel.eu-de-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.s-eu-de-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.na-us-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.na-us-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-jp-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-jp-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.na-us-3.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.na-ca-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.eu-nl-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-ae-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-sa-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-sa-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-cn-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - myMap["keppel.ap-au-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap" - var images []string = make([]string, 1) - - for replace, with := range myMap { - if strings.Contains(image, replace) { - image = strings.Replace(image, replace, with, 1) - } - } - //@todo Temporary End - - images[0] = image - listComponentVersionFilter := client.ComponentVersionFilter{ ComponentName: []string{image}, Version: []string{version}, From 871ef0fa5fd201090cedab14084d496e982d94dd Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 14 Oct 2024 13:47:26 +0200 Subject: [PATCH 04/25] Wip --- scanner/k8s-assets/processor/processor.go | 61 +++++++++++++++++------ 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index 57394be6..e961a2d7 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -264,13 +264,11 @@ func (p *Processor) ProcessContainer( podGroupName string, containerInfo UniqueContainerInfo, ) error { - // Find component version by container image hash - componentVersionId, err := p.getComponentVersion(ctx, containerInfo.ImageHash) - if err != nil { - // TODO: Create componentVersion if API call failed - return fmt.Errorf("Couldn't find ComponentVersion (imageHash: %s): %w", containerInfo.ImageHash, err) - } - + var ( + componentId string + componentVersionId string + componentInstanceId string + ) // Create new CCRN ccrn := CCRN{ Region: p.config.RegionName, @@ -283,7 +281,9 @@ func (p *Processor) ProcessContainer( Container: containerInfo.Name, } + // // Create new Component + // componentInput := &client.ComponentInput{ // TODO: Put name from container info Name: fmt.Sprintf("%s/%s/%s", "registry", "account.Name", "repository.Name"), @@ -297,20 +297,26 @@ func (p *Processor) ProcessContainer( "componentId": createComponentResp.CreateComponent.Id, }).Info("Component created") + componentId = createComponentResp.CreateComponent.Id + + // // Create new ComponentVersion - componentVersionInput := &client.ComponentVersionInput{ - Version: containerInfo.ImageHash, - ComponentId: createComponentResp.CreateComponent.Id, - } - createCompVersionResp, err := client.CreateComponentVersion(ctx, *p.Client, componentVersionInput) + // + componentVersionId, err = p.getComponentVersion(ctx, containerInfo.ImageHash) if err != nil { - return fmt.Errorf("failed to create ComponentVersion: %w", err) + log.WithFields(log.Fields{ + "id": containerInfo.ImageHash, + }).Info("ComponentVersion not found") + + componentVersionId, err := p.createComponentVersion(ctx, containerInfo.ImageHash, componentId) + if err != nil { + return fmt.Errorf("failed to create ComponentVersion: %w", err) + } } - log.WithFields(log.Fields{ - "componentId": createCompVersionResp.CreateComponentVersion.Id, - }).Info("ComponentVersion created") + // // Create new ComponentInstance + // componentInstanceInput := &client.ComponentInstanceInput{ Ccrn: ccrn.String(), Count: containerInfo.Count, @@ -334,3 +340,26 @@ func (p *Processor) ProcessContainer( return nil } + +// createComponentVersion create a new ComponentVersion based on a container image hash +func (p *Processor) createComponentVersion(ctx context.Context, imageHash string, componentId string) (string, error) { + componentVersionInput := &client.ComponentVersionInput{ + Version: imageHash, + ComponentId: componentId, + } + createCompVersionResp, err := client.CreateComponentVersion(ctx, *p.Client, componentVersionInput) + if err != nil { + return "", fmt.Errorf("failed to create ComponentVersion: %w", err) + } + log.WithFields(log.Fields{ + "componentId": createCompVersionResp.CreateComponentVersion.Id, + }).Info("ComponentVersion created") + + return createCompVersionResp.CreateComponentVersion.Id, nil +} + +// func (p *Processor) createComponentInstance(args) { + ... +func (p *Processor) createComponentInstance(args) { + +} From e2f50e6845f4d90f14a387f0284ba31b09759345 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 15 Oct 2024 09:24:47 +0200 Subject: [PATCH 05/25] Wip --- scanner/k8s-assets/processor/processor.go | 57 +++++++++++------- scanner/k8s-assets/scanner/scanner.go | 71 +++++++++++++++++++++-- 2 files changed, 101 insertions(+), 27 deletions(-) diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index e961a2d7..6e03e932 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -256,7 +256,8 @@ func (p *Processor) getComponentVersion(ctx context.Context, versionHash string) return "", fmt.Errorf("ListComponentVersion returned no ComponentVersion objects") } -// ProcessContainer creates a ComponentVersion and ComponentInstance for a container +// ProcessContainer is responsible for creating several entities based on the +// information at container level func (p *Processor) ProcessContainer( ctx context.Context, namespace string, @@ -284,20 +285,13 @@ func (p *Processor) ProcessContainer( // // Create new Component // - componentInput := &client.ComponentInput{ - // TODO: Put name from container info - Name: fmt.Sprintf("%s/%s/%s", "registry", "account.Name", "repository.Name"), + componentId, err := p.createComponent(ctx, &client.ComponentInput{ + Name: fmt.Sprintf("%s/%s/%s", containerInfo.ImageRegistry, containerInfo.ImageAccount, containerInfo.ImageRepository), Type: client.ComponentTypeValuesContainerimage, - } - createComponentResp, err := client.CreateComponent(ctx, *p.Client, componentInput) + }) if err != nil { - return fmt.Errorf("failed to create Component: %w", err) + return fmt.Errorf("failed to create Component. %w", err) } - log.WithFields(log.Fields{ - "componentId": createComponentResp.CreateComponent.Id, - }).Info("Component created") - - componentId = createComponentResp.CreateComponent.Id // // Create new ComponentVersion @@ -308,7 +302,7 @@ func (p *Processor) ProcessContainer( "id": containerInfo.ImageHash, }).Info("ComponentVersion not found") - componentVersionId, err := p.createComponentVersion(ctx, containerInfo.ImageHash, componentId) + componentVersionId, err = p.createComponentVersion(ctx, containerInfo.ImageHash, componentId) if err != nil { return fmt.Errorf("failed to create ComponentVersion: %w", err) } @@ -317,22 +311,20 @@ func (p *Processor) ProcessContainer( // // Create new ComponentInstance // - componentInstanceInput := &client.ComponentInstanceInput{ + componentInstanceId, err = p.createComponentInstance(ctx, &client.ComponentInstanceInput{ Ccrn: ccrn.String(), Count: containerInfo.Count, ComponentVersionId: componentVersionId, ServiceId: serviceID, - } - createCompInstResp, err := client.CreateComponentInstance(ctx, *p.Client, componentInstanceInput) + }) if err != nil { return fmt.Errorf("failed to create ComponentInstance: %w", err) } - componentInstanceID := createCompInstResp.CreateComponentInstance.Id // Do logging log.WithFields(log.Fields{ "componentVersionID": componentVersionId, - "componentInstanceID": componentInstanceID, + "componentInstanceID": componentInstanceId, "podGroup": podGroupName, "container": containerInfo.Name, "count": containerInfo.Count, @@ -341,6 +333,20 @@ func (p *Processor) ProcessContainer( return nil } +// createComponent creates a new Component +func (p *Processor) createComponent(ctx context.Context, input *client.ComponentInput) (string, error) { + createComponentResp, err := client.CreateComponent(ctx, *p.Client, input) + if err != nil { + return "", fmt.Errorf("failed to create Component: %w", err) + } + + log.WithFields(log.Fields{ + "componentId": createComponentResp.CreateComponent.Id, + }).Info("Component created") + + return createComponentResp.CreateComponent.Id, nil +} + // createComponentVersion create a new ComponentVersion based on a container image hash func (p *Processor) createComponentVersion(ctx context.Context, imageHash string, componentId string) (string, error) { componentVersionInput := &client.ComponentVersionInput{ @@ -351,6 +357,7 @@ func (p *Processor) createComponentVersion(ctx context.Context, imageHash string if err != nil { return "", fmt.Errorf("failed to create ComponentVersion: %w", err) } + log.WithFields(log.Fields{ "componentId": createCompVersionResp.CreateComponentVersion.Id, }).Info("ComponentVersion created") @@ -358,8 +365,16 @@ func (p *Processor) createComponentVersion(ctx context.Context, imageHash string return createCompVersionResp.CreateComponentVersion.Id, nil } -// func (p *Processor) createComponentInstance(args) { - ... -func (p *Processor) createComponentInstance(args) { +// createComponentInstance creates a new ComponentInstance +func (p *Processor) createComponentInstance(ctx context.Context, input *client.ComponentInstanceInput) (string, error) { + createCompInstResp, err := client.CreateComponentInstance(ctx, *p.Client, input) + if err != nil { + return "", fmt.Errorf("failed to create ComponentInstance: %w", err) + } + + log.WithFields(log.Fields{ + "componentInstanceId": createCompInstResp.CreateComponentInstance.Id, + }).Info("ComponentInstance created") + return createCompInstResp.CreateComponentInstance.Id, nil } diff --git a/scanner/k8s-assets/scanner/scanner.go b/scanner/k8s-assets/scanner/scanner.go index bac35138..3fd2772f 100644 --- a/scanner/k8s-assets/scanner/scanner.go +++ b/scanner/k8s-assets/scanner/scanner.go @@ -9,6 +9,7 @@ import ( "strings" + log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -42,10 +43,20 @@ type PodInfo struct { Containers []ContainerInfo } +type ImageInfo struct { + Registry string + Account string + Organization string + Repository string +} + type ContainerInfo struct { - Name string - Image string - ImageHash string + Name string + Image string + ImageHash string + ImageRegistry string + ImageRepository string + ImageAccount string } type PodLabels struct { @@ -112,6 +123,36 @@ func (s *Scanner) fetchImageId(pod v1.Pod, container v1.Container) string { return "" } +// extractImageInfo extracts image registry, image repository and the account name +// from a container image +func (s *Scanner) extractImageInfo(image string) (ImageInfo, error) { + // Split the string to remove the tag + parts := strings.Split(image, ":") + if len(parts) < 1 { + return ImageInfo{}, fmt.Errorf("invalid image") + } + + // Split the remaining string by '/' + components := strings.Split(parts[0], "/") + if len(components) < 3 || len(components) > 4 { + return ImageInfo{}, fmt.Errorf("invalid image string format: expected 3 or 4 components") + } + + info := ImageInfo{ + Registry: components[0], + Repository: components[len(components)-1], + } + + if len(components) == 3 { + info.Account = components[1] + } else { // len(components) == 4 + info.Account = components[1] + info.Organization = components[2] + } + + return info, nil +} + func (s *Scanner) GetPodInfo(pod v1.Pod) PodInfo { podInfo := PodInfo{ Name: pod.Name, @@ -123,10 +164,28 @@ func (s *Scanner) GetPodInfo(pod v1.Pod) PodInfo { } for _, containerStatus := range pod.Status.ContainerStatuses { + imageInfo, err := s.extractImageInfo(containerStatus.Image) + if err != nil { + log.WithFields(log.Fields{ + "image": containerStatus.Image, + }).Error("Couldn't extract information from image") + } + + // Also consider image repository with an organization + var imageRepository string + if len(imageInfo.Organization) > 0 { + imageRepository = fmt.Sprintf("%s/%s", imageInfo.Organization, imageInfo.Repository) + } else { + imageRepository = imageInfo.Repository + } + podInfo.Containers = append(podInfo.Containers, ContainerInfo{ - Name: containerStatus.Name, - Image: containerStatus.Image, - ImageHash: containerStatus.ImageID, + Name: containerStatus.Name, + Image: containerStatus.Image, + ImageHash: containerStatus.ImageID, + ImageRegistry: imageInfo.Registry, + ImageRepository: imageRepository, + ImageAccount: imageInfo.Account, }) } From 24a73a65756038de88415d5004c6128aa0b23aa9 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 15 Oct 2024 14:38:17 +0200 Subject: [PATCH 06/25] Wip --- scanner/k8s-assets/processor/processor.go | 49 +++++++++++++++++------ 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/scanner/k8s-assets/processor/processor.go b/scanner/k8s-assets/processor/processor.go index 6e03e932..dc21d691 100644 --- a/scanner/k8s-assets/processor/processor.go +++ b/scanner/k8s-assets/processor/processor.go @@ -34,6 +34,11 @@ type CCRN struct { Container string } +type ImageVersion struct { + Image string + Version string +} + // UniqueContainerInfo extends scanner.ContainerInfo to represent a unique container // configuration within a pod replica set, adding a count of occurrences. // It is used by the CollectUniqueContainers function to aggregate information about @@ -231,15 +236,7 @@ func (p *Processor) ProcessPodReplicaSet(ctx context.Context, namespace string, return nil } -func (p *Processor) getComponentVersion(ctx context.Context, versionHash string) (string, error) { - //separating image name and version hash - imageAndVersion := strings.SplitN(versionHash, "@", 2) - if len(imageAndVersion) < 2 { - return "", fmt.Errorf("Couldn't split image and version") - } - image := imageAndVersion[0] - version := imageAndVersion[1] - +func (p *Processor) getComponentVersion(ctx context.Context, image string, version string) (string, error) { listComponentVersionFilter := client.ComponentVersionFilter{ ComponentName: []string{image}, Version: []string{version}, @@ -256,6 +253,20 @@ func (p *Processor) getComponentVersion(ctx context.Context, versionHash string) return "", fmt.Errorf("ListComponentVersion returned no ComponentVersion objects") } +// extractVersion returns the hash part ia container image +func (p *Processor) extractImageVersion(versionHash string) (*ImageVersion, error) { + //separating image name and version hash + imageAndVersion := strings.SplitN(versionHash, "@", 2) + if len(imageAndVersion) < 2 { + return nil, fmt.Errorf("Couldn't split image and version") + } + imageVersion := &ImageVersion{ + Image: imageAndVersion[0], + Version: imageAndVersion[1], + } + return imageVersion, nil +} + // ProcessContainer is responsible for creating several entities based on the // information at container level func (p *Processor) ProcessContainer( @@ -285,6 +296,11 @@ func (p *Processor) ProcessContainer( // // Create new Component // + + // Check if we have everything we need + if len(containerInfo.ImageRegistry) == 0 || len(containerInfo.ImageAccount) == 0 || len(containerInfo.ImageRepository) == 0 { + return fmt.Errorf("cannot create Component (one or more containerInfo fields are empty)") + } componentId, err := p.createComponent(ctx, &client.ComponentInput{ Name: fmt.Sprintf("%s/%s/%s", containerInfo.ImageRegistry, containerInfo.ImageAccount, containerInfo.ImageRepository), Type: client.ComponentTypeValuesContainerimage, @@ -296,13 +312,20 @@ func (p *Processor) ProcessContainer( // // Create new ComponentVersion // - componentVersionId, err = p.getComponentVersion(ctx, containerInfo.ImageHash) + iv, err := p.extractImageVersion(containerInfo.ImageHash) + if err != nil { + log.WithFields(log.Fields{ + "imageHash": containerInfo.ImageHash, + }).Error("cannot extract image and version from imagehash") + return fmt.Errorf("cannot extract image version: %w", err) + } + componentVersionId, err = p.getComponentVersion(ctx, iv.Image, iv.Version) if err != nil { log.WithFields(log.Fields{ "id": containerInfo.ImageHash, }).Info("ComponentVersion not found") - componentVersionId, err = p.createComponentVersion(ctx, containerInfo.ImageHash, componentId) + componentVersionId, err = p.createComponentVersion(ctx, iv.Version, componentId) if err != nil { return fmt.Errorf("failed to create ComponentVersion: %w", err) } @@ -348,9 +371,9 @@ func (p *Processor) createComponent(ctx context.Context, input *client.Component } // createComponentVersion create a new ComponentVersion based on a container image hash -func (p *Processor) createComponentVersion(ctx context.Context, imageHash string, componentId string) (string, error) { +func (p *Processor) createComponentVersion(ctx context.Context, imageVersion string, componentId string) (string, error) { componentVersionInput := &client.ComponentVersionInput{ - Version: imageHash, + Version: imageVersion, ComponentId: componentId, } createCompVersionResp, err := client.CreateComponentVersion(ctx, *p.Client, componentVersionInput) From 950c8eeab43eb68f75b8d2cd39d8ccdb1b7c6642 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Wed, 16 Oct 2024 08:26:06 +0200 Subject: [PATCH 07/25] Wip --- scanner/keppel/client/generated.go | 25 ++-- .../query/listComponentVersions.graphql | 5 +- .../client/query/listComponents.graphql | 5 +- scanner/keppel/go.mod | 7 ++ scanner/keppel/go.sum | 16 +++ scanner/keppel/main.go | 115 ++++++++++-------- scanner/keppel/processor/processor.go | 24 ++-- 7 files changed, 114 insertions(+), 83 deletions(-) diff --git a/scanner/keppel/client/generated.go b/scanner/keppel/client/generated.go index 7af7b31d..b6c23023 100644 --- a/scanner/keppel/client/generated.go +++ b/scanner/keppel/client/generated.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors -// SPDX-License-Identifier: Apache-2.0 - // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client @@ -116,10 +113,18 @@ func (v *ComponentVersionConnectionEdgesComponentVersionEdge) GetNode() *Compone } type ComponentVersionFilter struct { - IssueId []string `json:"issueId"` - Version []string `json:"version"` + ComponentId []string `json:"componentId"` + ComponentName []string `json:"componentName"` + IssueId []string `json:"issueId"` + Version []string `json:"version"` } +// GetComponentId returns ComponentVersionFilter.ComponentId, and is useful for accessing the field via an interface. +func (v *ComponentVersionFilter) GetComponentId() []string { return v.ComponentId } + +// GetComponentName returns ComponentVersionFilter.ComponentName, and is useful for accessing the field via an interface. +func (v *ComponentVersionFilter) GetComponentName() []string { return v.ComponentName } + // GetIssueId returns ComponentVersionFilter.IssueId, and is useful for accessing the field via an interface. func (v *ComponentVersionFilter) GetIssueId() []string { return v.IssueId } @@ -337,15 +342,11 @@ func (v *__ListComponentVersionsInput) GetFirst() int { return v.First } // __ListComponentsInput is used internally by genqlient type __ListComponentsInput struct { Filter *ComponentFilter `json:"filter,omitempty"` - First int `json:"first"` } // GetFilter returns __ListComponentsInput.Filter, and is useful for accessing the field via an interface. func (v *__ListComponentsInput) GetFilter() *ComponentFilter { return v.Filter } -// GetFirst returns __ListComponentsInput.First, and is useful for accessing the field via an interface. -func (v *__ListComponentsInput) GetFirst() int { return v.First } - // __ListIssuesInput is used internally by genqlient type __ListIssuesInput struct { Filter *IssueFilter `json:"filter,omitempty"` @@ -555,8 +556,8 @@ func ListComponentVersions( // The query or mutation executed by ListComponents. const ListComponents_Operation = ` -query ListComponents ($filter: ComponentFilter, $first: Int) { - Components(filter: $filter, first: $first) { +query ListComponents ($filter: ComponentFilter) { + Components(filter: $filter) { edges { node { id @@ -572,14 +573,12 @@ func ListComponents( ctx_ context.Context, client_ graphql.Client, filter *ComponentFilter, - first int, ) (*ListComponentsResponse, error) { req_ := &graphql.Request{ OpName: "ListComponents", Query: ListComponents_Operation, Variables: &__ListComponentsInput{ Filter: filter, - First: first, }, } var err_ error diff --git a/scanner/keppel/client/query/listComponentVersions.graphql b/scanner/keppel/client/query/listComponentVersions.graphql index ac1fdb0d..4799a750 100644 --- a/scanner/keppel/client/query/listComponentVersions.graphql +++ b/scanner/keppel/client/query/listComponentVersions.graphql @@ -1,11 +1,10 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors # SPDX-License-Identifier: Apache-2.0 -query ListComponentVersions($filter: ComponentVersionFilter, $first: Int) { +query ListComponentVersions($filter: ComponentVersionFilter) { # @genqlient(typename: "ComponentVersionConnection") ComponentVersions ( filter: $filter, - first: $first, ) { edges { # @genqlient(typename: "ComponentVersion") @@ -16,4 +15,4 @@ query ListComponentVersions($filter: ComponentVersionFilter, $first: Int) { } } } -} \ No newline at end of file +} diff --git a/scanner/keppel/client/query/listComponents.graphql b/scanner/keppel/client/query/listComponents.graphql index 8e4063e9..a1ab56e1 100644 --- a/scanner/keppel/client/query/listComponents.graphql +++ b/scanner/keppel/client/query/listComponents.graphql @@ -1,11 +1,10 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors # SPDX-License-Identifier: Apache-2.0 -query ListComponents($filter: ComponentFilter, $first: Int) { +query ListComponents($filter: ComponentFilter) { # @genqlient(typename: "ComponentConnection") Components ( filter: $filter, - first: $first, ) { edges { # @genqlient(typename: "Component") @@ -16,4 +15,4 @@ query ListComponents($filter: ComponentFilter, $first: Int) { } } } -} \ No newline at end of file +} diff --git a/scanner/keppel/go.mod b/scanner/keppel/go.mod index 0329e11b..b641a39f 100644 --- a/scanner/keppel/go.mod +++ b/scanner/keppel/go.mod @@ -10,8 +10,15 @@ require ( ) require ( + github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/alexflint/go-arg v1.4.2 // indirect + github.com/alexflint/go-scalar v1.0.0 // indirect github.com/vektah/gqlparser/v2 v2.5.15 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect + golang.org/x/tools v0.23.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( diff --git a/scanner/keppel/go.sum b/scanner/keppel/go.sum index 9d8c52e9..b6d3c753 100644 --- a/scanner/keppel/go.sum +++ b/scanner/keppel/go.sum @@ -1,15 +1,23 @@ github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= +github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0= +github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= +github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70= +github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/aquasecurity/trivy v0.54.1 h1:/uNCF06PfdC69v5n3Zh4fXVf0xmXBml0c/ergf066SQ= github.com/aquasecurity/trivy v0.54.1/go.mod h1:i5S54WUtOEN9egFF0AHsxq6XT7QD11n9pSmIXhMJV0g= github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 h1:6/T8sFdNVG/AwOGoK6X55h7hF7LYqK8bsuPz8iEz8jM= github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04/go.mod h1:0T6oy2t1Iedt+yi3Ml5cpOYp5FZT4MI1/mx+3p+PIs8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0= @@ -30,13 +38,18 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vektah/gqlparser/v2 v2.5.15 h1:fYdnU8roQniJziV5TDiFPm/Ff7pE8xbVSOJqbsdl88A= github.com/vektah/gqlparser/v2 v2.5.15/go.mod h1:WQQjFc+I1YIzoPvZBhUQX7waZgg3pMLi0r8KymvAE2w= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -48,9 +61,12 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index c44d986c..8e4ee439 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -66,6 +66,16 @@ func main() { if err != nil { log.WithError(err).Fatal("Error during scanner setup") } + + // Get components + components, err := keppelProcessor.GetComponents() + fmt.Print(len(components)) + + // For each component get componentVersion + for _, comp := range components { + fmt.Print(comp) + } + accounts, err := keppelScanner.ListAccounts() if err != nil { log.WithError(err).Fatal("Error during ListAccounts") @@ -83,6 +93,7 @@ func main() { func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error { defer wg.Done() repositories, err := keppelScanner.ListRepositories(account.Name) + fmt.Print(len(repositories)) if err != nil { log.WithFields(log.Fields{ "account:": account.Name, @@ -90,62 +101,62 @@ func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.S return err } - for _, repository := range repositories { - HandleRepository(fqdn, account, repository, keppelScanner, keppelProcessor) - } + // for _, repository := range repositories { + // HandleRepository(fqdn, account, repository, keppelScanner, keppelProcessor) + // } return nil } -func HandleRepository(fqdn string, account models.Account, repository models.Repository, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { - component, err := keppelProcessor.ProcessRepository(fqdn, account, repository) - if err != nil { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during ProcessRepository") - component, err = keppelProcessor.GetComponent(fmt.Sprintf("%s/%s/%s", fqdn, account.Name, repository.Name)) - if err != nil { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during GetComponent") - } - } - - manifests, err := keppelScanner.ListManifests(account.Name, repository.Name) - if err != nil { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during ListManifests") - return - } - for _, manifest := range manifests { - if component == nil { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).Error("Component not found") - return - } - if manifest.VulnerabilityStatus == "Unsupported" { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType) - continue - } - if manifest.VulnerabilityStatus == "Clean" { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).Info("Manifest has no Vulnerabilities") - continue - } - HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor) - } -} +// func HandleRepository(fqdn string, account models.Account, repository models.Repository, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { +// component, err := keppelProcessor.ProcessRepository(fqdn, account, repository) +// if err != nil { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).WithError(err).Error("Error during ProcessRepository") +// component, err = keppelProcessor.GetComponent(fmt.Sprintf("%s/%s/%s", fqdn, account.Name, repository.Name)) +// if err != nil { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).WithError(err).Error("Error during GetComponent") +// } +// } + +// manifests, err := keppelScanner.ListManifests(account.Name, repository.Name) +// if err != nil { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).WithError(err).Error("Error during ListManifests") +// return +// } +// for _, manifest := range manifests { +// if component == nil { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).Error("Component not found") +// return +// } +// if manifest.VulnerabilityStatus == "Unsupported" { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType) +// continue +// } +// if manifest.VulnerabilityStatus == "Clean" { +// log.WithFields(log.Fields{ +// "account:": account.Name, +// "repository": repository.Name, +// }).Info("Manifest has no Vulnerabilities") +// continue +// } +// HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor) +// } +// } func HandleManifest(account models.Account, repository models.Repository, manifest models.Manifest, component *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { childManifests, err := keppelScanner.ListChildManifests(account.Name, repository.Name, manifest.Digest) diff --git a/scanner/keppel/processor/processor.go b/scanner/keppel/processor/processor.go index 86324651..79c05659 100644 --- a/scanner/keppel/processor/processor.go +++ b/scanner/keppel/processor/processor.go @@ -112,27 +112,27 @@ func (p *Processor) ProcessReport(report models.TrivyReport, componentVersionId } } -func (p *Processor) GetComponent(name string) (*client.Component, error) { - r, err := client.ListComponents(context.Background(), *p.Client, &client.ComponentFilter{ - ComponentName: []string{name}, - }, 1) - +// GetComponents returns a slice of all availble Components +func (p *Processor) GetComponents() ([]*client.Component, error) { + listComponentsResp, err := client.ListComponents(context.Background(), *p.Client, &client.ComponentFilter{}) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot list Components: %w", err) } - var component *client.Component - if len(r.Components.Edges) > 0 { - component = r.Components.Edges[0].GetNode() + var components []*client.Component + if len(listComponentsResp.Components.Edges) > 0 { + for _, comp := range listComponentsResp.Components.Edges { + components = append(components, comp.GetNode()) + } } - return component, nil + return components, nil } -func (p *Processor) GetComponentVersion(version string) (*client.ComponentVersion, error) { +func (p *Processor) GetComponentVersions(componentId string) ([]*client.ComponentVersion, error) { r, err := client.ListComponentVersions(context.Background(), *p.Client, &client.ComponentVersionFilter{ Version: []string{version}, - }, 1) + }) if err != nil { return nil, err From 993f5befe460cb4b64ded89d6b0de41ddff7056e Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Thu, 17 Oct 2024 06:37:36 +0200 Subject: [PATCH 08/25] Wip --- scanner/keppel/client/generated.go | 40 +++- .../client/query/listComponents.graphql | 31 ++- scanner/keppel/main.go | 222 +++++++++++------- scanner/keppel/processor/processor.go | 54 +++-- scanner/keppel/scanner/scanner.go | 45 ++++ 5 files changed, 260 insertions(+), 132 deletions(-) diff --git a/scanner/keppel/client/generated.go b/scanner/keppel/client/generated.go index b6c23023..d8f92f8a 100644 --- a/scanner/keppel/client/generated.go +++ b/scanner/keppel/client/generated.go @@ -36,20 +36,28 @@ func (v *Component) GetType() ComponentTypeValues { return v.Type } // ComponentConnection includes the requested fields of the GraphQL type ComponentConnection. type ComponentConnection struct { - Edges []*ComponentConnectionEdgesComponentEdge `json:"edges"` + TotalCount int `json:"totalCount"` + Edges []*ComponentConnectionEdgesComponentEdge `json:"edges"` } +// GetTotalCount returns ComponentConnection.TotalCount, and is useful for accessing the field via an interface. +func (v *ComponentConnection) GetTotalCount() int { return v.TotalCount } + // GetEdges returns ComponentConnection.Edges, and is useful for accessing the field via an interface. func (v *ComponentConnection) GetEdges() []*ComponentConnectionEdgesComponentEdge { return v.Edges } // ComponentConnectionEdgesComponentEdge includes the requested fields of the GraphQL type ComponentEdge. type ComponentConnectionEdgesComponentEdge struct { - Node *Component `json:"node"` + Node *Component `json:"node"` + Cursor string `json:"cursor"` } // GetNode returns ComponentConnectionEdgesComponentEdge.Node, and is useful for accessing the field via an interface. func (v *ComponentConnectionEdgesComponentEdge) GetNode() *Component { return v.Node } +// GetCursor returns ComponentConnectionEdgesComponentEdge.Cursor, and is useful for accessing the field via an interface. +func (v *ComponentConnectionEdgesComponentEdge) GetCursor() string { return v.Cursor } + type ComponentFilter struct { ComponentName []string `json:"componentName"` } @@ -330,23 +338,27 @@ func (v *__CreateIssueInput) GetInput() *IssueInput { return v.Input } // __ListComponentVersionsInput is used internally by genqlient type __ListComponentVersionsInput struct { Filter *ComponentVersionFilter `json:"filter,omitempty"` - First int `json:"first"` } // GetFilter returns __ListComponentVersionsInput.Filter, and is useful for accessing the field via an interface. func (v *__ListComponentVersionsInput) GetFilter() *ComponentVersionFilter { return v.Filter } -// GetFirst returns __ListComponentVersionsInput.First, and is useful for accessing the field via an interface. -func (v *__ListComponentVersionsInput) GetFirst() int { return v.First } - // __ListComponentsInput is used internally by genqlient type __ListComponentsInput struct { Filter *ComponentFilter `json:"filter,omitempty"` + First int `json:"first"` + After string `json:"after"` } // GetFilter returns __ListComponentsInput.Filter, and is useful for accessing the field via an interface. func (v *__ListComponentsInput) GetFilter() *ComponentFilter { return v.Filter } +// GetFirst returns __ListComponentsInput.First, and is useful for accessing the field via an interface. +func (v *__ListComponentsInput) GetFirst() int { return v.First } + +// GetAfter returns __ListComponentsInput.After, and is useful for accessing the field via an interface. +func (v *__ListComponentsInput) GetAfter() string { return v.After } + // __ListIssuesInput is used internally by genqlient type __ListIssuesInput struct { Filter *IssueFilter `json:"filter,omitempty"` @@ -513,8 +525,8 @@ func CreateIssue( // The query or mutation executed by ListComponentVersions. const ListComponentVersions_Operation = ` -query ListComponentVersions ($filter: ComponentVersionFilter, $first: Int) { - ComponentVersions(filter: $filter, first: $first) { +query ListComponentVersions ($filter: ComponentVersionFilter) { + ComponentVersions(filter: $filter) { edges { node { id @@ -530,14 +542,12 @@ func ListComponentVersions( ctx_ context.Context, client_ graphql.Client, filter *ComponentVersionFilter, - first int, ) (*ListComponentVersionsResponse, error) { req_ := &graphql.Request{ OpName: "ListComponentVersions", Query: ListComponentVersions_Operation, Variables: &__ListComponentVersionsInput{ Filter: filter, - First: first, }, } var err_ error @@ -556,14 +566,16 @@ func ListComponentVersions( // The query or mutation executed by ListComponents. const ListComponents_Operation = ` -query ListComponents ($filter: ComponentFilter) { - Components(filter: $filter) { +query ListComponents ($filter: ComponentFilter, $first: Int, $after: String) { + Components(filter: $filter, first: $first, after: $after) { + totalCount edges { node { id name type } + cursor } } } @@ -573,12 +585,16 @@ func ListComponents( ctx_ context.Context, client_ graphql.Client, filter *ComponentFilter, + first int, + after string, ) (*ListComponentsResponse, error) { req_ := &graphql.Request{ OpName: "ListComponents", Query: ListComponents_Operation, Variables: &__ListComponentsInput{ Filter: filter, + First: first, + After: after, }, } var err_ error diff --git a/scanner/keppel/client/query/listComponents.graphql b/scanner/keppel/client/query/listComponents.graphql index a1ab56e1..7733b814 100644 --- a/scanner/keppel/client/query/listComponents.graphql +++ b/scanner/keppel/client/query/listComponents.graphql @@ -1,18 +1,23 @@ # SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors # SPDX-License-Identifier: Apache-2.0 -query ListComponents($filter: ComponentFilter) { - # @genqlient(typename: "ComponentConnection") - Components ( - filter: $filter, - ) { - edges { - # @genqlient(typename: "Component") - node { - id - name - type - } - } +query ListComponents($filter: ComponentFilter, $first: Int, $after: String) { + # @genqlient(typename: "ComponentConnection") + Components ( + filter: $filter, + first: $first, + after: $after + + ) { + totalCount + edges { + # @genqlient(typename: "Component") + node { + id + name + type + } + cursor } + } } diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index 8e4ee439..7d0e902d 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -8,7 +8,6 @@ import ( "os" "sync" - "github.com/cloudoperators/heureka/scanners/keppel/client" "github.com/cloudoperators/heureka/scanners/keppel/models" "github.com/cloudoperators/heureka/scanners/keppel/processor" "github.com/cloudoperators/heureka/scanners/keppel/scanner" @@ -45,7 +44,7 @@ func init() { } func main() { - var wg sync.WaitGroup + // var wg sync.WaitGroup var scannerCfg scanner.Config err := envconfig.Process("heureka", &scannerCfg) if err != nil { @@ -53,7 +52,7 @@ func main() { } var processorCfg processor.Config - var fqdn = scannerCfg.KeppelFQDN + // var fqdn = scannerCfg.KeppelFQDN err = envconfig.Process("heureka", &processorCfg) if err != nil { log.WithError(err).Fatal("Error while reading env config for processor") @@ -68,26 +67,53 @@ func main() { } // Get components - components, err := keppelProcessor.GetComponents() - fmt.Print(len(components)) + components, err := keppelProcessor.GetAllComponents(nil, 100) + if err != nil { + log.WithError(err).Fatal("cannot list Components") + } - // For each component get componentVersion + // For each component get all component versions for _, comp := range components { - fmt.Print(comp) - } + imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) + if err != nil { + log.WithError(err).Error("Could't extract image information from component name") + } else { + fmt.Printf("imageInfo: %#v\n", imageInfo) + } - accounts, err := keppelScanner.ListAccounts() - if err != nil { - log.WithError(err).Fatal("Error during ListAccounts") - } + // Get component versions + compVersions, err := keppelProcessor.GetComponentVersions(comp.Id) + if err != nil { + log.WithError(err).Errorf("couldn't fetch component versions for componentId: %s", comp.Id) + } + + HandleRepository(comp.Id, compVersions[0].Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) - wg.Add(len(accounts)) + // Get trivy report + for _, cv := range compVersions { + trivyReport, err := keppelScanner.GetTrivyReport(imageInfo.Account, imageInfo.Repository, cv.Version) + if err != nil { + log.WithError(err).Errorf("couldn't fetch trivy report") + continue + } + fmt.Print(trivyReport) + } - for _, account := range accounts { - go HandleAccount(fqdn, account, keppelScanner, keppelProcessor, &wg) + fmt.Print(len(compVersions)) } - wg.Wait() + // accounts, err := keppelScanner.ListAccounts() + // if err != nil { + // log.WithError(err).Fatal("Error during ListAccounts") + // } + + // wg.Add(len(accounts)) + + // for _, account := range accounts { + // // go HandleAccount(fqdn, account, keppelScanner, keppelProcessor, &wg) + // } + + // wg.Wait() } func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error { @@ -108,90 +134,110 @@ func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.S return nil } -// func HandleRepository(fqdn string, account models.Account, repository models.Repository, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { -// component, err := keppelProcessor.ProcessRepository(fqdn, account, repository) -// if err != nil { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).WithError(err).Error("Error during ProcessRepository") -// component, err = keppelProcessor.GetComponent(fmt.Sprintf("%s/%s/%s", fqdn, account.Name, repository.Name)) -// if err != nil { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).WithError(err).Error("Error during GetComponent") -// } -// } - -// manifests, err := keppelScanner.ListManifests(account.Name, repository.Name) -// if err != nil { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).WithError(err).Error("Error during ListManifests") -// return -// } -// for _, manifest := range manifests { -// if component == nil { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).Error("Component not found") -// return -// } -// if manifest.VulnerabilityStatus == "Unsupported" { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType) -// continue -// } -// if manifest.VulnerabilityStatus == "Clean" { -// log.WithFields(log.Fields{ -// "account:": account.Name, -// "repository": repository.Name, -// }).Info("Manifest has no Vulnerabilities") -// continue -// } -// HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor) -// } -// } - -func HandleManifest(account models.Account, repository models.Repository, manifest models.Manifest, component *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { - childManifests, err := keppelScanner.ListChildManifests(account.Name, repository.Name, manifest.Digest) - +// HandleRepository does what ??? +func HandleRepository( + componentId string, + componentVersionId string, + account string, + repository string, + keppelScanner *scanner.Scanner, + keppelProcessor *processor.Processor, +) { + // componentId, err := keppelProcessor.ProcessRepository(fqdn, account, repository) + // if err != nil { + // log.WithFields(log.Fields{ + // "account:": account.Name, + // "repository": repository.Name, + // }).WithError(err).Error("Error during ProcessRepository") + // componentId, err = keppelProcessor.GetComponent(fmt.Sprintf("%s/%s/%s", fqdn, account.Name, repository.Name)) + // if err != nil { + // log.WithFields(log.Fields{ + // "account:": account.Name, + // "repository": repository.Name, + // }).WithError(err).Error("Error during GetComponent") + // } + // } + + manifests, err := keppelScanner.ListManifests(account, repository) if err != nil { log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during ListChildManifests") + "account:": account, + "repository": repository, + }).WithError(err).Error("Error during ListManifests") + return + } + for _, manifest := range manifests { + // TODO: What for is this needed? + // if componentId == nil { + // log.WithFields(log.Fields{ + // "account:": account.Name, + // "repository": repository.Name, + // }).Error("Component not found") + // return + // } + if manifest.VulnerabilityStatus == "Unsupported" { + log.WithFields(log.Fields{ + "account:": account, + "repository": repository, + }).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType) + continue + } + if manifest.VulnerabilityStatus == "Clean" { + log.WithFields(log.Fields{ + "account:": account, + "repository": repository, + }).Info("Manifest has no Vulnerabilities") + continue + } + HandleManifest(account, repository, manifest, componentId, componentVersionId, keppelScanner, keppelProcessor) } +} + +// HandleManifest does what ??? +func HandleManifest( + account string, + repository string, + manifest models.Manifest, + componentId string, + componentVersionId string, + keppelScanner *scanner.Scanner, + keppelProcessor *processor.Processor, +) { + childManifests, err := keppelScanner.ListChildManifests(account, repository, manifest.Digest) - componentVersion, err := keppelProcessor.ProcessManifest(manifest, component.Id) if err != nil { log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during ProcessManifest") - componentVersion, err = keppelProcessor.GetComponentVersion(manifest.Digest) - if err != nil || componentVersion == nil { - log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, - }).WithError(err).Error("Error during GetComponentVersion") - return - } + "account:": account, + "repository": repository, + }).WithError(err).Error("Error during ListChildManifests") } + // NOTE: Not really need because we already have the component versions (???) + // componentVersion, err := keppelProcessor.ProcessManifest(manifest, componentId) + // if err != nil { + // log.WithFields(log.Fields{ + // "account:": account.Name, + // "repository": repository.Name, + // }).WithError(err).Error("Error during ProcessManifest") + // componentVersions, err := keppelProcessor.GetComponentVersions(component.Id) + + // if err != nil || componentVersion == nil { + // log.WithFields(log.Fields{ + // "account:": account.Name, + // "repository": repository.Name, + // }).WithError(err).Error("Error during GetComponentVersion") + // return + // } + // } + childManifests = append(childManifests, manifest) for _, m := range childManifests { - trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, m.Digest) + trivyReport, err := keppelScanner.GetTrivyReport(account, repository, m.Digest) if err != nil { log.WithFields(log.Fields{ - "account:": account.Name, - "repository": repository.Name, + "account:": account, + "repository": repository, }).WithError(err).Error("Error during GetTrivyReport") return } @@ -200,6 +246,6 @@ func HandleManifest(account models.Account, repository models.Repository, manife return } - keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id) + keppelProcessor.ProcessReport(*trivyReport, componentVersionId) } } diff --git a/scanner/keppel/processor/processor.go b/scanner/keppel/processor/processor.go index 79c05659..8f3f803c 100644 --- a/scanner/keppel/processor/processor.go +++ b/scanner/keppel/processor/processor.go @@ -112,38 +112,54 @@ func (p *Processor) ProcessReport(report models.TrivyReport, componentVersionId } } -// GetComponents returns a slice of all availble Components -func (p *Processor) GetComponents() ([]*client.Component, error) { - listComponentsResp, err := client.ListComponents(context.Background(), *p.Client, &client.ComponentFilter{}) - if err != nil { - return nil, fmt.Errorf("cannot list Components: %w", err) - } +// GetAllComponents will fetch all available Components using pagination. +// pageSize specifies how many object we want to fetch in a row. Then we use cursor +// to fetch the next batch. +func (p *Processor) GetAllComponents(filter *client.ComponentFilter, pageSize int) ([]*client.Component, error) { + var allComponents []*client.Component + cursor := "0" // Set initial cursor to "0" + + for { + listComponentsResp, err := client.ListComponents(context.Background(), *p.Client, filter, pageSize, cursor) + if err != nil { + return nil, fmt.Errorf("cannot list Components: %w", err) + } - var components []*client.Component - if len(listComponentsResp.Components.Edges) > 0 { - for _, comp := range listComponentsResp.Components.Edges { - components = append(components, comp.GetNode()) + if len(listComponentsResp.Components.Edges) == 0 { + break + } + + for _, edge := range listComponentsResp.Components.Edges { + allComponents = append(allComponents, edge.Node) } - } - return components, nil + if len(listComponentsResp.Components.Edges) < pageSize { + break + } + + // Update cursor for the next iteration + cursor = listComponentsResp.Components.Edges[len(listComponentsResp.Components.Edges)-1].Cursor + } + return allComponents, nil } func (p *Processor) GetComponentVersions(componentId string) ([]*client.ComponentVersion, error) { - r, err := client.ListComponentVersions(context.Background(), *p.Client, &client.ComponentVersionFilter{ - Version: []string{version}, + listCompVersionResp, err := client.ListComponentVersions(context.Background(), *p.Client, &client.ComponentVersionFilter{ + ComponentId: []string{componentId}, }) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot list ComponentVersions: %w", err) } - var componentVersion *client.ComponentVersion - if len(r.ComponentVersions.Edges) > 0 { - componentVersion = r.ComponentVersions.Edges[0].GetNode() + var componentVersions []*client.ComponentVersion + if len(listCompVersionResp.ComponentVersions.Edges) > 0 { + for _, cv := range listCompVersionResp.GetComponentVersions().Edges { + componentVersions = append(componentVersions, cv.GetNode()) + } } - return componentVersion, nil + return componentVersions, nil } func (p *Processor) GetIssue(primaryName string) (*client.Issue, error) { diff --git a/scanner/keppel/scanner/scanner.go b/scanner/keppel/scanner/scanner.go index 367cfe78..69b0d4d8 100644 --- a/scanner/keppel/scanner/scanner.go +++ b/scanner/keppel/scanner/scanner.go @@ -17,6 +17,13 @@ import ( log "github.com/sirupsen/logrus" ) +type ImageInfo struct { + Registry string + Account string + Organization string + Repository string +} + type Scanner struct { KeppelBaseUrl string IdentityEndpoint string @@ -27,6 +34,14 @@ type Scanner struct { Project string } +func (i ImageInfo) FullRepository() string { + if len(i.Organization) > 0 { + return fmt.Sprintf("%s/%s", i.Organization, i.Repository) + } else { + return i.Repository + } +} + func NewScanner(cfg Config) *Scanner { return &Scanner{ KeppelBaseUrl: cfg.KeppelBaseUrl(), @@ -221,6 +236,36 @@ func (s *Scanner) GetTrivyReport(account string, repository string, manifest str } +// extractImageInfo extracts image registry, image repository and the account name +// from a container image +func (s *Scanner) ExtractImageInfo(image string) (ImageInfo, error) { + // Split the string to remove the tag + parts := strings.Split(image, ":") + if len(parts) < 1 { + return ImageInfo{}, fmt.Errorf("invalid image") + } + + // Split the remaining string by '/' + components := strings.Split(parts[0], "/") + if len(components) < 3 || len(components) > 4 { + return ImageInfo{}, fmt.Errorf("invalid image string format: expected 3 or 4 components") + } + + info := ImageInfo{ + Registry: components[0], + Repository: components[len(components)-1], + } + + if len(components) == 3 { + info.Account = components[1] + } else { // len(components) == 4 + info.Account = components[1] + info.Organization = components[2] + } + + return info, nil +} + func (s *Scanner) sendRequest(url string, token string) ([]byte, error) { client := new(http.Client) req, err := http.NewRequest("GET", url, nil) From 9ebdc0a90cc6d9c93abe9ec82bb7fd09d10a86d5 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Thu, 17 Oct 2024 14:07:47 +0200 Subject: [PATCH 09/25] Wip --- scanner/keppel/main.go | 73 ++---------------------------------------- 1 file changed, 2 insertions(+), 71 deletions(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index 7d0e902d..1bf6636e 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -6,7 +6,6 @@ package main import ( "fmt" "os" - "sync" "github.com/cloudoperators/heureka/scanners/keppel/models" "github.com/cloudoperators/heureka/scanners/keppel/processor" @@ -73,6 +72,7 @@ func main() { } // For each component get all component versions + // TODO: Make each call in a go routine for _, comp := range components { imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) if err != nil { @@ -88,18 +88,6 @@ func main() { } HandleRepository(comp.Id, compVersions[0].Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) - - // Get trivy report - for _, cv := range compVersions { - trivyReport, err := keppelScanner.GetTrivyReport(imageInfo.Account, imageInfo.Repository, cv.Version) - if err != nil { - log.WithError(err).Errorf("couldn't fetch trivy report") - continue - } - fmt.Print(trivyReport) - } - - fmt.Print(len(compVersions)) } // accounts, err := keppelScanner.ListAccounts() @@ -116,24 +104,6 @@ func main() { // wg.Wait() } -func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error { - defer wg.Done() - repositories, err := keppelScanner.ListRepositories(account.Name) - fmt.Print(len(repositories)) - if err != nil { - log.WithFields(log.Fields{ - "account:": account.Name, - }).WithError(err).Error("Error during listing ProcessRepository") - return err - } - - // for _, repository := range repositories { - // HandleRepository(fqdn, account, repository, keppelScanner, keppelProcessor) - // } - - return nil -} - // HandleRepository does what ??? func HandleRepository( componentId string, @@ -143,20 +113,6 @@ func HandleRepository( keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, ) { - // componentId, err := keppelProcessor.ProcessRepository(fqdn, account, repository) - // if err != nil { - // log.WithFields(log.Fields{ - // "account:": account.Name, - // "repository": repository.Name, - // }).WithError(err).Error("Error during ProcessRepository") - // componentId, err = keppelProcessor.GetComponent(fmt.Sprintf("%s/%s/%s", fqdn, account.Name, repository.Name)) - // if err != nil { - // log.WithFields(log.Fields{ - // "account:": account.Name, - // "repository": repository.Name, - // }).WithError(err).Error("Error during GetComponent") - // } - // } manifests, err := keppelScanner.ListManifests(account, repository) if err != nil { @@ -166,15 +122,8 @@ func HandleRepository( }).WithError(err).Error("Error during ListManifests") return } + for _, manifest := range manifests { - // TODO: What for is this needed? - // if componentId == nil { - // log.WithFields(log.Fields{ - // "account:": account.Name, - // "repository": repository.Name, - // }).Error("Component not found") - // return - // } if manifest.VulnerabilityStatus == "Unsupported" { log.WithFields(log.Fields{ "account:": account, @@ -212,24 +161,6 @@ func HandleManifest( }).WithError(err).Error("Error during ListChildManifests") } - // NOTE: Not really need because we already have the component versions (???) - // componentVersion, err := keppelProcessor.ProcessManifest(manifest, componentId) - // if err != nil { - // log.WithFields(log.Fields{ - // "account:": account.Name, - // "repository": repository.Name, - // }).WithError(err).Error("Error during ProcessManifest") - // componentVersions, err := keppelProcessor.GetComponentVersions(component.Id) - - // if err != nil || componentVersion == nil { - // log.WithFields(log.Fields{ - // "account:": account.Name, - // "repository": repository.Name, - // }).WithError(err).Error("Error during GetComponentVersion") - // return - // } - // } - childManifests = append(childManifests, manifest) for _, m := range childManifests { From d7766672a1bcc040200e49425518f9f5d88fae70 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Fri, 18 Oct 2024 14:23:00 +0200 Subject: [PATCH 10/25] Change naming --- scanner/keppel/main.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index 1bf6636e..6a537b4b 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -77,8 +77,7 @@ func main() { imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) if err != nil { log.WithError(err).Error("Could't extract image information from component name") - } else { - fmt.Printf("imageInfo: %#v\n", imageInfo) + continue } // Get component versions @@ -87,7 +86,11 @@ func main() { log.WithError(err).Errorf("couldn't fetch component versions for componentId: %s", comp.Id) } - HandleRepository(comp.Id, compVersions[0].Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + // Handle image manifests for each found component version + for _, cv := range compVersions { + HandleImageManifests(comp.Id, cv.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + } + } // accounts, err := keppelScanner.ListAccounts() @@ -104,8 +107,7 @@ func main() { // wg.Wait() } -// HandleRepository does what ??? -func HandleRepository( +func HandleImageManifests( componentId string, componentVersionId string, account string, @@ -138,12 +140,11 @@ func HandleRepository( }).Info("Manifest has no Vulnerabilities") continue } - HandleManifest(account, repository, manifest, componentId, componentVersionId, keppelScanner, keppelProcessor) + HandleChildManifests(account, repository, manifest, componentId, componentVersionId, keppelScanner, keppelProcessor) } } -// HandleManifest does what ??? -func HandleManifest( +func HandleChildManifests( account string, repository string, manifest models.Manifest, @@ -164,6 +165,7 @@ func HandleManifest( childManifests = append(childManifests, manifest) for _, m := range childManifests { + // Get Trivy report for a specific repository and image version (componentVersion) trivyReport, err := keppelScanner.GetTrivyReport(account, repository, m.Digest) if err != nil { log.WithFields(log.Fields{ From e51ead790ea4d035f54115460b90ded0e07e7282 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 21 Oct 2024 06:21:43 +0200 Subject: [PATCH 11/25] Add concurrency --- scanner/keppel/main.go | 80 +++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index 6a537b4b..9f8abbd7 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -4,14 +4,17 @@ package main import ( - "fmt" "os" + "context" + "github.com/cloudoperators/heureka/scanners/keppel/client" "github.com/cloudoperators/heureka/scanners/keppel/models" "github.com/cloudoperators/heureka/scanners/keppel/processor" "github.com/cloudoperators/heureka/scanners/keppel/scanner" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" + "runtime" + "sync" ) type Config struct { @@ -71,43 +74,63 @@ func main() { log.WithError(err).Fatal("cannot list Components") } - // For each component get all component versions - // TODO: Make each call in a go routine - for _, comp := range components { - imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) - if err != nil { - log.WithError(err).Error("Could't extract image information from component name") - continue - } - - // Get component versions - compVersions, err := keppelProcessor.GetComponentVersions(comp.Id) - if err != nil { - log.WithError(err).Errorf("couldn't fetch component versions for componentId: %s", comp.Id) - } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + processConcurrently(ctx, components, keppelScanner, keppelProcessor) +} - // Handle image manifests for each found component version - for _, cv := range compVersions { - HandleImageManifests(comp.Id, cv.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) - } +func processConcurrently(ctx context.Context, components []*client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { + maxWorkers := runtime.GOMAXPROCS(0) + componentCh := make(chan *client.Component, len(components)) + var wg sync.WaitGroup + + // Start worker goroutines + for i := 0; i < maxWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for comp := range componentCh { + select { + case <-ctx.Done(): + return + default: + processComponent(ctx, comp, keppelScanner, keppelProcessor) + } + } + }() + } + // Feed components to workers + for _, comp := range components { + componentCh <- comp } + close(componentCh) - // accounts, err := keppelScanner.ListAccounts() - // if err != nil { - // log.WithError(err).Fatal("Error during ListAccounts") - // } + // Wait for all workers to finish + wg.Wait() +} - // wg.Add(len(accounts)) +func processComponent(ctx context.Context, comp *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { + imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) + if err != nil { + log.WithError(err).Error("Couldn't extract image information from component name") + return + } - // for _, account := range accounts { - // // go HandleAccount(fqdn, account, keppelScanner, keppelProcessor, &wg) - // } + log.Infof("Processing component: %s", comp.Name) + compVersions, err := keppelProcessor.GetComponentVersions(comp.Id) + if err != nil { + log.WithError(err).Errorf("couldn't fetch component versions for componentId: %s", comp.Id) + return + } - // wg.Wait() + for _, cv := range compVersions { + HandleImageManifests(ctx, comp.Id, cv.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + } } func HandleImageManifests( + ctx context.Context, componentId string, componentVersionId string, account string, @@ -116,6 +139,7 @@ func HandleImageManifests( keppelProcessor *processor.Processor, ) { + log.Info("Handling manifest") manifests, err := keppelScanner.ListManifests(account, repository) if err != nil { log.WithFields(log.Fields{ From adc9d36e54c84e8992334d79a0fbbd289b96bf86 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 21 Oct 2024 06:40:57 +0200 Subject: [PATCH 12/25] Add READMEs --- scanner/k8s-assets/README.md | 62 ++++++++++++++++++++++++++++++++++++ scanner/keppel/README.md | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 scanner/k8s-assets/README.md create mode 100644 scanner/keppel/README.md diff --git a/scanner/k8s-assets/README.md b/scanner/k8s-assets/README.md new file mode 100644 index 00000000..c083f60e --- /dev/null +++ b/scanner/k8s-assets/README.md @@ -0,0 +1,62 @@ +# Kubernetes Assets Scanner + +The Kubernetes Assets Scanner is a tool designed to scan and collect information about services, pods, and containers running in a Kubernetes cluster. It processes the collected data and reports findings to a GraphQL API (presumably Heureka). + +## Prerequisites + +- Go 1.15 or later +- Access to a Kubernetes cluster +- Heureka system for reporting findings + +## Installation + +1. Clone the repository: + ``` + git clone https://github.com/cloudoperators/heureka.git + cd scanners/k8s-assets + ``` + +2. Install dependencies: + ``` + go mod tidy + ``` + +## Configuration + +The scanner is configured using environment variables. Set the following variables before running the scanner: + +- `HEUREKA_LOG_LEVEL`: Set the log level (default: "debug") +- `HEUREKA_KUBE_CONFIG_PATH`: Path to kubeconfig file (default: "~/.kube/config") +- `HEUREKA_KUBE_CONFIG_CONTEXT`: Kubernetes context to use +- `HEUREKA_KUBE_CONFIG_TYPE`: Type of Kubernetes config (default: "oidc") +- `HEUREKA_SUPPORT_GROUP_LABEL`: Label for support group (default: "ccloud/support-group") +- `HEUREKA_SERVICE_NAME_LABEL`: Label for service name (default: "ccloud/service") +- `HEUREKA_SCANNER_TIMEOUT`: Timeout for the scanner (default: "30m") +- `HEUREKA_HEUREKA_URL`: URL of the Heureka system for reporting findings +- `HEUREKA_CLUSTER_NAME`: Name of the cluster being scanned +- `HEUREKA_CLUSTER_REGION`: Region of the cluster being scanned + +Example: + +```bash +export HEUREKA_LOG_LEVEL=debug +export HEUREKA_KUBE_CONFIG_PATH=~/.kube/config +export HEUREKA_KUBE_CONFIG_CONTEXT=my-cluster-context +export HEUREKA_KUBE_CONFIG_TYPE=oidc +export HEUREKA_SUPPORT_GROUP_LABEL=ccloud/support-group +export HEUREKA_SERVICE_NAME_LABEL=ccloud/service +export HEUREKA_SCANNER_TIMEOUT=30m +export HEUREKA_HEUREKA_URL=https://heureka.example.com +export HEUREKA_CLUSTER_NAME=my-cluster +export HEUREKA_CLUSTER_REGION=us-west-1 +``` + +## Usage + +To run the Kubernetes Assets Scanner: + +```bash +go run main.go +``` + +The scanner will start processing namespaces, services, pods, and containers, and report findings to the configured Heureka system. diff --git a/scanner/keppel/README.md b/scanner/keppel/README.md new file mode 100644 index 00000000..3cfccd75 --- /dev/null +++ b/scanner/keppel/README.md @@ -0,0 +1,59 @@ +# Keppel Scanner + +The Keppel scanner is a tool designed to scan container images stored in a [Keppel](https://github.com/sapcc/keppel) registry. It retrieves information about accounts, repositories, and manifests, and processes vulnerability reports for each image. + +## Prerequisites + +- Go 1.16 or later +- Access to a Keppel registry +- Heureka system for reporting findings + +## Installation + +1. Clone the repository: + ``` + git clone https://github.com/cloudoperators/heureka.git + cd scanner/keppel + ``` + +2. Install dependencies: + ``` + go mod tidy + ``` + +## Configuration + +The scanner is configured using environment variables. Set the following variables before running the scanner: + +- `HEUREKA_LOG_LEVEL`: Set the log level (default: "debug") +- `HEUREKA_KEPPEL_FQDN`: Fully Qualified Domain Name of the Keppel registry +- `HEUREKA_KEPPEL_USERNAME`: Username for Keppel authentication +- `HEUREKA_KEPPEL_PASSWORD`: Password for Keppel authentication +- `HEUREKA_KEPPEL_DOMAIN`: Domain for Keppel authentication +- `HEUREKA_KEPPEL_PROJECT`: Project for Keppel authentication +- `HEUREKA_IDENTITY_ENDPOINT`: Identity endpoint for authentication +- `HEUREKA_HEUREKA_URL`: URL of the Heureka system for reporting findings + +Example: + +```bash +export HEUREKA_LOG_LEVEL=debug +export HEUREKA_KEPPEL_FQDN=keppel.example.com +export HEUREKA_KEPPEL_USERNAME=myusername +export HEUREKA_KEPPEL_PASSWORD=mypassword +export HEUREKA_KEPPEL_DOMAIN=mydomain +export HEUREKA_KEPPEL_PROJECT=myproject +export HEUREKA_IDENTITY_ENDPOINT=https://identity.example.com +export HEUREKA_HEUREKA_URL=https://heureka.example.com +``` + +## Usage + +To run the Keppel scanner: + +```bash +go run main.go +``` + +The scanner will start processing accounts, repositories, and manifests, and report findings to the configured Heureka system. + From 1b402b99898bd573edc37d448f3e7b7d47ef8b45 Mon Sep 17 00:00:00 2001 From: License Bot Date: Mon, 21 Oct 2024 04:42:50 +0000 Subject: [PATCH 13/25] Automatic application of license header --- scanner/k8s-assets/client/generated.go | 3 +++ scanner/keppel/client/generated.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/scanner/k8s-assets/client/generated.go b/scanner/k8s-assets/client/generated.go index 587c5698..dda1b2c4 100644 --- a/scanner/k8s-assets/client/generated.go +++ b/scanner/k8s-assets/client/generated.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client diff --git a/scanner/keppel/client/generated.go b/scanner/keppel/client/generated.go index d8f92f8a..588658c1 100644 --- a/scanner/keppel/client/generated.go +++ b/scanner/keppel/client/generated.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client From b12bf4b9f5c37cef40db058fb2d9cfd628953186 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 21 Oct 2024 13:36:57 +0200 Subject: [PATCH 14/25] Fix https://github.com/cloudoperators/heureka/pull/309#discussion_r1808570011 --- scanner/k8s-assets/scanner/scanner.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scanner/k8s-assets/scanner/scanner.go b/scanner/k8s-assets/scanner/scanner.go index 3fd2772f..dd9c4e72 100644 --- a/scanner/k8s-assets/scanner/scanner.go +++ b/scanner/k8s-assets/scanner/scanner.go @@ -133,21 +133,21 @@ func (s *Scanner) extractImageInfo(image string) (ImageInfo, error) { } // Split the remaining string by '/' - components := strings.Split(parts[0], "/") - if len(components) < 3 || len(components) > 4 { - return ImageInfo{}, fmt.Errorf("invalid image string format: expected 3 or 4 components") + tokens := strings.Split(parts[0], "/") + if len(tokens) < 3 || len(tokens) > 4 { + return ImageInfo{}, fmt.Errorf("invalid image string format: expected 3 or 4 tokens") } info := ImageInfo{ - Registry: components[0], - Repository: components[len(components)-1], + Registry: tokens[0], + Repository: tokens[len(tokens)-1], } - if len(components) == 3 { - info.Account = components[1] - } else { // len(components) == 4 - info.Account = components[1] - info.Organization = components[2] + if len(tokens) == 3 { + info.Account = tokens[1] + } else { // len(tokens) == 4 + info.Account = tokens[1] + info.Organization = tokens[2] } return info, nil From 369fa18594d33eae3fb222cf81a59fe59e953b2a Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 21 Oct 2024 13:38:37 +0200 Subject: [PATCH 15/25] Fix #discussion_r1808503568 --- scanner/keppel/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index 9f8abbd7..a5d844bb 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -54,7 +54,6 @@ func main() { } var processorCfg processor.Config - // var fqdn = scannerCfg.KeppelFQDN err = envconfig.Process("heureka", &processorCfg) if err != nil { log.WithError(err).Fatal("Error while reading env config for processor") From e1525f8e9d0b39b3c68f97987b6028bb8b5263bd Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Mon, 21 Oct 2024 13:43:16 +0200 Subject: [PATCH 16/25] Fix https://github.com/cloudoperators/heureka/pull/309#discussion_r1808514161 --- scanner/keppel/scanner/scanner.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scanner/keppel/scanner/scanner.go b/scanner/keppel/scanner/scanner.go index 69b0d4d8..a60b4c2d 100644 --- a/scanner/keppel/scanner/scanner.go +++ b/scanner/keppel/scanner/scanner.go @@ -22,6 +22,7 @@ type ImageInfo struct { Account string Organization string Repository string + Tag string } type Scanner struct { @@ -256,6 +257,12 @@ func (s *Scanner) ExtractImageInfo(image string) (ImageInfo, error) { Repository: components[len(components)-1], } + // Set tag + if len(parts) > 2 { + info.Tag = parts[1] + } + + // Set organization and registry if len(components) == 3 { info.Account = components[1] } else { // len(components) == 4 From 3c9503f2318670e67e87fb75b7fc1626e4edd919 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 06:26:25 +0200 Subject: [PATCH 17/25] Fix https://github.com/cloudoperators/heureka/pull/309#discussion_r1808505557 --- scanner/keppel/client/generated.go | 54 ++++++++++++++++--- ...phql => listComponentsAndVersions.graphql} | 13 ++++- scanner/keppel/main.go | 26 ++++----- scanner/keppel/processor/processor.go | 5 +- 4 files changed, 74 insertions(+), 24 deletions(-) rename scanner/keppel/client/query/{listComponents.graphql => listComponentsAndVersions.graphql} (58%) diff --git a/scanner/keppel/client/generated.go b/scanner/keppel/client/generated.go index 588658c1..7d801c79 100644 --- a/scanner/keppel/client/generated.go +++ b/scanner/keppel/client/generated.go @@ -1,6 +1,3 @@ -// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors -// SPDX-License-Identifier: Apache-2.0 - // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client @@ -37,6 +34,26 @@ func (v *Component) GetName() string { return v.Name } // GetType returns Component.Type, and is useful for accessing the field via an interface. func (v *Component) GetType() ComponentTypeValues { return v.Type } +// ComponentAggregate includes the requested fields of the GraphQL type Component. +type ComponentAggregate struct { + Id string `json:"id"` + Name string `json:"name"` + Type ComponentTypeValues `json:"type"` + ComponentVersions *ComponentVersions `json:"componentVersions"` +} + +// GetId returns ComponentAggregate.Id, and is useful for accessing the field via an interface. +func (v *ComponentAggregate) GetId() string { return v.Id } + +// GetName returns ComponentAggregate.Name, and is useful for accessing the field via an interface. +func (v *ComponentAggregate) GetName() string { return v.Name } + +// GetType returns ComponentAggregate.Type, and is useful for accessing the field via an interface. +func (v *ComponentAggregate) GetType() ComponentTypeValues { return v.Type } + +// GetComponentVersions returns ComponentAggregate.ComponentVersions, and is useful for accessing the field via an interface. +func (v *ComponentAggregate) GetComponentVersions() *ComponentVersions { return v.ComponentVersions } + // ComponentConnection includes the requested fields of the GraphQL type ComponentConnection. type ComponentConnection struct { TotalCount int `json:"totalCount"` @@ -51,12 +68,12 @@ func (v *ComponentConnection) GetEdges() []*ComponentConnectionEdgesComponentEdg // ComponentConnectionEdgesComponentEdge includes the requested fields of the GraphQL type ComponentEdge. type ComponentConnectionEdgesComponentEdge struct { - Node *Component `json:"node"` - Cursor string `json:"cursor"` + Node *ComponentAggregate `json:"node"` + Cursor string `json:"cursor"` } // GetNode returns ComponentConnectionEdgesComponentEdge.Node, and is useful for accessing the field via an interface. -func (v *ComponentConnectionEdgesComponentEdge) GetNode() *Component { return v.Node } +func (v *ComponentConnectionEdgesComponentEdge) GetNode() *ComponentAggregate { return v.Node } // GetCursor returns ComponentConnectionEdgesComponentEdge.Cursor, and is useful for accessing the field via an interface. func (v *ComponentConnectionEdgesComponentEdge) GetCursor() string { return v.Cursor } @@ -153,6 +170,22 @@ func (v *ComponentVersionInput) GetVersion() string { return v.Version } // GetComponentId returns ComponentVersionInput.ComponentId, and is useful for accessing the field via an interface. func (v *ComponentVersionInput) GetComponentId() string { return v.ComponentId } +// ComponentVersions includes the requested fields of the GraphQL type ComponentVersionConnection. +type ComponentVersions struct { + Edges []*ComponentVersionsEdgesComponentVersionEdge `json:"edges"` +} + +// GetEdges returns ComponentVersions.Edges, and is useful for accessing the field via an interface. +func (v *ComponentVersions) GetEdges() []*ComponentVersionsEdgesComponentVersionEdge { return v.Edges } + +// ComponentVersionsEdgesComponentVersionEdge includes the requested fields of the GraphQL type ComponentVersionEdge. +type ComponentVersionsEdgesComponentVersionEdge struct { + Node *ComponentVersion `json:"node"` +} + +// GetNode returns ComponentVersionsEdgesComponentVersionEdge.Node, and is useful for accessing the field via an interface. +func (v *ComponentVersionsEdgesComponentVersionEdge) GetNode() *ComponentVersion { return v.Node } + // CreateComponentResponse is returned by CreateComponent on success. type CreateComponentResponse struct { CreateComponent *Component `json:"createComponent"` @@ -577,6 +610,15 @@ query ListComponents ($filter: ComponentFilter, $first: Int, $after: String) { id name type + componentVersions { + edges { + node { + id + version + componentId + } + } + } } cursor } diff --git a/scanner/keppel/client/query/listComponents.graphql b/scanner/keppel/client/query/listComponentsAndVersions.graphql similarity index 58% rename from scanner/keppel/client/query/listComponents.graphql rename to scanner/keppel/client/query/listComponentsAndVersions.graphql index 7733b814..785c1929 100644 --- a/scanner/keppel/client/query/listComponents.graphql +++ b/scanner/keppel/client/query/listComponentsAndVersions.graphql @@ -11,11 +11,22 @@ query ListComponents($filter: ComponentFilter, $first: Int, $after: String) { ) { totalCount edges { - # @genqlient(typename: "Component") + # @genqlient(typename: "ComponentAggregate") node { id name type + # @genqlient(typename: "ComponentVersions") + componentVersions { + edges { + # @genqlient(typename: "ComponentVersion") + node { + id + version + componentId + } + } + } } cursor } diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index a5d844bb..e567a20d 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -7,14 +7,15 @@ import ( "os" "context" + "runtime" + "sync" + "github.com/cloudoperators/heureka/scanners/keppel/client" "github.com/cloudoperators/heureka/scanners/keppel/models" "github.com/cloudoperators/heureka/scanners/keppel/processor" "github.com/cloudoperators/heureka/scanners/keppel/scanner" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" - "runtime" - "sync" ) type Config struct { @@ -67,7 +68,7 @@ func main() { log.WithError(err).Fatal("Error during scanner setup") } - // Get components + // Get components and correponding componentVersions components, err := keppelProcessor.GetAllComponents(nil, 100) if err != nil { log.WithError(err).Fatal("cannot list Components") @@ -78,9 +79,9 @@ func main() { processConcurrently(ctx, components, keppelScanner, keppelProcessor) } -func processConcurrently(ctx context.Context, components []*client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { +func processConcurrently(ctx context.Context, components []*client.ComponentAggregate, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { maxWorkers := runtime.GOMAXPROCS(0) - componentCh := make(chan *client.Component, len(components)) + componentCh := make(chan *client.ComponentAggregate, len(components)) var wg sync.WaitGroup // Start worker goroutines @@ -109,22 +110,17 @@ func processConcurrently(ctx context.Context, components []*client.Component, ke wg.Wait() } -func processComponent(ctx context.Context, comp *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { +func processComponent(ctx context.Context, comp *client.ComponentAggregate, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { + log.Infof("Processing component: %s", comp.Name) + imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) if err != nil { log.WithError(err).Error("Couldn't extract image information from component name") return } - log.Infof("Processing component: %s", comp.Name) - compVersions, err := keppelProcessor.GetComponentVersions(comp.Id) - if err != nil { - log.WithError(err).Errorf("couldn't fetch component versions for componentId: %s", comp.Id) - return - } - - for _, cv := range compVersions { - HandleImageManifests(ctx, comp.Id, cv.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + for _, cv := range comp.ComponentVersions.Edges { + HandleImageManifests(ctx, comp.Id, cv.Node.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) } } diff --git a/scanner/keppel/processor/processor.go b/scanner/keppel/processor/processor.go index 8f3f803c..fedc64bc 100644 --- a/scanner/keppel/processor/processor.go +++ b/scanner/keppel/processor/processor.go @@ -115,11 +115,12 @@ func (p *Processor) ProcessReport(report models.TrivyReport, componentVersionId // GetAllComponents will fetch all available Components using pagination. // pageSize specifies how many object we want to fetch in a row. Then we use cursor // to fetch the next batch. -func (p *Processor) GetAllComponents(filter *client.ComponentFilter, pageSize int) ([]*client.Component, error) { - var allComponents []*client.Component +func (p *Processor) GetAllComponents(filter *client.ComponentFilter, pageSize int) ([]*client.ComponentAggregate, error) { + var allComponents []*client.ComponentAggregate cursor := "0" // Set initial cursor to "0" for { + // ListComponents also returns the ComponentVersions of each Component listComponentsResp, err := client.ListComponents(context.Background(), *p.Client, filter, pageSize, cursor) if err != nil { return nil, fmt.Errorf("cannot list Components: %w", err) From d665ad1f25e21fdbe7ea0ccc413fdbc481ea022e Mon Sep 17 00:00:00 2001 From: License Bot Date: Tue, 22 Oct 2024 04:27:31 +0000 Subject: [PATCH 18/25] Automatic application of license header --- scanner/keppel/client/generated.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scanner/keppel/client/generated.go b/scanner/keppel/client/generated.go index 7d801c79..738c3ab5 100644 --- a/scanner/keppel/client/generated.go +++ b/scanner/keppel/client/generated.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + // Code generated by github.com/Khan/genqlient, DO NOT EDIT. package client From 33b65edb68388ba945d2038d6ce25802756bfb32 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 06:50:55 +0200 Subject: [PATCH 19/25] Fix https://github.com/cloudoperators/heureka/pull/309#discussion_r1808509349 --- scanner/keppel/main.go | 12 ++++++------ scanner/keppel/scanner/scanner.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index e567a20d..b4b12ea3 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -120,14 +120,14 @@ func processComponent(ctx context.Context, comp *client.ComponentAggregate, kepp } for _, cv := range comp.ComponentVersions.Edges { - HandleImageManifests(ctx, comp.Id, cv.Node.Id, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + HandleImageManifests(ctx, comp.Id, cv.Node, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) } } func HandleImageManifests( ctx context.Context, componentId string, - componentVersionId string, + componentVersion *client.ComponentVersion, account string, repository string, keppelScanner *scanner.Scanner, @@ -135,7 +135,7 @@ func HandleImageManifests( ) { log.Info("Handling manifest") - manifests, err := keppelScanner.ListManifests(account, repository) + manifests, err := keppelScanner.GetManifest(account, repository, componentVersion.Version) if err != nil { log.WithFields(log.Fields{ "account:": account, @@ -159,7 +159,7 @@ func HandleImageManifests( }).Info("Manifest has no Vulnerabilities") continue } - HandleChildManifests(account, repository, manifest, componentId, componentVersionId, keppelScanner, keppelProcessor) + HandleChildManifests(account, repository, manifest, componentId, componentVersion, keppelScanner, keppelProcessor) } } @@ -168,7 +168,7 @@ func HandleChildManifests( repository string, manifest models.Manifest, componentId string, - componentVersionId string, + componentVersion *client.ComponentVersion, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, ) { @@ -198,6 +198,6 @@ func HandleChildManifests( return } - keppelProcessor.ProcessReport(*trivyReport, componentVersionId) + keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id) } } diff --git a/scanner/keppel/scanner/scanner.go b/scanner/keppel/scanner/scanner.go index a60b4c2d..558ec488 100644 --- a/scanner/keppel/scanner/scanner.go +++ b/scanner/keppel/scanner/scanner.go @@ -172,6 +172,29 @@ func (s *Scanner) ListManifests(account string, repository string) ([]models.Man return manifestResponse.Manifests, nil } +// GetManifest returns a single manifest from the image registry +func (s *Scanner) GetManifest(account string, repository string, manifest string) ([]models.Manifest, error) { + url := fmt.Sprintf("%s/v2/%s/%s/manifests/%s", s.KeppelBaseUrl, account, repository, manifest) + body, err := s.sendRequest(url, s.AuthToken) + if err != nil { + log.WithFields(log.Fields{ + "url": url, + }).WithError(err).Error("Error during request in GetManifest") + return nil, err + } + + var manifestResponse models.ManifestResponse + if err = json.Unmarshal(body, &manifestResponse); err != nil { + log.WithFields(log.Fields{ + "url": url, + "body": body, + }).WithError(err).Error("Error during unmarshal in GetManifest") + return nil, err + } + + return manifestResponse.Manifests, nil +} + // ListChildManifests is requred asa on Keppel not all Images are including vulnerability scan results directly on the // top layer of the image and rather have the scan results on the child manifests. An prime example of this are multi-arch // images where the scan results are available on the child manifests with the respective concrete architecture. From bc5b71959dc406a58e0e9b66a26a940753f34557 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 08:27:02 +0200 Subject: [PATCH 20/25] Refactoring --- scanner/keppel/main.go | 133 +++++++++++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 39 deletions(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index b4b12ea3..ebb3322c 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -4,6 +4,7 @@ package main import ( + "fmt" "os" "context" @@ -22,6 +23,23 @@ type Config struct { LogLevel string `envconfig:"LOG_LEVEL" default:"debug" required:"true" json:"-"` } +// ManifestInfo groups related manifest information +type ManifestInfo struct { + ComponentID string + ComponentVersion *client.ComponentVersion + Account string + Repository string +} + +// ChildManifestInfo groups related child manifest information +type ChildManifestInfo struct { + Account string + Repository string + Manifest models.Manifest + ComponentID string + ComponentVersion *client.ComponentVersion +} + func init() { // Log as JSON instead of the default ASCII formatter. log.SetFormatter(&log.JSONFormatter{}) @@ -76,28 +94,39 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - processConcurrently(ctx, components, keppelScanner, keppelProcessor) + + if err := processConcurrently(ctx, components, keppelScanner, keppelProcessor); err != nil { + log.WithError(err).Error("Error during concurrent processing") + } } -func processConcurrently(ctx context.Context, components []*client.ComponentAggregate, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { +func processConcurrently( + ctx context.Context, + components []*client.ComponentAggregate, + keppelScanner *scanner.Scanner, + keppelProcessor *processor.Processor, +) error { maxWorkers := runtime.GOMAXPROCS(0) componentCh := make(chan *client.ComponentAggregate, len(components)) + errors := make(chan error, maxWorkers) var wg sync.WaitGroup // Start worker goroutines for i := 0; i < maxWorkers; i++ { wg.Add(1) - go func() { + go func(errors chan error) { defer wg.Done() for comp := range componentCh { select { case <-ctx.Done(): return default: - processComponent(ctx, comp, keppelScanner, keppelProcessor) + if err := processComponent(ctx, comp, keppelScanner, keppelProcessor); err != nil { + errors <- err + } } } - }() + }(errors) } // Feed components to workers @@ -108,88 +137,114 @@ func processConcurrently(ctx context.Context, components []*client.ComponentAggr // Wait for all workers to finish wg.Wait() + close(errors) + + // Check for errors + for err := range errors { + if err != nil { + return err + } + } + + return nil } -func processComponent(ctx context.Context, comp *client.ComponentAggregate, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) { +func processComponent( + ctx context.Context, + comp *client.ComponentAggregate, + keppelScanner *scanner.Scanner, + keppelProcessor *processor.Processor, +) error { log.Infof("Processing component: %s", comp.Name) imageInfo, err := keppelScanner.ExtractImageInfo(comp.Name) if err != nil { - log.WithError(err).Error("Couldn't extract image information from component name") - return + return fmt.Errorf("Cannot extract image information from component name: %w", err) } for _, cv := range comp.ComponentVersions.Edges { - HandleImageManifests(ctx, comp.Id, cv.Node, imageInfo.Account, imageInfo.FullRepository(), keppelScanner, keppelProcessor) + manifestInfo := ManifestInfo{ + ComponentID: comp.Id, + ComponentVersion: cv.Node, + Account: imageInfo.Account, + Repository: imageInfo.FullRepository(), + } + + if err := HandleImageManifests(ctx, manifestInfo, keppelScanner, keppelProcessor); err != nil { + return err + } } + return nil } func HandleImageManifests( ctx context.Context, - componentId string, - componentVersion *client.ComponentVersion, - account string, - repository string, + info ManifestInfo, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, -) { +) error { log.Info("Handling manifest") - manifests, err := keppelScanner.GetManifest(account, repository, componentVersion.Version) + manifests, err := keppelScanner.GetManifest(info.Account, info.Repository, info.ComponentVersion.Version) if err != nil { log.WithFields(log.Fields{ - "account:": account, - "repository": repository, - }).WithError(err).Error("Error during ListManifests") - return + "account:": info.Account, + "repository": info.Repository, + }).WithError(err).Error("Error during GetManifest") + return fmt.Errorf("Couldn't get manifest: %w", err) } for _, manifest := range manifests { if manifest.VulnerabilityStatus == "Unsupported" { log.WithFields(log.Fields{ - "account:": account, - "repository": repository, + "account:": info.Account, + "repository": info.Repository, }).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType) continue } if manifest.VulnerabilityStatus == "Clean" { log.WithFields(log.Fields{ - "account:": account, - "repository": repository, + "account:": info.Account, + "repository": info.Repository, }).Info("Manifest has no Vulnerabilities") continue } - HandleChildManifests(account, repository, manifest, componentId, componentVersion, keppelScanner, keppelProcessor) + + childInfo := ChildManifestInfo{ + Account: info.Account, + Repository: info.Repository, + Manifest: manifest, + ComponentID: info.ComponentID, + ComponentVersion: info.ComponentVersion, + } + HandleChildManifests(ctx, childInfo, keppelScanner, keppelProcessor) } + return nil } func HandleChildManifests( - account string, - repository string, - manifest models.Manifest, - componentId string, - componentVersion *client.ComponentVersion, + ctx context.Context, + info ChildManifestInfo, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, ) { - childManifests, err := keppelScanner.ListChildManifests(account, repository, manifest.Digest) - + childManifests, err := keppelScanner.ListChildManifests(info.Account, info.Repository, info.Manifest.Digest) if err != nil { log.WithFields(log.Fields{ - "account:": account, - "repository": repository, + "account:": info.Account, + "repository": info.Repository, }).WithError(err).Error("Error during ListChildManifests") } - childManifests = append(childManifests, manifest) - + childManifests = append(childManifests, info.Manifest) for _, m := range childManifests { + // Get Trivy report for a specific repository and image version (componentVersion) - trivyReport, err := keppelScanner.GetTrivyReport(account, repository, m.Digest) + trivyReport, err := keppelScanner.GetTrivyReport(info.Account, info.Repository, m.Digest) if err != nil { log.WithFields(log.Fields{ - "account:": account, - "repository": repository, + "account:": info.Account, + "repository": info.Repository, }).WithError(err).Error("Error during GetTrivyReport") return } @@ -198,6 +253,6 @@ func HandleChildManifests( return } - keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id) + keppelProcessor.ProcessReport(*trivyReport, info.ComponentVersion.Id) } } From c5e5669b5b1f08897b460aa6ec24a3c0faa9c2bf Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 08:46:38 +0200 Subject: [PATCH 21/25] Update dependencies --- scanner/keppel/go.mod | 22 ++++++++++-------- scanner/keppel/go.sum | 53 +++++++++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/scanner/keppel/go.mod b/scanner/keppel/go.mod index b641a39f..a1337b2a 100644 --- a/scanner/keppel/go.mod +++ b/scanner/keppel/go.mod @@ -7,18 +7,22 @@ require ( github.com/aquasecurity/trivy v0.54.1 github.com/gophercloud/gophercloud v1.14.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/onsi/ginkgo/v2 v2.20.2 + github.com/onsi/gomega v1.34.1 ) require ( - github.com/agnivade/levenshtein v1.1.1 // indirect - github.com/alexflint/go-arg v1.4.2 // indirect - github.com/alexflint/go-scalar v1.0.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/vektah/gqlparser/v2 v2.5.15 // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/tools v0.23.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/tools v0.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -27,6 +31,6 @@ require ( github.com/package-url/packageurl-go v0.1.3 // indirect github.com/samber/lo v1.46.0 // indirect github.com/sirupsen/logrus v1.9.3 - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect ) diff --git a/scanner/keppel/go.sum b/scanner/keppel/go.sum index b6d3c753..5eed80a7 100644 --- a/scanner/keppel/go.sum +++ b/scanner/keppel/go.sum @@ -1,36 +1,47 @@ github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= -github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= -github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= -github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0= -github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= -github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70= -github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/aquasecurity/trivy v0.54.1 h1:/uNCF06PfdC69v5n3Zh4fXVf0xmXBml0c/ergf066SQ= github.com/aquasecurity/trivy v0.54.1/go.mod h1:i5S54WUtOEN9egFF0AHsxq6XT7QD11n9pSmIXhMJV0g= github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 h1:6/T8sFdNVG/AwOGoK6X55h7hF7LYqK8bsuPz8iEz8jM= github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04/go.mod h1:0T6oy2t1Iedt+yi3Ml5cpOYp5FZT4MI1/mx+3p+PIs8= -github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0= github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= +github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ= github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= @@ -38,35 +49,37 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vektah/gqlparser/v2 v2.5.15 h1:fYdnU8roQniJziV5TDiFPm/Ff7pE8xbVSOJqbsdl88A= github.com/vektah/gqlparser/v2 v2.5.15/go.mod h1:WQQjFc+I1YIzoPvZBhUQX7waZgg3pMLi0r8KymvAE2w= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 5e8793132fa322f8c2f2ed6ab3027559f0605314 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 09:06:31 +0200 Subject: [PATCH 22/25] Fix logic for ExtractImageInfo --- scanner/keppel/scanner/scanner.go | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/scanner/keppel/scanner/scanner.go b/scanner/keppel/scanner/scanner.go index 558ec488..d9ce3ecd 100644 --- a/scanner/keppel/scanner/scanner.go +++ b/scanner/keppel/scanner/scanner.go @@ -264,33 +264,26 @@ func (s *Scanner) GetTrivyReport(account string, repository string, manifest str // from a container image func (s *Scanner) ExtractImageInfo(image string) (ImageInfo, error) { // Split the string to remove the tag - parts := strings.Split(image, ":") - if len(parts) < 1 { + imageAndTag := strings.Split(image, ":") + if len(imageAndTag) < 1 { return ImageInfo{}, fmt.Errorf("invalid image") } // Split the remaining string by '/' - components := strings.Split(parts[0], "/") - if len(components) < 3 || len(components) > 4 { - return ImageInfo{}, fmt.Errorf("invalid image string format: expected 3 or 4 components") + parts := strings.Split(imageAndTag[0], "/") + if len(parts) < 3 { + return ImageInfo{}, fmt.Errorf("invalid image string format: at least registry, account and repository required") } info := ImageInfo{ - Registry: components[0], - Repository: components[len(components)-1], + Registry: parts[0], + Account: parts[1], + Repository: strings.Join(parts[2:], "/"), } - // Set tag - if len(parts) > 2 { - info.Tag = parts[1] - } - - // Set organization and registry - if len(components) == 3 { - info.Account = components[1] - } else { // len(components) == 4 - info.Account = components[1] - info.Organization = components[2] + // Set tag if present + if len(imageAndTag) > 1 { + info.Tag = imageAndTag[1] } return info, nil From cd9f8a0c2562a45fd6e90605a8977149f69c19ca Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 09:07:37 +0200 Subject: [PATCH 23/25] Add unit tests for ExtractImageInfo --- scanner/keppel/scanner/scanner_test.go | 138 +++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 scanner/keppel/scanner/scanner_test.go diff --git a/scanner/keppel/scanner/scanner_test.go b/scanner/keppel/scanner/scanner_test.go new file mode 100644 index 00000000..68cd93fc --- /dev/null +++ b/scanner/keppel/scanner/scanner_test.go @@ -0,0 +1,138 @@ +package scanner_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + + "github.com/cloudoperators/heureka/scanners/keppel/scanner" +) + +func TestScanner(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Scanner.Keppel Scanner Suite") +} + +var _ = Describe("Scanner", func() { + var ( + testScanner *scanner.Scanner + ) + + BeforeEach(func() { + testScanner = scanner.NewScanner(scanner.Config{}) + }) + + Describe("ExtractImageInfo", func() { + Context("with valid image names", func() { + DescribeTable("should correctly parse", + func(image string, expected scanner.ImageInfo) { + info, err := testScanner.ExtractImageInfo(image) + Expect(err).NotTo(HaveOccurred()) + Expect(info).To(Equal(expected)) + }, + Entry("three components without tag", + "registry.example.com/account/repo", + scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "repo", + }, + ), + Entry("with organization", + "registry.example.com/account/org/repo", + scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "org/repo", + }, + ), + Entry("with tag", + "registry.example.com/account/repo:latest", + scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "repo", + Tag: "latest", + }, + ), + Entry("with organization and tag", + "registry.example.com/account/org/repo:v1.2.3", + scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "org/repo", + Tag: "v1.2.3", + }, + ), + Entry("with complex tag", + "registry.example.com/account/repo:tag-with-hyphen.123", + scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "repo", + Tag: "tag-with-hyphen.123", + }, + ), + Entry("with dots in registry and repo names", + "k8s.gcr.io/account/nginx.web:1.14.2", + scanner.ImageInfo{ + Registry: "k8s.gcr.io", + Account: "account", + Repository: "nginx.web", + Tag: "1.14.2", + }, + ), + ) + }) + + Context("with invalid image names", func() { + When("image string is empty", func() { + It("should return an error", func() { + _, err := testScanner.ExtractImageInfo("") + Expect(err).To(HaveOccurred()) + }) + }) + + When("image has only registry", func() { + It("should return an error", func() { + _, err := testScanner.ExtractImageInfo("registry.example.com") + Expect(err).To(HaveOccurred()) + }) + }) + + When("image has only registry and account", func() { + It("should return an error", func() { + _, err := testScanner.ExtractImageInfo("registry.example.com/account") + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Context("with edge cases", func() { + When("image has multiple colons", func() { + It("should use the first colon for tag separation", func() { + info, err := testScanner.ExtractImageInfo("registry.example.com/account/org/repo:tag:extra") + Expect(err).NotTo(HaveOccurred()) + Expect(info).To(Equal(scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "org/repo", + Tag: "tag", + })) + }) + }) + + When("image has trailing slash", func() { + It("should handle it correctly", func() { + info, err := testScanner.ExtractImageInfo("registry.example.com/account/repo/") + Expect(err).NotTo(HaveOccurred()) + Expect(info).To(Equal(scanner.ImageInfo{ + Registry: "registry.example.com", + Account: "account", + Repository: "repo/", + })) + }) + }) + }) + }) +}) From 57c883903aa9dbc4df6c215281520d0c01bcad99 Mon Sep 17 00:00:00 2001 From: License Bot Date: Tue, 22 Oct 2024 07:08:46 +0000 Subject: [PATCH 24/25] Automatic application of license header --- scanner/keppel/scanner/scanner_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scanner/keppel/scanner/scanner_test.go b/scanner/keppel/scanner/scanner_test.go index 68cd93fc..1a60e211 100644 --- a/scanner/keppel/scanner/scanner_test.go +++ b/scanner/keppel/scanner/scanner_test.go @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors +// SPDX-License-Identifier: Apache-2.0 + package scanner_test import ( From 0e669bac062d1a3066854ec943989f5e0ba56c20 Mon Sep 17 00:00:00 2001 From: Victor Dorneanu Date: Tue, 22 Oct 2024 11:07:09 +0200 Subject: [PATCH 25/25] Clean-up --- scanner/keppel/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/scanner/keppel/main.go b/scanner/keppel/main.go index ebb3322c..31bb7d43 100644 --- a/scanner/keppel/main.go +++ b/scanner/keppel/main.go @@ -65,7 +65,6 @@ func init() { } func main() { - // var wg sync.WaitGroup var scannerCfg scanner.Config err := envconfig.Process("heureka", &scannerCfg) if err != nil {