Skip to content

Commit

Permalink
fix: fetch sub-manifests from keppel (#275)
Browse files Browse the repository at this point in the history
* fix: fetch sub-manifests from keppel

* refactor: renamed method and added comment

* refactor: combined two error cases into one condition
  • Loading branch information
drochow authored Oct 9, 2024
1 parent 95e1c1e commit f04feab
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 34 deletions.
67 changes: 47 additions & 20 deletions scanner/k8s-assets/processor/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,14 @@ func (p *Processor) ProcessService(ctx context.Context, serviceInfo scanner.Serv
supportGroupId = _supportGroupId
}

_, _ = client.AddServiceToSupportGroup(ctx, *p.Client, serviceId, supportGroupId)
_, err = client.AddServiceToSupportGroup(ctx, *p.Client, supportGroupId, serviceId)

if err != nil {
log.WithError(err).WithFields(log.Fields{
"serviceName": serviceInfo.Name,
"supportGroup": serviceInfo.SupportGroup,
}).Warning("Failed adding service to support group")
}

return serviceId, nil
}
Expand All @@ -141,7 +148,7 @@ func (p *Processor) getSupportGroup(ctx context.Context, serviceInfo scanner.Ser
}

// Return the first item
if listSupportGroupsResp.SupportGroups.TotalCount > 0 {
if listSupportGroupsResp.SupportGroups.TotalCount > 0 && len(listSupportGroupsResp.SupportGroups.Edges) > 0 {
supportGroupId = listSupportGroupsResp.SupportGroups.Edges[0].Node.Id
} else {
return "", fmt.Errorf("ListSupportGroups returned no SupportGroupID")
Expand All @@ -152,8 +159,6 @@ func (p *Processor) getSupportGroup(ctx context.Context, serviceInfo scanner.Ser

// getService returns (if any) a ServiceID
func (p *Processor) getService(ctx context.Context, serviceInfo scanner.ServiceInfo) (string, error) {
var serviceId string

listServicesFilter := client.ServiceFilter{
ServiceName: []string{serviceInfo.Name},
}
Expand All @@ -164,15 +169,10 @@ func (p *Processor) getService(ctx context.Context, serviceInfo scanner.ServiceI

// Return the first item
if listServicesResp.Services.TotalCount > 0 {
for _, s := range listServicesResp.Services.Edges {
serviceId = s.Node.Id
break
}
} else {
return "", fmt.Errorf("ListServices returned no ServiceID")
return listServicesResp.Services.Edges[0].Node.Id, nil
}

return serviceId, nil
return "", fmt.Errorf("ListServices returned no ServiceID")
}

// CollectUniqueContainers processes a PodReplicaSet and returns a slice of
Expand Down Expand Up @@ -232,8 +232,6 @@ func (p *Processor) ProcessPodReplicaSet(ctx context.Context, namespace string,
}

func (p *Processor) getComponentVersion(ctx context.Context, versionHash string) (string, error) {
var componentVersionId string

//separating image name and version hash
imageAndVersion := strings.SplitN(versionHash, "@", 2)
if len(imageAndVersion) < 2 {
Expand All @@ -242,6 +240,39 @@ func (p *Processor) getComponentVersion(ctx context.Context, versionHash string)
image := imageAndVersion[0]
version := imageAndVersion[1]

//@todo Temporary Start
// AS we do not scan all the individual registries, we replace the registry string
// this is a temporary "hack" we need to move this to the heureka core with a registry configuration
//so that the respective versions are created correctly during version creation

var myMap map[string]string = make(map[string]string)
myMap["keppel.global.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.qa-de-1.cloud.sap/ccloud-mirror"] = "keppel.eu-de-1.cloud.sap/ccloud"
myMap["keppel.eu-de-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.s-eu-de-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.na-us-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.na-us-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-jp-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-jp-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.na-us-3.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.na-ca-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.eu-nl-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-ae-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-sa-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-sa-2.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-cn-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
myMap["keppel.ap-au-1.cloud.sap"] = "keppel.eu-de-1.cloud.sap"
var images []string = make([]string, 1)

for replace, with := range myMap {
if strings.Contains(image, replace) {
image = strings.Replace(image, replace, with, 1)
}
}
//@todo Temporary End

images[0] = image

listComponentVersionFilter := client.ComponentVersionFilter{
ComponentName: []string{image},
Version: []string{version},
Expand All @@ -252,14 +283,10 @@ func (p *Processor) getComponentVersion(ctx context.Context, versionHash string)
}

if listCompoVersResp.ComponentVersions.TotalCount > 0 {
for _, cv := range listCompoVersResp.ComponentVersions.Edges {
componentVersionId = cv.Node.Id
break
}
} else {
return "", fmt.Errorf("ListComponentVersion returned no ComponentVersion objects")
return listCompoVersResp.ComponentVersions.Edges[0].Node.Id, nil
}
return componentVersionId, nil

return "", fmt.Errorf("ListComponentVersion returned no ComponentVersion objects")
}

// ProcessContainer creates a ComponentVersion and ComponentInstance for a container
Expand Down
57 changes: 43 additions & 14 deletions scanner/keppel/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func HandleAccount(fqdn string, account models.Account, keppelScanner *scanner.S
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
}).WithError(err).Error("Error during ProcessRepository")
}).WithError(err).Error("Error during listing ProcessRepository")
return err
}

Expand Down Expand Up @@ -129,37 +129,66 @@ func HandleRepository(fqdn string, account models.Account, repository models.Rep
}).Error("Component not found")
return
}
if manifest.VulnerabilityStatus == "Unsupported" {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).Warn("Manifest has UNSUPPORTED type: " + manifest.MediaType)
continue
}
if manifest.VulnerabilityStatus == "Clean" {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).Info("Manifest has no Vulnerabilities")
continue
}
HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor)
}
}

func HandleManifest(account models.Account, repository models.Repository, manifest models.Manifest, component *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) {
childManifests, err := keppelScanner.ListChildManifests(account.Name, repository.Name, manifest.Digest)

if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ListChildManifests")
}

componentVersion, err := keppelProcessor.ProcessManifest(manifest, component.Id)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ProcessManifest")
componentVersion, err = keppelProcessor.GetComponentVersion(manifest.Digest)
if err != nil {
if err != nil || componentVersion == nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetComponentVersion")
return
}
}
trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, manifest.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetTrivyReport")
return
}

if trivyReport == nil {
return
}
childManifests = append(childManifests, manifest)

for _, m := range childManifests {
trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, m.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetTrivyReport")
return
}

keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id)
if trivyReport == nil {
return
}

keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id)
}
}
1 change: 1 addition & 0 deletions scanner/keppel/processor/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ func (p *Processor) ProcessManifest(manifest models.Manifest, componentId string
})

if err != nil {
log.WithError(err).Error("Error while creating component")
return nil, err
}

Expand Down
39 changes: 39 additions & 0 deletions scanner/keppel/scanner/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,36 @@ func (s *Scanner) ListManifests(account string, repository string) ([]models.Man
return manifestResponse.Manifests, nil
}

// ListChildManifests is requred asa on Keppel not all Images are including vulnerability scan results directly on the
// top layer of the image and rather have the scan results on the child manifests. An prime example of this are multi-arch
// images where the scan results are available on the child manifests with the respective concrete architecture.
// This method is using the v2 API endpoint as on the v1 of the API the child manifests listing is not available.
//
// Note: The v2 API does return slightly different results and therefore some of the fileds of models.Manifest are unset.
// This fact is accepted and no additional struct for parsing all information is implemented at this point in time
// as the additional available information is currently not utilized.
func (s *Scanner) ListChildManifests(account string, repository string, manifest string) ([]models.Manifest, error) {
url := fmt.Sprintf("%s/v2/%s/%s/manifests/%s", s.KeppelBaseUrl, account, repository, manifest)
body, err := s.sendRequest(url, s.AuthToken)
if err != nil {
log.WithFields(log.Fields{
"url": url,
}).WithError(err).Error("Error during request in ListManifests")
return nil, err
}

var manifestResponse models.ManifestResponse
if err = json.Unmarshal(body, &manifestResponse); err != nil {
log.WithFields(log.Fields{
"url": url,
"body": body,
}).WithError(err).Error("Error during unmarshal in ListManifests")
return nil, err
}

return manifestResponse.Manifests, nil
}

func (s *Scanner) GetTrivyReport(account string, repository string, manifest string) (*models.TrivyReport, error) {
url := fmt.Sprintf("%s/keppel/v1/accounts/%s/repositories/%s/_manifests/%s/trivy_report", s.KeppelBaseUrl, account, repository, manifest)
body, err := s.sendRequest(url, s.AuthToken)
Expand All @@ -172,6 +202,14 @@ func (s *Scanner) GetTrivyReport(account string, repository string, manifest str

var trivyReport models.TrivyReport
if err = json.Unmarshal(body, &trivyReport); err != nil {
if strings.Contains(string(body), "not") {
log.WithFields(log.Fields{
"url": url,
"body": body,
}).Info("Trivy report not found")
return nil, fmt.Errorf("Trivy report not found")
}

log.WithFields(log.Fields{
"url": url,
"body": body,
Expand All @@ -195,6 +233,7 @@ func (s *Scanner) sendRequest(url string, token string) ([]byte, error) {
}

resp, err := client.Do(req)

if err != nil {
return nil, err
}
Expand Down

0 comments on commit f04feab

Please sign in to comment.