Skip to content

Commit

Permalink
support updates
Browse files Browse the repository at this point in the history 
Signed-off-by: David Wertenteil <[email protected]>
  • Loading branch information
David Wertenteil committed Jan 15, 2024
1 parent 06756ec commit e4cff75
Show file tree
Hide file tree
Showing 10 changed files with 233 additions and 146 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/kinbiko/jsonassert v1.1.1
github.com/kubescape/backend v0.0.16
github.com/kubescape/go-logger v0.0.22
github.com/kubescape/k8s-interface v0.0.156
github.com/kubescape/k8s-interface v0.0.157
github.com/kubescape/storage v0.0.48-0.20231220083733-77c05196c95c
github.com/panjf2000/ants/v2 v2.9.0
github.com/spf13/viper v1.18.2
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -492,8 +492,8 @@ github.com/kubescape/backend v0.0.16 h1:bkQGY39GSoNIeFfnAJ2zlcrGEyXk6LGYv1/MgS51
github.com/kubescape/backend v0.0.16/go.mod h1:ug9NFmmxT4DcQx3sgdLRzlLPWMKGHE/fpbcYUm5G5Qo=
github.com/kubescape/go-logger v0.0.22 h1:gle7wH6emOiGv9ljdpVi82pWLQ3jGucrUucvil6JXHE=
github.com/kubescape/go-logger v0.0.22/go.mod h1:x3HBpZo3cMT/WIdy18BxvVVd5D0e/PWFVk/HiwBNu3g=
github.com/kubescape/k8s-interface v0.0.156 h1:rzKxOgCdfmOkztO2cOQiOwNOHIynSFo4MpICB4Xz/ZM=
github.com/kubescape/k8s-interface v0.0.156/go.mod h1:5sz+5Cjvo98lTbTVDiDA4MmlXxeHSVMW/wR0V3hV4K8=
github.com/kubescape/k8s-interface v0.0.157 h1:3IbGpoPtuQ3KzEQcbCTaMOJL15LkF70xmwHWxI+dvWM=
github.com/kubescape/k8s-interface v0.0.157/go.mod h1:5sz+5Cjvo98lTbTVDiDA4MmlXxeHSVMW/wR0V3hV4K8=
github.com/kubescape/storage v0.0.48-0.20231220083733-77c05196c95c h1:BTtIYOv9a5ojJ8whNYFkv9aZTcWK+dU0Dn/Pj9Noh9A=
github.com/kubescape/storage v0.0.48-0.20231220083733-77c05196c95c/go.mod h1:YYoMvVq7iHIfFhDgRFF1yQ0ItaSKtl5483fLJehZ6Wk=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
Expand Down
36 changes: 31 additions & 5 deletions pkg/relevancymanager/v1/relevancy_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ func (rm *RelevancyManager) getContainerInfo(namespace, podName, containerName s
if err != nil {
return imageID, imageTag, parentWlid, parentResourceVersion, podTemplateHash, instanceID, fmt.Errorf("WLID of parent workload is not in the right %s in namespace %s with error: %v", pod.GetName(), pod.GetNamespace(), err)
}

// find imageTag
containers, err := pod.GetContainers()
if err != nil {
Expand All @@ -117,6 +118,20 @@ func (rm *RelevancyManager) getContainerInfo(namespace, podName, containerName s
imageTag = containers[i].Image
}
}

if imageTag == "" {
// search in init containers
initContainers, err := pod.GetInitContainers()
if err != nil {
return imageID, imageTag, parentWlid, parentResourceVersion, podTemplateHash, instanceID, fmt.Errorf("fail to get containers for pod %s in namespace %s with error: %v", podName, namespace, err)
}
for i := range initContainers {
if initContainers[i].Name == containerName {
imageTag = containers[i].Image
}
}
}

if imageTag == "" {
return imageID, imageTag, parentWlid, parentResourceVersion, podTemplateHash, instanceID, fmt.Errorf("fail to find container %s in pod %s in namespace %s", containerName, podName, namespace)
}
Expand All @@ -130,9 +145,18 @@ func (rm *RelevancyManager) getContainerInfo(namespace, podName, containerName s
imageID = status.ContainerStatuses[i].ImageID
}
}
if imageID == "" {
// search in init containers
for i := range status.InitContainerStatuses {
if status.InitContainerStatuses[i].Name == containerName {
imageID = status.InitContainerStatuses[i].ImageID
}
}
}
if imageID == "" {
return imageID, imageTag, parentWlid, parentResourceVersion, podTemplateHash, instanceID, fmt.Errorf("fail to find container status %s in pod %s in namespace %s", containerName, podName, namespace)
}

// find instanceID
instanceIDs, err := instanceidhandlerV1.GenerateInstanceID(pod)
if err != nil {
Expand Down Expand Up @@ -205,11 +229,13 @@ func (rm *RelevancyManager) startRelevancyProcess(ctx context.Context, container
defer span.End()

watchedContainer := &utils.WatchedContainerData{
ContainerID: container.Runtime.ContainerID,
UpdateDataTicker: time.NewTicker(rm.cfg.InitialDelay),
SyncChannel: make(chan error, 10),
K8sContainerID: k8sContainerID,
RelevantSyftFilesByIdentifier: make(map[string]bool),
ContainerID: container.Runtime.ContainerID,
UpdateDataTicker: time.NewTicker(rm.cfg.InitialDelay),
SyncChannel: make(chan error, 10),
K8sContainerID: k8sContainerID,
RelevantRelationshipsArtifactsByIdentifier: make(map[string]bool),
RelevantArtifactsFilesByIdentifier: make(map[string]bool),
RelevantRealtimeFilesByIdentifier: make(map[string]bool),
}
rm.watchedContainerChannels.Set(watchedContainer.ContainerID, watchedContainer.SyncChannel)

Expand Down
194 changes: 113 additions & 81 deletions pkg/sbomhandler/syfthandler/syft_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,26 +29,10 @@ func CreateSyftSBOMHandler(sc storage.StorageClient) *SyftHandler {
func (sc *SyftHandler) FilterSBOM(watchedContainer *utils.WatchedContainerData, sbomFileRelevantMap map[string]bool) error {

if watchedContainer.InstanceID == nil {
return fmt.Errorf("instance id is nil")
}

filteredSBOMKey, err := watchedContainer.InstanceID.GetSlug()
if err != nil {
return err
return nil
}

filteredSyftSBOM := v1beta1.SBOMSyftFiltered{
ObjectMeta: metav1.ObjectMeta{
Name: filteredSBOMKey,
Annotations: map[string]string{
instanceidhandler.WlidMetadataKey: watchedContainer.Wlid,
instanceidhandler.InstanceIDMetadataKey: watchedContainer.InstanceID.GetStringFormatted(),
instanceidhandler.ContainerNameMetadataKey: watchedContainer.InstanceID.GetContainerName(),
instanceidhandler.ImageIDMetadataKey: watchedContainer.ImageID,
},
Labels: utils.GetLabels(watchedContainer, false),
},
}
newRelevantData := false

// retrieve SBOM from storage
SBOMKey, err := names.ImageInfoToSlug(watchedContainer.ImageTag, watchedContainer.ImageID)
Expand All @@ -60,111 +44,155 @@ func (sc *SyftHandler) FilterSBOM(watchedContainer *utils.WatchedContainerData,
if err != nil {
return err
}
if syftData == nil {
return nil
}

// check SBOM is complete
if status, ok := syftData.Annotations[instanceidhandler.StatusMetadataKey]; ok {
if status == instanceidhandler.Incomplete {
watchedContainer.SyncChannel <- utils.IncompleteSBOMError
if syftData.Annotations != nil {
if status, ok := syftData.Annotations[instanceidhandler.StatusMetadataKey]; ok {
if status == instanceidhandler.Incomplete {
watchedContainer.SyncChannel <- utils.IncompleteSBOMError
}
}
}

newRelevantData := false
if watchedContainer.SBOMSyftFiltered == nil {
filteredSBOMKey, err := watchedContainer.InstanceID.GetSlug()
if err != nil {
return err
}

if filteredSBOM, err := sc.storageClient.GetFilteredSBOM(filteredSBOMKey); err != nil {
logger.L().Debug("filtered SBOM not found, creating new one", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID), helpers.String("filtered SBOM", filteredSBOMKey))
watchedContainer.SBOMSyftFiltered = &v1beta1.SBOMSyftFiltered{
ObjectMeta: metav1.ObjectMeta{
Name: filteredSBOMKey,
Annotations: map[string]string{
instanceidhandler.WlidMetadataKey: watchedContainer.Wlid,
instanceidhandler.InstanceIDMetadataKey: watchedContainer.InstanceID.GetStringFormatted(),
instanceidhandler.ContainerNameMetadataKey: watchedContainer.InstanceID.GetContainerName(),
instanceidhandler.ImageIDMetadataKey: watchedContainer.ImageID,
},
Labels: utils.GetLabels(watchedContainer, false),
},
}
} else {
logger.L().Debug("filtered SBOM found, using it", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID), helpers.String("filtered SBOM", filteredSBOMKey))
filteredSBOM.ObjectMeta.ResourceVersion = ""
filteredSBOM.ObjectMeta.CreationTimestamp = metav1.Time{}
filteredSBOM.ObjectMeta.UID = ""

for i := range filteredSBOM.Spec.Syft.ArtifactRelationships {
watchedContainer.RelevantRelationshipsArtifactsByIdentifier[getRelationshipID(filteredSBOM.Spec.Syft.ArtifactRelationships[i])] = true
}
for i := range filteredSBOM.Spec.Syft.Files {
watchedContainer.RelevantRealtimeFilesByIdentifier[filteredSBOM.Spec.Syft.Files[i].ID] = true
}
for i := range filteredSBOM.Spec.Syft.Artifacts {
watchedContainer.RelevantArtifactsFilesByIdentifier[filteredSBOM.Spec.Syft.Artifacts[i].ID] = true
}

watchedContainer.SBOMSyftFiltered = filteredSBOM
}
}

// if resource version is higher than in-memory, delete saved relevancy info
if syftData != nil && utils.Atoi(syftData.ResourceVersion) > watchedContainer.SBOMResourceVersion {
logger.L().Debug("SBOM resource version is higher than the one in memory, parsing SBOM", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID))
if utils.Atoi(syftData.ResourceVersion) > watchedContainer.SBOMResourceVersion {
logger.L().Debug("SBOM resource version is higher than the one in memory, parsing SBOM", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID), helpers.Int("in memory", watchedContainer.SBOMResourceVersion), helpers.Int("in storage", utils.Atoi(syftData.ResourceVersion)))

for _, file := range syftData.Spec.Syft.Files {
watchedContainer.RelevantSyftFilesByIdentifier[file.ID] = false
if _, ok := watchedContainer.RelevantRealtimeFilesByIdentifier[file.ID]; ok {
watchedContainer.RelevantRealtimeFilesByIdentifier[file.ID] = false
}
}
// save files, packages and relationships
files := watchedContainer.SBOMSyftFiltered.Spec.Syft.Files
packages := watchedContainer.SBOMSyftFiltered.Spec.Syft.Artifacts
relationships := watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships

// copy spec and status
watchedContainer.SBOMSyftFiltered.Spec = syftData.Spec
watchedContainer.SBOMSyftFiltered.Status = syftData.Status

// restore files, packages and relationships
watchedContainer.SBOMSyftFiltered.Spec.Syft.Files = files
watchedContainer.SBOMSyftFiltered.Spec.Syft.Artifacts = packages
watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships = relationships

// update resource version
watchedContainer.SBOMResourceVersion = utils.Atoi(syftData.ResourceVersion)
newRelevantData = true
}

// check if there are new relevant files
for _, file := range syftData.Spec.Syft.Files {
if sbomFileRelevantMap[file.Location.RealPath] && !watchedContainer.RelevantSyftFilesByIdentifier[file.ID] {
newRelevantData = true
break
}
}
filterRelevantFilesInSBOM(watchedContainer, syftData.Spec.Syft, sbomFileRelevantMap, &newRelevantData)

if !newRelevantData {
return nil
}

filteredSyftDoc := filterRelevantFilesInSBOM(syftData.Spec.Syft, sbomFileRelevantMap)

// update relevant files internally
watchedContainer.RelevantSyftFilesByIdentifier = make(map[string]bool, 0)
for _, file := range filteredSyftDoc.Files {
watchedContainer.RelevantSyftFilesByIdentifier[file.ID] = true
}

filteredSyftSBOM.Spec.Syft = filteredSyftDoc

if err = sc.storageClient.CreateFilteredSBOM(&filteredSyftSBOM); err != nil {
if err = sc.storageClient.CreateFilteredSBOM(watchedContainer.SBOMSyftFiltered); err != nil {
return err
}
logger.L().Info("filtered SBOM has been stored successfully", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID))
logger.L().Info("filtered SBOM has been stored successfully", helpers.String("containerID", watchedContainer.ContainerID), helpers.String("k8s workload", watchedContainer.K8sContainerID), helpers.String("filtered SBOM", watchedContainer.SBOMSyftFiltered.Name))

return nil
}

func filterRelevantFilesInSBOM(syftDoc v1beta1.SyftDocument, sbomFileRelevantMap map[string]bool) v1beta1.SyftDocument {
relevantSBOM := v1beta1.SyftDocument{
SyftSource: syftDoc.SyftSource,
Distro: syftDoc.Distro,
SyftDescriptor: syftDoc.SyftDescriptor,
Schema: syftDoc.Schema,
}
func filterRelevantFilesInSBOM(watchedContainer *utils.WatchedContainerData, syftDoc v1beta1.SyftDocument, sbomFileRelevantMap map[string]bool, newRelevantData *bool) {

// build map of relevant file IDs
relevantFileIdentifiers := make(map[string]bool, 0)
relevantFiles := make([]v1beta1.SyftFile, 0)
for _, file := range syftDoc.Files {
if sbomFileRelevantMap[file.Location.RealPath] {
relevantFileIdentifiers[file.ID] = true
relevantFiles = append(relevantFiles, file)
// filter relevant file list
for i := range syftDoc.Files {
// the .location.realPath is not the ID of the file, that's why the map identifier is the ID and not the path
if _, k := watchedContainer.RelevantRealtimeFilesByIdentifier[syftDoc.Files[i].ID]; !k {
if _, ok := sbomFileRelevantMap[syftDoc.Files[i].Location.RealPath]; ok {
watchedContainer.SBOMSyftFiltered.Spec.Syft.Files = append(watchedContainer.SBOMSyftFiltered.Spec.Syft.Files, syftDoc.Files[i])
watchedContainer.RelevantRealtimeFilesByIdentifier[syftDoc.Files[i].ID] = true
*(newRelevantData) = true
}
}
}

// filter relevant relationships. A relationship is relevant if the child is a relevant file or artifact
relevantRelationshipsArtifactsByIdentifier := make(map[string]bool, 0)
relevantRelationships := make([]v1beta1.SyftRelationship, 0)
for _, relationship := range syftDoc.ArtifactRelationships {
if _, ok := relevantFileIdentifiers[relationship.Child]; ok {
relevantRelationshipsArtifactsByIdentifier[relationship.Parent] = true
if !*newRelevantData {
return
}

relevantRelationships = append(relevantRelationships, relationship)
// filter relevant relationships. A relationship is relevant if the child is a relevant file
relationshipsArtifacts := make(map[string]bool, 0)
for _, relationship := range syftDoc.ArtifactRelationships {
if _, ok := watchedContainer.RelevantRelationshipsArtifactsByIdentifier[getRelationshipID(relationship)]; ok {
continue
}
if _, ok := watchedContainer.RelevantRealtimeFilesByIdentifier[relationship.Child]; ok { // if the child is a relevant file
relationshipsArtifacts[relationship.Parent] = true
watchedContainer.RelevantRelationshipsArtifactsByIdentifier[getRelationshipID(relationship)] = true
watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships = append(watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships, relationship)
}
}

// loop again so we can add the artifacts relationships where the child is an relevant artifact
// Add children of relevant relationships (that the parent is not relevant)
for _, relationship := range syftDoc.ArtifactRelationships {
if _, ok := relevantFileIdentifiers[relationship.Child]; ok {
if _, ok := relevantRelationshipsArtifactsByIdentifier[relationship.Parent]; ok {
continue
}
relevantRelationshipsArtifactsByIdentifier[relationship.Parent] = true

relevantRelationships = append(relevantRelationships, relationship)
if _, ok := watchedContainer.RelevantRelationshipsArtifactsByIdentifier[getRelationshipID(relationship)]; ok {
continue
}
if _, ok := relationshipsArtifacts[relationship.Child]; ok {
relationshipsArtifacts[relationship.Parent] = true
watchedContainer.RelevantRelationshipsArtifactsByIdentifier[getRelationshipID(relationship)] = true
watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships = append(watchedContainer.SBOMSyftFiltered.Spec.Syft.ArtifactRelationships, relationship)
}
}

relevantArtifacts := make([]v1beta1.SyftPackage, 0)
// filter relevant artifacts. An artifact is relevant if it is in the relevant relationships
for _, artifact := range syftDoc.Artifacts {
if _, ok := relevantRelationshipsArtifactsByIdentifier[artifact.ID]; ok {
relevantArtifacts = append(relevantArtifacts, artifact)
if _, ok := watchedContainer.RelevantArtifactsFilesByIdentifier[artifact.ID]; ok {
continue
}
if _, ok := relationshipsArtifacts[artifact.ID]; ok {
watchedContainer.SBOMSyftFiltered.Spec.Syft.Artifacts = append(watchedContainer.SBOMSyftFiltered.Spec.Syft.Artifacts, artifact)
watchedContainer.RelevantArtifactsFilesByIdentifier[artifact.ID] = true
}
}

relevantSBOM.Files = relevantFiles
relevantSBOM.Artifacts = relevantArtifacts
relevantSBOM.ArtifactRelationships = relevantRelationships

return relevantSBOM
}

func (sc *SyftHandler) IncrementImageUse(imageID string) {
Expand All @@ -173,3 +201,7 @@ func (sc *SyftHandler) IncrementImageUse(imageID string) {
func (sc *SyftHandler) DecrementImageUse(imageID string) {

}

func getRelationshipID(relationship v1beta1.SyftRelationship) string {
return fmt.Sprintf("%s/%s/%s", relationship.Parent, relationship.Child, relationship.Type)
}
Loading

0 comments on commit e4cff75

Please sign in to comment.