From 9636ec8c82003157e70f330ef377a98734a060ea Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Wed, 13 Nov 2024 08:57:00 +0000 Subject: [PATCH 1/4] Fixing socket path Signed-off-by: Amit Schendel --- pkg/utils/utils.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e78f99f8..cfa99f49 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -6,6 +6,7 @@ import ( "crypto/sha1" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "hash" @@ -46,6 +47,7 @@ import ( igtypes "github.com/inspektor-gadget/inspektor-gadget/pkg/types" apitypes "github.com/armosec/armoapi-go/armotypes" + kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" ) var ( @@ -745,6 +747,11 @@ func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterfac if runtimeConfig.Name == igtypes.RuntimeNameUnknown { return nil, fmt.Errorf("unknown container runtime: %s", node.Status.NodeInfo.ContainerRuntimeVersion) } + realSocketPath, err := getContainerRuntimeSocketPath(k8sClient, nodeName) + if err != nil { + return nil, fmt.Errorf("failed to get container runtime socket path from Kubelet configz: %v", err) + } + runtimeConfig.SocketPath = realSocketPath return runtimeConfig, nil } @@ -783,3 +790,48 @@ func parseRuntimeInfo(version string) *containerutilsTypes.RuntimeConfig { } } } + +func getContainerRuntimeSocketPath(clientset *k8sinterface.KubernetesApi, nodeName string) (string, error) { + kubeletConfig, err := getCurrentKubeletConfig(clientset, nodeName) + if err != nil { + return "", fmt.Errorf("getting /configz: %w", err) + } + socketPath, found := strings.CutPrefix(kubeletConfig.ContainerRuntimeEndpoint, "unix://") + if !found { + return "", fmt.Errorf("socket path does not start with unix://") + } + logger.L().Info("using the detected container runtime socket path from Kubelet's config", helpers.String("socketPath", socketPath)) + return socketPath, nil +} + +// The /configz endpoint isn't officially documented. It was introduced in Kubernetes 1.26 and been around for a long time +// as stated in https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/component-base/configz/OWNERS +func getCurrentKubeletConfig(clientset *k8sinterface.KubernetesApi, nodeName string) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + resp, err := clientset.GetKubernetesClient().CoreV1().RESTClient().Get().Resource("nodes"). + Name(nodeName).Suffix("proxy", "configz").DoRaw(context.TODO()) + if err != nil { + return nil, fmt.Errorf("fetching /configz from %q: %w", nodeName, err) + } + kubeCfg, err := decodeConfigz(resp) + if err != nil { + return nil, err + } + return kubeCfg, nil +} + +// Decodes the http response from /configz and returns the kubelet configuration +func decodeConfigz(respBody []byte) (*kubeletconfigv1beta1.KubeletConfiguration, error) { + // This hack because /configz reports the following structure: + // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} + type configzWrapper struct { + ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` + } + + configz := configzWrapper{} + err := json.Unmarshal(respBody, &configz) + if err != nil { + return nil, err + } + + return &configz.ComponentConfig, nil +} From e965ff8129b3fb3990724fbf59c27828def61a0e Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Wed, 13 Nov 2024 09:12:28 +0000 Subject: [PATCH 2/4] go mod tidy + fixing test chart Signed-off-by: Amit Schendel --- go.mod | 2 +- tests/chart/templates/node-agent/clusterrole.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 053666f2..7f15c6e7 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( k8s.io/apimachinery v0.31.1 k8s.io/client-go v0.31.1 k8s.io/kubectl v0.31.0 + k8s.io/kubelet v0.31.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/yaml v1.4.0 ) @@ -252,7 +253,6 @@ require ( k8s.io/cri-api v0.31.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect - k8s.io/kubelet v0.31.1 // indirect oras.land/oras-go/v2 v2.4.0 // indirect sigs.k8s.io/controller-runtime v0.19.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index 480a37f1..04bcae9d 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -6,7 +6,7 @@ metadata: kubescape.io/ignore: "true" rules: - apiGroups: [""] - resources: ["pods", "nodes", "services", "endpoints", "namespaces"] + resources: ["pods", "nodes", "nodes/proxy", "services", "endpoints", "namespaces"] verbs: ["get", "watch", "list"] - apiGroups: [""] resources: ["events"] From fcf682a7158835310134c3437bc0b39a9bd952e8 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Wed, 13 Nov 2024 12:46:11 +0000 Subject: [PATCH 3/4] Adding pod informer instead of enriching using the container runtime Signed-off-by: Amit Schendel --- main.go | 10 +------- pkg/containerwatcher/v1/container_watcher.go | 6 +---- .../v1/container_watcher_private.go | 23 +++++++++++-------- pkg/containerwatcher/v1/open_test.go | 2 +- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index 34e0318b..d3f55a55 100644 --- a/main.go +++ b/main.go @@ -132,14 +132,6 @@ func main() { nodeName := os.Getenv(config.NodeNameEnvVar) - // Detect the container containerRuntime of the node - containerRuntime, err := utils.DetectContainerRuntimeViaK8sAPI(ctx, k8sClient, nodeName) - if err != nil { - logger.L().Ctx(ctx).Fatal("error detecting the container runtime", helpers.Error(err)) - } - - logger.L().Ctx(ctx).Info("Detected container runtime", helpers.String("containerRuntime", containerRuntime.Name.String())) - // Create watchers dWatcher := dynamicwatcher.NewWatchHandler(k8sClient, storageClient.StorageClient, cfg.SkipNamespace) // create k8sObject cache @@ -283,7 +275,7 @@ func main() { } // Create the container handler - mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, relevancyManager, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, preRunningContainersIDs, &ruleBindingNotify, containerRuntime, nil, processManager) + mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, relevancyManager, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, preRunningContainersIDs, &ruleBindingNotify, nil, processManager) if err != nil { logger.L().Ctx(ctx).Fatal("error creating the container watcher", helpers.Error(err)) } diff --git a/pkg/containerwatcher/v1/container_watcher.go b/pkg/containerwatcher/v1/container_watcher.go index 4e1af499..5841f2c8 100644 --- a/pkg/containerwatcher/v1/container_watcher.go +++ b/pkg/containerwatcher/v1/container_watcher.go @@ -8,7 +8,6 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/goradd/maps" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" - containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" tracerseccomp "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/advise/seccomp/tracer" tracercapabilities "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/capabilities/tracer" tracercapabilitiestype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/capabilities/types" @@ -152,15 +151,13 @@ type IGContainerWatcher struct { metrics metricsmanager.MetricsManager // cache ruleBindingPodNotify *chan rulebinding.RuleBindingNotify - // container runtime - runtime *containerutilsTypes.RuntimeConfig // process manager processManager processmanager.ProcessManagerClient } var _ containerwatcher.ContainerWatcher = (*IGContainerWatcher)(nil) -func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager applicationprofilemanager.ApplicationProfileManagerClient, k8sClient *k8sinterface.KubernetesApi, relevancyManager relevancymanager.RelevancyManagerClient, networkManagerClient networkmanager.NetworkManagerClient, dnsManagerClient dnsmanager.DNSManagerClient, metrics metricsmanager.MetricsManager, ruleManager rulemanager.RuleManagerClient, malwareManager malwaremanager.MalwareManagerClient, preRunningContainers mapset.Set[string], ruleBindingPodNotify *chan rulebinding.RuleBindingNotify, runtime *containerutilsTypes.RuntimeConfig, thirdPartyEventReceivers *maps.SafeMap[utils.EventType, mapset.Set[containerwatcher.EventReceiver]], processManager processmanager.ProcessManagerClient) (*IGContainerWatcher, error) { +func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager applicationprofilemanager.ApplicationProfileManagerClient, k8sClient *k8sinterface.KubernetesApi, relevancyManager relevancymanager.RelevancyManagerClient, networkManagerClient networkmanager.NetworkManagerClient, dnsManagerClient dnsmanager.DNSManagerClient, metrics metricsmanager.MetricsManager, ruleManager rulemanager.RuleManagerClient, malwareManager malwaremanager.MalwareManagerClient, preRunningContainers mapset.Set[string], ruleBindingPodNotify *chan rulebinding.RuleBindingNotify, thirdPartyEventReceivers *maps.SafeMap[utils.EventType, mapset.Set[containerwatcher.EventReceiver]], processManager processmanager.ProcessManagerClient) (*IGContainerWatcher, error) { // Use container collection to get notified for new containers containerCollection := &containercollection.ContainerCollection{} // Create a tracer collection instance @@ -450,7 +447,6 @@ func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager appli ruleBindingPodNotify: ruleBindingPodNotify, timeBasedContainers: mapset.NewSet[string](), ruleManagedPods: mapset.NewSet[string](), - runtime: runtime, thirdPartyTracers: mapset.NewSet[containerwatcher.CustomTracer](), thirdPartyContainerReceivers: mapset.NewSet[containerwatcher.ContainerReceiver](), processManager: processManager, diff --git a/pkg/containerwatcher/v1/container_watcher_private.go b/pkg/containerwatcher/v1/container_watcher_private.go index 55e6451d..f703a333 100644 --- a/pkg/containerwatcher/v1/container_watcher_private.go +++ b/pkg/containerwatcher/v1/container_watcher_private.go @@ -107,16 +107,14 @@ func (ch *IGContainerWatcher) startContainerCollection(ctx context.Context) erro // Enrich events with Linux namespaces information, it is needed for per container filtering containercollection.WithLinuxNamespaceEnrichment(), - // Get containers created with container runtimes - containercollection.WithContainerRuntimeEnrichment(ch.runtime), - - // Get containers created with ebpf (works also if hostPid=false) - containercollection.WithContainerFanotifyEbpf(), + // Enrich those containers with data from the Kubernetes API + containercollection.WithKubernetesEnrichment(ch.nodeName, ch.k8sClient.K8SConfig), + // WithTracerCollection enables the interation between the TracerCollection and ContainerCollection packages. containercollection.WithTracerCollection(ch.tracerCollection), - // Enrich those containers with data from the Kubernetes API - containercollection.WithKubernetesEnrichment(ch.nodeName, ch.k8sClient.K8SConfig), + // WithPodInformer uses a pod informer to get both initial containers and the stream of container events. + containercollection.WithPodInformer(ch.nodeName), } // Initialize the container collection @@ -124,7 +122,10 @@ func (ch *IGContainerWatcher) startContainerCollection(ctx context.Context) erro return fmt.Errorf("initializing container collection: %w", err) } - // add containers that are already running + // Add pre-running containers to preRunningContainersIDs and ruleManagedPods, this is needed for the following reasons: + // 1. For runtime threat detection we want to keep monitoring the containers and not stop monitoring them after the max sniffing time. + // 2. To know in different parts of the code if a container "learning" is partial or complete. + // In addition, this routine will keep monitoring for rule bindings notifications for the entire lifecycle of the node-agent. go ch.startRunningContainers() return nil @@ -182,7 +183,11 @@ func (ch *IGContainerWatcher) addRunningContainers(k8sClient IGK8sClient, notf * helpers.String("containerName", runningContainers[i].K8s.ContainerName)) ch.preRunningContainersIDs.Add(runningContainers[i].Runtime.ContainerID) - ch.containerCollection.AddContainer(&runningContainers[i]) + if ch.containerCollection.GetContainer(runningContainers[i].Runtime.ContainerID) == nil { + // This should not happen, but just in case (it should be added by the podInformer). + logger.L().Info("adding container - pod managed by rules", helpers.String("ContainerName", runningContainers[i].K8s.ContainerName)) + ch.containerCollection.AddContainer(&runningContainers[i]) + } } } diff --git a/pkg/containerwatcher/v1/open_test.go b/pkg/containerwatcher/v1/open_test.go index 7fb4e084..7c91dd6f 100644 --- a/pkg/containerwatcher/v1/open_test.go +++ b/pkg/containerwatcher/v1/open_test.go @@ -23,7 +23,7 @@ func BenchmarkIGContainerWatcher_openEventCallback(b *testing.B) { assert.NoError(b, err) mockExporter := metricsmanager.NewMetricsMock() - mainHandler, err := CreateIGContainerWatcher(cfg, nil, nil, relevancyManager, nil, nil, mockExporter, nil, nil, nil, nil, nil, nil, nil) + mainHandler, err := CreateIGContainerWatcher(cfg, nil, nil, relevancyManager, nil, nil, mockExporter, nil, nil, nil, nil, nil, nil) assert.NoError(b, err) event := &traceropentype.Event{ Event: types.Event{ From 088ed863cb0748bc70e6f53cc40d372e0390833c Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Wed, 13 Nov 2024 15:58:47 +0000 Subject: [PATCH 4/4] Removing memory leak test Signed-off-by: Amit Schendel --- .github/workflows/component-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index 2c741b92..1b1f9444 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -44,7 +44,7 @@ jobs: Test_02_AllAlertsFromMaliciousApp, Test_03_BasicLoadActivities, # Test_04_MemoryLeak, - Test_05_MemoryLeak_10K_Alerts, + # Test_05_MemoryLeak_10K_Alerts, Test_06_KillProcessInTheMiddle, Test_07_RuleBindingApplyTest, Test_08_ApplicationProfilePatching,