From 4ce8423e8e804a1dfb4a6c46d088ee619fcf1285 Mon Sep 17 00:00:00 2001 From: Qiyue Yao Date: Mon, 29 Apr 2024 11:26:50 -0700 Subject: [PATCH] Add E2E test for L7 NetworkPolicy Logging This PR adds an E2E test for L7 NetworkPolicy logging. It checks both allowed and dropped HTTP event logs under l7engine directory. Signed-off-by: Qiyue Yao --- test/e2e/antreaipam_test.go | 2 +- test/e2e/framework.go | 15 +++- test/e2e/l7networkpolicy_test.go | 135 ++++++++++++++++++++++++------- test/e2e/traceflow_test.go | 2 +- test/e2e/trafficcontrol_test.go | 4 +- 5 files changed, 124 insertions(+), 34 deletions(-) diff --git a/test/e2e/antreaipam_test.go b/test/e2e/antreaipam_test.go index be2e35e5d28..0f1810b571b 100644 --- a/test/e2e/antreaipam_test.go +++ b/test/e2e/antreaipam_test.go @@ -446,7 +446,7 @@ func testAntreaIPAMStatefulSet(t *testing.T, data *TestData, dedicatedIPPoolKey if dedicatedIPPoolKey != nil { podAnnotations[annotation.AntreaIPAMAnnotationKey] = ipPoolName } - err = NewPodBuilder(podName, testAntreaIPAMNamespace, agnhostImage).OnNode(controlPlaneNodeName()).WithCommand([]string{"sleep", "3600"}).WithAnnotations(podAnnotations).Create(data) + err = NewPodBuilder(podName, testAntreaIPAMNamespace, agnhostImage).OnNode(controlPlaneNodeName()).WithAnnotations(podAnnotations).Create(data) if err != nil { t.Fatalf("Error when creating Pod '%s': %v", podName, err) } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 140555f0041..274c8ba4efa 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -314,6 +314,17 @@ func (p *PodIPs) hasSameIP(p1 *PodIPs) bool { return false } +func (p *PodIPs) getPodIPs() []*net.IP { + var ips []*net.IP + if p.IPv4 != nil { + ips = append(ips, p.IPv4) + } + if p.IPv6 != nil { + ips = append(ips, p.IPv6) + } + return ips +} + // workerNodeName returns an empty string if there is no worker Node with the provided idx // (including if idx is 0, which is reserved for the control-plane Node) func workerNodeName(idx int) string { @@ -2852,14 +2863,14 @@ func (data *TestData) copyNodeFiles(nodeName string, fileName string, covDir str // createAgnhostPodOnNode creates a Pod in the test namespace with a single agnhost container. The // Pod will be scheduled on the specified Node (if nodeName is not empty). func (data *TestData) createAgnhostPodOnNode(name string, ns string, nodeName string, hostNetwork bool) error { - return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) + return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithHostNetwork(hostNetwork).Create(data) } // createAgnhostPodWithSAOnNode creates a Pod in the test namespace with a single // agnhost container and a specific ServiceAccount. The Pod will be scheduled on // the specified Node (if nodeName is not empty). func (data *TestData) createAgnhostPodWithSAOnNode(name string, ns string, nodeName string, hostNetwork bool, serviceAccountName string) error { - return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).WithServiceAccountName(serviceAccountName).Create(data) + return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithHostNetwork(hostNetwork).WithServiceAccountName(serviceAccountName).Create(data) } func (data *TestData) createDaemonSet(name string, ns string, ctrName string, image string, cmd []string, args []string) (*appsv1.DaemonSet, func() error, error) { diff --git a/test/e2e/l7networkpolicy_test.go b/test/e2e/l7networkpolicy_test.go index 4a21ff4e9c6..20d110ed8b9 100644 --- a/test/e2e/l7networkpolicy_test.go +++ b/test/e2e/l7networkpolicy_test.go @@ -16,8 +16,11 @@ package e2e import ( "context" + "encoding/json" "fmt" "net" + "path" + "slices" "strings" "testing" "time" @@ -25,6 +28,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" agentconfig "antrea.io/antrea/pkg/config/agent" @@ -59,6 +63,9 @@ func TestL7NetworkPolicy(t *testing.T) { t.Run("TLS", func(t *testing.T) { testL7NetworkPolicyTLS(t, data) }) + t.Run("Logging", func(t *testing.T) { + testL7NetworkPolicyLogging(t, data) + }) } func createL7NetworkPolicy(t *testing.T, @@ -189,31 +196,19 @@ func probeL7NetworkPolicyTLS(t *testing.T, data *TestData, clientPodName string, func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) { clientPodName := "test-l7-http-client-selected" clientPodLabels := map[string]string{"test-l7-http-e2e": "client"} - cmd := []string{"bash", "-c", "sleep 3600"} // Create a client Pod which will be selected by test L7 NetworkPolices. - require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data)) - if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil { - t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err) - } - require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace)) + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace) + require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName) serverPodName := "test-l7-http-server" serverPodLabels := map[string]string{"test-l7-http-e2e": "server"} - cmd = []string{"bash", "-c", "/agnhost netexec --http-port=8080"} + cmd := []string{"/agnhost", "netexec", "--http-port=8080"} require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(serverPodLabels).Create(data)) podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace) - if err != nil { - t.Fatalf("Error when waiting for IP for Pod '%s': %v", serverPodName, err) - } - require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace)) - var serverIPs []*net.IP - if podIPs.IPv4 != nil { - serverIPs = append(serverIPs, podIPs.IPv4) - } - if podIPs.IPv6 != nil { - serverIPs = append(serverIPs, podIPs.IPv6) - } + require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName) + serverIPs := podIPs.getPodIPs() l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{ { @@ -238,8 +233,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) { // Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that, // the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod // selector labels. - createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname) - createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath) + createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname) + createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath) time.Sleep(networkPolicyDelay) // HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy @@ -265,8 +260,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) { // Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that, // the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod // selector labels. - createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname) - createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath) + createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname) + createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath) time.Sleep(networkPolicyDelay) // HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy @@ -289,14 +284,11 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) { func testL7NetworkPolicyTLS(t *testing.T, data *TestData) { clientPodName := "test-l7-tls-client-selected" clientPodLabels := map[string]string{"test-l7-tls-e2e": "client"} - cmd := []string{"bash", "-c", "sleep 3600"} // Create a client Pod which will be selected by test L7 NetworkPolices. - require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data)) - if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil { - t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err) - } - require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace)) + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace) + require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName) l7ProtocolAllowsGoogle := []crdv1beta1.L7Protocol{ { @@ -333,3 +325,90 @@ func testL7NetworkPolicyTLS(t *testing.T, data *TestData) { probeL7NetworkPolicyTLS(t, data, clientPodName, "apis.google.com", false) probeL7NetworkPolicyTLS(t, data, clientPodName, "www.facebook.com", true) } + +func testL7NetworkPolicyLogging(t *testing.T, data *TestData) { + l7LoggingNode := nodeName(0) + + clientPodName := "test-l7-logging-client-selected" + clientPodLabels := map[string]string{"test-l7-logging-e2e": "client"} + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithLabels(clientPodLabels).Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace) + require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName) + + serverPodName := "test-l7-logging-server" + serverPodLabels := map[string]string{"test-l7-logging-e2e": "server"} + cmd := []string{"/agnhost", "netexec", "--http-port=8080"} + require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithCommand(cmd).WithLabels(serverPodLabels).Create(data)) + podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace) + require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName) + serverIPs := podIPs.getPodIPs() + + policyAllowPathHostname := "test-l7-http-allow-path-hostname" + l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{ + { + HTTP: &crdv1beta1.HTTPProtocol{ + Method: "GET", + Path: "/host*", + }, + }, + } + // Create one L7 NetworkPolicy that allows HTTP path 'hostname', and probe twice + // where HTTP path 'hostname' is allowed yet 'clientip' will be rejected. + createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname) + time.Sleep(networkPolicyDelay) + probeL7NetworkPolicyHTTP(t, data, serverPodName, clientPodName, serverIPs, true, false) + + // Define log matchers for expected L7 NetworkPolicies log entries. + var l7LogMatchers []L7LogEntry + for _, ip := range serverIPs { + clientMatcher := L7LogEntry{EventType: "alert", Protocol: "TCP", Http: L7LogHttpEntry{Hostname: ip.String(), Port: 8080, Url: "/clientip"}} + hostMatcher := L7LogEntry{EventType: "http", Protocol: "TCP", Http: L7LogHttpEntry{Hostname: ip.String(), Port: 8080, Url: "/hostname"}} + l7LogMatchers = append(l7LogMatchers, clientMatcher, hostMatcher) + } + + checkL7LoggingResult(t, data, l7LoggingNode, l7LogMatchers) +} + +// Partial entries of L7 NetworkPolicy logging necessary for testing. +type L7LogHttpEntry struct { + Hostname string `json:"hostname"` + Port int32 `json:"http_port"` + Url string `json:"url"` +} + +type L7LogEntry struct { + EventType string `json:"event_type"` + Protocol string `json:"proto"` + Http L7LogHttpEntry `json:"http"` +} + +func checkL7LoggingResult(t *testing.T, data *TestData, nodeName string, matchers []L7LogEntry) { + // Filename base on generated Suricata config [https://github.com/antrea-io/antrea/blob/main/pkg/agent/controller/networkpolicy/l7engine/reconciler.go]. + l7LogFile := path.Join(logDir, "l7engine", fmt.Sprintf("eve-%s.json", time.Now().Format(time.DateOnly))) + antreaPodName, err := data.getAntreaPodOnNode(nodeName) + require.NoError(t, err, "Error occurred when trying to get the Antrea Agent Pod running on Node %s", nodeName) + cmd := []string{"cat", l7LogFile} + + if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-agent", cmd) + if err != nil || stderr != "" { + // file may not exist yet + t.Logf("Error when printing the audit log file, err: %v, stderr: %v", err, stderr) + return false, nil + } + + var gotLogs []L7LogEntry + dec := json.NewDecoder(strings.NewReader(stdout)) + for dec.More() { + var log L7LogEntry + assert.NoError(t, dec.Decode(&log)) + if slices.Contains(matchers, log) { + gotLogs = append(gotLogs, log) + } + } + assert.ElementsMatch(t, gotLogs, matchers) + return true, nil + }); err != nil { + t.Errorf("Error when polling l7 log files for required entries: %v", err) + } +} diff --git a/test/e2e/traceflow_test.go b/test/e2e/traceflow_test.go index f8df547836e..b81de7c8e2e 100644 --- a/test/e2e/traceflow_test.go +++ b/test/e2e/traceflow_test.go @@ -1134,7 +1134,7 @@ func testTraceflowInterNode(t *testing.T, data *TestData) { // Create Service backend Pod. The "hairpin" testcases require the Service to have a single backend Pod, // and no more, in order to be deterministic. agnhostPodName := "agnhost" - require.NoError(t, NewPodBuilder(agnhostPodName, data.testNamespace, agnhostImage).OnNode(node2).WithCommand([]string{"sleep", "3600"}).WithLabels(map[string]string{"app": "agnhost-server"}).Create(data)) + require.NoError(t, NewPodBuilder(agnhostPodName, data.testNamespace, agnhostImage).OnNode(node2).WithLabels(map[string]string{"app": "agnhost-server"}).Create(data)) agnhostIP, err := data.podWaitForIPs(defaultTimeout, agnhostPodName, data.testNamespace) require.NoError(t, err) diff --git a/test/e2e/trafficcontrol_test.go b/test/e2e/trafficcontrol_test.go index 77a534ffc05..db5a4c83c42 100644 --- a/test/e2e/trafficcontrol_test.go +++ b/test/e2e/trafficcontrol_test.go @@ -98,7 +98,7 @@ func createTrafficControlTestPod(t *testing.T, data *TestData, podName string) { } func createTrafficControlPacketsCollectorPod(t *testing.T, data *TestData, podName string) { - require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).WithCommand([]string{"sleep", "3600"}).Privileged().Create(data)) + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).Privileged().Create(data)) ips, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) if err != nil { t.Fatalf("Error when waiting for IP for Pod '%s': %v", podName, err) @@ -299,7 +299,7 @@ func testRedirectToLocal(t *testing.T, data *TestData) { ip link add dev %[1]s type veth peer name %[2]s && \ ip link set dev %[1]s up && \ ip link set dev %[2]s up`, targetPortName, returnPortName) - if err := NewPodBuilder(tempPodName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Privileged().Create(data); err != nil { + if err := NewPodBuilder(tempPodName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).InHostNetwork().Privileged().Create(data); err != nil { t.Fatalf("Failed to create Pod %s: %v", tempPodName, err) } require.NoError(t, data.podWaitForRunning(defaultTimeout, tempPodName, data.testNamespace))