Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add E2E test for L7 NetworkPolicy Logging #6275

Merged
merged 1 commit into from
May 14, 2024
Merged

Add E2E test for L7 NetworkPolicy Logging #6275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/e2e/antreaipam_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ func testAntreaIPAMStatefulSet(t *testing.T, data *TestData, dedicatedIPPoolKey
if dedicatedIPPoolKey != nil {
podAnnotations[annotation.AntreaIPAMAnnotationKey] = ipPoolName
}
err = NewPodBuilder(podName, testAntreaIPAMNamespace, agnhostImage).OnNode(controlPlaneNodeName()).WithCommand([]string{"sleep", "3600"}).WithAnnotations(podAnnotations).Create(data)
err = NewPodBuilder(podName, testAntreaIPAMNamespace, agnhostImage).OnNode(controlPlaneNodeName()).WithAnnotations(podAnnotations).Create(data)
if err != nil {
t.Fatalf("Error when creating Pod '%s': %v", podName, err)
}
Expand Down
15 changes: 13 additions & 2 deletions test/e2e/framework.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,17 @@ func (p *PodIPs) hasSameIP(p1 *PodIPs) bool {
return false
}

func (p *PodIPs) AsSlice() []*net.IP {
var ips []*net.IP
if p.IPv4 != nil {
ips = append(ips, p.IPv4)
}
if p.IPv6 != nil {
ips = append(ips, p.IPv6)
}
return ips
}

// workerNodeName returns an empty string if there is no worker Node with the provided idx
// (including if idx is 0, which is reserved for the control-plane Node)
func workerNodeName(idx int) string {
Expand Down Expand Up @@ -2853,14 +2864,14 @@ func (data *TestData) copyNodeFiles(nodeName string, fileName string, covDir str
// createAgnhostPodOnNode creates a Pod in the test namespace with a single agnhost container. The
// Pod will be scheduled on the specified Node (if nodeName is not empty).
func (data *TestData) createAgnhostPodOnNode(name string, ns string, nodeName string, hostNetwork bool) error {
return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data)
return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithHostNetwork(hostNetwork).Create(data)
}

// createAgnhostPodWithSAOnNode creates a Pod in the test namespace with a single
// agnhost container and a specific ServiceAccount. The Pod will be scheduled on
// the specified Node (if nodeName is not empty).
func (data *TestData) createAgnhostPodWithSAOnNode(name string, ns string, nodeName string, hostNetwork bool, serviceAccountName string) error {
return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).WithServiceAccountName(serviceAccountName).Create(data)
return NewPodBuilder(name, ns, agnhostImage).OnNode(nodeName).WithHostNetwork(hostNetwork).WithServiceAccountName(serviceAccountName).Create(data)
}

func (data *TestData) createDaemonSet(name string, ns string, ctrName string, image string, cmd []string, args []string) (*appsv1.DaemonSet, func() error, error) {
Expand Down
136 changes: 108 additions & 28 deletions test/e2e/l7networkpolicy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,19 @@ package e2e

import (
"context"
"encoding/json"
"fmt"
"net"
"path"
"slices"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"

crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1"
agentconfig "antrea.io/antrea/pkg/config/agent"
Expand Down Expand Up @@ -59,6 +63,9 @@ func TestL7NetworkPolicy(t *testing.T) {
t.Run("TLS", func(t *testing.T) {
testL7NetworkPolicyTLS(t, data)
})
t.Run("Logging", func(t *testing.T) {
testL7NetworkPolicyLogging(t, data)
})
}

func createL7NetworkPolicy(t *testing.T,
Expand Down Expand Up @@ -189,31 +196,19 @@ func probeL7NetworkPolicyTLS(t *testing.T, data *TestData, clientPodName string,
func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
clientPodName := "test-l7-http-client-selected"
clientPodLabels := map[string]string{"test-l7-http-e2e": "client"}
cmd := []string{"bash", "-c", "sleep 3600"}

// Create a client Pod which will be selected by test L7 NetworkPolices.
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data))
if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)

serverPodName := "test-l7-http-server"
serverPodLabels := map[string]string{"test-l7-http-e2e": "server"}
cmd = []string{"bash", "-c", "/agnhost netexec --http-port=8080"}
cmd := []string{"/agnhost", "netexec", "--http-port=8080"}
require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(serverPodLabels).Create(data))
podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace)
if err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", serverPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace))
var serverIPs []*net.IP
if podIPs.IPv4 != nil {
serverIPs = append(serverIPs, podIPs.IPv4)
}
if podIPs.IPv6 != nil {
serverIPs = append(serverIPs, podIPs.IPv6)
}
require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName)
serverIPs := podIPs.AsSlice()

l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
Expand All @@ -238,8 +233,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
Expand All @@ -265,8 +260,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
hongliangl marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -289,14 +284,11 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
func testL7NetworkPolicyTLS(t *testing.T, data *TestData) {
clientPodName := "test-l7-tls-client-selected"
clientPodLabels := map[string]string{"test-l7-tls-e2e": "client"}
cmd := []string{"bash", "-c", "sleep 3600"}

// Create a client Pod which will be selected by test L7 NetworkPolices.
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data))
if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)

l7ProtocolAllowsGoogle := []crdv1beta1.L7Protocol{
{
Expand Down Expand Up @@ -333,3 +325,91 @@ func testL7NetworkPolicyTLS(t *testing.T, data *TestData) {
probeL7NetworkPolicyTLS(t, data, clientPodName, "apis.google.com", false)
probeL7NetworkPolicyTLS(t, data, clientPodName, "www.facebook.com", true)
}

func testL7NetworkPolicyLogging(t *testing.T, data *TestData) {
l7LoggingNode := nodeName(0)

clientPodName := "test-l7-logging-client-selected"
clientPodLabels := map[string]string{"test-l7-logging-e2e": "client"}
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)

serverPodName := "test-l7-logging-server"
serverPodLabels := map[string]string{"test-l7-logging-e2e": "server"}
cmd := []string{"/agnhost", "netexec", "--http-port=8080"}
require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithCommand(cmd).WithLabels(serverPodLabels).Create(data))
podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName)
serverIPs := podIPs.AsSlice()

policyAllowPathHostname := "test-l7-http-allow-path-hostname"
l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
HTTP: &crdv1beta1.HTTPProtocol{
Method: "GET",
Path: "/host*",
},
},
}
// Create one L7 NetworkPolicy that allows HTTP path 'hostname', and probe twice
// where HTTP path 'hostname' is allowed yet 'clientip' will be rejected.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
time.Sleep(networkPolicyDelay)
probeL7NetworkPolicyHTTP(t, data, serverPodName, clientPodName, serverIPs, true, false)

// Define log matchers for expected L7 NetworkPolicies log entries.
var l7LogMatchers []L7LogEntry
for _, ip := range serverIPs {
clientMatcher := L7LogEntry{EventType: "alert", Protocol: "TCP", Http: L7LogHttpEntry{Hostname: ip.String(), Port: 8080, Url: "/clientip"}}
hostMatcher := L7LogEntry{EventType: "http", Protocol: "TCP", Http: L7LogHttpEntry{Hostname: ip.String(), Port: 8080, Url: "/hostname"}}
l7LogMatchers = append(l7LogMatchers, clientMatcher, hostMatcher)
}

checkL7LoggingResult(t, data, l7LoggingNode, l7LogMatchers)
}

// Partial entries of L7 NetworkPolicy logging necessary for testing.
type L7LogHttpEntry struct {
Hostname string `json:"hostname"`
Port int32 `json:"http_port"`
Url string `json:"url"`
}

type L7LogEntry struct {
EventType string `json:"event_type"`
Protocol string `json:"proto"`
Http L7LogHttpEntry `json:"http"`
}

func checkL7LoggingResult(t *testing.T, data *TestData, nodeName string, matchers []L7LogEntry) {
// Filename base on generated Suricata config [https://github.com/antrea-io/antrea/blob/main/pkg/agent/controller/networkpolicy/l7engine/reconciler.go].
l7LogFile := path.Join(logDir, "l7engine", fmt.Sprintf("eve-%s.json", time.Now().Format(time.DateOnly)))
antreaPodName, err := data.getAntreaPodOnNode(nodeName)
require.NoError(t, err, "Error occurred when trying to get the Antrea Agent Pod running on Node %s", nodeName)
cmd := []string{"cat", l7LogFile}

if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) {
stdout, _, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-agent", cmd)
if err != nil {
// file may not exist yet
return false, nil
}

var gotLogs []L7LogEntry
dec := json.NewDecoder(strings.NewReader(stdout))
for dec.More() {
var log L7LogEntry
if err := dec.Decode(&log); err != nil {
// log format error, fail immediately
return false, err
}
if slices.Contains(matchers, log) {
gotLogs = append(gotLogs, log)
}
}
return slices.Equal(gotLogs, matchers), nil
}); err != nil {
t.Errorf("Error when polling L7 audit log files for required entries: %v", err)
}
}
2 changes: 1 addition & 1 deletion test/e2e/traceflow_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1166,7 +1166,7 @@ func testTraceflowInterNode(t *testing.T, data *TestData) {
// Create Service backend Pod. The "hairpin" testcases require the Service to have a single backend Pod,
// and no more, in order to be deterministic.
agnhostPodName := "agnhost"
require.NoError(t, NewPodBuilder(agnhostPodName, data.testNamespace, agnhostImage).OnNode(node2).WithCommand([]string{"sleep", "3600"}).WithLabels(map[string]string{"app": "agnhost-server"}).Create(data))
require.NoError(t, NewPodBuilder(agnhostPodName, data.testNamespace, agnhostImage).OnNode(node2).WithLabels(map[string]string{"app": "agnhost-server"}).Create(data))
agnhostIP, err := data.podWaitForIPs(defaultTimeout, agnhostPodName, data.testNamespace)
require.NoError(t, err)

Expand Down
4 changes: 2 additions & 2 deletions test/e2e/trafficcontrol_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func createTrafficControlTestPod(t *testing.T, data *TestData, podName string) {
}

func createTrafficControlPacketsCollectorPod(t *testing.T, data *TestData, podName string) {
require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).WithCommand([]string{"sleep", "3600"}).Privileged().Create(data))
require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).Privileged().Create(data))
ips, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace)
if err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", podName, err)
Expand Down Expand Up @@ -299,7 +299,7 @@ func testRedirectToLocal(t *testing.T, data *TestData) {
ip link add dev %[1]s type veth peer name %[2]s && \
ip link set dev %[1]s up && \
ip link set dev %[2]s up`, targetPortName, returnPortName)
if err := NewPodBuilder(tempPodName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Privileged().Create(data); err != nil {
if err := NewPodBuilder(tempPodName, data.testNamespace, agnhostImage).OnNode(tcTestConfig.nodeName).InHostNetwork().Privileged().Create(data); err != nil {
t.Fatalf("Failed to create Pod %s: %v", tempPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, tempPodName, data.testNamespace))
Expand Down
Loading