Skip to content

Commit

Permalink
Add E2E test for L7 NetworkPolicy Logging
Browse files Browse the repository at this point in the history 
This PR adds an E2E test for L7 NetworkPolicy
logging. It checks both allowed and dropped HTTP
event logs under l7engine directory.

Signed-off-by: Qiyue Yao <[email protected]>
  • Loading branch information
@qiyueyao
qiyueyao committed May 1, 2024
1 parent 572790d commit 7b85297
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 31 deletions.
13 changes: 7 additions & 6 deletions test/e2e/antreapolicy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net"
"path"
"regexp"
"strconv"
"strings"
Expand Down Expand Up @@ -52,6 +53,7 @@ var (
p8081 int32 = 8081
p8082 int32 = 8082
p8085 int32 = 8085
logFileName = path.Join(logDir, "np.log")
allPods []Pod
podsByNamespace map[string][]Pod
k8sUtils *KubernetesUtils
Expand All @@ -73,7 +75,6 @@ const (
timeout = 10 * time.Second
// audit log directory on Antrea Agent
logDir = "/var/log/antrea/networkpolicy/"
logfileName = "np.log"
defaultTierName = "application"
)

Expand Down Expand Up @@ -2575,7 +2576,7 @@ func testAuditLoggingBasic(t *testing.T, data *TestData) {

// nodeName is guaranteed to be set at this stage, since the framework waits for all Pods to be in Running phase
nodeName := podXA.Spec.NodeName
checkAuditLoggingResult(t, data, nodeName, npRef, matcher.Matchers())
checkAuditLoggingResult(t, data, nodeName, logFileName, npRef, matcher.Matchers())

failOnError(k8sUtils.CleanACNPs(), t)
}
Expand Down Expand Up @@ -2627,7 +2628,7 @@ func testAuditLoggingEnableK8s(t *testing.T, data *TestData) {

// nodeName is guaranteed to be set at this stage, since the framework waits for all Pods to be in Running phase
nodeName := podXA.Spec.NodeName
checkAuditLoggingResult(t, data, nodeName, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))
checkAuditLoggingResult(t, data, nodeName, logFileName, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))

failOnError(k8sUtils.DeleteNetworkPolicy(getNS("x"), "allow-x-b-to-x-a"), t)
failOnError(data.UpdateNamespace(getNS("x"), func(namespace *v1.Namespace) {
Expand Down Expand Up @@ -2699,7 +2700,7 @@ func testAuditLoggingK8sService(t *testing.T, data *TestData) {
oneProbe(getNS("x"), "b", matcher2)
wg.Wait()

checkAuditLoggingResult(t, data, serverNode, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))
checkAuditLoggingResult(t, data, serverNode, logFileName, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))

failOnError(k8sUtils.DeleteNetworkPolicy(getNS("x"), npName), t)
failOnError(data.UpdateNamespace(getNS("x"), func(namespace *v1.Namespace) {
Expand Down Expand Up @@ -4052,12 +4053,12 @@ func testACNPMulticastEgress(t *testing.T, data *TestData, acnpName, caseName, g

// the matchers parameter is a list of regular expressions which will be matched against the
// contents of the audit logs. The call will "succeed" if all matches are successful.
func checkAuditLoggingResult(t *testing.T, data *TestData, nodeName, logLocator string, matchers []*regexp.Regexp) {
func checkAuditLoggingResult(t *testing.T, data *TestData, nodeName, logFileDir, logLocator string, matchers []*regexp.Regexp) {
antreaPodName, err := data.getAntreaPodOnNode(nodeName)
if err != nil {
t.Errorf("Error occurred when trying to get the Antrea Agent Pod running on Node %s: %v", nodeName, err)
}
cmd := []string{"cat", logDir + logfileName}
cmd := []string{"cat", logFileDir}

if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) {
stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-agent", cmd)
Expand Down
106 changes: 81 additions & 25 deletions test/e2e/l7networkpolicy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"context"
"fmt"
"net"
"path"
"regexp"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -59,6 +61,9 @@ func TestL7NetworkPolicy(t *testing.T) {
t.Run("TLS", func(t *testing.T) {
testL7NetworkPolicyTLS(t, data)
})
t.Run("Logging", func(t *testing.T) {
testL7NetworkPolicyLogging(t, data)
})
}

func createL7NetworkPolicy(t *testing.T,
Expand Down Expand Up @@ -123,6 +128,17 @@ func createL7NetworkPolicy(t *testing.T,
assert.NoError(t, err)
}

func getPodIPs(podIPs *PodIPs) []*net.IP {
var ips []*net.IP
if podIPs.IPv4 != nil {
ips = append(ips, podIPs.IPv4)
}
if podIPs.IPv6 != nil {
ips = append(ips, podIPs.IPv6)
}
return ips
}

func probeL7NetworkPolicyHTTP(t *testing.T, data *TestData, serverPodName, clientPodName string, serverIPs []*net.IP, allowHTTPPathHostname, allowHTTPPathClientIP bool) {
for _, ip := range serverIPs {
baseURL := net.JoinHostPort(ip.String(), "8080")
Expand Down Expand Up @@ -189,31 +205,21 @@ func probeL7NetworkPolicyTLS(t *testing.T, data *TestData, clientPodName string,
func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
clientPodName := "test-l7-http-client-selected"
clientPodLabels := map[string]string{"test-l7-http-e2e": "client"}
cmd := []string{"bash", "-c", "sleep 3600"}

// Create a client Pod which will be selected by test L7 NetworkPolices.
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data))
if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err)
}
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))

serverPodName := "test-l7-http-server"
serverPodLabels := map[string]string{"test-l7-http-e2e": "server"}
cmd = []string{"bash", "-c", "/agnhost netexec --http-port=8080"}
cmd := []string{"/agnhost", "netexec", "--http-port=8080"}
require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(serverPodLabels).Create(data))
podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace)
if err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", serverPodName, err)
}
require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName)
require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace))
var serverIPs []*net.IP
if podIPs.IPv4 != nil {
serverIPs = append(serverIPs, podIPs.IPv4)
}
if podIPs.IPv6 != nil {
serverIPs = append(serverIPs, podIPs.IPv6)
}
serverIPs := getPodIPs(podIPs)

l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
Expand All @@ -238,8 +244,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
Expand All @@ -265,8 +271,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
Expand All @@ -289,13 +295,11 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
func testL7NetworkPolicyTLS(t *testing.T, data *TestData) {
clientPodName := "test-l7-tls-client-selected"
clientPodLabels := map[string]string{"test-l7-tls-e2e": "client"}
cmd := []string{"bash", "-c", "sleep 3600"}

// Create a client Pod which will be selected by test L7 NetworkPolices.
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithCommand(cmd).WithLabels(clientPodLabels).Create(data))
if _, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err)
}
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(nodeName(0)).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))

l7ProtocolAllowsGoogle := []crdv1beta1.L7Protocol{
Expand Down Expand Up @@ -333,3 +337,55 @@ func testL7NetworkPolicyTLS(t *testing.T, data *TestData) {
probeL7NetworkPolicyTLS(t, data, clientPodName, "apis.google.com", false)
probeL7NetworkPolicyTLS(t, data, clientPodName, "www.facebook.com", true)
}

func testL7NetworkPolicyLogging(t *testing.T, data *TestData) {
l7LoggingNode := nodeName(0)
// L7 audit logging filename is defined in Suricata config in /etc/suricata/antrea.yaml as "eve-%%Y-%%m-%%d.json".
l7LogDir := path.Join(logDir, fmt.Sprintf("l7engine/eve-%s.json", time.Now().Format(time.DateOnly)))

clientPodName := "test-l7-logging-client-selected"
clientPodLabels := map[string]string{"test-l7-logging-e2e": "client"}
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", clientPodName)
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))

serverPodName := "test-l7-logging-server"
serverPodLabels := map[string]string{"test-l7-logging-e2e": "server"}
cmd := []string{"/agnhost", "netexec", "--http-port=8080"}
require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithCommand(cmd).WithLabels(serverPodLabels).Create(data))
podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace)
require.NoError(t, err, "Expected IP for Pod '%s'", serverPodName)
require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace))
serverIPs := getPodIPs(podIPs)

policyAllowPathHostname := "test-l7-http-allow-path-hostname"
l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
HTTP: &crdv1beta1.HTTPProtocol{
Method: "GET",
Path: "/host*",
},
},
}
// Create one L7 NetworkPolicy that allows HTTP path 'hostname', and probe twice
// where HTTP path 'hostname' is allowed yet 'clientip' will be rejected.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
time.Sleep(networkPolicyDelay)
probeL7NetworkPolicyHTTP(t, data, serverPodName, clientPodName, serverIPs, true, false)

// Define log matchers for expected L7 NetworkPolicies log entries.
formatChecks := func(typeCheck, pathCheck string) *regexp.Regexp {
matchers := []string{"{", typeCheck}
for _, ip := range serverIPs {
matchers = append(matchers, `("hostname":"`+ip.String()+`)`)
}
matchers = append(matchers, `("http_port":8080)`, pathCheck, `("protocol":"HTTP)`, `}`)
return regexp.MustCompile(strings.Join(matchers, ".*"))
}
clientChecks := formatChecks(`("event_type":"alert")`, `("url":"/clientip")`)
hostChecks := formatChecks(`("event_type":"http")`, `("url":"/hostname")`)
l7LogMatchers := []*regexp.Regexp{clientChecks, hostChecks}

checkAuditLoggingResult(t, data, l7LoggingNode, l7LogDir, "http", l7LogMatchers)
}

0 comments on commit 7b85297

Please sign in to comment.