Skip to content

Commit

Permalink
Add E2E test for L7 NetworkPolicy Logging
Browse files Browse the repository at this point in the history 
This PR adds an E2E test for L7 NetworkPolicy
logging. It checks both allowed and dropped HTTP
event logs under l7engine directory.

Signed-off-by: Qiyue Yao <[email protected]>
  • Loading branch information
@qiyueyao
qiyueyao committed Apr 29, 2024
1 parent b52d946 commit a6e1ca7
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 20 deletions.
14 changes: 7 additions & 7 deletions test/e2e/antreapolicy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ const (
policyRealizedTimeout = 5 * time.Second
// Verification of deleting/creating resources timed out.
timeout = 10 * time.Second
// audit log directory on Antrea Agent
// audit log file directory on Antrea Agent
logDir = "/var/log/antrea/networkpolicy/"
logfileName = "np.log"
logFileDir = logDir + "np.log"
defaultTierName = "application"
)

Expand Down Expand Up @@ -2566,7 +2566,7 @@ func testAuditLoggingBasic(t *testing.T, data *TestData) {

// nodeName is guaranteed to be set at this stage, since the framework waits for all Pods to be in Running phase
nodeName := podXA.Spec.NodeName
checkAuditLoggingResult(t, data, nodeName, npRef, matcher.Matchers())
checkAuditLoggingResult(t, data, nodeName, logFileDir, npRef, matcher.Matchers())

failOnError(k8sUtils.CleanACNPs(), t)
}
Expand Down Expand Up @@ -2618,7 +2618,7 @@ func testAuditLoggingEnableK8s(t *testing.T, data *TestData) {

// nodeName is guaranteed to be set at this stage, since the framework waits for all Pods to be in Running phase
nodeName := podXA.Spec.NodeName
checkAuditLoggingResult(t, data, nodeName, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))
checkAuditLoggingResult(t, data, nodeName, logFileDir, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))

failOnError(k8sUtils.DeleteNetworkPolicy(getNS("x"), "allow-x-b-to-x-a"), t)
failOnError(data.UpdateNamespace(getNS("x"), func(namespace *v1.Namespace) {
Expand Down Expand Up @@ -2690,7 +2690,7 @@ func testAuditLoggingK8sService(t *testing.T, data *TestData) {
oneProbe(getNS("x"), "b", matcher2)
wg.Wait()

checkAuditLoggingResult(t, data, serverNode, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))
checkAuditLoggingResult(t, data, serverNode, logFileDir, "K8sNetworkPolicy", append(matcher1.Matchers(), matcher2.Matchers()...))

failOnError(k8sUtils.DeleteNetworkPolicy(getNS("x"), npName), t)
failOnError(data.UpdateNamespace(getNS("x"), func(namespace *v1.Namespace) {
Expand Down Expand Up @@ -4043,12 +4043,12 @@ func testACNPMulticastEgress(t *testing.T, data *TestData, acnpName, caseName, g

// the matchers parameter is a list of regular expressions which will be matched against the
// contents of the audit logs. The call will "succeed" if all matches are successful.
func checkAuditLoggingResult(t *testing.T, data *TestData, nodeName, logLocator string, matchers []*regexp.Regexp) {
func checkAuditLoggingResult(t *testing.T, data *TestData, nodeName, logFileDir, logLocator string, matchers []*regexp.Regexp) {
antreaPodName, err := data.getAntreaPodOnNode(nodeName)
if err != nil {
t.Errorf("Error occurred when trying to get the Antrea Agent Pod running on Node %s: %v", nodeName, err)
}
cmd := []string{"cat", logDir + logfileName}
cmd := []string{"cat", logFileDir}

if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) {
stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-agent", cmd)
Expand Down
100 changes: 87 additions & 13 deletions test/e2e/l7networkpolicy_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"
"net"
"regexp"
"strings"
"testing"
"time"
Expand All @@ -32,6 +33,11 @@ import (
. "antrea.io/antrea/test/e2e/utils"
)

var (
clientipPath = "clientip"
hostnamePath = "hostname"
)

func TestL7NetworkPolicy(t *testing.T) {
skipIfHasWindowsNodes(t)
skipIfFeatureDisabled(t, features.L7NetworkPolicy, true, true)
Expand Down Expand Up @@ -59,6 +65,9 @@ func TestL7NetworkPolicy(t *testing.T) {
t.Run("TLS", func(t *testing.T) {
testL7NetworkPolicyTLS(t, data)
})
t.Run("Logging", func(t *testing.T) {
testL7NetworkPolicyLogging(t, data)
})
}

func createL7NetworkPolicy(t *testing.T,
Expand Down Expand Up @@ -123,13 +132,24 @@ func createL7NetworkPolicy(t *testing.T,
assert.NoError(t, err)
}

func getPodIPs(podIPs *PodIPs) []*net.IP {
var concatenatedIPs []*net.IP
if podIPs.IPv4 != nil {
concatenatedIPs = append(concatenatedIPs, podIPs.IPv4)
}
if podIPs.IPv6 != nil {
concatenatedIPs = append(concatenatedIPs, podIPs.IPv6)
}
return concatenatedIPs
}

func probeL7NetworkPolicyHTTP(t *testing.T, data *TestData, serverPodName, clientPodName string, serverIPs []*net.IP, allowHTTPPathHostname, allowHTTPPathClientIP bool) {
for _, ip := range serverIPs {
baseURL := net.JoinHostPort(ip.String(), "8080")

// Verify that access to path /clientip is as expected.
assert.Eventually(t, func() bool {
cmd := []string{"wget", "-O", "-", fmt.Sprintf("%s/%s", baseURL, "clientip"), "-T", "1"}
cmd := []string{"wget", "-O", "-", fmt.Sprintf("%s/%s", baseURL, clientipPath), "-T", "1"}
_, _, err := data.RunCommandFromPod(data.testNamespace, clientPodName, agnhostContainerName, cmd)
if (allowHTTPPathClientIP && err != nil) || (!allowHTTPPathClientIP && err == nil) {
return false
Expand All @@ -139,7 +159,7 @@ func probeL7NetworkPolicyHTTP(t *testing.T, data *TestData, serverPodName, clien

// Verify that access to path /hostname is as expected.
assert.Eventually(t, func() bool {
cmd := []string{"wget", "-O", "-", fmt.Sprintf("%s/%s", baseURL, "hostname"), "-T", "1"}
cmd := []string{"wget", "-O", "-", fmt.Sprintf("%s/%s", baseURL, hostnamePath), "-T", "1"}
hostname, _, err := data.RunCommandFromPod(data.testNamespace, clientPodName, agnhostContainerName, cmd)
if (allowHTTPPathHostname && err != nil) || (!allowHTTPPathHostname && err == nil) {
return false
Expand Down Expand Up @@ -207,13 +227,7 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", serverPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace))
var serverIPs []*net.IP
if podIPs.IPv4 != nil {
serverIPs = append(serverIPs, podIPs.IPv4)
}
if podIPs.IPv6 != nil {
serverIPs = append(serverIPs, podIPs.IPv6)
}
serverIPs := getPodIPs(podIPs)

l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
Expand All @@ -238,8 +252,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, true, policyAllowAnyPath, 2, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
Expand All @@ -265,8 +279,8 @@ func testL7NetworkPolicyHTTP(t *testing.T, data *TestData) {
// Create two L7 NetworkPolicies, one allows HTTP path 'hostname', the other allows any HTTP path. Note that,
// the priority of the first one is higher than the second one, and they have the same appliedTo labels and Pod
// selector labels.
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, 8080, l7ProtocolAllowsAnyPath)
createL7NetworkPolicy(t, data, false, policyAllowPathHostname, 1, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
createL7NetworkPolicy(t, data, false, policyAllowAnyPath, 2, serverPodLabels, clientPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsAnyPath)
time.Sleep(networkPolicyDelay)

// HTTP path 'hostname' is allowed by the first L7 NetworkPolicy, and the priority of the second L7 NetworkPolicy
Expand Down Expand Up @@ -333,3 +347,63 @@ func testL7NetworkPolicyTLS(t *testing.T, data *TestData) {
probeL7NetworkPolicyTLS(t, data, clientPodName, "apis.google.com", false)
probeL7NetworkPolicyTLS(t, data, clientPodName, "www.facebook.com", true)
}

func testL7NetworkPolicyLogging(t *testing.T, data *TestData) {
l7LoggingNode := nodeName(0)
l7LogDir := logDir + "l7engine/eve-" + time.Now().Format(time.DateOnly) + ".json"

clientPodName := "test-l7-logging-client-selected"
clientPodLabels := map[string]string{"test-l7-logging-e2e": "client"}
cmd := []string{"bash", "-c", "sleep 3600"}
require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithCommand(cmd).WithLabels(clientPodLabels).Create(data))
_, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace)
if err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", clientPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))

serverPodName := "test-l7-logging-server"
serverPodLabels := map[string]string{"test-l7-logging-e2e": "server"}
cmd = []string{"bash", "-c", "/agnhost netexec --http-port=8080"}
require.NoError(t, NewPodBuilder(serverPodName, data.testNamespace, agnhostImage).OnNode(l7LoggingNode).WithCommand(cmd).WithLabels(serverPodLabels).Create(data))
podIPs, err := data.podWaitForIPs(defaultTimeout, serverPodName, data.testNamespace)
if err != nil {
t.Fatalf("Error when waiting for IP for Pod '%s': %v", serverPodName, err)
}
require.NoError(t, data.podWaitForRunning(defaultTimeout, serverPodName, data.testNamespace))
serverIPs := getPodIPs(podIPs)

policyAllowPathHostname := "test-l7-http-allow-path-hostname"
l7ProtocolAllowsPathHostname := []crdv1beta1.L7Protocol{
{
HTTP: &crdv1beta1.HTTPProtocol{
Method: "GET",
Path: "/host*",
},
},
}
// Create an L7 NetworkPolicies that allows HTTP path 'hostname', and probe twice
// where HTTP path 'hostname' is allowed yet 'clientip' will be rejected.
createL7NetworkPolicy(t, data, true, policyAllowPathHostname, 1, clientPodLabels, serverPodLabels, ProtocolTCP, p8080, l7ProtocolAllowsPathHostname)
time.Sleep(networkPolicyDelay)
probeL7NetworkPolicyHTTP(t, data, serverPodName, clientPodName, serverIPs, true, false)

// Define log matchers for expected L7 NetworkPolicies log entries based on probe.
formatChecks := func(typeCheck, pathCheck string) *regexp.Regexp {
matchers := []string{"{", typeCheck}
for _, ip := range serverIPs {
matchers = append(matchers, "(\"hostname\":\""+ip.String()+")")
}
matchers = append(matchers, "(\"http_port\":8080)", pathCheck, "(\"protocol\":\"HTTP)", "}")
return regexp.MustCompile(strings.Join(matchers, ".*"))
}
clientChecks := formatChecks("(\"event_type\":\"alert\")", "(\"url\":\"/clientip\")")
hostChecks := formatChecks("(\"event_type\":\"http\")", "(\"url\":\"/hostname\")")
l7LogMatchers := []*regexp.Regexp{clientChecks, hostChecks}

checkAuditLoggingResult(t, data, l7LoggingNode, l7LogDir, "http", l7LogMatchers)

// Delete the L7 NetworkPolicy that only allows HTTP path 'hostname'.
data.crdClient.CrdV1beta1().NetworkPolicies(data.testNamespace).Delete(context.TODO(), policyAllowPathHostname, metav1.DeleteOptions{})
time.Sleep(networkPolicyDelay)
}

0 comments on commit a6e1ca7

Please sign in to comment.