From 7acc1c2e0a404cff70cfc71a3be58225ffa703a5 Mon Sep 17 00:00:00 2001 From: Hongliang Liu <75655411+hongliangl@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:11:24 +0800 Subject: [PATCH] Fix NodeNetworkPolicy e2e test failure (#6172) In NodeNetworkPolicy e2e tests, we have the following cases: - Node to Node. We deploy two hostNetwork Pods on different Nodes. - Node to Pods. We deploy a hostNetwork Pod on a Node and two non-hostNetwork Pods on different Nodes. For the case of Node to Pods, after creating test Pods, a full mesh probing is run to ensure that all Pods can be reachable from each other. However, the UDP or SCTP probing from a non-hostNetwork Pod, using the Node external IP (the IP in the hostNetwork Pod object) as destination IP, to the hostNetwork Pod within on the same Node will get a failure. The reason is that due to UDP's connectionless nature, the reply traffic may use a source IP address determined by routing decisions or outgoing interfaces, which means that the local Antrea gateway IPs will be chosen as source IP address, rather than the destination IP address used in request traffic. As a result, the probing will get a failure because the source IP address of reply traffic is unexpected. To resolve the issue, when the target Pod is a hostNetwork Pod, we use the local Antrea gateway IPs as the destination IP, rather than the hostNetwork Pod IP (the Node external IP). Signed-off-by: Hongliang Liu --- test/e2e/fixtures.go | 10 +++++++++ test/e2e/k8s_util.go | 36 +++++++++++++++++++++++++----- test/e2e/nodenetworkpolicy_test.go | 10 +++++---- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 762560cd69a..71fa872759e 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -131,6 +131,16 @@ func skipIfEncapModeIsNot(tb testing.TB, data *TestData, encapMode config.Traffi } } +func skipIfEncapModeIs(tb testing.TB, data *TestData, encapMode config.TrafficEncapModeType) { + currentEncapMode, err := data.GetEncapMode() + if err != nil { + tb.Fatalf("Failed to get encap mode: %v", err) + } + if currentEncapMode == encapMode { + tb.Skipf("Skipping test for encap mode '%s'", encapMode.String()) + } +} + func skipIfHasWindowsNodes(tb testing.TB) { if len(clusterInfo.windowsNodes) != 0 { tb.Skipf("Skipping test as the cluster has Windows Nodes") diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 2dc2261a1f2..a614f83b1c7 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -430,12 +430,38 @@ func (k *KubernetesUtils) probeAndDecideConnectivity(fromPod, toPod v1.Pod, fromPodName, toPodName string, port int32, protocol utils.AntreaPolicyProtocol, expectedResult *PodConnectivityMark) (PodConnectivityMark, error) { // Both IPv4 and IPv6 address should be tested. connectivity := Unknown - for _, eachIP := range toPod.Status.PodIPs { - toIP := eachIP.IP - // If it's an IPv6 address, add "[]" around it. - if strings.Contains(toIP, ":") { - toIP = fmt.Sprintf("[%s]", toIP) + var toIPs []string + if toPod.Spec.HostNetwork { + // When probing UDP or SCTP from a non-hostNetwork Pod to a hostNetwork Pod within the same Node, the local + // Antrea gateway IPs should be used as the destination IP, rather than the Node external IPs. If using the Node + // external IPs as destination IPs when probing, the UDP or SCTP reply traffic from hostNetwork Pod will choose + // a source IP address based on the routing decision or outgoing interface, which means that the local Antrea + // gateway IPs will be chosen as the source IP address. As a result, the probing will get a failure because the + // source IP address of reply traffic is unexpected. To accommodate with this case, when the target Pod is a + // hostNetwork Pod, the local Antrea gateway IPs are used. + nodeInfo := getNodeByName(toPod.Spec.NodeName) + if nodeInfo == nil { + return connectivity, fmt.Errorf("failed to get Node information by name %s", toPod.Spec.NodeName) + } + gwIPv4, gwIPv6 := nodeGatewayIPs(nodeInfo.idx) + if gwIPv4 != "" { + toIPs = append(toIPs, gwIPv4) + } + if gwIPv6 != "" { + toIPs = append(toIPs, fmt.Sprintf("[%s]", gwIPv6)) + } + } else { + for _, eachIP := range toPod.Status.PodIPs { + toIP := eachIP.IP + // If it's an IPv6 address, add "[]" around it. + if strings.Contains(toIP, ":") { + toIP = fmt.Sprintf("[%s]", toIP) + } + toIPs = append(toIPs, toIP) } + } + + for _, toIP := range toIPs { // HACK: inferring container name as c80, c81 etc., for simplicity. containerName := fmt.Sprintf("c%v", port) curConnectivity := k.probe(&fromPod, fromPodName, containerName, toIP, toPodName, port, protocol, expectedResult) diff --git a/test/e2e/nodenetworkpolicy_test.go b/test/e2e/nodenetworkpolicy_test.go index 5564fd37329..c8e20a0d446 100644 --- a/test/e2e/nodenetworkpolicy_test.go +++ b/test/e2e/nodenetworkpolicy_test.go @@ -22,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "antrea.io/antrea/pkg/agent/config" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" "antrea.io/antrea/pkg/features" . "antrea.io/antrea/test/e2e/utils" @@ -45,17 +46,17 @@ func initializeAntreaNodeNetworkPolicy(t *testing.T, data *TestData, toHostNetwo } } nodes = make(map[string]string) - nodes["x"] = controlPlaneNodeName() - nodes["y"] = workerNodeName(1) + nodes["x"] = nodeName(0) + nodes["y"] = nodeName(1) + nodes["z"] = nodeName(0) hostNetworks := make(map[string]bool) hostNetworks["x"] = true if toHostNetworkPod { hostNetworks["y"] = true } else { hostNetworks["y"] = false - nodes["z"] = workerNodeName(1) - hostNetworks["z"] = false } + hostNetworks["z"] = false allPods = []Pod{} for _, podName := range podsPerNamespace { @@ -88,6 +89,7 @@ func TestAntreaNodeNetworkPolicy(t *testing.T) { t.Fatalf("Error when setting up test: %v", err) } defer teardownTest(t, data) + skipIfEncapModeIs(t, data, config.TrafficEncapModeNetworkPolicyOnly) initializeAntreaNodeNetworkPolicy(t, data, true)