From 7e48349f2506c98d84185151fb0a3068ee9fb947 Mon Sep 17 00:00:00 2001 From: Quan Tian Date: Fri, 1 Mar 2024 14:45:07 +0800 Subject: [PATCH] Fix IPv6 e2e test caused by image update wget in toolbox image expects the protocol to be specified when IPv6 address is used in the url, otherwise it would use ftp protocol. This patch fixes it and unifies how we get url from IP, port and path. Signed-off-by: Quan Tian --- test/e2e/egress_test.go | 42 +++++++++++++---------------- test/e2e/flowaggregator_test.go | 11 +------- test/e2e/prometheus_test.go | 3 +-- test/e2e/proxy_test.go | 6 ++--- test/e2e/service_externalip_test.go | 13 +++++---- test/e2e/service_test.go | 4 +-- test/e2e/util.go | 18 +++++++++++++ 7 files changed, 49 insertions(+), 48 deletions(-) diff --git a/test/e2e/egress_test.go b/test/e2e/egress_test.go index 63ea8bfc16e..15abfedb5ab 100644 --- a/test/e2e/egress_test.go +++ b/test/e2e/egress_test.go @@ -149,15 +149,10 @@ func testEgressClientIP(t *testing.T, data *TestData) { t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", remotePod) } - serverIPStr := tt.serverIP - if utilnet.IsIPv6String(tt.localIP0) { - serverIPStr = fmt.Sprintf("[%s]", tt.serverIP) - } - // As the fake server runs in a netns of the Egress Node, only egress Node can reach the server, Pods running on // other Nodes cannot reach it before Egress is added. - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Logf("Creating an Egress applying to all e2e Pods") matchExpressions := []metav1.LabelSelectorRequirement{ @@ -168,8 +163,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { } egress := data.createEgress(t, "egress-", matchExpressions, nil, "", egressNodeIP, nil) defer data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP) - assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP) + assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP) var err error err = wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) { @@ -187,7 +182,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if utilnet.IsIPv6String(clientIPStr) { clientIPStr = fmt.Sprintf("[%s]", clientIPStr) } - cmd = fmt.Sprintf("wget -T 3 -O - %s:8080/clientip | grep %s:", serverIPStr, clientIPStr) + url := getURL(tt.serverIP, "8080", "clientip") + cmd = fmt.Sprintf("wget -T 3 -O - %s | grep %s:", url, clientIPStr) if err := NewPodBuilder(initialIPChecker, data.testNamespace, agnhostImage).OnNode(egressNode).WithCommand([]string{"sh", "-c", cmd}).Create(data); err != nil { t.Fatalf("Failed to create Pod initial-ip-checker: %v", err) } @@ -210,8 +206,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP) t.Log("Updating the Egress's AppliedTo to localPod only") egress.Spec.AppliedTo = v1beta1.AppliedTo{ @@ -223,8 +219,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Logf("Updating the Egress's EgressIP to %s", tt.localIP1) egress.Spec.EgressIP = tt.localIP1 @@ -232,16 +228,16 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Log("Deleting the Egress") err = data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) if err != nil { t.Fatalf("Failed to delete Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) }) } } @@ -933,7 +929,7 @@ func setupIPNeighborChecker(data *TestData, t *testing.T, observerNode, node1, n // Before the Node actually connects to the Egress IP, we expect that the lladdr either matches the Egress Node's MAC address or is empty. check(true) // The protocol must be present when using wget with IPv6 address. - cmd := []string{"wget", fmt.Sprintf("http://%s", net.JoinHostPort(ip, "80")), "-T", "1", "-t", "1"} + cmd := []string{"wget", getURL(ip), "-T", "1", "-t", "1"} // We don't care whether it succeeds, just make it connect to the Egress IP to learn its MAC address. data.RunCommandFromPod(antreaNamespace, antreaPodName, agentContainerName, cmd) // After the Node tries to connect to the Egress IP, we expect that the lladdr matches the Egress Node's MAC address. @@ -997,11 +993,11 @@ func (data *TestData) waitForEgressRealized(egress *v1beta1.Egress) (*v1beta1.Eg } // assertClientIP asserts the Pod is translated to the provided client IP. -func assertClientIP(data *TestData, t *testing.T, pod, container, server string, clientIPs ...string) { +func assertClientIP(data *TestData, t *testing.T, pod, container, serverIP string, clientIPs ...string) { var exeErr error var stdout, stderr string err := wait.Poll(100*time.Millisecond, 5*time.Second, func() (done bool, err error) { - url := fmt.Sprintf("%s:8080/clientip", server) + url := getURL(serverIP, "8080", "clientip") stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, container, url, 5) if exeErr != nil { return false, nil @@ -1023,11 +1019,11 @@ func assertClientIP(data *TestData, t *testing.T, pod, container, server string, } // assertConnError asserts the Pod is not able to access the API that replies the request's client IP. -func assertConnError(data *TestData, t *testing.T, pod, container, server string) { +func assertConnError(data *TestData, t *testing.T, pod, container, serverIP string) { var exeErr error var stdout, stderr string err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { - url := fmt.Sprintf("%s:8080/clientip", server) + url := getURL(serverIP, "8080", "clientip") stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, url, container, 5) if exeErr != nil { return true, nil diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go index dc9ebe048e5..b7c0d4f97d8 100644 --- a/test/e2e/flowaggregator_test.go +++ b/test/e2e/flowaggregator_test.go @@ -1904,23 +1904,14 @@ func testL7FlowExporterController(t *testing.T, data *TestData, isIPv6 bool) { testFlow1 := testFlow{ srcPodName: clientPodName, } - var cmd []string if !isIPv6 { testFlow1.srcIP = clientPodIPs.IPv4.String() testFlow1.dstIP = serverIPs.IPv4.String() - cmd = []string{ - "curl", - fmt.Sprintf("http://%s:%d", serverIPs.IPv4.String(), serverPodPort), - } } else { testFlow1.srcIP = clientPodIPs.IPv6.String() testFlow1.dstIP = serverIPs.IPv6.String() - cmd = []string{ - "curl", - "-6", - fmt.Sprintf("http://[%s]:%d", serverIPs.IPv6.String(), serverPodPort), - } } + cmd := []string{"curl", getURL(testFlow1.dstIP, fmt.Sprint(serverPodPort))} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, testFlow1.srcPodName, "l7flowexporter", cmd) require.NoErrorf(t, err, "Error when running curl command, stdout: %s, stderr: %s", stdout, stderr) _, recordSlices := getCollectorOutput(t, testFlow1.srcIP, testFlow1.dstIP, "", false, true, isIPv6, data, "") diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go index 25ce5354ba0..b1dbcbbf5f3 100644 --- a/test/e2e/prometheus_test.go +++ b/test/e2e/prometheus_test.go @@ -281,8 +281,7 @@ func testMetricsFromPrometheusServer(t *testing.T, data *TestData, prometheusJob // Target metadata API(/api/v1/targets/metadata) has been available since Prometheus v2.4.0. // This API is still experimental in Prometheus v2.46.0. path := url.PathEscape("match_target={job=\"" + prometheusJob + "\"}") - address := net.JoinHostPort(hostIP, fmt.Sprint(nodePort)) - queryURL := fmt.Sprintf("http://%s/api/v1/targets/metadata?%s", address, path) + queryURL := getURL(hostIP, fmt.Sprint(nodePort), fmt.Sprintf("api/v1/targets/metadata?%s", path)) client := &http.Client{} var output prometheusServerOutput diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index a1643f8b412..3b9be8dc093 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -625,10 +625,10 @@ func testProxyServiceSessionAffinity(ipFamily *corev1.IPFamily, ingressIPs []str require.NoError(t, data.createToolboxPodOnNode(toolboxPod, data.testNamespace, nodeName, false)) defer data.DeletePodAndWait(defaultTimeout, toolboxPod, data.testNamespace) require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPod, data.testNamespace)) - stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, svc.Spec.ClusterIP, 5) + stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getURL(svc.Spec.ClusterIP), 5) require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr)) for _, ingressIP := range ingressIPs { - stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, ingressIP, 5) + stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getURL(ingressIP), 5) require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr)) } @@ -1175,7 +1175,7 @@ func TestProxyLoadBalancerModeDSR(t *testing.T) { defer data.deleteServiceAndWait(defaultTimeout, serviceName, data.testNamespace) curlServiceWithPath := func(clientPod, clientNetns, path string) string { - testURL := fmt.Sprintf("http://%s:%d/%s", lbIP, service.Spec.Ports[0].Port, path) + testURL := getURL(lbIP, fmt.Sprint(service.Spec.Ports[0].Port), path) cmd = fmt.Sprintf("curl --connect-timeout 1 --retry 5 --retry-connrefused %s", testURL) if clientNetns != "" { cmd = fmt.Sprintf("ip netns exec %s %s", clientNetns, cmd) diff --git a/test/e2e/service_externalip_test.go b/test/e2e/service_externalip_test.go index aaae30c6b8f..4f32bd4eb38 100644 --- a/test/e2e/service_externalip_test.go +++ b/test/e2e/service_externalip_test.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "net" - "strconv" "strings" "testing" "time" @@ -633,14 +632,14 @@ func testExternalIPAccess(t *testing.T, data *TestData) { // Create a pod in a different netns with the same subnet of the external IP to mock as another Node in the same subnet. cmd, netns := getCommandInFakeExternalNetwork("sleep 3600", tt.clientIPMaskLen, tt.clientIP, tt.localIP) - baseUrl := net.JoinHostPort(externalIP, strconv.FormatInt(int64(port), 10)) + baseURL := getURL(externalIP, fmt.Sprint(port)) require.NoError(t, NewPodBuilder(tt.clientName, data.testNamespace, agnhostImage).OnNode(host).WithCommand([]string{"sh", "-c", cmd}).InHostNetwork().Privileged().WithMutateFunc(func(pod *v1.Pod) { delete(pod.Labels, "app") // curl will exit immediately if the destination IP is unreachable and will NOT retry despite having retry flags set. // Use an exec readiness probe to ensure the route is configured to the interface. // Refer to https://github.com/curl/curl/issues/1603. - probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseUrl), " ") + probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseURL), " ") pod.Spec.Containers[0].ReadinessProbe = &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ @@ -663,8 +662,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) { require.NoError(t, err) defer data.DeletePodAndWait(defaultTimeout, tt.clientName, data.testNamespace) - hostNameUrl := fmt.Sprintf("%s/%s", baseUrl, "hostname") - probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameUrl) + hostNameURL := fmt.Sprintf("%s/hostname", baseURL) + probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameURL) hostname, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeCmd}) assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr) @@ -674,8 +673,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) { assert.Equal(t, agnhosts[idx], hostname, "Hostname should match when ExternalTrafficPolicy setting to Local") } } - clientIPUrl := fmt.Sprintf("%s/clientip", baseUrl) - probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPUrl) + clientIPURL := fmt.Sprintf("%s/clientip", baseURL) + probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPURL) clientIPPort, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeClientIPCmd}) assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr) clientIP, _, err := net.SplitHostPort(clientIPPort) diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 61d39751070..73e8024b58a 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -164,9 +164,7 @@ func (data *TestData) testNodePort(t *testing.T, isWindows bool, clientNamespace require.NoError(t, err) t.Logf("Created client Pod IPs %v", podIPs.IPStrings) - nodeIP := clusterInfo.nodes[0].ip() - nodePort := int(svc.Spec.Ports[0].NodePort) - url := fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + url := getURL(clusterInfo.nodes[0].ip(), fmt.Sprint(svc.Spec.Ports[0].NodePort)) stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(clientName, clientNamespace, url, 5) if err != nil { diff --git a/test/e2e/util.go b/test/e2e/util.go index c26693b0555..ed88d11e098 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -56,3 +56,21 @@ func IPFamily(ip string) string { return "" } } + +// getURL returns a HTTP url based on the IP and the addition port and path if provided. +// Examples: +// getURL("1.2.3.4") == "http://1.2.3.4" +// getURL("1.2.3.4", "8080") == "http://1.2.3.4:8080" +// getURL("1.2.3.4", "8080", "clientip") == "http://1.2.3.4:8080/clientip" +// getURL("fd74:ca9b:172::b4e", "8080") == "http://[fd74:ca9b:172::b4e]:8080" +func getURL(ip string, args ...string) string { + port := "80" + if len(args) > 0 { + port = args[0] + } + url := "http://" + net.JoinHostPort(ip, port) + if len(args) > 1 { + url += "/" + args[1] + } + return url +}