From 6e81d26728392f50f4ca2d8fc6e4dab71616171f Mon Sep 17 00:00:00 2001 From: Quan Tian Date: Fri, 1 Mar 2024 14:45:07 +0800 Subject: [PATCH] Fix IPv6 e2e test caused by image update wget in toolbox image expects the protocol to be specified when IPv6 address is used in the url, otherwise it would use ftp protocol. This patch fixes it and unifies how we get url from IP, port and path. Signed-off-by: Quan Tian --- test/e2e/egress_test.go | 42 +++++++++++++---------------- test/e2e/flowaggregator_test.go | 11 +------- test/e2e/prometheus_test.go | 3 +-- test/e2e/proxy_test.go | 6 ++--- test/e2e/service_externalip_test.go | 13 +++++---- test/e2e/service_test.go | 4 +-- test/e2e/util.go | 16 +++++++++++ 7 files changed, 47 insertions(+), 48 deletions(-) diff --git a/test/e2e/egress_test.go b/test/e2e/egress_test.go index 63ea8bfc16e..fc674d1506d 100644 --- a/test/e2e/egress_test.go +++ b/test/e2e/egress_test.go @@ -149,15 +149,10 @@ func testEgressClientIP(t *testing.T, data *TestData) { t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", remotePod) } - serverIPStr := tt.serverIP - if utilnet.IsIPv6String(tt.localIP0) { - serverIPStr = fmt.Sprintf("[%s]", tt.serverIP) - } - // As the fake server runs in a netns of the Egress Node, only egress Node can reach the server, Pods running on // other Nodes cannot reach it before Egress is added. - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Logf("Creating an Egress applying to all e2e Pods") matchExpressions := []metav1.LabelSelectorRequirement{ @@ -168,8 +163,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { } egress := data.createEgress(t, "egress-", matchExpressions, nil, "", egressNodeIP, nil) defer data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP) - assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP) + assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP) var err error err = wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) { @@ -187,7 +182,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if utilnet.IsIPv6String(clientIPStr) { clientIPStr = fmt.Sprintf("[%s]", clientIPStr) } - cmd = fmt.Sprintf("wget -T 3 -O - %s:8080/clientip | grep %s:", serverIPStr, clientIPStr) + url := getHTTPURLFromIPPort(tt.serverIP, 8080, "clientip") + cmd = fmt.Sprintf("wget -T 3 -O - %s | grep %s:", url, clientIPStr) if err := NewPodBuilder(initialIPChecker, data.testNamespace, agnhostImage).OnNode(egressNode).WithCommand([]string{"sh", "-c", cmd}).Create(data); err != nil { t.Fatalf("Failed to create Pod initial-ip-checker: %v", err) } @@ -210,8 +206,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP) t.Log("Updating the Egress's AppliedTo to localPod only") egress.Spec.AppliedTo = v1beta1.AppliedTo{ @@ -223,8 +219,8 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Logf("Updating the Egress's EgressIP to %s", tt.localIP1) egress.Spec.EgressIP = tt.localIP1 @@ -232,16 +228,16 @@ func testEgressClientIP(t *testing.T, data *TestData) { if err != nil { t.Fatalf("Failed to update Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) t.Log("Deleting the Egress") err = data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{}) if err != nil { t.Fatalf("Failed to delete Egress %v: %v", egress, err) } - assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1) - assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr) + assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1) + assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP) }) } } @@ -933,7 +929,7 @@ func setupIPNeighborChecker(data *TestData, t *testing.T, observerNode, node1, n // Before the Node actually connects to the Egress IP, we expect that the lladdr either matches the Egress Node's MAC address or is empty. check(true) // The protocol must be present when using wget with IPv6 address. - cmd := []string{"wget", fmt.Sprintf("http://%s", net.JoinHostPort(ip, "80")), "-T", "1", "-t", "1"} + cmd := []string{"wget", getHTTPURLFromIPPort(ip, 80), "-T", "1", "-t", "1"} // We don't care whether it succeeds, just make it connect to the Egress IP to learn its MAC address. data.RunCommandFromPod(antreaNamespace, antreaPodName, agentContainerName, cmd) // After the Node tries to connect to the Egress IP, we expect that the lladdr matches the Egress Node's MAC address. @@ -997,11 +993,11 @@ func (data *TestData) waitForEgressRealized(egress *v1beta1.Egress) (*v1beta1.Eg } // assertClientIP asserts the Pod is translated to the provided client IP. -func assertClientIP(data *TestData, t *testing.T, pod, container, server string, clientIPs ...string) { +func assertClientIP(data *TestData, t *testing.T, pod, container, serverIP string, clientIPs ...string) { var exeErr error var stdout, stderr string err := wait.Poll(100*time.Millisecond, 5*time.Second, func() (done bool, err error) { - url := fmt.Sprintf("%s:8080/clientip", server) + url := getHTTPURLFromIPPort(serverIP, 8080, "clientip") stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, container, url, 5) if exeErr != nil { return false, nil @@ -1023,11 +1019,11 @@ func assertClientIP(data *TestData, t *testing.T, pod, container, server string, } // assertConnError asserts the Pod is not able to access the API that replies the request's client IP. -func assertConnError(data *TestData, t *testing.T, pod, container, server string) { +func assertConnError(data *TestData, t *testing.T, pod, container, serverIP string) { var exeErr error var stdout, stderr string err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) { - url := fmt.Sprintf("%s:8080/clientip", server) + url := getHTTPURLFromIPPort(serverIP, 8080, "clientip") stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, url, container, 5) if exeErr != nil { return true, nil diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go index dc9ebe048e5..848b739b401 100644 --- a/test/e2e/flowaggregator_test.go +++ b/test/e2e/flowaggregator_test.go @@ -1904,23 +1904,14 @@ func testL7FlowExporterController(t *testing.T, data *TestData, isIPv6 bool) { testFlow1 := testFlow{ srcPodName: clientPodName, } - var cmd []string if !isIPv6 { testFlow1.srcIP = clientPodIPs.IPv4.String() testFlow1.dstIP = serverIPs.IPv4.String() - cmd = []string{ - "curl", - fmt.Sprintf("http://%s:%d", serverIPs.IPv4.String(), serverPodPort), - } } else { testFlow1.srcIP = clientPodIPs.IPv6.String() testFlow1.dstIP = serverIPs.IPv6.String() - cmd = []string{ - "curl", - "-6", - fmt.Sprintf("http://[%s]:%d", serverIPs.IPv6.String(), serverPodPort), - } } + cmd := []string{"curl", getHTTPURLFromIPPort(testFlow1.dstIP, serverPodPort)} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, testFlow1.srcPodName, "l7flowexporter", cmd) require.NoErrorf(t, err, "Error when running curl command, stdout: %s, stderr: %s", stdout, stderr) _, recordSlices := getCollectorOutput(t, testFlow1.srcIP, testFlow1.dstIP, "", false, true, isIPv6, data, "") diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go index 25ce5354ba0..577826bb960 100644 --- a/test/e2e/prometheus_test.go +++ b/test/e2e/prometheus_test.go @@ -281,8 +281,7 @@ func testMetricsFromPrometheusServer(t *testing.T, data *TestData, prometheusJob // Target metadata API(/api/v1/targets/metadata) has been available since Prometheus v2.4.0. // This API is still experimental in Prometheus v2.46.0. path := url.PathEscape("match_target={job=\"" + prometheusJob + "\"}") - address := net.JoinHostPort(hostIP, fmt.Sprint(nodePort)) - queryURL := fmt.Sprintf("http://%s/api/v1/targets/metadata?%s", address, path) + queryURL := getHTTPURLFromIPPort(hostIP, nodePort, "api/v1/targets/metadata?", path) client := &http.Client{} var output prometheusServerOutput diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index a1643f8b412..58dd59c61b8 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -625,10 +625,10 @@ func testProxyServiceSessionAffinity(ipFamily *corev1.IPFamily, ingressIPs []str require.NoError(t, data.createToolboxPodOnNode(toolboxPod, data.testNamespace, nodeName, false)) defer data.DeletePodAndWait(defaultTimeout, toolboxPod, data.testNamespace) require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPod, data.testNamespace)) - stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, svc.Spec.ClusterIP, 5) + stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getHTTPURLFromIPPort(svc.Spec.ClusterIP, 80), 5) require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr)) for _, ingressIP := range ingressIPs { - stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, ingressIP, 5) + stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getHTTPURLFromIPPort(ingressIP, 80), 5) require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr)) } @@ -1175,7 +1175,7 @@ func TestProxyLoadBalancerModeDSR(t *testing.T) { defer data.deleteServiceAndWait(defaultTimeout, serviceName, data.testNamespace) curlServiceWithPath := func(clientPod, clientNetns, path string) string { - testURL := fmt.Sprintf("http://%s:%d/%s", lbIP, service.Spec.Ports[0].Port, path) + testURL := getHTTPURLFromIPPort(lbIP, service.Spec.Ports[0].Port, path) cmd = fmt.Sprintf("curl --connect-timeout 1 --retry 5 --retry-connrefused %s", testURL) if clientNetns != "" { cmd = fmt.Sprintf("ip netns exec %s %s", clientNetns, cmd) diff --git a/test/e2e/service_externalip_test.go b/test/e2e/service_externalip_test.go index aaae30c6b8f..57692e7ce6f 100644 --- a/test/e2e/service_externalip_test.go +++ b/test/e2e/service_externalip_test.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "net" - "strconv" "strings" "testing" "time" @@ -633,14 +632,14 @@ func testExternalIPAccess(t *testing.T, data *TestData) { // Create a pod in a different netns with the same subnet of the external IP to mock as another Node in the same subnet. cmd, netns := getCommandInFakeExternalNetwork("sleep 3600", tt.clientIPMaskLen, tt.clientIP, tt.localIP) - baseUrl := net.JoinHostPort(externalIP, strconv.FormatInt(int64(port), 10)) + baseURL := getHTTPURLFromIPPort(externalIP, port) require.NoError(t, NewPodBuilder(tt.clientName, data.testNamespace, agnhostImage).OnNode(host).WithCommand([]string{"sh", "-c", cmd}).InHostNetwork().Privileged().WithMutateFunc(func(pod *v1.Pod) { delete(pod.Labels, "app") // curl will exit immediately if the destination IP is unreachable and will NOT retry despite having retry flags set. // Use an exec readiness probe to ensure the route is configured to the interface. // Refer to https://github.com/curl/curl/issues/1603. - probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseUrl), " ") + probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseURL), " ") pod.Spec.Containers[0].ReadinessProbe = &v1.Probe{ ProbeHandler: v1.ProbeHandler{ Exec: &v1.ExecAction{ @@ -663,8 +662,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) { require.NoError(t, err) defer data.DeletePodAndWait(defaultTimeout, tt.clientName, data.testNamespace) - hostNameUrl := fmt.Sprintf("%s/%s", baseUrl, "hostname") - probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameUrl) + hostNameURL := fmt.Sprintf("%s/hostname", baseURL) + probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameURL) hostname, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeCmd}) assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr) @@ -674,8 +673,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) { assert.Equal(t, agnhosts[idx], hostname, "Hostname should match when ExternalTrafficPolicy setting to Local") } } - clientIPUrl := fmt.Sprintf("%s/clientip", baseUrl) - probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPUrl) + clientIPURL := fmt.Sprintf("%s/clientip", baseURL) + probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPURL) clientIPPort, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeClientIPCmd}) assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr) clientIP, _, err := net.SplitHostPort(clientIPPort) diff --git a/test/e2e/service_test.go b/test/e2e/service_test.go index 61d39751070..12b146d802d 100644 --- a/test/e2e/service_test.go +++ b/test/e2e/service_test.go @@ -164,9 +164,7 @@ func (data *TestData) testNodePort(t *testing.T, isWindows bool, clientNamespace require.NoError(t, err) t.Logf("Created client Pod IPs %v", podIPs.IPStrings) - nodeIP := clusterInfo.nodes[0].ip() - nodePort := int(svc.Spec.Ports[0].NodePort) - url := fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + url := getHTTPURLFromIPPort(clusterInfo.nodes[0].ip(), svc.Spec.Ports[0].NodePort) stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(clientName, clientNamespace, url, 5) if err != nil { diff --git a/test/e2e/util.go b/test/e2e/util.go index c26693b0555..a3be8100f98 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -15,9 +15,11 @@ package e2e import ( + "fmt" "io" "net" "os" + "strings" "time" "k8s.io/klog/v2" @@ -56,3 +58,17 @@ func IPFamily(ip string) string { return "" } } + +// getHTTPURLFromIPPort returns a HTTP url based on the IP, port and the additional paths if provided. +// Examples: +// getHTTPURLFromIPPort("1.2.3.4", 80) == "http://1.2.3.4:80" +// getHTTPURLFromIPPort("1.2.3.4", 8080, "clientip") == "http://1.2.3.4:8080/clientip" +// getHTTPURLFromIPPort("1.2.3.4", 8080, "api/v1/metadata?", "foo=bar") == "http://1.2.3.4:8080/api/v1/metadata?foo=bar" +// getHTTPURLFromIPPort("fd74:ca9b:172::b4e", 8080) == "http://[fd74:ca9b:172::b4e]:8080" +func getHTTPURLFromIPPort(ip string, port int32, paths ...string) string { + url := "http://" + net.JoinHostPort(ip, fmt.Sprint(port)) + if len(paths) > 0 { + url += "/" + strings.Join(paths, "") + } + return url +}