Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix IPv6 e2e test caused by image update #6045

Merged
merged 1 commit into from
Mar 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 19 additions & 23 deletions test/e2e/egress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,15 +149,10 @@ func testEgressClientIP(t *testing.T, data *TestData) {
t.Fatalf("Error when waiting for Pod '%s' to be in the Running state", remotePod)
}

serverIPStr := tt.serverIP
if utilnet.IsIPv6String(tt.localIP0) {
serverIPStr = fmt.Sprintf("[%s]", tt.serverIP)
}

// As the fake server runs in a netns of the Egress Node, only egress Node can reach the server, Pods running on
// other Nodes cannot reach it before Egress is added.
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP)

t.Logf("Creating an Egress applying to all e2e Pods")
matchExpressions := []metav1.LabelSelectorRequirement{
Expand All @@ -168,8 +163,8 @@ func testEgressClientIP(t *testing.T, data *TestData) {
}
egress := data.createEgress(t, "egress-", matchExpressions, nil, "", egressNodeIP, nil)
defer data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{})
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP)
assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP)
assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP)

var err error
err = wait.Poll(time.Millisecond*100, time.Second, func() (bool, error) {
Expand All @@ -187,7 +182,8 @@ func testEgressClientIP(t *testing.T, data *TestData) {
if utilnet.IsIPv6String(clientIPStr) {
clientIPStr = fmt.Sprintf("[%s]", clientIPStr)
}
cmd = fmt.Sprintf("wget -T 3 -O - %s:8080/clientip | grep %s:", serverIPStr, clientIPStr)
url := getHTTPURLFromIPPort(tt.serverIP, 8080, "clientip")
cmd = fmt.Sprintf("wget -T 3 -O - %s | grep %s:", url, clientIPStr)
if err := NewPodBuilder(initialIPChecker, data.testNamespace, agnhostImage).OnNode(egressNode).WithCommand([]string{"sh", "-c", cmd}).Create(data); err != nil {
t.Fatalf("Failed to create Pod initial-ip-checker: %v", err)
}
Expand All @@ -210,8 +206,8 @@ func testEgressClientIP(t *testing.T, data *TestData) {
if err != nil {
t.Fatalf("Failed to update Egress %v: %v", egress, err)
}
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1)
assertClientIP(data, t, remotePod, toolboxContainerName, serverIPStr, egressNodeIP)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1)
assertClientIP(data, t, remotePod, toolboxContainerName, tt.serverIP, egressNodeIP)

t.Log("Updating the Egress's AppliedTo to localPod only")
egress.Spec.AppliedTo = v1beta1.AppliedTo{
Expand All @@ -223,25 +219,25 @@ func testEgressClientIP(t *testing.T, data *TestData) {
if err != nil {
t.Fatalf("Failed to update Egress %v: %v", egress, err)
}
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, egressNodeIP)
assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, egressNodeIP)
assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP)

t.Logf("Updating the Egress's EgressIP to %s", tt.localIP1)
egress.Spec.EgressIP = tt.localIP1
egress, err = data.crdClient.CrdV1beta1().Egresses().Update(context.TODO(), egress, metav1.UpdateOptions{})
if err != nil {
t.Fatalf("Failed to update Egress %v: %v", egress, err)
}
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP)

t.Log("Deleting the Egress")
err = data.crdClient.CrdV1beta1().Egresses().Delete(context.TODO(), egress.Name, metav1.DeleteOptions{})
if err != nil {
t.Fatalf("Failed to delete Egress %v: %v", egress, err)
}
assertClientIP(data, t, localPod, toolboxContainerName, serverIPStr, tt.localIP0, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, serverIPStr)
assertClientIP(data, t, localPod, toolboxContainerName, tt.serverIP, tt.localIP0, tt.localIP1)
assertConnError(data, t, remotePod, toolboxContainerName, tt.serverIP)
})
}
}
Expand Down Expand Up @@ -933,7 +929,7 @@ func setupIPNeighborChecker(data *TestData, t *testing.T, observerNode, node1, n
// Before the Node actually connects to the Egress IP, we expect that the lladdr either matches the Egress Node's MAC address or is empty.
check(true)
// The protocol must be present when using wget with IPv6 address.
cmd := []string{"wget", fmt.Sprintf("http://%s", net.JoinHostPort(ip, "80")), "-T", "1", "-t", "1"}
cmd := []string{"wget", getHTTPURLFromIPPort(ip, 80), "-T", "1", "-t", "1"}
// We don't care whether it succeeds, just make it connect to the Egress IP to learn its MAC address.
data.RunCommandFromPod(antreaNamespace, antreaPodName, agentContainerName, cmd)
// After the Node tries to connect to the Egress IP, we expect that the lladdr matches the Egress Node's MAC address.
Expand Down Expand Up @@ -997,11 +993,11 @@ func (data *TestData) waitForEgressRealized(egress *v1beta1.Egress) (*v1beta1.Eg
}

// assertClientIP asserts the Pod is translated to the provided client IP.
func assertClientIP(data *TestData, t *testing.T, pod, container, server string, clientIPs ...string) {
func assertClientIP(data *TestData, t *testing.T, pod, container, serverIP string, clientIPs ...string) {
var exeErr error
var stdout, stderr string
err := wait.Poll(100*time.Millisecond, 5*time.Second, func() (done bool, err error) {
url := fmt.Sprintf("%s:8080/clientip", server)
url := getHTTPURLFromIPPort(serverIP, 8080, "clientip")
stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, container, url, 5)
if exeErr != nil {
return false, nil
Expand All @@ -1023,11 +1019,11 @@ func assertClientIP(data *TestData, t *testing.T, pod, container, server string,
}

// assertConnError asserts the Pod is not able to access the API that replies the request's client IP.
func assertConnError(data *TestData, t *testing.T, pod, container, server string) {
func assertConnError(data *TestData, t *testing.T, pod, container, serverIP string) {
var exeErr error
var stdout, stderr string
err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (done bool, err error) {
url := fmt.Sprintf("%s:8080/clientip", server)
url := getHTTPURLFromIPPort(serverIP, 8080, "clientip")
stdout, stderr, exeErr = data.runWgetCommandFromTestPodWithRetry(pod, data.testNamespace, url, container, 5)
if exeErr != nil {
return true, nil
Expand Down
11 changes: 1 addition & 10 deletions test/e2e/flowaggregator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1904,23 +1904,14 @@ func testL7FlowExporterController(t *testing.T, data *TestData, isIPv6 bool) {
testFlow1 := testFlow{
srcPodName: clientPodName,
}
var cmd []string
if !isIPv6 {
testFlow1.srcIP = clientPodIPs.IPv4.String()
testFlow1.dstIP = serverIPs.IPv4.String()
cmd = []string{
"curl",
fmt.Sprintf("http://%s:%d", serverIPs.IPv4.String(), serverPodPort),
}
} else {
testFlow1.srcIP = clientPodIPs.IPv6.String()
testFlow1.dstIP = serverIPs.IPv6.String()
cmd = []string{
"curl",
"-6",
fmt.Sprintf("http://[%s]:%d", serverIPs.IPv6.String(), serverPodPort),
}
}
cmd := []string{"curl", getHTTPURLFromIPPort(testFlow1.dstIP, serverPodPort)}
stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, testFlow1.srcPodName, "l7flowexporter", cmd)
require.NoErrorf(t, err, "Error when running curl command, stdout: %s, stderr: %s", stdout, stderr)
_, recordSlices := getCollectorOutput(t, testFlow1.srcIP, testFlow1.dstIP, "", false, true, isIPv6, data, "")
Expand Down
3 changes: 1 addition & 2 deletions test/e2e/prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,7 @@ func testMetricsFromPrometheusServer(t *testing.T, data *TestData, prometheusJob
// Target metadata API(/api/v1/targets/metadata) has been available since Prometheus v2.4.0.
// This API is still experimental in Prometheus v2.46.0.
path := url.PathEscape("match_target={job=\"" + prometheusJob + "\"}")
address := net.JoinHostPort(hostIP, fmt.Sprint(nodePort))
queryURL := fmt.Sprintf("http://%s/api/v1/targets/metadata?%s", address, path)
queryURL := getHTTPURLFromIPPort(hostIP, nodePort, "api/v1/targets/metadata?", path)

client := &http.Client{}
var output prometheusServerOutput
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/proxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -625,10 +625,10 @@ func testProxyServiceSessionAffinity(ipFamily *corev1.IPFamily, ingressIPs []str
require.NoError(t, data.createToolboxPodOnNode(toolboxPod, data.testNamespace, nodeName, false))
defer data.DeletePodAndWait(defaultTimeout, toolboxPod, data.testNamespace)
require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPod, data.testNamespace))
stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, svc.Spec.ClusterIP, 5)
stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getHTTPURLFromIPPort(svc.Spec.ClusterIP, 80), 5)
require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr))
for _, ingressIP := range ingressIPs {
stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, ingressIP, 5)
stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(toolboxPod, data.testNamespace, getHTTPURLFromIPPort(ingressIP, 80), 5)
require.NoError(t, err, fmt.Sprintf("ipFamily: %v\nstdout: %s\nstderr: %s\n", *ipFamily, stdout, stderr))
}

Expand Down Expand Up @@ -1175,7 +1175,7 @@ func TestProxyLoadBalancerModeDSR(t *testing.T) {
defer data.deleteServiceAndWait(defaultTimeout, serviceName, data.testNamespace)

curlServiceWithPath := func(clientPod, clientNetns, path string) string {
testURL := fmt.Sprintf("http://%s:%d/%s", lbIP, service.Spec.Ports[0].Port, path)
testURL := getHTTPURLFromIPPort(lbIP, service.Spec.Ports[0].Port, path)
cmd = fmt.Sprintf("curl --connect-timeout 1 --retry 5 --retry-connrefused %s", testURL)
if clientNetns != "" {
cmd = fmt.Sprintf("ip netns exec %s %s", clientNetns, cmd)
Expand Down
13 changes: 6 additions & 7 deletions test/e2e/service_externalip_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"encoding/json"
"fmt"
"net"
"strconv"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -633,14 +632,14 @@ func testExternalIPAccess(t *testing.T, data *TestData) {
// Create a pod in a different netns with the same subnet of the external IP to mock as another Node in the same subnet.
cmd, netns := getCommandInFakeExternalNetwork("sleep 3600", tt.clientIPMaskLen, tt.clientIP, tt.localIP)

baseUrl := net.JoinHostPort(externalIP, strconv.FormatInt(int64(port), 10))
baseURL := getHTTPURLFromIPPort(externalIP, port)

require.NoError(t, NewPodBuilder(tt.clientName, data.testNamespace, agnhostImage).OnNode(host).WithCommand([]string{"sh", "-c", cmd}).InHostNetwork().Privileged().WithMutateFunc(func(pod *v1.Pod) {
delete(pod.Labels, "app")
// curl will exit immediately if the destination IP is unreachable and will NOT retry despite having retry flags set.
// Use an exec readiness probe to ensure the route is configured to the interface.
// Refer to https://github.com/curl/curl/issues/1603.
probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseUrl), " ")
probeCmd := strings.Split(fmt.Sprintf("ip netns exec %s curl -s %s", netns, baseURL), " ")
pod.Spec.Containers[0].ReadinessProbe = &v1.Probe{
ProbeHandler: v1.ProbeHandler{
Exec: &v1.ExecAction{
Expand All @@ -663,8 +662,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) {
require.NoError(t, err)
defer data.DeletePodAndWait(defaultTimeout, tt.clientName, data.testNamespace)

hostNameUrl := fmt.Sprintf("%s/%s", baseUrl, "hostname")
probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameUrl)
hostNameURL := fmt.Sprintf("%s/hostname", baseURL)
probeCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, hostNameURL)
hostname, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeCmd})
assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr)

Expand All @@ -674,8 +673,8 @@ func testExternalIPAccess(t *testing.T, data *TestData) {
assert.Equal(t, agnhosts[idx], hostname, "Hostname should match when ExternalTrafficPolicy setting to Local")
}
}
clientIPUrl := fmt.Sprintf("%s/clientip", baseUrl)
probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPUrl)
clientIPURL := fmt.Sprintf("%s/clientip", baseURL)
probeClientIPCmd := fmt.Sprintf("ip netns exec %s curl --connect-timeout 1 --retry 5 --retry-connrefused %s", netns, clientIPURL)
clientIPPort, stderr, err := data.RunCommandFromPod(data.testNamespace, tt.clientName, "", []string{"sh", "-c", probeClientIPCmd})
assert.NoError(t, err, "External IP should be able to be connected from remote: %s", stderr)
clientIP, _, err := net.SplitHostPort(clientIPPort)
Expand Down
4 changes: 1 addition & 3 deletions test/e2e/service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,7 @@ func (data *TestData) testNodePort(t *testing.T, isWindows bool, clientNamespace
require.NoError(t, err)
t.Logf("Created client Pod IPs %v", podIPs.IPStrings)

nodeIP := clusterInfo.nodes[0].ip()
nodePort := int(svc.Spec.Ports[0].NodePort)
url := fmt.Sprintf("http://%s:%d", nodeIP, nodePort)
url := getHTTPURLFromIPPort(clusterInfo.nodes[0].ip(), svc.Spec.Ports[0].NodePort)

stdout, stderr, err := data.runWgetCommandOnToolboxWithRetry(clientName, clientNamespace, url, 5)
if err != nil {
Expand Down
16 changes: 16 additions & 0 deletions test/e2e/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
package e2e

import (
"fmt"
"io"
"net"
"os"
"strings"
"time"

"k8s.io/klog/v2"
Expand Down Expand Up @@ -56,3 +58,17 @@ func IPFamily(ip string) string {
return ""
}
}

// getHTTPURLFromIPPort returns a HTTP url based on the IP, port and the additional paths if provided.
// Examples:
// getHTTPURLFromIPPort("1.2.3.4", 80) == "http://1.2.3.4:80"
// getHTTPURLFromIPPort("1.2.3.4", 8080, "clientip") == "http://1.2.3.4:8080/clientip"
// getHTTPURLFromIPPort("1.2.3.4", 8080, "api/v1/metadata?", "foo=bar") == "http://1.2.3.4:8080/api/v1/metadata?foo=bar"
// getHTTPURLFromIPPort("fd74:ca9b:172::b4e", 8080) == "http://[fd74:ca9b:172::b4e]:8080"
func getHTTPURLFromIPPort(ip string, port int32, paths ...string) string {
url := "http://" + net.JoinHostPort(ip, fmt.Sprint(port))
if len(paths) > 0 {
url += "/" + strings.Join(paths, "")
}
return url
}
Loading