From ab093405c2e7ea201728902c8c26cb2e85b5ac20 Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 27 Sep 2024 03:36:33 +0530 Subject: [PATCH 01/17] initial commit for e2e test for #6229 Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 459 ++++++++++++++++++++++++++++++++ test/e2e/framework.go | 15 +- test/e2e/k8s_util.go | 233 ++++++++++++++++ 3 files changed, 706 insertions(+), 1 deletion(-) create mode 100644 test/e2e/fqdn_dns_cache_test.go diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go new file mode 100644 index 00000000000..af58c7e84aa --- /dev/null +++ b/test/e2e/fqdn_dns_cache_test.go @@ -0,0 +1,459 @@ +package e2e + +import ( + crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" + "antrea.io/antrea/test/e2e/utils" + "context" + "encoding/json" + "fmt" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + v12 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + "testing" + "time" +) + +/* +1) Create the KIND cluster. +2) Once cluster is up , create a service. +3) Get the IP of above service and configure the same in antrea configMap. +4) Update antrea configMap. + +5) Create NGINX deployment. +6) Get IP of one of the pods of nginx. +7) create and configure the custom CoreDNS configMap with the IP received above. +8) Create custom CoreDNS deployment. +9) Create and apply antrea FQDN policy. +10) Deploy antrea-toolbox. + + +---------- tic +11) curl the FQDN from within toolbox. +12) imitate caching the IP belonging to above FQDN resolution by keeping it in a variable. +13) edit configmap with the other IP. +14) wait for new IP to get updated in configMap and let the changes be reflected in dns pod. +15) curl the FQDN again with IP , simulating usage of cache -- and it must fail with no connectivity. +*/ + +func TestFQDNPolicyWithCachedDNS(t *testing.T) { + skipIfAntreaPolicyDisabled(t) + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + + testFqdn := "nginx-test-pod.lfx.test" + + //TODO: Check for IPv6 ? + ipFamily := v1.IPv4Protocol + + // Create the service . + //TODO: Should the names be put up as constants instead of direct strings here? + customDnsService, err := testData.CreateUDPService("custom-dns-service", data.testNamespace, 53, 53, map[string]string{"app": "custom-dns"}, false, false, v1.ServiceTypeClusterIP, &ipFamily) + if err != nil { + t.Fatalf("Error when creating custom DNS service: %v", err) + } + require.NoError(t, err) + + // get the IP + customCoreDnsServiceObject, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.Background(), customDnsService.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error when getting custom DNS service object : %v", err) + } + require.NoError(t, err) + + // Print the ClusterIP + t.Logf("ClusterIP of the service: %s\n", customCoreDnsServiceObject.Spec.ClusterIP) + + // Get Antrea ConfigMap + cm, err := data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + t.Fatalf("Error when getting custom DNS configMap : %v", err) + } + require.NoError(t, err) + + // Read current value of dnsServer + var agentConf agentconfig.AgentConfig + + if err := yaml.Unmarshal([]byte(cm.Data["antrea-agent.conf"]), &agentConf); err != nil { + t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) + } + require.NoError(t, err) + + //Set up customDNS server IP in Antrea configmap. + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = customCoreDnsServiceObject.Spec.ClusterIP + } + err = data.mutateAntreaConfigMap(nil, agentChanges, false, true) + if err != nil { + t.Fatalf("Error when setting up customDNS server IP in Antrea configmap : %v", err) + } + + cm2, err := data.GetAntreaConfigMap(antreaNamespace) + if err != nil { + t.Fatalf("Error when getting custom DNS configMap : %v", err) + } + require.NoError(t, err) + + // Read current value of dnsServer + var agentConfChanged agentconfig.AgentConfig + if err := yaml.Unmarshal([]byte(cm2.Data["antrea-agent.conf"]), &agentConfChanged); err != nil { + t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) + } + require.NoError(t, err) + + t.Logf("dns server value set to %+v in antrea \n", agentConfChanged.DNSServerOverride) + + // Set up nginx server + nginxConfig := `events {} + +http { + server { + listen 80; + + location / { + return 200 "Pod hostname: $hostname\n"; + add_header Content-Type text/plain; + } + } +}` + + configData := map[string]string{ + "nginx.conf": nginxConfig, + } + nginxConfiMapObject, err := data.CreateConfigMap(data.testNamespace, "nginx-config", configData, nil, false) + if err != nil { + t.Fatalf("failed to create nginx ConfigMap: %v", err) + } + require.NoError(t, err) + + deploymentLabels := map[string]string{ + "app": "nginx", + } + + nginxDeployObject, err := data.CreateNginxDeploymentForTest("nginx-deployment", data.testNamespace, nginxConfiMapObject.Name, 2, deploymentLabels) + if err != nil { + t.Fatalf("failed to create nginx deployment: %v", err) + } + require.NoError(t, err) + + // Though this is used in vmagent_test.go but i think i needed it here to check for the deployment. + //TODO: Time of 15 seconds is still large for a timeout. + err = data.waitForDeploymentReady(t, data.testNamespace, nginxDeployObject.Name, 15*time.Second) + if err != nil { + t.Fatalf("error while waiting for nginx deployment to be ready : %v", err) + } + require.NoError(t, err) + + k8sUtils, err = NewKubernetesUtils(data) + if err != nil { + t.Fatalf("error getting k8s utils %+v", err) + } + require.NoError(t, err) + + nginxPods, err := k8sUtils.GetPodsByLabel(data.testNamespace, "app", "nginx") + if err != nil { + t.Fatalf("error getting Pods by label %+v", err) + } + require.NoError(t, err) + + //domainMapping holds whether the IP is mapped to Domain or not. + domainMapping := make(map[string]bool) + + // pick an IP to be added in config + var ipForConfig string + for idx, pod := range nginxPods { + //TODO: Following wait time change ? + _, err = data.podWaitForIPs(10*time.Second, pod.Name, data.testNamespace) + if err != nil { + t.Fatalf("error waiting for nginx pods to get IPs %+v", err) + } + require.NoError(t, err) + + ipStr := strings.TrimSpace(pod.Status.PodIP) + domainMapping[ipStr] = false + //pick last IP for config + if idx == len(nginxPods)-1 { + ipForConfig = ipStr + } + } + + // Create and update the custom DNS configMap + customDNSconfig := fmt.Sprintf(`lfx.test:53 { + errors + t + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 60 + } + hosts { + %s %s + no_reverse + pods verified + ttl 10 + } + loop + reload +}`, ipForConfig, testFqdn) + + customDNSconfigData := map[string]string{ + "Corefile": customDNSconfig, + } + + customDNSconfigMapObject, err := data.CreateConfigMap(data.testNamespace, "custom-dns-config", customDNSconfigData, nil, false) + + if err != nil { + t.Fatalf("failed to create custom dns ConfigMap: %v", err) + } + require.NoError(t, err) + domainMapping[ipForConfig] = true + + // create supporting SA, Role and Role Binding for DNS deployment. + saSpec := data.BuildServiceAccount("custom-dns-service-account", data.testNamespace, nil) + sa, err := data.CreateOrUpdateServiceAccount(saSpec) + if err != nil { + t.Fatalf("failed to create service acount for custom dns : %v", err) + } + require.NoError(t, err) + + clusterRoleSpec := &v12.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role"}, + Rules: []v12.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services"}, + Verbs: []string{"list", "watch"}, + }, + { + APIGroups: []string{"discovery.k8s.io"}, + Resources: []string{"endpointslices"}, + Verbs: []string{"list", "watch"}, + }, + }, + } + // TODO: Delete role on teardown. + role, err := data.CreateRole(clusterRoleSpec) + if err != nil { + t.Fatalf("failed to create cluster role for custom dns : %v", err) + } + require.NoError(t, err) + + clusterRoleBinding := &v12.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role-binding"}, + Subjects: []v12.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: data.testNamespace, + }, + }, + RoleRef: v12.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: role.Name, + }, + } + + err = data.CreateRoleBinding(clusterRoleBinding) + if err != nil { + t.Fatalf("failed to create cluster role binding for custom dns : %v", err) + } + require.NoError(t, err) + + // Create custom DNS deployment + dnsDeploymentLabels := map[string]string{ + "app": "custom-dns", + } + dnsDeploymentObj, err := data.CreateCustomDnsDeployment("custom-dns-deployment", data.testNamespace, customDNSconfigMapObject.Name, sa.Name, dnsDeploymentLabels, 1) + if err != nil { + t.Fatalf("failed to create custom dns deployment : %v", err) + } + require.NoError(t, err) + + err = data.waitForDeploymentReady(t, data.testNamespace, dnsDeploymentObj.Name, 120*time.Second) + if err != nil { + t.Fatalf("error while waiting for custom dns deployment to be ready : %v", err) + } + require.NoError(t, err) + + // Create policy + npPodSelectorLabel := map[string]string{ + "app": "fqdn-cache-test", + } + port := int32(80) + builder := &utils.ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName("test-acnp-fqdn"). + SetTier("application"). + SetPriority(1.0). + SetAppliedToGroup([]utils.ACNPAppliedToSpec{{PodSelector: npPodSelectorLabel}}) + builder.AddFQDNRule(testFqdn, "TCP", &port, nil, nil, "r1", nil, crdv1beta1.RuleActionAllow) + builder.AddEgress("UDP", nil, nil, nil, nil, nil, nil, nil, nil, dnsDeploymentLabels, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "", nil) + builder.AddEgress("TCP", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionReject, "", "", nil) + + acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) + failOnError(err, t) + failOnError(waitForResourceReady(t, 30*time.Second, acnp), t) + + //TODO: deletion of a namespace deletes all resources under it, do we still need to explicitly delete resources created under that namespace during a test ? + defer tearDownFQDN(t, data, builder) + + // create toolbox using their framework + toolBoxLabel := npPodSelectorLabel + pb := NewPodBuilder("toolbox", data.testNamespace, ToolboxImage) + pb.WithLabels(toolBoxLabel) + pb.WithContainerName("toolbox-container") + mutateSpecForAddingCustomDNS := func(pod *v1.Pod) { + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &v1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{customCoreDnsServiceObject.Spec.ClusterIP} + + } + pb.WithMutateFunc(mutateSpecForAddingCustomDNS) + err = pb.Create(data) + if err != nil { + t.Fatalf("failed to create antrea toolbox : %v", err) + } + require.NoError(t, err) + + /* ------ Actual test ------ */ + + // idea copied from antctl_test.go:284 + // getEndpointStatus will return "Success", "Failure", or the empty string when out is not a + // marshalled metav1.Status object. + getEndpointStatus := func(out []byte) string { + var status metav1.Status + if err := json.Unmarshal(out, &status); err != nil { + // Output is not JSON or does not encode a metav1.Status object. + return "" + } + return status.Status + } + + checkFQDNaccess := func(podName, containerName, fqdn string, checkStatus bool) error { + t.Logf("trying to curl the fqdn %v", fqdn) + cmd := []string{"curl", fqdn} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + if err != nil { + return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, err, stdout, stderr) + } + if checkStatus && getEndpointStatus([]byte(stdout)) == "Failure" { + return fmt.Errorf("failure status when accessing endpoint: <%v>", stdout) + } + fmt.Printf(" curl FQDN | running command '%s' on Pod '%s', stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, stdout, stderr) + return nil + } + + err = checkFQDNaccess(pb.Name, pb.ContainerName, testFqdn, true) + if err != nil { + t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) + } + require.NoError(t, err) + + // DIG to get actual IP , to be sure. + fqdnIp, err := k8sUtils.digDnSCustom(pb.Name, pb.Namespace, testFqdn, false) + if err != nil { + t.Fatalf("failed to get IP of FQDN using DIG from toolbox pod : %v", err) + } + require.NoError(t, err) + fqdnIp = strings.TrimSpace(fqdnIp) + + t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) + + var newIP string + for ip, mapped := range domainMapping { + if ip != fqdnIp && mapped == false { + newIP = ip + } + } + + t.Logf("New IP to update to DNS %v", newIP) + + // Curl the old ip and it should be a success. + err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) + if err != nil { + t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) + } + require.NoError(t, err) + + // Create and update the custom DNS configMap + UpdatedCustomDNSconfig := fmt.Sprintf(`lfx.test:53 { + errors + t + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 60 + } + hosts { + %s %s + no_reverse + pods verified + ttl 10 + } + loop + reload +}`, newIP, testFqdn) + + // edit configmap with this IP + dnsConfigMap, err := data.GetConfigMap(data.testNamespace, customDNSconfigMapObject.Name) + if err != nil { + t.Fatalf("failed to get configmap to replace IP : %v", err) + } + require.NoError(t, err) + + dnsConfigMap.Data["Corefile"] = UpdatedCustomDNSconfig + err = data.UpdateConfigMap(dnsConfigMap) + if err != nil { + t.Fatalf("failed to update configmap with new IP : %v", err) + } + require.NoError(t, err) + + // Update the annotation + if dnsDeploymentObj.Annotations == nil { + dnsDeploymentObj.Annotations = make(map[string]string) + } + dnsDeploymentObj.Annotations["baar"] = "foo" + + // Get the Deployment + updatedDnsDeploymentObj, err := data.clientset.AppsV1().Deployments(data.testNamespace).Get(context.TODO(), dnsDeploymentObj.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Error getting Deployment: %s", err) + } + + // Update the Deployment + _, err = data.clientset.AppsV1().Deployments(data.testNamespace).Update(context.TODO(), updatedDnsDeploymentObj, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Error updating Deployment: %s", err) + } + require.NoError(t, err) + + for { + err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) + if err != nil { + t.Logf("curl to ip filed. Error %v", err) + break + } + // Wait for 1 second before retrying + time.Sleep(1 * time.Second) + } + + // Ensuring that the test checks that an error occurred. + require.Error(t, err) + +} + +func tearDownFQDN(t *testing.T, data *TestData, builder *utils.ClusterNetworkPolicySpecBuilder) { + // cleanup test resources + teardownTest(t, data) + failOnError(k8sUtils.DeleteACNP(builder.Name), t) +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a497b49b53f..00927f8b2c9 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1543,9 +1543,15 @@ func (b *PodBuilder) Create(data *TestData) error { if b.MutateFunc != nil { b.MutateFunc(pod) } - if _, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { + + podObj, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { return err } + _, err = data.PodWaitFor(10*time.Second, podObj.Name, podObj.Namespace, func(pod2 *corev1.Pod) (bool, error) { + return pod2.Status.Phase == corev1.PodRunning, nil + }) + return nil } @@ -1988,6 +1994,13 @@ func (data *TestData) CreateService(serviceName, namespace string, port, targetP return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolTCP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) } +// CreateUDPService creates a service with a UDP port and targetPort. +func (data *TestData) CreateUDPService(serviceName, namespace string, port, targetPort int32, selector map[string]string, affinity, nodeLocalExternal bool, + serviceType corev1.ServiceType, ipFamily *corev1.IPFamily) (*corev1.Service, error) { + annotation := make(map[string]string) + return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolUDP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) +} + // CreateServiceWithAnnotations creates a service with Annotation func (data *TestData) CreateServiceWithAnnotations(serviceName, namespace string, port, targetPort int32, protocol corev1.Protocol, selector map[string]string, affinity, nodeLocalExternal bool, serviceType corev1.ServiceType, ipFamily *corev1.IPFamily, annotations map[string]string, mutators ...func(service *corev1.Service)) (*corev1.Service, error) { diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index d665cce4c82..35df16de6cb 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + v12 "k8s.io/api/rbac/v1" "strconv" "strings" "sync" @@ -103,6 +104,7 @@ type TestNamespaceMeta struct { // GetPodByLabel returns a Pod with the matching Namespace and "pod" label if it's found. // If the pod is not found, GetPodByLabel returns "ErrPodNotFound". func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) { + //TODO: Query why this is hard coded as "pod" pods, err := k.getPodsUncached(ns, "pod", name) if err != nil { return nil, fmt.Errorf("unable to get Pod in Namespace %s with label pod=%s: %w", ns, name, err) @@ -332,6 +334,68 @@ func decidePingProbeResult(stdout string, probeNum int) PodConnectivityMark { } return Error } +func (k *KubernetesUtils) digDnSCustom( + podName string, + podNamespace string, + dstAddr string, + useTCP bool) (string, error) { + + // Get the Pod + pod, err := k.clientset.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + log.Fatalf("Error getting pod: %v", err) + } + + digCmd := fmt.Sprintf("dig %s", dstAddr) + if useTCP { + digCmd += " +tcp" + } + cmd := []string{ + "/bin/sh", + "-c", + digCmd, + } + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) + log.Tracef("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) + stdout, stderr, err := k.RunCommandFromPod(pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, cmd) + fmt.Printf("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) + log.Tracef("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) + //========DiG command stdout example======== + //; <<>> DiG 9.16.6 <<>> github.com +tcp + //;; global options: +cmd + //;; Got answer: + //;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 21816 + //;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 + // + //;; OPT PSEUDOSECTION: + //; EDNS: version: 0, flags:; udp: 4096 + //; COOKIE: 2d7fe493ea37c430 (echoed) + //;; QUESTION SECTION: + //;github.com. IN A + // + //;; ANSWER SECTION: + //github.com. 6 IN A 140.82.113.3 + // + //;; Query time: 0 msec + //;; SERVER: 10.96.0.10#53(10.96.0.10) + //;; WHEN: Tue Feb 14 22:34:23 UTC 2023 + //;; MSG SIZE rcvd: 77 + //========================================== + answerMarkIdx := strings.Index(stdout, ";; ANSWER SECTION:") + if answerMarkIdx == -1 { + return "", fmt.Errorf("failed to parse dig response") + } + splitResp := strings.Split(stdout[answerMarkIdx:], "\n") + if len(splitResp) < 2 { + return "", fmt.Errorf("failed to parse dig response") + } + ipLine := splitResp[1] + lastTab := strings.LastIndex(ipLine, "\t") + if lastTab == -1 { + return "", fmt.Errorf("failed to parse dig response") + } + return ipLine[lastTab:], nil +} func (k *KubernetesUtils) digDNS( podName string, @@ -667,6 +731,162 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } +func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string]byte, immutable bool) (*v1.ConfigMap, error) { + configMap := &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Immutable: nil, + Data: configData, + BinaryData: nil, + } + + configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMap, metav1.CreateOptions{}) + return configMapObject, err +} + +func (data *TestData) CreateNginxDeploymentForTest(name, namespace, nginxConfigMapName string, replicas int32, labels map[string]string) (*appsv1.Deployment, error) { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:alpine", + Ports: []v1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "html-volume", + MountPath: "/etc/nginx/nginx.conf", + SubPath: "nginx.conf", + }, + }, + Env: []v1.EnvVar{ + { + Name: "HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "html-volume", // This may be given a name here instead of passing as a parameter. + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: nginxConfigMapName, + }, + }, + }, + }, + }, + }, + }, + }, + } + + nginxDeploymentObject, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + return nginxDeploymentObject, err + +} + +func (data *TestData) CreateCustomDnsDeployment(name, namespace, configMapName, serviceAccountName string, labels map[string]string, replicas int32) (*appsv1.Deployment, error) { + + customDNSdeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: v1.PodSpec{ + ServiceAccountName: serviceAccountName, + Containers: []v1.Container{ + { + Name: "monitor", + Image: "busybox", + Command: []string{"/bin/sh", "-c", "sleep 1d"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + }, + }, + { + Name: "coredns", + Image: "coredns/coredns:latest", + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{"-conf", "/etc/coredns/Corefile"}, + VolumeMounts: []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "config-volume", // This may be given a name here instead of passing as a parameter. + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configMapName, + }, + Items: []v1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + dnsDeploymentObj, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), customDNSdeployment, metav1.CreateOptions{}) + return dnsDeploymentObj, err + +} + // DeleteService is a convenience function for deleting a Service by Namespace and name. func (data *TestData) DeleteService(ns, name string) error { log.Infof("Deleting Service %s in ns %s", name, ns) @@ -705,6 +925,19 @@ func (data *TestData) BuildServiceAccount(name, ns string, labels map[string]str return serviceAccount } +func (data *TestData) CreateRole(clusterRole *v12.ClusterRole) (*v12.ClusterRole, error) { + role, err := data.clientset.RbacV1().ClusterRoles().Create(context.Background(), clusterRole, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return role, nil +} + +func (data *TestData) CreateRoleBinding(roleBinding *v12.ClusterRoleBinding) error { + _, err := data.clientset.RbacV1().ClusterRoleBindings().Create(context.Background(), roleBinding, metav1.CreateOptions{}) + return err +} + // CreateOrUpdateServiceAccount is a convenience function for updating/creating ServiceAccount. func (data *TestData) CreateOrUpdateServiceAccount(sa *v1.ServiceAccount) (*v1.ServiceAccount, error) { log.Infof("Creating/updating ServiceAccount %s in ns %s", sa.Name, sa.Namespace) From e63de57ce735d2fce97307024ae08704731b4f69 Mon Sep 17 00:00:00 2001 From: HEMANT KUMAR Date: Fri, 27 Sep 2024 05:54:30 +0000 Subject: [PATCH 02/17] Refactored to remove nginx and include agnHost. Also Updated dig command to use short flag and added annotation update for dns pod to attempt to make configmap changes reflect early. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 593 ++++++++++++++------------------ test/e2e/framework.go | 15 +- test/e2e/k8s_util.go | 195 +---------- 3 files changed, 279 insertions(+), 524 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index af58c7e84aa..bd86fa81cc9 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -9,9 +9,9 @@ import ( "fmt" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" - v1 "k8s.io/api/core/v1" - v12 "k8s.io/api/rbac/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strconv" "strings" "testing" "time" @@ -19,26 +19,49 @@ import ( /* 1) Create the KIND cluster. -2) Once cluster is up , create a service. -3) Get the IP of above service and configure the same in antrea configMap. -4) Update antrea configMap. +2) Once the cluster is up , create a service for custom DNS. +3) Configure and update Antrea configmap with above service's IP. -5) Create NGINX deployment. -6) Get IP of one of the pods of nginx. +5) Create agnHost 2 pods (deployment?). +6) Get IP of one of the pods of agnHost. 7) create and configure the custom CoreDNS configMap with the IP received above. -8) Create custom CoreDNS deployment. +8) Create a custom DNS pod and configure it to use above configMap with agnHost pod's IP. 9) Create and apply antrea FQDN policy. 10) Deploy antrea-toolbox. - ----------- tic 11) curl the FQDN from within toolbox. 12) imitate caching the IP belonging to above FQDN resolution by keeping it in a variable. -13) edit configmap with the other IP. +13) change the existing IP in dns configmap with the other agnHost pod's IP. 14) wait for new IP to get updated in configMap and let the changes be reflected in dns pod. -15) curl the FQDN again with IP , simulating usage of cache -- and it must fail with no connectivity. +15) curl the FQDN again with old IP , simulating usage of cache -- and it must fail with no connectivity. */ +const ( + testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" + + customDnsServiceName = "custom-dns-service" + customDnsConfigName = "custom-dns-config" + customDnsPort = 53 + customDnsTargetPort = 53 + + customDnsImage = "coredns/coredns:latest" + customDnsPodName = "custom-dns-server" + customDnsContainerName = "coredns" + customDnsLabelKey = "app" + customDnsLabelValue = "custom-dns" + customDnsVolume = "config-volume" + + fqdnPolicyName = "test-acnp-fqdn" + fqdnPodSelectorLabelKey = "app" + fqdnPodSelectorLabelValue = "fqdn-cache-test" + toolBoxPodName = "toolbox" + + agnHostPort = 80 + agnHostLabelKey = "app" + agnHostLabelValue = "agnhost" + agnHostPodNamePreFix = "agnhost-" +) + func TestFQDNPolicyWithCachedDNS(t *testing.T) { skipIfAntreaPolicyDisabled(t) data, err := setupTest(t) @@ -46,285 +69,50 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Fatalf("Error when setting up test: %v", err) } - testFqdn := "nginx-test-pod.lfx.test" - - //TODO: Check for IPv6 ? - ipFamily := v1.IPv4Protocol - - // Create the service . - //TODO: Should the names be put up as constants instead of direct strings here? - customDnsService, err := testData.CreateUDPService("custom-dns-service", data.testNamespace, 53, 53, map[string]string{"app": "custom-dns"}, false, false, v1.ServiceTypeClusterIP, &ipFamily) - if err != nil { - t.Fatalf("Error when creating custom DNS service: %v", err) - } - require.NoError(t, err) - - // get the IP - customCoreDnsServiceObject, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.Background(), customDnsService.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Error when getting custom DNS service object : %v", err) - } - require.NoError(t, err) - - // Print the ClusterIP - t.Logf("ClusterIP of the service: %s\n", customCoreDnsServiceObject.Spec.ClusterIP) - - // Get Antrea ConfigMap - cm, err := data.GetAntreaConfigMap(antreaNamespace) - if err != nil { - t.Fatalf("Error when getting custom DNS configMap : %v", err) - } - require.NoError(t, err) - - // Read current value of dnsServer - var agentConf agentconfig.AgentConfig - - if err := yaml.Unmarshal([]byte(cm.Data["antrea-agent.conf"]), &agentConf); err != nil { - t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) - } - require.NoError(t, err) - - //Set up customDNS server IP in Antrea configmap. - agentChanges := func(config *agentconfig.AgentConfig) { - config.DNSServerOverride = customCoreDnsServiceObject.Spec.ClusterIP - } - err = data.mutateAntreaConfigMap(nil, agentChanges, false, true) - if err != nil { - t.Fatalf("Error when setting up customDNS server IP in Antrea configmap : %v", err) - } - - cm2, err := data.GetAntreaConfigMap(antreaNamespace) - if err != nil { - t.Fatalf("Error when getting custom DNS configMap : %v", err) - } - require.NoError(t, err) - - // Read current value of dnsServer - var agentConfChanged agentconfig.AgentConfig - if err := yaml.Unmarshal([]byte(cm2.Data["antrea-agent.conf"]), &agentConfChanged); err != nil { - t.Fatalf("failed to unmarshal Agent config from ConfigMap: %v", err) - } - require.NoError(t, err) - - t.Logf("dns server value set to %+v in antrea \n", agentConfChanged.DNSServerOverride) - - // Set up nginx server - nginxConfig := `events {} - -http { - server { - listen 80; - - location / { - return 200 "Pod hostname: $hostname\n"; - add_header Content-Type text/plain; - } - } -}` - - configData := map[string]string{ - "nginx.conf": nginxConfig, - } - nginxConfiMapObject, err := data.CreateConfigMap(data.testNamespace, "nginx-config", configData, nil, false) - if err != nil { - t.Fatalf("failed to create nginx ConfigMap: %v", err) - } - require.NoError(t, err) + customDnsService, err := createCustomDnsService(data) + require.NoError(t, err, "Error when creating custom DNS service: %v", err) - deploymentLabels := map[string]string{ - "app": "nginx", - } + dnsServiceIP := getCustomDnsServiceIP(t, data, customDnsService) - nginxDeployObject, err := data.CreateNginxDeploymentForTest("nginx-deployment", data.testNamespace, nginxConfiMapObject.Name, 2, deploymentLabels) - if err != nil { - t.Fatalf("failed to create nginx deployment: %v", err) - } - require.NoError(t, err) + setDnsServerAddressInAntrea(t, data, dnsServiceIP) - // Though this is used in vmagent_test.go but i think i needed it here to check for the deployment. - //TODO: Time of 15 seconds is still large for a timeout. - err = data.waitForDeploymentReady(t, data.testNamespace, nginxDeployObject.Name, 15*time.Second) - if err != nil { - t.Fatalf("error while waiting for nginx deployment to be ready : %v", err) - } - require.NoError(t, err) + // I think creating 2 pods with a deployment might be better. + createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) + createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) k8sUtils, err = NewKubernetesUtils(data) - if err != nil { - t.Fatalf("error getting k8s utils %+v", err) - } - require.NoError(t, err) + require.NoError(t, err, "error getting k8s utils %+v", err) - nginxPods, err := k8sUtils.GetPodsByLabel(data.testNamespace, "app", "nginx") - if err != nil { - t.Fatalf("error getting Pods by label %+v", err) - } - require.NoError(t, err) + agnHostPods, err := k8sUtils.GetPodsByLabel(data.testNamespace, agnHostLabelKey, agnHostLabelValue) + require.NoError(t, err, "error getting Pods by label %+v", err) //domainMapping holds whether the IP is mapped to Domain or not. domainMapping := make(map[string]bool) // pick an IP to be added in config - var ipForConfig string - for idx, pod := range nginxPods { - //TODO: Following wait time change ? - _, err = data.podWaitForIPs(10*time.Second, pod.Name, data.testNamespace) - if err != nil { - t.Fatalf("error waiting for nginx pods to get IPs %+v", err) - } - require.NoError(t, err) - + var ipForDnsConfig string + for idx, pod := range agnHostPods { ipStr := strings.TrimSpace(pod.Status.PodIP) domainMapping[ipStr] = false //pick last IP for config - if idx == len(nginxPods)-1 { - ipForConfig = ipStr + if idx == len(agnHostPods)-1 { + ipForDnsConfig = ipStr } } - // Create and update the custom DNS configMap - customDNSconfig := fmt.Sprintf(`lfx.test:53 { - errors - t - health - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 60 - } - hosts { - %s %s - no_reverse - pods verified - ttl 10 - } - loop - reload -}`, ipForConfig, testFqdn) - - customDNSconfigData := map[string]string{ - "Corefile": customDNSconfig, - } - - customDNSconfigMapObject, err := data.CreateConfigMap(data.testNamespace, "custom-dns-config", customDNSconfigData, nil, false) - - if err != nil { - t.Fatalf("failed to create custom dns ConfigMap: %v", err) - } - require.NoError(t, err) - domainMapping[ipForConfig] = true - - // create supporting SA, Role and Role Binding for DNS deployment. - saSpec := data.BuildServiceAccount("custom-dns-service-account", data.testNamespace, nil) - sa, err := data.CreateOrUpdateServiceAccount(saSpec) - if err != nil { - t.Fatalf("failed to create service acount for custom dns : %v", err) - } - require.NoError(t, err) - - clusterRoleSpec := &v12.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role"}, - Rules: []v12.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"namespaces", "services"}, - Verbs: []string{"list", "watch"}, - }, - { - APIGroups: []string{"discovery.k8s.io"}, - Resources: []string{"endpointslices"}, - Verbs: []string{"list", "watch"}, - }, - }, - } - // TODO: Delete role on teardown. - role, err := data.CreateRole(clusterRoleSpec) - if err != nil { - t.Fatalf("failed to create cluster role for custom dns : %v", err) - } - require.NoError(t, err) - - clusterRoleBinding := &v12.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "custom-dns-role-binding"}, - Subjects: []v12.Subject{ - { - Kind: "ServiceAccount", - Name: sa.Name, - Namespace: data.testNamespace, - }, - }, - RoleRef: v12.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: role.Name, - }, - } - - err = data.CreateRoleBinding(clusterRoleBinding) - if err != nil { - t.Fatalf("failed to create cluster role binding for custom dns : %v", err) - } - require.NoError(t, err) - - // Create custom DNS deployment - dnsDeploymentLabels := map[string]string{ - "app": "custom-dns", - } - dnsDeploymentObj, err := data.CreateCustomDnsDeployment("custom-dns-deployment", data.testNamespace, customDNSconfigMapObject.Name, sa.Name, dnsDeploymentLabels, 1) - if err != nil { - t.Fatalf("failed to create custom dns deployment : %v", err) - } - require.NoError(t, err) - - err = data.waitForDeploymentReady(t, data.testNamespace, dnsDeploymentObj.Name, 120*time.Second) - if err != nil { - t.Fatalf("error while waiting for custom dns deployment to be ready : %v", err) - } - require.NoError(t, err) - - // Create policy - npPodSelectorLabel := map[string]string{ - "app": "fqdn-cache-test", - } - port := int32(80) - builder := &utils.ClusterNetworkPolicySpecBuilder{} - builder = builder.SetName("test-acnp-fqdn"). - SetTier("application"). - SetPriority(1.0). - SetAppliedToGroup([]utils.ACNPAppliedToSpec{{PodSelector: npPodSelectorLabel}}) - builder.AddFQDNRule(testFqdn, "TCP", &port, nil, nil, "r1", nil, crdv1beta1.RuleActionAllow) - builder.AddEgress("UDP", nil, nil, nil, nil, nil, nil, nil, nil, dnsDeploymentLabels, nil, - nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "", nil) - builder.AddEgress("TCP", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionReject, "", "", nil) + dnsConfigData := createDnsConfig(ipForDnsConfig) - acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) - failOnError(err, t) - failOnError(waitForResourceReady(t, 30*time.Second, acnp), t) + customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) + require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) + domainMapping[ipForDnsConfig] = true - //TODO: deletion of a namespace deletes all resources under it, do we still need to explicitly delete resources created under that namespace during a test ? - defer tearDownFQDN(t, data, builder) + createDnsPod(t, data) - // create toolbox using their framework - toolBoxLabel := npPodSelectorLabel - pb := NewPodBuilder("toolbox", data.testNamespace, ToolboxImage) - pb.WithLabels(toolBoxLabel) - pb.WithContainerName("toolbox-container") - mutateSpecForAddingCustomDNS := func(pod *v1.Pod) { - if pod.Spec.DNSConfig == nil { - pod.Spec.DNSConfig = &v1.PodDNSConfig{} - } - pod.Spec.DNSConfig.Nameservers = []string{customCoreDnsServiceObject.Spec.ClusterIP} + builder := buildFqdnPolicy(t) - } - pb.WithMutateFunc(mutateSpecForAddingCustomDNS) - err = pb.Create(data) - if err != nil { - t.Fatalf("failed to create antrea toolbox : %v", err) - } - require.NoError(t, err) + defer tearDown(t, data, builder) - /* ------ Actual test ------ */ + createToolBoxPod(t, data, dnsServiceIP) // idea copied from antctl_test.go:284 // getEndpointStatus will return "Success", "Failure", or the empty string when out is not a @@ -338,32 +126,27 @@ http { return status.Status } - checkFQDNaccess := func(podName, containerName, fqdn string, checkStatus bool) error { + curlFqdn := func(podName, containerName, fqdn string, checkStatus bool) error { t.Logf("trying to curl the fqdn %v", fqdn) cmd := []string{"curl", fqdn} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) if err != nil { - return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, err, stdout, stderr) + return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), podName, err, stdout, stderr) } if checkStatus && getEndpointStatus([]byte(stdout)) == "Failure" { return fmt.Errorf("failure status when accessing endpoint: <%v>", stdout) } - fmt.Printf(" curl FQDN | running command '%s' on Pod '%s', stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, stdout, stderr) + t.Logf(stdout) return nil } - err = checkFQDNaccess(pb.Name, pb.ContainerName, testFqdn, true) - if err != nil { - t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) - } - require.NoError(t, err) + err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName, true) + require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) // DIG to get actual IP , to be sure. - fqdnIp, err := k8sUtils.digDnSCustom(pb.Name, pb.Namespace, testFqdn, false) - if err != nil { - t.Fatalf("failed to get IP of FQDN using DIG from toolbox pod : %v", err) - } - require.NoError(t, err) + fqdnIp, err := k8sUtils.digUsingShort(toolBoxPodName, data.testNamespace, testFullyQualifiedDomainName, false, dnsServiceIP) + require.NoError(t, err, "failed to get IP of FQDN using DIG from toolbox pod : %v", err) fqdnIp = strings.TrimSpace(fqdnIp) t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) @@ -378,22 +161,94 @@ http { t.Logf("New IP to update to DNS %v", newIP) // Curl the old ip and it should be a success. - err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) - if err != nil { - t.Fatalf("failed to curl FQDN from antrea toolbox on initial run : %v", err) + err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp, true) + require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) + + // Update the custom DNS configMap + UpdatedCustomDNSconfig := createDnsConfig(newIP) + + customDnsConfigMapObject.Data = UpdatedCustomDNSconfig + err = data.UpdateConfigMap(customDnsConfigMapObject) + require.NoError(t, err, "failed to update configmap with new IP : %v", err) + t.Logf("successfully updated dns configMap with new IP : %+v", newIP) + + updateDnsPodAnnotations(t, data) + + for { + err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp, true) + require.NoError(t, err, "curl to ip failed with error : %v", err) + + // Wait for 1 second before retrying + time.Sleep(1 * time.Second) } - require.NoError(t, err) - // Create and update the custom DNS configMap - UpdatedCustomDNSconfig := fmt.Sprintf(`lfx.test:53 { +} + +func tearDown(t *testing.T, data *TestData, builder *utils.ClusterNetworkPolicySpecBuilder) { + // cleanup test resources + teardownTest(t, data) + failOnError(k8sUtils.DeleteACNP(builder.Name), t) +} + +func getCustomDnsServerPodLabel() map[string]string { + return map[string]string{"app": "custom-dns"} +} + +func createCustomDnsService(data *TestData) (*corev1.Service, error) { + return testData.CreateUDPService(customDnsServiceName, data.testNamespace, customDnsPort, customDnsTargetPort, + getCustomDnsServerPodLabel(), false, false, corev1.ServiceTypeClusterIP, getIPFamily()) +} + +// CreateUDPService creates a service with a UDP port and targetPort. +func (data *TestData) CreateUDPService(serviceName, namespace string, port, targetPort int32, selector map[string]string, + affinity, nodeLocalExternal bool, + serviceType corev1.ServiceType, ipFamily *corev1.IPFamily) (*corev1.Service, error) { + annotation := make(map[string]string) + return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolUDP, selector, + affinity, nodeLocalExternal, serviceType, ipFamily, annotation) +} + +func getCustomDnsServiceIP(t *testing.T, data *TestData, customDnsService *corev1.Service) string { + // Usually the IP gets assigned quickly to a service , but maybe it's a safe idea to use Get() for getting the IP. + customCoreDnsServiceObject, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.Background(), + customDnsService.Name, metav1.GetOptions{}) + require.NoError(t, err, "Error when getting custom DNS service object : %v", err) + + if customCoreDnsServiceObject.Spec.ClusterIP == "" { + require.Fail(t, "ClusterIP is empty for the custom DNS service") + } + + t.Logf("ClusterIP of the dns service: %s\n", customCoreDnsServiceObject.Spec.ClusterIP) + return customCoreDnsServiceObject.Spec.ClusterIP +} + +func getIPFamily() *corev1.IPFamily { + ipFamily := corev1.IPv4Protocol + return &ipFamily +} + +func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { + var agentConf agentconfig.AgentConfig + cm, err := data.GetAntreaConfigMap(antreaNamespace) + require.NoError(t, err, "Error when getting antrea configMap : %v", err) + + err = yaml.Unmarshal([]byte(cm.Data["antrea-agent.conf"]), &agentConf) + require.NoError(t, err, "failed to unmarshal antrea agent config from ConfigMap: %v", err) + + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = dnsServiceIP + } + err = data.mutateAntreaConfigMap(nil, agentChanges, false, true) + require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) + + t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + +} + +func createDnsConfig(ipForConfig string) map[string]string { + config := fmt.Sprintf(`lfx.test:53 { errors - t health - kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure - fallthrough in-addr.arpa ip6.arpa - ttl 60 - } hosts { %s %s no_reverse @@ -402,58 +257,134 @@ http { } loop reload -}`, newIP, testFqdn) +}`, ipForConfig, testFullyQualifiedDomainName) - // edit configmap with this IP - dnsConfigMap, err := data.GetConfigMap(data.testNamespace, customDNSconfigMapObject.Name) - if err != nil { - t.Fatalf("failed to get configmap to replace IP : %v", err) + configData := map[string]string{ + "Corefile": config, } - require.NoError(t, err) - dnsConfigMap.Data["Corefile"] = UpdatedCustomDNSconfig - err = data.UpdateConfigMap(dnsConfigMap) - if err != nil { - t.Fatalf("failed to update configmap with new IP : %v", err) + return configData +} + +func createDnsPod(t *testing.T, data *TestData) { + volume := []corev1.Volume{ + { + Name: customDnsVolume, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: customDnsConfigName, + }, + Items: []corev1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, } - require.NoError(t, err) - // Update the annotation - if dnsDeploymentObj.Annotations == nil { - dnsDeploymentObj.Annotations = make(map[string]string) + volumeMount := []corev1.VolumeMount{ + { + Name: customDnsVolume, + MountPath: "/etc/coredns", + }, } - dnsDeploymentObj.Annotations["baar"] = "foo" - // Get the Deployment - updatedDnsDeploymentObj, err := data.clientset.AppsV1().Deployments(data.testNamespace).Get(context.TODO(), dnsDeploymentObj.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Error getting Deployment: %s", err) + label := map[string]string{customDnsLabelKey: customDnsLabelValue} + pb := NewPodBuilder(customDnsPodName, data.testNamespace, customDnsImage) + pb.WithLabels(label) + pb.WithAnnotations(map[string]string{"Foo": "Bar"}) + pb.WithContainerName(customDnsContainerName) + pb.WithArgs([]string{"-conf", "/etc/coredns/Corefile"}) + pb.AddVolume(volume) + pb.AddVolumeMount(volumeMount) + + require.NoError(t, pb.Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, customDnsPodName, data.testNamespace) + require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) + +} + +func buildFqdnPolicy(t *testing.T) *utils.ClusterNetworkPolicySpecBuilder { + podSelectorLabel := map[string]string{ + fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue, } + port := int32(80) + builder := &utils.ClusterNetworkPolicySpecBuilder{} + builder = builder.SetName(fqdnPolicyName). + SetTier("application"). + SetPriority(1.0). + SetAppliedToGroup([]utils.ACNPAppliedToSpec{{PodSelector: podSelectorLabel}}) + builder.AddFQDNRule(testFullyQualifiedDomainName, "TCP", &port, nil, nil, "r1", nil, + crdv1beta1.RuleActionAllow) + builder.AddEgress("UDP", nil, nil, nil, nil, nil, nil, nil, + nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, nil, + nil, nil, nil, nil, nil, nil, + crdv1beta1.RuleActionAllow, "", "", nil) + builder.AddEgress("TCP", nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, + nil, nil, nil, nil, nil, nil, + crdv1beta1.RuleActionReject, "", "", nil) + + acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) + require.NoError(t, err, "error while deploying antrea policy %+v", err) + failOnError(err, t) + failOnError(waitForResourceReady(t, 30*time.Second, acnp), t) + + return builder +} + +func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { + toolBoxLabel := map[string]string{fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue} + pb := NewPodBuilder(toolBoxPodName, data.testNamespace, ToolboxImage) + pb.WithLabels(toolBoxLabel) + pb.WithContainerName(toolboxContainerName) + mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &corev1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} - // Update the Deployment - _, err = data.clientset.AppsV1().Deployments(data.testNamespace).Update(context.TODO(), updatedDnsDeploymentObj, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Error updating Deployment: %s", err) } + pb.WithMutateFunc(mutateSpecForAddingCustomDNS) + require.NoError(t, pb.Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, toolBoxPodName, data.testNamespace) require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolBoxPodName, data.testNamespace)) +} - for { - err = checkFQDNaccess(pb.Name, pb.ContainerName, fqdnIp, true) - if err != nil { - t.Logf("curl to ip filed. Error %v", err) - break - } - // Wait for 1 second before retrying - time.Sleep(1 * time.Second) +func updateDnsPodAnnotations(t *testing.T, data *TestData) { + dnsPod, err := data.clientset.CoreV1().Pods(data.testNamespace).Get(context.TODO(), customDnsPodName, metav1.GetOptions{}) + require.NoError(t, err, "Error getting DNS pod for annotation update.") + + if dnsPod.Annotations == nil { + dnsPod.Annotations = make(map[string]string) } + dnsPod.Annotations["Bar"] = "Foo" + delete(dnsPod.Annotations, "Foo") - // Ensuring that the test checks that an error occurred. - require.Error(t, err) + updatedPod, err := data.clientset.CoreV1().Pods(data.testNamespace).Update(context.TODO(), dnsPod, metav1.UpdateOptions{}) + require.NoError(t, err, "error updating dns pod annotations.") + t.Logf("updated dns pod annotations %+v\n", updatedPod.Annotations) } -func tearDownFQDN(t *testing.T, data *TestData, builder *utils.ClusterNetworkPolicySpecBuilder) { - // cleanup test resources - teardownTest(t, data) - failOnError(k8sUtils.DeleteACNP(builder.Name), t) +func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) { + args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: agnHostPort, + Protocol: corev1.ProtocolTCP, + }, + } + + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data)) + _, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, podName, data.testNamespace)) } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 00927f8b2c9..a497b49b53f 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1543,15 +1543,9 @@ func (b *PodBuilder) Create(data *TestData) error { if b.MutateFunc != nil { b.MutateFunc(pod) } - - podObj, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) - if err != nil { + if _, err := data.clientset.CoreV1().Pods(b.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}); err != nil { return err } - _, err = data.PodWaitFor(10*time.Second, podObj.Name, podObj.Namespace, func(pod2 *corev1.Pod) (bool, error) { - return pod2.Status.Phase == corev1.PodRunning, nil - }) - return nil } @@ -1994,13 +1988,6 @@ func (data *TestData) CreateService(serviceName, namespace string, port, targetP return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolTCP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) } -// CreateUDPService creates a service with a UDP port and targetPort. -func (data *TestData) CreateUDPService(serviceName, namespace string, port, targetPort int32, selector map[string]string, affinity, nodeLocalExternal bool, - serviceType corev1.ServiceType, ipFamily *corev1.IPFamily) (*corev1.Service, error) { - annotation := make(map[string]string) - return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolUDP, selector, affinity, nodeLocalExternal, serviceType, ipFamily, annotation) -} - // CreateServiceWithAnnotations creates a service with Annotation func (data *TestData) CreateServiceWithAnnotations(serviceName, namespace string, port, targetPort int32, protocol corev1.Protocol, selector map[string]string, affinity, nodeLocalExternal bool, serviceType corev1.ServiceType, ipFamily *corev1.IPFamily, annotations map[string]string, mutators ...func(service *corev1.Service)) (*corev1.Service, error) { diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 35df16de6cb..4c6a1571780 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" v12 "k8s.io/api/rbac/v1" + "net" "strconv" "strings" "sync" @@ -334,11 +335,13 @@ func decidePingProbeResult(stdout string, probeNum int) PodConnectivityMark { } return Error } -func (k *KubernetesUtils) digDnSCustom( + +func (k *KubernetesUtils) digUsingShort( podName string, podNamespace string, dstAddr string, - useTCP bool) (string, error) { + useTCP bool, + dnsServiceIP string) (string, error) { // Get the Pod pod, err := k.clientset.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) @@ -346,7 +349,7 @@ func (k *KubernetesUtils) digDnSCustom( log.Fatalf("Error getting pod: %v", err) } - digCmd := fmt.Sprintf("dig %s", dstAddr) + digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) if useTCP { digCmd += " +tcp" } @@ -356,45 +359,19 @@ func (k *KubernetesUtils) digDnSCustom( digCmd, } fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) - log.Tracef("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) stdout, stderr, err := k.RunCommandFromPod(pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, cmd) - fmt.Printf("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) - log.Tracef("%s -> %s: error when running command: err - %v /// stdout - %s /// stderr - %s", podName, dstAddr, err, stdout, stderr) - //========DiG command stdout example======== - //; <<>> DiG 9.16.6 <<>> github.com +tcp - //;; global options: +cmd - //;; Got answer: - //;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 21816 - //;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 - // - //;; OPT PSEUDOSECTION: - //; EDNS: version: 0, flags:; udp: 4096 - //; COOKIE: 2d7fe493ea37c430 (echoed) - //;; QUESTION SECTION: - //;github.com. IN A - // - //;; ANSWER SECTION: - //github.com. 6 IN A 140.82.113.3 - // - //;; Query time: 0 msec - //;; SERVER: 10.96.0.10#53(10.96.0.10) - //;; WHEN: Tue Feb 14 22:34:23 UTC 2023 - //;; MSG SIZE rcvd: 77 - //========================================== - answerMarkIdx := strings.Index(stdout, ";; ANSWER SECTION:") - if answerMarkIdx == -1 { - return "", fmt.Errorf("failed to parse dig response") - } - splitResp := strings.Split(stdout[answerMarkIdx:], "\n") - if len(splitResp) < 2 { - return "", fmt.Errorf("failed to parse dig response") + + isValidIPv4 := func(ip string) bool { + parsedIP := net.ParseIP(ip) + return parsedIP != nil && parsedIP.To4() != nil } - ipLine := splitResp[1] - lastTab := strings.LastIndex(ipLine, "\t") - if lastTab == -1 { - return "", fmt.Errorf("failed to parse dig response") + + out := strings.TrimSpace(stdout) + if isValidIPv4(out) { + return out, nil } - return ipLine[lastTab:], nil + + return "", fmt.Errorf("error running dig command %v", stderr) } func (k *KubernetesUtils) digDNS( @@ -747,146 +724,6 @@ func (data *TestData) CreateConfigMap(namespace, name string, configData map[str return configMapObject, err } -func (data *TestData) CreateNginxDeploymentForTest(name, namespace, nginxConfigMapName string, replicas int32, labels map[string]string) (*appsv1.Deployment, error) { - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - Image: "nginx:alpine", - Ports: []v1.ContainerPort{ - { - ContainerPort: 80, - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "html-volume", - MountPath: "/etc/nginx/nginx.conf", - SubPath: "nginx.conf", - }, - }, - Env: []v1.EnvVar{ - { - Name: "HOSTNAME", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "html-volume", // This may be given a name here instead of passing as a parameter. - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: nginxConfigMapName, - }, - }, - }, - }, - }, - }, - }, - }, - } - - nginxDeploymentObject, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), deployment, metav1.CreateOptions{}) - return nginxDeploymentObject, err - -} - -func (data *TestData) CreateCustomDnsDeployment(name, namespace, configMapName, serviceAccountName string, labels map[string]string, replicas int32) (*appsv1.Deployment, error) { - - customDNSdeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: labels, - Annotations: map[string]string{"foo": "bar"}, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: map[string]string{"foo": "bar"}, - }, - Spec: v1.PodSpec{ - ServiceAccountName: serviceAccountName, - Containers: []v1.Container{ - { - Name: "monitor", - Image: "busybox", - Command: []string{"/bin/sh", "-c", "sleep 1d"}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "config-volume", - MountPath: "/etc/coredns", - }, - }, - }, - { - Name: "coredns", - Image: "coredns/coredns:latest", - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string{"-conf", "/etc/coredns/Corefile"}, - VolumeMounts: []v1.VolumeMount{ - { - Name: "config-volume", - MountPath: "/etc/coredns", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "config-volume", // This may be given a name here instead of passing as a parameter. - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: configMapName, - }, - Items: []v1.KeyToPath{ - { - Key: "Corefile", - Path: "Corefile", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - dnsDeploymentObj, err := data.clientset.AppsV1().Deployments(namespace).Create(context.Background(), customDNSdeployment, metav1.CreateOptions{}) - return dnsDeploymentObj, err - -} - // DeleteService is a convenience function for deleting a Service by Namespace and name. func (data *TestData) DeleteService(ns, name string) error { log.Infof("Deleting Service %s in ns %s", name, ns) From 34a0a15e5d498d1fe3e26d6ba46aa7229672b034 Mon Sep 17 00:00:00 2001 From: Hemant Date: Tue, 1 Oct 2024 02:26:10 +0530 Subject: [PATCH 03/17] refactored with changes provided in review Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 140 ++++++++++------------------ test/e2e/framework.go | 32 +++++++ test/e2e/k8s_util.go | 53 ----------- test/e2e/utils/annp_spec_builder.go | 26 ++++++ 4 files changed, 106 insertions(+), 145 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index bd86fa81cc9..5191d6236c8 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -5,12 +5,12 @@ import ( agentconfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/test/e2e/utils" "context" - "encoding/json" "fmt" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "os" "strconv" "strings" "testing" @@ -51,7 +51,7 @@ const ( customDnsLabelValue = "custom-dns" customDnsVolume = "config-volume" - fqdnPolicyName = "test-acnp-fqdn" + fqdnPolicyName = "test-anp-fqdn" fqdnPodSelectorLabelKey = "app" fqdnPodSelectorLabelValue = "fqdn-cache-test" toolBoxPodName = "toolbox" @@ -68,13 +68,14 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { if err != nil { t.Fatalf("Error when setting up test: %v", err) } + defer teardown(t, data) - customDnsService, err := createCustomDnsService(data) + ipFamily := corev1.IPv4Protocol + customDnsService, err := data.CreateServiceWithAnnotations(customDnsServiceName, data.testNamespace, customDnsPort, customDnsTargetPort, corev1.ProtocolUDP, + map[string]string{customDnsLabelKey: customDnsLabelValue}, false, false, corev1.ServiceTypeClusterIP, &ipFamily, map[string]string{}) require.NoError(t, err, "Error when creating custom DNS service: %v", err) - dnsServiceIP := getCustomDnsServiceIP(t, data, customDnsService) - - setDnsServerAddressInAntrea(t, data, dnsServiceIP) + setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) // I think creating 2 pods with a deployment might be better. createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) @@ -108,25 +109,11 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { createDnsPod(t, data) - builder := buildFqdnPolicy(t) + _ = buildFqdnPolicy(t, data) - defer tearDown(t, data, builder) + createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) - createToolBoxPod(t, data, dnsServiceIP) - - // idea copied from antctl_test.go:284 - // getEndpointStatus will return "Success", "Failure", or the empty string when out is not a - // marshalled metav1.Status object. - getEndpointStatus := func(out []byte) string { - var status metav1.Status - if err := json.Unmarshal(out, &status); err != nil { - // Output is not JSON or does not encode a metav1.Status object. - return "" - } - return status.Status - } - - curlFqdn := func(podName, containerName, fqdn string, checkStatus bool) error { + curlFqdn := func(podName, containerName, fqdn string) error { t.Logf("trying to curl the fqdn %v", fqdn) cmd := []string{"curl", fqdn} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) @@ -134,20 +121,16 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, err, stdout, stderr) } - if checkStatus && getEndpointStatus([]byte(stdout)) == "Failure" { - return fmt.Errorf("failure status when accessing endpoint: <%v>", stdout) - } - t.Logf(stdout) return nil } - err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName, true) + err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) // DIG to get actual IP , to be sure. - fqdnIp, err := k8sUtils.digUsingShort(toolBoxPodName, data.testNamespace, testFullyQualifiedDomainName, false, dnsServiceIP) + digResponse, err := k8sUtils.runDNSQuery(toolBoxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) require.NoError(t, err, "failed to get IP of FQDN using DIG from toolbox pod : %v", err) - fqdnIp = strings.TrimSpace(fqdnIp) + fqdnIp := digResponse.String() t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) @@ -161,7 +144,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Logf("New IP to update to DNS %v", newIP) // Curl the old ip and it should be a success. - err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp, true) + err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) // Update the custom DNS configMap @@ -175,7 +158,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { updateDnsPodAnnotations(t, data) for { - err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp, true) + err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) require.NoError(t, err, "curl to ip failed with error : %v", err) // Wait for 1 second before retrying @@ -184,49 +167,6 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } -func tearDown(t *testing.T, data *TestData, builder *utils.ClusterNetworkPolicySpecBuilder) { - // cleanup test resources - teardownTest(t, data) - failOnError(k8sUtils.DeleteACNP(builder.Name), t) -} - -func getCustomDnsServerPodLabel() map[string]string { - return map[string]string{"app": "custom-dns"} -} - -func createCustomDnsService(data *TestData) (*corev1.Service, error) { - return testData.CreateUDPService(customDnsServiceName, data.testNamespace, customDnsPort, customDnsTargetPort, - getCustomDnsServerPodLabel(), false, false, corev1.ServiceTypeClusterIP, getIPFamily()) -} - -// CreateUDPService creates a service with a UDP port and targetPort. -func (data *TestData) CreateUDPService(serviceName, namespace string, port, targetPort int32, selector map[string]string, - affinity, nodeLocalExternal bool, - serviceType corev1.ServiceType, ipFamily *corev1.IPFamily) (*corev1.Service, error) { - annotation := make(map[string]string) - return data.CreateServiceWithAnnotations(serviceName, namespace, port, targetPort, corev1.ProtocolUDP, selector, - affinity, nodeLocalExternal, serviceType, ipFamily, annotation) -} - -func getCustomDnsServiceIP(t *testing.T, data *TestData, customDnsService *corev1.Service) string { - // Usually the IP gets assigned quickly to a service , but maybe it's a safe idea to use Get() for getting the IP. - customCoreDnsServiceObject, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.Background(), - customDnsService.Name, metav1.GetOptions{}) - require.NoError(t, err, "Error when getting custom DNS service object : %v", err) - - if customCoreDnsServiceObject.Spec.ClusterIP == "" { - require.Fail(t, "ClusterIP is empty for the custom DNS service") - } - - t.Logf("ClusterIP of the dns service: %s\n", customCoreDnsServiceObject.Spec.ClusterIP) - return customCoreDnsServiceObject.Spec.ClusterIP -} - -func getIPFamily() *corev1.IPFamily { - ipFamily := corev1.IPv4Protocol - return &ipFamily -} - func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { var agentConf agentconfig.AgentConfig cm, err := data.GetAntreaConfigMap(antreaNamespace) @@ -309,31 +249,32 @@ func createDnsPod(t *testing.T, data *TestData) { } -func buildFqdnPolicy(t *testing.T) *utils.ClusterNetworkPolicySpecBuilder { +func buildFqdnPolicy(t *testing.T, data *TestData) *utils.AntreaNetworkPolicySpecBuilder { podSelectorLabel := map[string]string{ fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue, } port := int32(80) - builder := &utils.ClusterNetworkPolicySpecBuilder{} - builder = builder.SetName(fqdnPolicyName). - SetTier("application"). + udpPort := int32(53) + builder := &utils.AntreaNetworkPolicySpecBuilder{} + builder = builder.SetName(data.testNamespace, fqdnPolicyName). + SetTier(defaultTierName). SetPriority(1.0). - SetAppliedToGroup([]utils.ACNPAppliedToSpec{{PodSelector: podSelectorLabel}}) - builder.AddFQDNRule(testFullyQualifiedDomainName, "TCP", &port, nil, nil, "r1", nil, + SetAppliedToGroup([]utils.ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) + builder.AddFQDNRule(testFullyQualifiedDomainName, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, crdv1beta1.RuleActionAllow) - builder.AddEgress("UDP", nil, nil, nil, nil, nil, nil, nil, - nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, nil, - nil, nil, nil, nil, nil, nil, - crdv1beta1.RuleActionAllow, "", "", nil) - builder.AddEgress("TCP", nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, - nil, nil, nil, nil, nil, nil, - crdv1beta1.RuleActionReject, "", "", nil) - - acnp, err := k8sUtils.CreateOrUpdateACNP(builder.Get()) + builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, + nil, nil, nil, nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, + map[string]string{"ns": data.testNamespace}, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") + builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + map[string]string{"ns": data.testNamespace}, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") + + annp, err := k8sUtils.CreateOrUpdateANNP(builder.Get()) require.NoError(t, err, "error while deploying antrea policy %+v", err) failOnError(err, t) - failOnError(waitForResourceReady(t, 30*time.Second, acnp), t) + failOnError(waitForResourceReady(t, 30*time.Second, annp), t) return builder } @@ -344,6 +285,7 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { pb.WithLabels(toolBoxLabel) pb.WithContainerName(toolboxContainerName) mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { + pod.Spec.DNSPolicy = corev1.DNSNone if pod.Spec.DNSConfig == nil { pod.Spec.DNSConfig = &corev1.PodDNSConfig{} } @@ -388,3 +330,17 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel require.NoError(t, err) require.NoError(t, data.podWaitForRunning(defaultTimeout, podName, data.testNamespace)) } + +func teardown(t *testing.T, data *TestData) { + // delete the namespace, hence removing all resources attached to it. + // Also remove the dnsServerOverRide value. + exportLogs(t, data, "beforeTeardown", true) + if empty, _ := IsDirEmpty(data.logsDirForTestCase); empty { + _ = os.Remove(data.logsDirForTestCase) + } + t.Logf("Deleting '%s' K8s Namespace", testData.testNamespace) + if err := data.DeleteNamespace(testData.testNamespace, -1); err != nil { + t.Logf("Error when tearing down test: %v", err) + } + setDnsServerAddressInAntrea(t, data, "") +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a497b49b53f..4e353110de0 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3220,3 +3220,35 @@ func (data *TestData) GetPodLogs(ctx context.Context, namespace, name, container } return b.String(), nil } + +func (data *TestData) runDNSQuery( + podName string, + containerName string, + podNamespace string, + dstAddr string, + useTCP bool, + dnsServiceIP string) (net.IP, error) { + + digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) + if useTCP { + digCmd += " +tcp" + } + cmd := []string{ + "/bin/sh", + "-c", + digCmd, + } + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(cmd, " ")) + stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, cmd) + if err != nil { + return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) + } + + ipAddress := net.ParseIP(strings.TrimSpace(stdout)) + fmt.Printf("---- IP = [%+v]", ipAddress) + if ipAddress != nil { + return ipAddress, nil + } else { + return nil, fmt.Errorf("invalid IP adress found %v", stdout) + } +} diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 4c6a1571780..93e3f7c2106 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -18,8 +18,6 @@ import ( "context" "errors" "fmt" - v12 "k8s.io/api/rbac/v1" - "net" "strconv" "strings" "sync" @@ -336,44 +334,6 @@ func decidePingProbeResult(stdout string, probeNum int) PodConnectivityMark { return Error } -func (k *KubernetesUtils) digUsingShort( - podName string, - podNamespace string, - dstAddr string, - useTCP bool, - dnsServiceIP string) (string, error) { - - // Get the Pod - pod, err := k.clientset.CoreV1().Pods(podNamespace).Get(context.TODO(), podName, metav1.GetOptions{}) - if err != nil { - log.Fatalf("Error getting pod: %v", err) - } - - digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) - if useTCP { - digCmd += " +tcp" - } - cmd := []string{ - "/bin/sh", - "-c", - digCmd, - } - fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", pod.Name, pod.Spec.Containers[0].Name, pod.Namespace, strings.Join(cmd, " ")) - stdout, stderr, err := k.RunCommandFromPod(pod.Namespace, pod.Name, pod.Spec.Containers[0].Name, cmd) - - isValidIPv4 := func(ip string) bool { - parsedIP := net.ParseIP(ip) - return parsedIP != nil && parsedIP.To4() != nil - } - - out := strings.TrimSpace(stdout) - if isValidIPv4(out) { - return out, nil - } - - return "", fmt.Errorf("error running dig command %v", stderr) -} - func (k *KubernetesUtils) digDNS( podName string, podNamespace string, @@ -762,19 +722,6 @@ func (data *TestData) BuildServiceAccount(name, ns string, labels map[string]str return serviceAccount } -func (data *TestData) CreateRole(clusterRole *v12.ClusterRole) (*v12.ClusterRole, error) { - role, err := data.clientset.RbacV1().ClusterRoles().Create(context.Background(), clusterRole, metav1.CreateOptions{}) - if err != nil { - return nil, err - } - return role, nil -} - -func (data *TestData) CreateRoleBinding(roleBinding *v12.ClusterRoleBinding) error { - _, err := data.clientset.RbacV1().ClusterRoleBindings().Create(context.Background(), roleBinding, metav1.CreateOptions{}) - return err -} - // CreateOrUpdateServiceAccount is a convenience function for updating/creating ServiceAccount. func (data *TestData) CreateOrUpdateServiceAccount(sa *v1.ServiceAccount) (*v1.ServiceAccount, error) { log.Infof("Creating/updating ServiceAccount %s in ns %s", sa.Name, sa.Namespace) diff --git a/test/e2e/utils/annp_spec_builder.go b/test/e2e/utils/annp_spec_builder.go index 670aa584ca4..f3eb1a1b651 100644 --- a/test/e2e/utils/annp_spec_builder.go +++ b/test/e2e/utils/annp_spec_builder.go @@ -214,3 +214,29 @@ func (b *AntreaNetworkPolicySpecBuilder) AddEgressLogging(logLabel string) *Antr } return b } + +func (b *AntreaNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, + protoc AntreaPolicyProtocol, port *int32, portName *string, endPort *int32, name string, + specs []ANNPAppliedToSpec, action crdv1beta1.RuleAction) *AntreaNetworkPolicySpecBuilder { + var appliedTos []crdv1beta1.AppliedTo + + for _, at := range specs { + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, + at.PodSelectorMatchExp, + at.ExternalEntitySelector, + at.ExternalEntitySelectorMatchExp, + at.Group)) + } + + policyPeer := []crdv1beta1.NetworkPolicyPeer{{FQDN: fqdn}} + ports, _ := GenPortsOrProtocols(protoc, port, portName, endPort, nil, nil, nil, nil, nil, nil) + newRule := crdv1beta1.Rule{ + To: policyPeer, + Ports: ports, + Action: &action, + Name: name, + AppliedTo: appliedTos, + } + b.Spec.Egress = append(b.Spec.Egress, newRule) + return b +} From 5412d1fb44aae9235dcef175e9828847a4d8e757 Mon Sep 17 00:00:00 2001 From: Hemant Date: Tue, 1 Oct 2024 23:05:48 +0530 Subject: [PATCH 04/17] fix: update policy build in code to ensure tests run successfully - Corrected issues in the policy build - Formatted imports for consistency - Fixed spelling errors Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 34 ++++++++++++++++++++++++--------- test/e2e/framework.go | 3 +-- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 5191d6236c8..8bfbb8d6313 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -1,20 +1,36 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package e2e import ( - crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" - agentconfig "antrea.io/antrea/pkg/config/agent" - "antrea.io/antrea/test/e2e/utils" "context" "fmt" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "os" "strconv" "strings" "testing" "time" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" + "antrea.io/antrea/test/e2e/utils" ) /* @@ -264,11 +280,11 @@ func buildFqdnPolicy(t *testing.T, data *TestData) *utils.AntreaNetworkPolicySpe crdv1beta1.RuleActionAllow) builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, - map[string]string{"ns": data.testNamespace}, nil, nil, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - map[string]string{"ns": data.testNamespace}, nil, nil, nil, + nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") annp, err := k8sUtils.CreateOrUpdateANNP(builder.Get()) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 4e353110de0..d4fd05586bc 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3245,10 +3245,9 @@ func (data *TestData) runDNSQuery( } ipAddress := net.ParseIP(strings.TrimSpace(stdout)) - fmt.Printf("---- IP = [%+v]", ipAddress) if ipAddress != nil { return ipAddress, nil } else { - return nil, fmt.Errorf("invalid IP adress found %v", stdout) + return nil, fmt.Errorf("invalid IP address found %v", stdout) } } From 5fe77ef519b011c9e7fd42eca199fd2b07777dae Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 4 Oct 2024 02:39:59 +0530 Subject: [PATCH 05/17] used eventual assertions and removed using podwait for running status of pods. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 94 ++++++++++++--------------------- 1 file changed, 34 insertions(+), 60 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 8bfbb8d6313..18854070570 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -17,14 +17,14 @@ package e2e import ( "context" "fmt" - "os" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" "strconv" "strings" "testing" "time" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -60,7 +60,7 @@ const ( customDnsPort = 53 customDnsTargetPort = 53 - customDnsImage = "coredns/coredns:latest" + customDnsImage = "coredns/coredns:1.11.3" customDnsPodName = "custom-dns-server" customDnsContainerName = "coredns" customDnsLabelKey = "app" @@ -76,6 +76,8 @@ const ( agnHostLabelKey = "app" agnHostLabelValue = "agnhost" agnHostPodNamePreFix = "agnhost-" + + eventualWaitTime = 20 * time.Second ) func TestFQDNPolicyWithCachedDNS(t *testing.T) { @@ -84,53 +86,49 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { if err != nil { t.Fatalf("Error when setting up test: %v", err) } - defer teardown(t, data) + defer teardownTest(t, data) - ipFamily := corev1.IPv4Protocol customDnsService, err := data.CreateServiceWithAnnotations(customDnsServiceName, data.testNamespace, customDnsPort, customDnsTargetPort, corev1.ProtocolUDP, - map[string]string{customDnsLabelKey: customDnsLabelValue}, false, false, corev1.ServiceTypeClusterIP, &ipFamily, map[string]string{}) + map[string]string{customDnsLabelKey: customDnsLabelValue}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) require.NoError(t, err, "Error when creating custom DNS service: %v", err) setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) - // I think creating 2 pods with a deployment might be better. - createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) - createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) - - k8sUtils, err = NewKubernetesUtils(data) - require.NoError(t, err, "error getting k8s utils %+v", err) - - agnHostPods, err := k8sUtils.GetPodsByLabel(data.testNamespace, agnHostLabelKey, agnHostLabelValue) - require.NoError(t, err, "error getting Pods by label %+v", err) - + var agnHostPodIps []*PodIPs + podCount := 2 + for i := 0; i < podCount; i++ { + podIp := createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) + agnHostPodIps = append(agnHostPodIps, podIp) + } //domainMapping holds whether the IP is mapped to Domain or not. domainMapping := make(map[string]bool) // pick an IP to be added in config var ipForDnsConfig string - for idx, pod := range agnHostPods { - ipStr := strings.TrimSpace(pod.Status.PodIP) + for idx, podIp := range agnHostPodIps { + ipStr := podIp.IPv4.String() domainMapping[ipStr] = false //pick last IP for config - if idx == len(agnHostPods)-1 { + if idx == len(agnHostPodIps)-1 { ipForDnsConfig = ipStr } } dnsConfigData := createDnsConfig(ipForDnsConfig) - customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) domainMapping[ipForDnsConfig] = true createDnsPod(t, data) + k8sUtils, err = NewKubernetesUtils(data) + require.NoError(t, err, "error getting k8s utils %+v", err) + _ = buildFqdnPolicy(t, data) createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) curlFqdn := func(podName, containerName, fqdn string) error { - t.Logf("trying to curl the fqdn %v", fqdn) cmd := []string{"curl", fqdn} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) if err != nil { @@ -140,8 +138,11 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { return nil } - err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) - require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) + assert.Eventually(t, func() bool { + t.Logf("trying to curl the fqdn %v", testFullyQualifiedDomainName) + err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) + return assert.NoError(t, err) + }, 2*time.Second, 100*time.Millisecond) // DIG to get actual IP , to be sure. digResponse, err := k8sUtils.runDNSQuery(toolBoxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) @@ -159,10 +160,6 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Logf("New IP to update to DNS %v", newIP) - // Curl the old ip and it should be a success. - err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) - require.NoError(t, err, "failed to curl FQDN from antrea toolbox on initial run : %v", err) - // Update the custom DNS configMap UpdatedCustomDNSconfig := createDnsConfig(newIP) @@ -173,28 +170,21 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { updateDnsPodAnnotations(t, data) - for { - err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) - require.NoError(t, err, "curl to ip failed with error : %v", err) + defer setDnsServerAddressInAntrea(t, data, "") - // Wait for 1 second before retrying - time.Sleep(1 * time.Second) - } + assert.Eventually(t, func() bool { + t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) + err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) + return assert.Error(t, err) + }, eventualWaitTime, 1*time.Second) } func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { - var agentConf agentconfig.AgentConfig - cm, err := data.GetAntreaConfigMap(antreaNamespace) - require.NoError(t, err, "Error when getting antrea configMap : %v", err) - - err = yaml.Unmarshal([]byte(cm.Data["antrea-agent.conf"]), &agentConf) - require.NoError(t, err, "failed to unmarshal antrea agent config from ConfigMap: %v", err) - agentChanges := func(config *agentconfig.AgentConfig) { config.DNSServerOverride = dnsServiceIP } - err = data.mutateAntreaConfigMap(nil, agentChanges, false, true) + err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) @@ -261,7 +251,6 @@ func createDnsPod(t *testing.T, data *TestData) { require.NoError(t, pb.Create(data)) _, err := data.podWaitForIPs(defaultTimeout, customDnsPodName, data.testNamespace) require.NoError(t, err) - require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) } @@ -312,7 +301,6 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { require.NoError(t, pb.Create(data)) _, err := data.podWaitForIPs(defaultTimeout, toolBoxPodName, data.testNamespace) require.NoError(t, err) - require.NoError(t, data.podWaitForRunning(defaultTimeout, toolBoxPodName, data.testNamespace)) } func updateDnsPodAnnotations(t *testing.T, data *TestData) { @@ -331,7 +319,7 @@ func updateDnsPodAnnotations(t *testing.T, data *TestData) { t.Logf("updated dns pod annotations %+v\n", updatedPod.Annotations) } -func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) { +func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} ports := []corev1.ContainerPort{ { @@ -342,21 +330,7 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel } require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data)) - _, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + podIP, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) require.NoError(t, err) - require.NoError(t, data.podWaitForRunning(defaultTimeout, podName, data.testNamespace)) -} - -func teardown(t *testing.T, data *TestData) { - // delete the namespace, hence removing all resources attached to it. - // Also remove the dnsServerOverRide value. - exportLogs(t, data, "beforeTeardown", true) - if empty, _ := IsDirEmpty(data.logsDirForTestCase); empty { - _ = os.Remove(data.logsDirForTestCase) - } - t.Logf("Deleting '%s' K8s Namespace", testData.testNamespace) - if err := data.DeleteNamespace(testData.testNamespace, -1); err != nil { - t.Logf("Error when tearing down test: %v", err) - } - setDnsServerAddressInAntrea(t, data, "") + return podIP } From 008101036e8914709ec3638abb53c91a6afc3fcf Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 4 Oct 2024 02:46:55 +0530 Subject: [PATCH 06/17] removed podWaitForIPs where not needed. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 18854070570..6c5e604d160 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -249,8 +249,7 @@ func createDnsPod(t *testing.T, data *TestData) { pb.AddVolumeMount(volumeMount) require.NoError(t, pb.Create(data)) - _, err := data.podWaitForIPs(defaultTimeout, customDnsPodName, data.testNamespace) - require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) } @@ -299,8 +298,7 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { } pb.WithMutateFunc(mutateSpecForAddingCustomDNS) require.NoError(t, pb.Create(data)) - _, err := data.podWaitForIPs(defaultTimeout, toolBoxPodName, data.testNamespace) - require.NoError(t, err) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolBoxPodName, data.testNamespace)) } func updateDnsPodAnnotations(t *testing.T, data *TestData) { From d1a82275ee4db4c2acd39e94d027a0c334a5850e Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 4 Oct 2024 23:45:36 +0530 Subject: [PATCH 07/17] added function to annotate pods. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 26 ++++----------------- test/e2e/framework.go | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 6c5e604d160..b6039919ab0 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -15,18 +15,16 @@ package e2e import ( - "context" "fmt" - "github.com/stretchr/testify/assert" - "k8s.io/utils/ptr" "strconv" "strings" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" agentconfig "antrea.io/antrea/pkg/config/agent" @@ -168,7 +166,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { require.NoError(t, err, "failed to update configmap with new IP : %v", err) t.Logf("successfully updated dns configMap with new IP : %+v", newIP) - updateDnsPodAnnotations(t, data) + require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to update custom dns pod annotation.") defer setDnsServerAddressInAntrea(t, data, "") @@ -242,7 +240,6 @@ func createDnsPod(t *testing.T, data *TestData) { label := map[string]string{customDnsLabelKey: customDnsLabelValue} pb := NewPodBuilder(customDnsPodName, data.testNamespace, customDnsImage) pb.WithLabels(label) - pb.WithAnnotations(map[string]string{"Foo": "Bar"}) pb.WithContainerName(customDnsContainerName) pb.WithArgs([]string{"-conf", "/etc/coredns/Corefile"}) pb.AddVolume(volume) @@ -250,6 +247,7 @@ func createDnsPod(t *testing.T, data *TestData) { require.NoError(t, pb.Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) + require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to annotate the custom dns pod.") } @@ -301,22 +299,6 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { require.NoError(t, data.podWaitForRunning(defaultTimeout, toolBoxPodName, data.testNamespace)) } -func updateDnsPodAnnotations(t *testing.T, data *TestData) { - dnsPod, err := data.clientset.CoreV1().Pods(data.testNamespace).Get(context.TODO(), customDnsPodName, metav1.GetOptions{}) - require.NoError(t, err, "Error getting DNS pod for annotation update.") - - if dnsPod.Annotations == nil { - dnsPod.Annotations = make(map[string]string) - } - dnsPod.Annotations["Bar"] = "Foo" - delete(dnsPod.Annotations, "Foo") - - updatedPod, err := data.clientset.CoreV1().Pods(data.testNamespace).Update(context.TODO(), dnsPod, metav1.UpdateOptions{}) - require.NoError(t, err, "error updating dns pod annotations.") - - t.Logf("updated dns pod annotations %+v\n", updatedPod.Annotations) -} - func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} ports := []corev1.ContainerPort{ diff --git a/test/e2e/framework.go b/test/e2e/framework.go index d4fd05586bc..bee8ae3def1 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -140,6 +140,7 @@ const ( defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" + randomPatchAnnotationKey = "test.antrea.io/random-value" iperfPort = 5201 iperfSvcPort = 9999 @@ -3251,3 +3252,43 @@ func (data *TestData) runDNSQuery( return nil, fmt.Errorf("invalid IP address found %v", stdout) } } + +// patchPodAnnotation Patches an annotation for a given pod in a given namespace. If no annotation is +// provided as a parameter then it generates a random string value for a predefined key and adds it as annotation. +// The patchType used is MergePatchType since we intend to make updates to annotations in pods instead of updating fields in complex k8s +// resources like deployments. +func (data *TestData) patchPodAnnotation(namespace, podName string, annotation map[string]string) error { + numberOfRunes := 8 + annotationPatch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": make(map[string]string), + }, + } + + if annotation == nil { + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[randomPatchAnnotationKey] = randSeq(numberOfRunes) + } else { + for key, value := range annotation { + if key == randomPatchAnnotationKey { + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = randSeq(numberOfRunes) + } else { + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value + } + } + } + + patchData, err := json.Marshal(annotationPatch) + if err != nil { + log.Infof("Error marshalling annotation: %+v", err) + return err + } + + _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err) + return err + } + + log.Infof("Successfully patched pod %s in namespace %s with provided annotation", podName, namespace) + return nil +} From e2e5466b7545b2cf29ac2c83ea0b483390c788db Mon Sep 17 00:00:00 2001 From: Hemant Date: Wed, 9 Oct 2024 21:28:33 +0530 Subject: [PATCH 08/17] accomodate ipv4 and ipv6 and change annotation patch function Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 128 ++++++++++++++++++++------------ test/e2e/framework.go | 62 ++++++++++------ test/e2e/k8s_util.go | 15 ++-- 3 files changed, 129 insertions(+), 76 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index b6039919ab0..e8fad3d736c 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -15,10 +15,12 @@ package e2e import ( + "bytes" "fmt" "strconv" "strings" "testing" + "text/template" "time" "github.com/stretchr/testify/assert" @@ -92,30 +94,17 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) - var agnHostPodIps []*PodIPs podCount := 2 + agnHostPodIps := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { - podIp := createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) - agnHostPodIps = append(agnHostPodIps, podIp) - } - //domainMapping holds whether the IP is mapped to Domain or not. - domainMapping := make(map[string]bool) - - // pick an IP to be added in config - var ipForDnsConfig string - for idx, podIp := range agnHostPodIps { - ipStr := podIp.IPv4.String() - domainMapping[ipStr] = false - //pick last IP for config - if idx == len(agnHostPodIps)-1 { - ipForDnsConfig = ipStr - } + agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) } - dnsConfigData := createDnsConfig(ipForDnsConfig) + ipv4, ipv6 := extractIPs(agnHostPodIps[0]) + + dnsConfigData := createDnsConfig(t, ipv4, ipv6) customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) - domainMapping[ipForDnsConfig] = true createDnsPod(t, data) @@ -149,30 +138,35 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) - var newIP string - for ip, mapped := range domainMapping { - if ip != fqdnIp && mapped == false { - newIP = ip - } + ipv4, ipv6 = extractIPs(agnHostPodIps[1]) + if ipv6 != "" { + t.Logf("New IPs to update to DNS | ipv4 : %v, ipv6 : %+v", ipv4, ipv6) + } else { + t.Logf("New IPs to update to DNS | ipv4 : %v", ipv4) } - t.Logf("New IP to update to DNS %v", newIP) - // Update the custom DNS configMap - UpdatedCustomDNSconfig := createDnsConfig(newIP) + UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6) customDnsConfigMapObject.Data = UpdatedCustomDNSconfig err = data.UpdateConfigMap(customDnsConfigMapObject) require.NoError(t, err, "failed to update configmap with new IP : %v", err) - t.Logf("successfully updated dns configMap with new IP : %+v", newIP) + if ipv6 != "" { + t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v, ipv6 :%+v", ipv4, ipv6) + } else { + t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v", ipv4) + } - require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to update custom dns pod annotation.") + require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to update custom dns pod annotation.") defer setDnsServerAddressInAntrea(t, data, "") assert.Eventually(t, func() bool { t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) + if err != nil { + t.Logf("The test failed because of error %+v", err) + } return assert.Error(t, err) }, eventualWaitTime, 1*time.Second) @@ -185,31 +179,59 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) - t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + if dnsServiceIP == "" { + t.Logf("removing dns server IP from antrea agent as part of teardown") + } else { + t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + } } -func createDnsConfig(ipForConfig string) map[string]string { - config := fmt.Sprintf(`lfx.test:53 { - errors - health - hosts { - %s %s - no_reverse - pods verified - ttl 10 - } - loop - reload -}`, ipForConfig, testFullyQualifiedDomainName) - +func createDnsConfig(t *testing.T, ipv4Address, ipv6Address string) map[string]string { + configMapData, _ := generateCoreFile(t, ipv4Address, ipv6Address) configData := map[string]string{ - "Corefile": config, + "Corefile": configMapData, } return configData } +func generateCoreFile(t *testing.T, ipv4Address, ipv6Address string) (string, error) { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} + {{ if .IPv6 }}{{ .IPv6 }} {{ $.FQDN }}{{ end }} + no_reverse + pods verified + ttl 10 + } + loop + reload + }` + + data := struct { + IPv4 string + IPv6 string + FQDN string + }{ + IPv4: ipv4Address, + IPv6: ipv6Address, + FQDN: testFullyQualifiedDomainName, + } + + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + require.NoError(t, err, "error while creating template ", err) + + var output bytes.Buffer + err = tmpl.Execute(&output, data) + require.NoError(t, err, "error while executing config map template", err) + + return strings.TrimSpace(output.String()), nil +} + func createDnsPod(t *testing.T, data *TestData) { volume := []corev1.Volume{ { @@ -247,7 +269,7 @@ func createDnsPod(t *testing.T, data *TestData) { require.NoError(t, pb.Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) - require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to annotate the custom dns pod.") + require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to annotate the custom dns pod.") } @@ -310,7 +332,19 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel } require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data)) - podIP, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) require.NoError(t, err) - return podIP + return podIPs +} + +func extractIPs(podIPs *PodIPs) (string, string) { + var ipv4, ipv6 string + res := podIPs.AsSlice() + if len(res) > 0 { + ipv4 = res[0].String() + if len(res) == 2 { + ipv6 = res[1].String() + } + } + return ipv4, ipv6 } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index bee8ae3def1..7093ae9eea5 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -141,9 +141,9 @@ const ( statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" randomPatchAnnotationKey = "test.antrea.io/random-value" - - iperfPort = 5201 - iperfSvcPort = 9999 + annotationValueLen = 8 + iperfPort = 5201 + iperfSvcPort = 9999 ) type ClusterNode struct { @@ -3230,17 +3230,14 @@ func (data *TestData) runDNSQuery( useTCP bool, dnsServiceIP string) (net.IP, error) { - digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) + digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) if useTCP { - digCmd += " +tcp" - } - cmd := []string{ - "/bin/sh", - "-c", - digCmd, + digCmdStr += " +tcp" } - fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(cmd, " ")) - stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, cmd) + + digCmd := strings.Fields(digCmdStr) + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(digCmd, " ")) + stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd) if err != nil { return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) } @@ -3253,12 +3250,8 @@ func (data *TestData) runDNSQuery( } } -// patchPodAnnotation Patches an annotation for a given pod in a given namespace. If no annotation is -// provided as a parameter then it generates a random string value for a predefined key and adds it as annotation. -// The patchType used is MergePatchType since we intend to make updates to annotations in pods instead of updating fields in complex k8s -// resources like deployments. +// patchPodAnnotation Patches a pod with given map of keys and values. func (data *TestData) patchPodAnnotation(namespace, podName string, annotation map[string]string) error { - numberOfRunes := 8 annotationPatch := map[string]interface{}{ "metadata": map[string]interface{}{ "annotations": make(map[string]string), @@ -3266,14 +3259,10 @@ func (data *TestData) patchPodAnnotation(namespace, podName string, annotation m } if annotation == nil { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[randomPatchAnnotationKey] = randSeq(numberOfRunes) + return fmt.Errorf("no annotations were provided") } else { for key, value := range annotation { - if key == randomPatchAnnotationKey { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = randSeq(numberOfRunes) - } else { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value - } + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value } } @@ -3292,3 +3281,30 @@ func (data *TestData) patchPodAnnotation(namespace, podName string, annotation m log.Infof("Successfully patched pod %s in namespace %s with provided annotation", podName, namespace) return nil } + +// setPodAnnotationToRandomValue Patches a pod by adding an annotation with a specified key and a randomly generated string as the value. +func (data *TestData) setPodAnnotationToRandomValue(namespace, podName, annotationKey string) error { + annotationPatch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": make(map[string]string), + }, + } + + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = randSeq(annotationValueLen) + + patchData, err := json.Marshal(annotationPatch) + if err != nil { + log.Infof("Error marshalling annotation: %+v", err) + return err + } + + _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err) + return err + } + + log.Infof("Successfully patched pod %s in namespace %s with a random value annotation", podName, namespace) + return nil + +} diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 93e3f7c2106..366365783ed 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -103,7 +103,6 @@ type TestNamespaceMeta struct { // GetPodByLabel returns a Pod with the matching Namespace and "pod" label if it's found. // If the pod is not found, GetPodByLabel returns "ErrPodNotFound". func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) { - //TODO: Query why this is hard coded as "pod" pods, err := k.getPodsUncached(ns, "pod", name) if err != nil { return nil, fmt.Errorf("unable to get Pod in Namespace %s with label pod=%s: %w", ns, name, err) @@ -668,20 +667,24 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } -func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string]byte, immutable bool) (*v1.ConfigMap, error) { +func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string][]byte, immutable bool) (*v1.ConfigMap, error) { configMap := &v1.ConfigMap{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, - Immutable: nil, + Immutable: &immutable, Data: configData, - BinaryData: nil, + BinaryData: binaryData, } - configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMap, metav1.CreateOptions{}) - return configMapObject, err + configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + log.Infof("Unable to create configMap : %s", err) + return nil, err + } + return configMapObject, nil } // DeleteService is a convenience function for deleting a Service by Namespace and name. From bc919644141394f3f9aa9ac2192cf9b2e4c3270f Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 11 Oct 2024 21:59:23 +0530 Subject: [PATCH 09/17] merge generateCoreFile to createDnsConfig making use of closures. assert fqdn ip returned from dns server, for both ipv4 and ipv6. usage of EventuallyWithT rename buildFqdnPolicy removed certain constants. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 106 ++++++++++++++++---------------- test/e2e/framework.go | 38 +----------- 2 files changed, 57 insertions(+), 87 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index e8fad3d736c..22c7c23648e 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -53,12 +53,8 @@ import ( */ const ( - testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" - customDnsServiceName = "custom-dns-service" customDnsConfigName = "custom-dns-config" - customDnsPort = 53 - customDnsTargetPort = 53 customDnsImage = "coredns/coredns:1.11.3" customDnsPodName = "custom-dns-server" @@ -76,11 +72,12 @@ const ( agnHostLabelKey = "app" agnHostLabelValue = "agnhost" agnHostPodNamePreFix = "agnhost-" - - eventualWaitTime = 20 * time.Second ) func TestFQDNPolicyWithCachedDNS(t *testing.T) { + const testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" + const eventualWaitTime = 60 * time.Second + const dnsPort = 53 skipIfAntreaPolicyDisabled(t) data, err := setupTest(t) if err != nil { @@ -88,11 +85,12 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } defer teardownTest(t, data) - customDnsService, err := data.CreateServiceWithAnnotations(customDnsServiceName, data.testNamespace, customDnsPort, customDnsTargetPort, corev1.ProtocolUDP, + customDnsService, err := data.CreateServiceWithAnnotations(customDnsServiceName, data.testNamespace, dnsPort, dnsPort, corev1.ProtocolUDP, map[string]string{customDnsLabelKey: customDnsLabelValue}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) require.NoError(t, err, "Error when creating custom DNS service: %v", err) setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) + defer setDnsServerAddressInAntrea(t, data, "") podCount := 2 agnHostPodIps := make([]*PodIPs, podCount) @@ -102,7 +100,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { ipv4, ipv6 := extractIPs(agnHostPodIps[0]) - dnsConfigData := createDnsConfig(t, ipv4, ipv6) + dnsConfigData := createDnsConfig(t, ipv4, ipv6, testFullyQualifiedDomainName) customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) @@ -111,11 +109,11 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { k8sUtils, err = NewKubernetesUtils(data) require.NoError(t, err, "error getting k8s utils %+v", err) - _ = buildFqdnPolicy(t, data) + createFqdnPolicyInNamespace(t, data, testFullyQualifiedDomainName) createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) - curlFqdn := func(podName, containerName, fqdn string) error { + curlTarget := func(podName, containerName, fqdn string) error { cmd := []string{"curl", fqdn} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) if err != nil { @@ -125,18 +123,23 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { return nil } - assert.Eventually(t, func() bool { - t.Logf("trying to curl the fqdn %v", testFullyQualifiedDomainName) - err = curlFqdn(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) - return assert.NoError(t, err) - }, 2*time.Second, 100*time.Millisecond) + assert.EventuallyWithT(t, func(collect *assert.CollectT) { + err = curlTarget(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) + assert.NoError(t, err) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFullyQualifiedDomainName) // DIG to get actual IP , to be sure. digResponse, err := k8sUtils.runDNSQuery(toolBoxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) require.NoError(t, err, "failed to get IP of FQDN using DIG from toolbox pod : %v", err) - fqdnIp := digResponse.String() - t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) + // Use an assertion to check if the received IPs are same. + fqdnIp := digResponse.String() + if digResponse.To4() == nil { + require.Equalf(t, ipv6, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %+v instead of %+v", fqdnIp, ipv6) + } else { + require.Equalf(t, ipv4, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %+v instead of %+v", fqdnIp, ipv4) + } + t.Logf("Successfully received the expected IP %+v using the dig command against the FQDN", fqdnIp) ipv4, ipv6 = extractIPs(agnHostPodIps[1]) if ipv6 != "" { @@ -146,7 +149,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } // Update the custom DNS configMap - UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6) + UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6, testFullyQualifiedDomainName) customDnsConfigMapObject.Data = UpdatedCustomDNSconfig err = data.UpdateConfigMap(customDnsConfigMapObject) @@ -157,13 +160,11 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v", ipv4) } - require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to update custom dns pod annotation.") - - defer setDnsServerAddressInAntrea(t, data, "") + require.NoError(t, data.setPodAnnotation(data.testNamespace, customDnsPodName, randomPatchAnnotationKey, randSeq(annotationValueLen)), "failed to update custom dns pod annotation.") assert.Eventually(t, func() bool { t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) - err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) + err = curlTarget(toolBoxPodName, toolboxContainerName, fqdnIp) if err != nil { t.Logf("The test failed because of error %+v", err) } @@ -187,16 +188,7 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri } -func createDnsConfig(t *testing.T, ipv4Address, ipv6Address string) map[string]string { - configMapData, _ := generateCoreFile(t, ipv4Address, ipv6Address) - configData := map[string]string{ - "Corefile": configMapData, - } - - return configData -} - -func generateCoreFile(t *testing.T, ipv4Address, ipv6Address string) (string, error) { +func createDnsConfig(t *testing.T, ipv4Address, ipv6Address, domainName string) map[string]string { const coreFileTemplate = `lfx.test:53 { errors log @@ -212,24 +204,36 @@ func generateCoreFile(t *testing.T, ipv4Address, ipv6Address string) (string, er reload }` - data := struct { - IPv4 string - IPv6 string - FQDN string - }{ - IPv4: ipv4Address, - IPv6: ipv6Address, - FQDN: testFullyQualifiedDomainName, - } + generateConfigData := func() (string, error) { + data := struct { + IPv4 string + IPv6 string + FQDN string + }{ + IPv4: ipv4Address, + IPv6: ipv6Address, + FQDN: domainName, + } - tmpl, err := template.New("configMapData").Parse(coreFileTemplate) - require.NoError(t, err, "error while creating template ", err) + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + if err != nil { + return "", err + } + var output bytes.Buffer + err = tmpl.Execute(&output, data) + if err != nil { + return "", err + } + return strings.TrimSpace(output.String()), nil + } - var output bytes.Buffer - err = tmpl.Execute(&output, data) - require.NoError(t, err, "error while executing config map template", err) + configMapData, err := generateConfigData() + require.NoError(t, err, "error processing configData template for DNS, ", err) + configData := map[string]string{ + "Corefile": configMapData, + } - return strings.TrimSpace(output.String()), nil + return configData } func createDnsPod(t *testing.T, data *TestData) { @@ -269,11 +273,11 @@ func createDnsPod(t *testing.T, data *TestData) { require.NoError(t, pb.Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) - require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to annotate the custom dns pod.") + require.NoError(t, data.setPodAnnotation(data.testNamespace, customDnsPodName, randomPatchAnnotationKey, randSeq(annotationValueLen)), "failed to annotate the custom dns pod.") } -func buildFqdnPolicy(t *testing.T, data *TestData) *utils.AntreaNetworkPolicySpecBuilder { +func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string) { podSelectorLabel := map[string]string{ fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue, } @@ -284,7 +288,7 @@ func buildFqdnPolicy(t *testing.T, data *TestData) *utils.AntreaNetworkPolicySpe SetTier(defaultTierName). SetPriority(1.0). SetAppliedToGroup([]utils.ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) - builder.AddFQDNRule(testFullyQualifiedDomainName, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, + builder.AddFQDNRule(domainName, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, crdv1beta1.RuleActionAllow) builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, @@ -299,8 +303,6 @@ func buildFqdnPolicy(t *testing.T, data *TestData) *utils.AntreaNetworkPolicySpe require.NoError(t, err, "error while deploying antrea policy %+v", err) failOnError(err, t) failOnError(waitForResourceReady(t, 30*time.Second, annp), t) - - return builder } func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 7093ae9eea5..2ce8f6a759f 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3250,47 +3250,15 @@ func (data *TestData) runDNSQuery( } } -// patchPodAnnotation Patches a pod with given map of keys and values. -func (data *TestData) patchPodAnnotation(namespace, podName string, annotation map[string]string) error { +// setPodAnnotation Patches a pod by adding an annotation with a specified key and a randomly generated string as the value. +func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error { annotationPatch := map[string]interface{}{ "metadata": map[string]interface{}{ "annotations": make(map[string]string), }, } - if annotation == nil { - return fmt.Errorf("no annotations were provided") - } else { - for key, value := range annotation { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value - } - } - - patchData, err := json.Marshal(annotationPatch) - if err != nil { - log.Infof("Error marshalling annotation: %+v", err) - return err - } - - _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) - if err != nil { - log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err) - return err - } - - log.Infof("Successfully patched pod %s in namespace %s with provided annotation", podName, namespace) - return nil -} - -// setPodAnnotationToRandomValue Patches a pod by adding an annotation with a specified key and a randomly generated string as the value. -func (data *TestData) setPodAnnotationToRandomValue(namespace, podName, annotationKey string) error { - annotationPatch := map[string]interface{}{ - "metadata": map[string]interface{}{ - "annotations": make(map[string]string), - }, - } - - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = randSeq(annotationValueLen) + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = annotationValue patchData, err := json.Marshal(annotationPatch) if err != nil { From 2ef3fd1fc8de9593071c1a54758498284a2268fc Mon Sep 17 00:00:00 2001 From: Hemant Date: Mon, 14 Oct 2024 23:28:16 +0530 Subject: [PATCH 10/17] Removed not needed globally defined constants. Removed function to build pod using PodBuilder from framework.go Added usage of --dns-server flag in curl command. Added additional logic to set DNS ip in resolv.conf. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 88 +++++++++++++++++---------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 22c7c23648e..7adbf8af30c 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -53,31 +53,26 @@ import ( */ const ( - customDnsServiceName = "custom-dns-service" - customDnsConfigName = "custom-dns-config" + testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" + customDnsConfigName = "custom-dns-config" + customDnsPodName = "custom-dns-server" - customDnsImage = "coredns/coredns:1.11.3" - customDnsPodName = "custom-dns-server" - customDnsContainerName = "coredns" - customDnsLabelKey = "app" - customDnsLabelValue = "custom-dns" - customDnsVolume = "config-volume" + customDnsLabelKey = "app" + customDnsLabelValue = "custom-dns" - fqdnPolicyName = "test-anp-fqdn" fqdnPodSelectorLabelKey = "app" fqdnPodSelectorLabelValue = "fqdn-cache-test" - toolBoxPodName = "toolbox" - - agnHostPort = 80 - agnHostLabelKey = "app" - agnHostLabelValue = "agnhost" - agnHostPodNamePreFix = "agnhost-" ) func TestFQDNPolicyWithCachedDNS(t *testing.T) { - const testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" - const eventualWaitTime = 60 * time.Second - const dnsPort = 53 + const ( + agnHostLabelKey = "app" + agnHostLabelValue = "agnhost" + agnHostPodNamePreFix = "agnhost-" + + dnsPort = 53 + customDnsServiceName = "custom-dns-service" + ) skipIfAntreaPolicyDisabled(t) data, err := setupTest(t) if err != nil { @@ -111,7 +106,12 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { createFqdnPolicyInNamespace(t, data, testFullyQualifiedDomainName) - createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) + t.Logf("Creating Toolbox pod...") + require.NoError(t, data.createToolboxPodOnNode(toolboxPodName, data.testNamespace, "", false), "Error creating toolbox pod") + _, err = data.PodWaitFor(defaultTimeout, toolboxPodName, data.testNamespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == corev1.PodRunning, nil + }) + require.NoError(t, err, "Error while waiting for Toolbox Pod to be in Running state") curlTarget := func(podName, containerName, fqdn string) error { cmd := []string{"curl", fqdn} @@ -123,13 +123,28 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { return nil } + //curl does not seem to support --dns-server flag unless we provide DNS IP in resolv.conf, same with --resolve flag. + setDnsInPod := func(podName, containerName, fqdn string) error { + cmd := []string{"bash", "-c", "echo 'nameserver " + customDnsService.Spec.ClusterIP + "' > /etc/resolv.conf"} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + if err != nil { + return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), podName, err, stdout, stderr) + } + return nil + } + + t.Logf("Setting custom DNS IP in resolv.conf of Toolbox pod...") + require.NoError(t, setDnsInPod(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName), "Error setting custom DNS in toolbox") + + t.Logf("Trying to curl FQDN ...") assert.EventuallyWithT(t, func(collect *assert.CollectT) { - err = curlTarget(toolBoxPodName, toolboxContainerName, testFullyQualifiedDomainName) + err = curlTarget(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName) assert.NoError(t, err) }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFullyQualifiedDomainName) - // DIG to get actual IP , to be sure. - digResponse, err := k8sUtils.runDNSQuery(toolBoxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) + t.Logf("Trying to DIG the FQDN to simulate caching the current IP inside Toolbox pod...") + digResponse, err := k8sUtils.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) require.NoError(t, err, "failed to get IP of FQDN using DIG from toolbox pod : %v", err) // Use an assertion to check if the received IPs are same. @@ -164,12 +179,12 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { assert.Eventually(t, func() bool { t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) - err = curlTarget(toolBoxPodName, toolboxContainerName, fqdnIp) + err = curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) if err != nil { t.Logf("The test failed because of error %+v", err) } return assert.Error(t, err) - }, eventualWaitTime, 1*time.Second) + }, 15*time.Second, 1*time.Second) } @@ -237,6 +252,11 @@ func createDnsConfig(t *testing.T, ipv4Address, ipv6Address, domainName string) } func createDnsPod(t *testing.T, data *TestData) { + const ( + customDnsImage = "coredns/coredns:1.11.3" + customDnsContainerName = "coredns" + customDnsVolume = "config-volume" + ) volume := []corev1.Volume{ { Name: customDnsVolume, @@ -278,6 +298,7 @@ func createDnsPod(t *testing.T, data *TestData) { } func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string) { + const fqdnPolicyName = "test-anp-fqdn" podSelectorLabel := map[string]string{ fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue, } @@ -305,25 +326,8 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string failOnError(waitForResourceReady(t, 30*time.Second, annp), t) } -func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { - toolBoxLabel := map[string]string{fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue} - pb := NewPodBuilder(toolBoxPodName, data.testNamespace, ToolboxImage) - pb.WithLabels(toolBoxLabel) - pb.WithContainerName(toolboxContainerName) - mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { - pod.Spec.DNSPolicy = corev1.DNSNone - if pod.Spec.DNSConfig == nil { - pod.Spec.DNSConfig = &corev1.PodDNSConfig{} - } - pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} - - } - pb.WithMutateFunc(mutateSpecForAddingCustomDNS) - require.NoError(t, pb.Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, toolBoxPodName, data.testNamespace)) -} - func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { + const agnHostPort = 80 args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} ports := []corev1.ContainerPort{ { From df67bb55776975348f9f5d13ef85f84c104fea14 Mon Sep 17 00:00:00 2001 From: Hemant Date: Tue, 15 Oct 2024 21:33:25 +0530 Subject: [PATCH 11/17] using pod builder to create toolbox and added some logs for verbosity while running Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 55 +++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 7adbf8af30c..cd8a20dac51 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -107,40 +107,23 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { createFqdnPolicyInNamespace(t, data, testFullyQualifiedDomainName) t.Logf("Creating Toolbox pod...") - require.NoError(t, data.createToolboxPodOnNode(toolboxPodName, data.testNamespace, "", false), "Error creating toolbox pod") - _, err = data.PodWaitFor(defaultTimeout, toolboxPodName, data.testNamespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == corev1.PodRunning, nil - }) - require.NoError(t, err, "Error while waiting for Toolbox Pod to be in Running state") + createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) - curlTarget := func(podName, containerName, fqdn string) error { + curlTarget := func(podName, containerName, fqdn string) (string, error) { cmd := []string{"curl", fqdn} stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) if err != nil { - return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", strings.Join(cmd, " "), podName, err, stdout, stderr) } - return nil + return stdout, nil } - //curl does not seem to support --dns-server flag unless we provide DNS IP in resolv.conf, same with --resolve flag. - setDnsInPod := func(podName, containerName, fqdn string) error { - cmd := []string{"bash", "-c", "echo 'nameserver " + customDnsService.Spec.ClusterIP + "' > /etc/resolv.conf"} - stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) - if err != nil { - return fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", - strings.Join(cmd, " "), podName, err, stdout, stderr) - } - return nil - } - - t.Logf("Setting custom DNS IP in resolv.conf of Toolbox pod...") - require.NoError(t, setDnsInPod(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName), "Error setting custom DNS in toolbox") - t.Logf("Trying to curl FQDN ...") assert.EventuallyWithT(t, func(collect *assert.CollectT) { - err = curlTarget(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName) + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName) assert.NoError(t, err) + t.Logf("response of curl to FQDN - %+v", stdout) }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFullyQualifiedDomainName) t.Logf("Trying to DIG the FQDN to simulate caching the current IP inside Toolbox pod...") @@ -178,10 +161,12 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { require.NoError(t, data.setPodAnnotation(data.testNamespace, customDnsPodName, randomPatchAnnotationKey, randSeq(annotationValueLen)), "failed to update custom dns pod annotation.") assert.Eventually(t, func() bool { - t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) - err = curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) + t.Logf("Trying to curl the existing cached IP of the domain - %v", fqdnIp) + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) if err != nil { - t.Logf("The test failed because of error %+v", err) + t.Logf("The test failed because of error : %+v", err) + } else { + t.Logf("response of curl to cached IP - %+v", stdout) } return assert.Error(t, err) }, 15*time.Second, 1*time.Second) @@ -326,6 +311,24 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string failOnError(waitForResourceReady(t, 30*time.Second, annp), t) } +func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { + toolBoxLabel := map[string]string{fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue} + pb := NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage) + pb.WithLabels(toolBoxLabel) + pb.WithContainerName(toolboxContainerName) + mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { + pod.Spec.DNSPolicy = corev1.DNSNone + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &corev1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} + + } + pb.WithMutateFunc(mutateSpecForAddingCustomDNS) + require.NoError(t, pb.Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) +} + func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { const agnHostPort = 80 args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} From 36102bae0cf074aa6c64f3b55826f4805e86cad8 Mon Sep 17 00:00:00 2001 From: Hemant Date: Thu, 17 Oct 2024 00:41:36 +0530 Subject: [PATCH 12/17] reduced verbosity and refactored createConfigMap function Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 31 ++++++++++++------------------- test/e2e/framework.go | 14 ++++++-------- test/e2e/k8s_util.go | 16 ++-------------- 3 files changed, 20 insertions(+), 41 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index cd8a20dac51..e502bd09e48 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" @@ -96,7 +97,14 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { ipv4, ipv6 := extractIPs(agnHostPodIps[0]) dnsConfigData := createDnsConfig(t, ipv4, ipv6, testFullyQualifiedDomainName) - customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: customDnsConfigName, + Namespace: data.testNamespace, + }, + Data: dnsConfigData, + } + customDnsConfigMapObject, err := data.CreateConfigMap(configMap) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) createDnsPod(t, data) @@ -268,18 +276,8 @@ func createDnsPod(t *testing.T, data *TestData) { }, } - label := map[string]string{customDnsLabelKey: customDnsLabelValue} - pb := NewPodBuilder(customDnsPodName, data.testNamespace, customDnsImage) - pb.WithLabels(label) - pb.WithContainerName(customDnsContainerName) - pb.WithArgs([]string{"-conf", "/etc/coredns/Corefile"}) - pb.AddVolume(volume) - pb.AddVolumeMount(volumeMount) - - require.NoError(t, pb.Create(data)) + require.NoError(t, NewPodBuilder(customDnsPodName, data.testNamespace, customDnsImage).WithLabels(map[string]string{customDnsLabelKey: customDnsLabelValue}).WithContainerName(customDnsContainerName).WithArgs([]string{"-conf", "/etc/coredns/Corefile"}).AddVolume(volume).AddVolumeMount(volumeMount).WithAnnotations(map[string]string{randomPatchAnnotationKey: randSeq(annotationValueLen)}).Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) - require.NoError(t, data.setPodAnnotation(data.testNamespace, customDnsPodName, randomPatchAnnotationKey, randSeq(annotationValueLen)), "failed to annotate the custom dns pod.") - } func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string) { @@ -307,15 +305,11 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string annp, err := k8sUtils.CreateOrUpdateANNP(builder.Get()) require.NoError(t, err, "error while deploying antrea policy %+v", err) - failOnError(err, t) - failOnError(waitForResourceReady(t, 30*time.Second, annp), t) + require.NoError(t, waitForResourceReady(t, 30*time.Second, annp)) } func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { toolBoxLabel := map[string]string{fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue} - pb := NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage) - pb.WithLabels(toolBoxLabel) - pb.WithContainerName(toolboxContainerName) mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { pod.Spec.DNSPolicy = corev1.DNSNone if pod.Spec.DNSConfig == nil { @@ -324,8 +318,7 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} } - pb.WithMutateFunc(mutateSpecForAddingCustomDNS) - require.NoError(t, pb.Create(data)) + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage).WithLabels(toolBoxLabel).WithContainerName(toolboxContainerName).WithMutateFunc(mutateSpecForAddingCustomDNS).Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 2ce8f6a759f..e7d4c7647c8 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3250,29 +3250,27 @@ func (data *TestData) runDNSQuery( } } -// setPodAnnotation Patches a pod by adding an annotation with a specified key and a randomly generated string as the value. +// setPodAnnotation Patches a pod by adding an annotation with a specified key and value. func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error { + annotations := map[string]string{ + annotationKey: annotationValue, + } annotationPatch := map[string]interface{}{ "metadata": map[string]interface{}{ - "annotations": make(map[string]string), + "annotations": annotations, }, } - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = annotationValue - patchData, err := json.Marshal(annotationPatch) if err != nil { - log.Infof("Error marshalling annotation: %+v", err) return err } _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) if err != nil { - log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err) return err } - log.Infof("Successfully patched pod %s in namespace %s with a random value annotation", podName, namespace) + log.Infof("Successfully patched pod %s in namespace %s", podName, namespace) return nil - } diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 366365783ed..6fe96c27801 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -667,21 +667,9 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } -func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string][]byte, immutable bool) (*v1.ConfigMap, error) { - configMap := &v1.ConfigMap{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Immutable: &immutable, - Data: configData, - BinaryData: binaryData, - } - - configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) +func (data *TestData) CreateConfigMap(configMap *v1.ConfigMap) (*v1.ConfigMap, error) { + configMapObject, err := data.clientset.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { - log.Infof("Unable to create configMap : %s", err) return nil, err } return configMapObject, nil From 894ff3253cc07e71a61bcd3bbdde9994138f5634 Mon Sep 17 00:00:00 2001 From: Hemant Date: Wed, 23 Oct 2024 21:10:10 +0530 Subject: [PATCH 13/17] Made significant changes to the e2e test by improving readability and maintainability. Added a AsStrings() function to PodIPs type to return ipv4 and ipv6 addresses as strings. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 307 ++++++++++++++++++-------------- test/e2e/framework.go | 10 ++ 2 files changed, 183 insertions(+), 134 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index e502bd09e48..afbb7a5e919 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -53,157 +53,190 @@ import ( 15) curl the FQDN again with old IP , simulating usage of cache -- and it must fail with no connectivity. */ -const ( - testFullyQualifiedDomainName = "fqdn-test-pod.lfx.test" - customDnsConfigName = "custom-dns-config" - customDnsPodName = "custom-dns-server" - - customDnsLabelKey = "app" - customDnsLabelValue = "custom-dns" - - fqdnPodSelectorLabelKey = "app" - fqdnPodSelectorLabelValue = "fqdn-cache-test" -) +// agnHostPodIpv4Addresses type stores the IPv4 addresses of the 2 agnHost pods created for the test. +type agnHostPodIpv4Addresses struct { + pod1Ipv4 string + pod2Ipv4 string +} func TestFQDNPolicyWithCachedDNS(t *testing.T) { - const ( - agnHostLabelKey = "app" - agnHostLabelValue = "agnhost" - agnHostPodNamePreFix = "agnhost-" + const testFQDN = "fqdn-test-pod.lfx.test" - dnsPort = 53 - customDnsServiceName = "custom-dns-service" - ) skipIfAntreaPolicyDisabled(t) + skipIfNotIPv4Cluster(t) + skipIfIPv6Cluster(t) + data, err := setupTest(t) if err != nil { t.Fatalf("Error when setting up test: %v", err) } defer teardownTest(t, data) - customDnsService, err := data.CreateServiceWithAnnotations(customDnsServiceName, data.testNamespace, dnsPort, dnsPort, corev1.ProtocolUDP, - map[string]string{customDnsLabelKey: customDnsLabelValue}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) - require.NoError(t, err, "Error when creating custom DNS service: %v", err) + dnsServiceIP := setCustomDnsServerAddressInAntrea(t, data, false) + defer setCustomDnsServerAddressInAntrea(t, data, true) - setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) - defer setDnsServerAddressInAntrea(t, data, "") + agnHostPodIp := createAgnHostPods(t, data) - podCount := 2 - agnHostPodIps := make([]*PodIPs, podCount) - for i := 0; i < podCount; i++ { - agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) - } - - ipv4, ipv6 := extractIPs(agnHostPodIps[0]) - - dnsConfigData := createDnsConfig(t, ipv4, ipv6, testFullyQualifiedDomainName) - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: customDnsConfigName, - Namespace: data.testNamespace, - }, - Data: dnsConfigData, - } - customDnsConfigMapObject, err := data.CreateConfigMap(configMap) + customDnsConfigMap, err := createAndDeployCustomDnsConfigMap(t, data, agnHostPodIp.pod1Ipv4, testFQDN) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) createDnsPod(t, data) + createFqdnPolicyInNamespace(t, data, testFQDN) + + t.Logf("Creating Toolbox pod...") + createToolBoxPod(t, data, dnsServiceIP) - k8sUtils, err = NewKubernetesUtils(data) - require.NoError(t, err, "error getting k8s utils %+v", err) + fqdnIp, err := curlAndVerifyFQDN(t, data, testFQDN, dnsServiceIP) + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod : %v", err) + require.Equalf(t, agnHostPodIp.pod1Ipv4, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIp, agnHostPodIp.pod1Ipv4) - createFqdnPolicyInNamespace(t, data, testFullyQualifiedDomainName) + t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIp) - t.Logf("Creating Toolbox pod...") - createToolBoxPod(t, data, customDnsService.Spec.ClusterIP) + t.Logf("New IP to update to DNS | ipv4 : %s", agnHostPodIp.pod2Ipv4) + err = updateCustomDnsConfigMap(t, data, agnHostPodIp.pod2Ipv4, testFQDN, customDnsConfigMap) + require.NoError(t, err, "failed to update configmap with new IP : %v", err) + t.Logf("successfully updated dns configMap with new IPs | ipv4 : %s", agnHostPodIp.pod2Ipv4) - curlTarget := func(podName, containerName, fqdn string) (string, error) { - cmd := []string{"curl", fqdn} - stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) - if err != nil { - return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", - strings.Join(cmd, " "), podName, err, stdout, stderr) - } - return stdout, nil - } + require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", randomPatchAnnotationKey, + randSeq(annotationValueLen)), "failed to update custom dns pod annotation.") - t.Logf("Trying to curl FQDN ...") - assert.EventuallyWithT(t, func(collect *assert.CollectT) { - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFullyQualifiedDomainName) - assert.NoError(t, err) - t.Logf("response of curl to FQDN - %+v", stdout) - }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFullyQualifiedDomainName) - - t.Logf("Trying to DIG the FQDN to simulate caching the current IP inside Toolbox pod...") - digResponse, err := k8sUtils.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFullyQualifiedDomainName, false, customDnsService.Spec.ClusterIP) - require.NoError(t, err, "failed to get IP of FQDN using DIG from toolbox pod : %v", err) - - // Use an assertion to check if the received IPs are same. - fqdnIp := digResponse.String() - if digResponse.To4() == nil { - require.Equalf(t, ipv6, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %+v instead of %+v", fqdnIp, ipv6) - } else { - require.Equalf(t, ipv4, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %+v instead of %+v", fqdnIp, ipv4) - } - t.Logf("Successfully received the expected IP %+v using the dig command against the FQDN", fqdnIp) + cachedIpTest(t, fqdnIp, data) +} - ipv4, ipv6 = extractIPs(agnHostPodIps[1]) - if ipv6 != "" { - t.Logf("New IPs to update to DNS | ipv4 : %v, ipv6 : %+v", ipv4, ipv6) - } else { - t.Logf("New IPs to update to DNS | ipv4 : %v", ipv4) - } +// createAgnHostPods creates two agnHost pods and returns their IPv4 addresses filled in agnHostPodIpv4Addresses type. +func createAgnHostPods(t *testing.T, data *TestData) *agnHostPodIpv4Addresses { + const agnHostPodNamePreFix = "agnhost-" - // Update the custom DNS configMap - UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6, testFullyQualifiedDomainName) + var agnHostPodIpv4AddressesObj agnHostPodIpv4Addresses - customDnsConfigMapObject.Data = UpdatedCustomDNSconfig - err = data.UpdateConfigMap(customDnsConfigMapObject) - require.NoError(t, err, "failed to update configmap with new IP : %v", err) - if ipv6 != "" { - t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v, ipv6 :%+v", ipv4, ipv6) - } else { - t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v", ipv4) + podCount := 2 + agnHostPodIps := make([]*PodIPs, podCount) + for i := 0; i < podCount; i++ { + agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{"app": "agnhost"}) } - require.NoError(t, data.setPodAnnotation(data.testNamespace, customDnsPodName, randomPatchAnnotationKey, randSeq(annotationValueLen)), "failed to update custom dns pod annotation.") + agnHostPodIpv4AddressesObj.pod1Ipv4, _ = agnHostPodIps[0].AsStrings() + agnHostPodIpv4AddressesObj.pod2Ipv4, _ = agnHostPodIps[1].AsStrings() + return &agnHostPodIpv4AddressesObj +} + +// curlTarget runs a curl command from a specified pod to the given FQDN and returns the output. +func curlTarget(podName, containerName, fqdn string, data *TestData) (string, error) { + cmd := []string{"curl", fqdn} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + if err != nil { + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), podName, err, stdout, stderr) + } + return stdout, nil +} + +// cachedIpTest verifies that curling the previously cached IP fails after DNS update. +func cachedIpTest(t *testing.T, fqdnIp string, data *TestData) { assert.Eventually(t, func() bool { t.Logf("Trying to curl the existing cached IP of the domain - %v", fqdnIp) - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp, data) if err != nil { t.Logf("The test failed because of error : %+v", err) } else { t.Logf("response of curl to cached IP - %+v", stdout) } return assert.Error(t, err) - }, 15*time.Second, 1*time.Second) + }, 20*time.Second, 1*time.Second) +} + +// curlAndVerifyFQDN curls the specified FQDN and verifies its resolution to the expected IP. +func curlAndVerifyFQDN(t *testing.T, data *TestData, testFQDN string, dnsServiceIP string) (string, error) { + t.Logf("Trying to curl FQDN ...") + assert.EventuallyWithT(t, func(collect *assert.CollectT) { + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFQDN, data) + assert.NoError(t, err) + t.Logf("response of curl to FQDN - %s", stdout) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFQDN) + t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") + resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) + + if err != nil { + return "", err + } + + return resolvedIP.String(), nil } -func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { +// createAndDeployCustomDnsConfigMap creates and deploys the custom DNS ConfigMap with the specified IP and FQDN. +func createAndDeployCustomDnsConfigMap(t *testing.T, data *TestData, ipv4 string, testFQDN string) (*corev1.ConfigMap, error) { + const customDnsConfigName = "custom-dns-config" + dnsConfigData := createDnsConfig(t, ipv4, testFQDN) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: customDnsConfigName, + Namespace: data.testNamespace, + }, + Data: dnsConfigData, + } + return data.CreateConfigMap(configMap) + +} + +// updateCustomDnsConfigMap updates the existing custom DNS ConfigMap with a new IP for the specified FQDN. +func updateCustomDnsConfigMap(t *testing.T, data *TestData, ipv4 string, testFullyQualifiedDomainName string, customDnsConfigMap *corev1.ConfigMap) error { + customDnsConfigMap.Data = createDnsConfig(t, ipv4, testFullyQualifiedDomainName) + return data.UpdateConfigMap(customDnsConfigMap) +} + +// setCustomDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. +func setCustomDnsServerAddressInAntrea(t *testing.T, data *TestData, teardown bool) string { + var dnsServiceIP string + + // Create or reset DNS service based on teardown flag + if !teardown { + var err error + dnsServiceIP, err = createCustomDnsService(data) + require.NoError(t, err, "Error when creating custom DNS service: %v", err) + } + + err := updateAntreaConfigMap(data, dnsServiceIP) + require.NoError(t, err, "Error when updating Antrea configmap with custom dns ip : %v", err) + + if teardown { + t.Logf("removing dns server IP from antrea agent during teardown") + return "" + } + t.Logf("Set DNS server IP in Antrea : %s", dnsServiceIP) + return dnsServiceIP +} + +// updateAntreaConfigMap updates the Antrea configuration with the specified DNS server IP. +func updateAntreaConfigMap(data *TestData, dnsServiceIP string) error { agentChanges := func(config *agentconfig.AgentConfig) { config.DNSServerOverride = dnsServiceIP } - err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) - require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) + return data.mutateAntreaConfigMap(nil, agentChanges, false, true) +} - if dnsServiceIP == "" { - t.Logf("removing dns server IP from antrea agent as part of teardown") - } else { - t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) +// createCustomDnsService creates the custom DNS service and returns its ClusterIP as ipv4 address. +func createCustomDnsService(data *TestData) (string, error) { + const dnsPort = 53 + customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, + false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) + if err != nil { + return "", err } + return customDnsService.Spec.ClusterIP, nil } -func createDnsConfig(t *testing.T, ipv4Address, ipv6Address, domainName string) map[string]string { +// createDnsConfig generates a DNS configuration for the specified IP address and domain name. +func createDnsConfig(t *testing.T, ipv4Address, domainName string) map[string]string { const coreFileTemplate = `lfx.test:53 { errors log health hosts { {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} - {{ if .IPv6 }}{{ .IPv6 }} {{ $.FQDN }}{{ end }} no_reverse pods verified ttl 10 @@ -215,11 +248,9 @@ func createDnsConfig(t *testing.T, ipv4Address, ipv6Address, domainName string) generateConfigData := func() (string, error) { data := struct { IPv4 string - IPv6 string FQDN string }{ IPv4: ipv4Address, - IPv6: ipv6Address, FQDN: domainName, } @@ -244,19 +275,17 @@ func createDnsConfig(t *testing.T, ipv4Address, ipv6Address, domainName string) return configData } +// createDnsPod creates the CoreDNS pod configured to use the custom DNS ConfigMap. func createDnsPod(t *testing.T, data *TestData) { - const ( - customDnsImage = "coredns/coredns:1.11.3" - customDnsContainerName = "coredns" - customDnsVolume = "config-volume" - ) + const customDnsImage = "coredns/coredns:1.11.3" + volume := []corev1.Volume{ { - Name: customDnsVolume, + Name: "config-volume", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: customDnsConfigName, + Name: "custom-dns-config", }, Items: []corev1.KeyToPath{ { @@ -271,19 +300,31 @@ func createDnsPod(t *testing.T, data *TestData) { volumeMount := []corev1.VolumeMount{ { - Name: customDnsVolume, + Name: "config-volume", MountPath: "/etc/coredns", }, } - require.NoError(t, NewPodBuilder(customDnsPodName, data.testNamespace, customDnsImage).WithLabels(map[string]string{customDnsLabelKey: customDnsLabelValue}).WithContainerName(customDnsContainerName).WithArgs([]string{"-conf", "/etc/coredns/Corefile"}).AddVolume(volume).AddVolumeMount(volumeMount).WithAnnotations(map[string]string{randomPatchAnnotationKey: randSeq(annotationValueLen)}).Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) + require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, customDnsImage). + WithLabels(map[string]string{"app": "custom-dns"}). + WithContainerName("coredns"). + WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). + AddVolume(volume).AddVolumeMount(volumeMount). + WithAnnotations(map[string]string{randomPatchAnnotationKey: randSeq(annotationValueLen)}). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) } -func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string) { - const fqdnPolicyName = "test-anp-fqdn" +// createFqdnPolicyInNamespace creates an FQDN policy in the specified namespace. +func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string) { + const ( + fqdnPolicyName = "test-anp-fqdn" + customDnsLabelValue = "custom-dns" + fqdnPodSelectorLabelValue = "fqdn-cache-test" + ) + podSelectorLabel := map[string]string{ - fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue, + "app": fqdnPodSelectorLabelValue, } port := int32(80) udpPort := int32(53) @@ -292,10 +333,10 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string SetTier(defaultTierName). SetPriority(1.0). SetAppliedToGroup([]utils.ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) - builder.AddFQDNRule(domainName, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, + builder.AddFQDNRule(testFQDN, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, crdv1beta1.RuleActionAllow) builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, - nil, nil, nil, nil, map[string]string{customDnsLabelKey: customDnsLabelValue}, + nil, nil, nil, nil, map[string]string{"app": customDnsLabelValue}, nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, @@ -303,13 +344,14 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, domainName string nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") - annp, err := k8sUtils.CreateOrUpdateANNP(builder.Get()) + annp, err := data.CreateOrUpdateANNP(builder.Get()) require.NoError(t, err, "error while deploying antrea policy %+v", err) - require.NoError(t, waitForResourceReady(t, 30*time.Second, annp)) + require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) } +// createToolBoxPod creates the toolbox pod with custom DNS settings for test purpose. func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { - toolBoxLabel := map[string]string{fqdnPodSelectorLabelKey: fqdnPodSelectorLabelValue} + toolBoxLabel := map[string]string{"app": "fqdn-cache-test"} mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { pod.Spec.DNSPolicy = corev1.DNSNone if pod.Spec.DNSConfig == nil { @@ -318,10 +360,15 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} } - require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage).WithLabels(toolBoxLabel).WithContainerName(toolboxContainerName).WithMutateFunc(mutateSpecForAddingCustomDNS).Create(data)) + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). + WithLabels(toolBoxLabel). + WithContainerName(toolboxContainerName). + WithMutateFunc(mutateSpecForAddingCustomDNS). + Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) } +// createHttpAgnhostPod creates an agnHost pod that serves HTTP requests and returns the IP of pod created. func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { const agnHostPort = 80 args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} @@ -333,20 +380,12 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel }, } - require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data)) + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). + WithArgs(args). + WithPorts(ports). + WithLabels(agnLabels). + Create(data)) podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) require.NoError(t, err) return podIPs } - -func extractIPs(podIPs *PodIPs) (string, string) { - var ipv4, ipv6 string - res := podIPs.AsSlice() - if len(res) > 0 { - ipv4 = res[0].String() - if len(res) == 2 { - ipv6 = res[1].String() - } - } - return ipv4, ipv6 -} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e7d4c7647c8..a5171cd9e1a 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -332,6 +332,16 @@ func (p *PodIPs) AsSlice() []*net.IP { return ips } +func (p *PodIPs) AsStrings() (ipv4, ipv6 string) { + if p.IPv4 != nil { + ipv4 = p.IPv4.String() + } + if p.IPv6 != nil { + ipv6 = p.IPv6.String() + } + return +} + // workerNodeName returns an empty string if there is no worker Node with the provided idx // (including if idx is 0, which is reserved for the control-plane Node) func workerNodeName(idx int) string { From 839f6909434b430e33aeb68c63ad78fc553d1897 Mon Sep 17 00:00:00 2001 From: Hemant Date: Thu, 24 Oct 2024 20:27:50 +0530 Subject: [PATCH 14/17] reduced helper functions Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 411 +++++++++++++------------------- test/e2e/framework.go | 2 - 2 files changed, 169 insertions(+), 244 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index afbb7a5e919..0892aa9bd3b 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -34,33 +34,11 @@ import ( "antrea.io/antrea/test/e2e/utils" ) -/* -1) Create the KIND cluster. -2) Once the cluster is up , create a service for custom DNS. -3) Configure and update Antrea configmap with above service's IP. - -5) Create agnHost 2 pods (deployment?). -6) Get IP of one of the pods of agnHost. -7) create and configure the custom CoreDNS configMap with the IP received above. -8) Create a custom DNS pod and configure it to use above configMap with agnHost pod's IP. -9) Create and apply antrea FQDN policy. -10) Deploy antrea-toolbox. - -11) curl the FQDN from within toolbox. -12) imitate caching the IP belonging to above FQDN resolution by keeping it in a variable. -13) change the existing IP in dns configmap with the other agnHost pod's IP. -14) wait for new IP to get updated in configMap and let the changes be reflected in dns pod. -15) curl the FQDN again with old IP , simulating usage of cache -- and it must fail with no connectivity. -*/ - -// agnHostPodIpv4Addresses type stores the IPv4 addresses of the 2 agnHost pods created for the test. -type agnHostPodIpv4Addresses struct { - pod1Ipv4 string - pod2Ipv4 string -} - func TestFQDNPolicyWithCachedDNS(t *testing.T) { - const testFQDN = "fqdn-test-pod.lfx.test" + const ( + testFQDN = "fqdn-test-pod.lfx.test" + dnsPort = 53 + ) skipIfAntreaPolicyDisabled(t) skipIfNotIPv4Cluster(t) @@ -72,247 +50,109 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } defer teardownTest(t, data) - dnsServiceIP := setCustomDnsServerAddressInAntrea(t, data, false) - defer setCustomDnsServerAddressInAntrea(t, data, true) - - agnHostPodIp := createAgnHostPods(t, data) - - customDnsConfigMap, err := createAndDeployCustomDnsConfigMap(t, data, agnHostPodIp.pod1Ipv4, testFQDN) - require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) - - createDnsPod(t, data) - createFqdnPolicyInNamespace(t, data, testFQDN) - - t.Logf("Creating Toolbox pod...") - createToolBoxPod(t, data, dnsServiceIP) - - fqdnIp, err := curlAndVerifyFQDN(t, data, testFQDN, dnsServiceIP) - require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod : %v", err) - require.Equalf(t, agnHostPodIp.pod1Ipv4, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIp, agnHostPodIp.pod1Ipv4) - - t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIp) - - t.Logf("New IP to update to DNS | ipv4 : %s", agnHostPodIp.pod2Ipv4) - err = updateCustomDnsConfigMap(t, data, agnHostPodIp.pod2Ipv4, testFQDN, customDnsConfigMap) - require.NoError(t, err, "failed to update configmap with new IP : %v", err) - t.Logf("successfully updated dns configMap with new IPs | ipv4 : %s", agnHostPodIp.pod2Ipv4) - - require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", randomPatchAnnotationKey, - randSeq(annotationValueLen)), "failed to update custom dns pod annotation.") - - cachedIpTest(t, fqdnIp, data) -} - -// createAgnHostPods creates two agnHost pods and returns their IPv4 addresses filled in agnHostPodIpv4Addresses type. -func createAgnHostPods(t *testing.T, data *TestData) *agnHostPodIpv4Addresses { - const agnHostPodNamePreFix = "agnhost-" - - var agnHostPodIpv4AddressesObj agnHostPodIpv4Addresses - + // create two agnHost pods and get their IPv4 addresses. The IP of these pods will be mapped against the FQDN. podCount := 2 agnHostPodIps := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { - agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{"app": "agnhost"}) + agnHostPodIps[i] = createHttpAgnHostPod(t, data) } - agnHostPodIpv4AddressesObj.pod1Ipv4, _ = agnHostPodIps[0].AsStrings() - agnHostPodIpv4AddressesObj.pod2Ipv4, _ = agnHostPodIps[1].AsStrings() - - return &agnHostPodIpv4AddressesObj -} - -// curlTarget runs a curl command from a specified pod to the given FQDN and returns the output. -func curlTarget(podName, containerName, fqdn string, data *TestData) (string, error) { - cmd := []string{"curl", fqdn} - stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) - if err != nil { - return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", - strings.Join(cmd, " "), podName, err, stdout, stderr) - } - return stdout, nil -} - -// cachedIpTest verifies that curling the previously cached IP fails after DNS update. -func cachedIpTest(t *testing.T, fqdnIp string, data *TestData) { - assert.Eventually(t, func() bool { - t.Logf("Trying to curl the existing cached IP of the domain - %v", fqdnIp) - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp, data) - if err != nil { - t.Logf("The test failed because of error : %+v", err) - } else { - t.Logf("response of curl to cached IP - %+v", stdout) - } - return assert.Error(t, err) - }, 20*time.Second, 1*time.Second) -} - -// curlAndVerifyFQDN curls the specified FQDN and verifies its resolution to the expected IP. -func curlAndVerifyFQDN(t *testing.T, data *TestData, testFQDN string, dnsServiceIP string) (string, error) { - t.Logf("Trying to curl FQDN ...") - assert.EventuallyWithT(t, func(collect *assert.CollectT) { - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFQDN, data) - assert.NoError(t, err) - t.Logf("response of curl to FQDN - %s", stdout) - }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFQDN) - - t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") - resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) - - if err != nil { - return "", err - } + // get IPv4 addresses of the agnHost pods created. + agnHostPodOneIP, _ := agnHostPodIps[0].AsStrings() + agnHostPodTwoIP, _ := agnHostPodIps[1].AsStrings() - return resolvedIP.String(), nil -} + // create customDNS service and get its ClusterIP. + customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, + false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) + require.NoError(t, err, "Error creating customDNS service: %+v", err) + dnsServiceIP := customDnsService.Spec.ClusterIP -// createAndDeployCustomDnsConfigMap creates and deploys the custom DNS ConfigMap with the specified IP and FQDN. -func createAndDeployCustomDnsConfigMap(t *testing.T, data *TestData, ipv4 string, testFQDN string) (*corev1.ConfigMap, error) { - const customDnsConfigName = "custom-dns-config" - dnsConfigData := createDnsConfig(t, ipv4, testFQDN) + // create a ConfigMap for the custom DNS server, mapping IP of agnHost pod 1 to the FQDN. + dnsConfigData := createDnsConfig(t, map[string]string{"ipAddress": agnHostPodOneIP, "domainName": testFQDN}) configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: customDnsConfigName, + Name: "custom-dns-config", Namespace: data.testNamespace, }, Data: dnsConfigData, } - return data.CreateConfigMap(configMap) - -} - -// updateCustomDnsConfigMap updates the existing custom DNS ConfigMap with a new IP for the specified FQDN. -func updateCustomDnsConfigMap(t *testing.T, data *TestData, ipv4 string, testFullyQualifiedDomainName string, customDnsConfigMap *corev1.ConfigMap) error { - customDnsConfigMap.Data = createDnsConfig(t, ipv4, testFullyQualifiedDomainName) - return data.UpdateConfigMap(customDnsConfigMap) -} - -// setCustomDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. -func setCustomDnsServerAddressInAntrea(t *testing.T, data *TestData, teardown bool) string { - var dnsServiceIP string + customDnsConfigMap, err := data.CreateConfigMap(configMap) + require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) - // Create or reset DNS service based on teardown flag - if !teardown { - var err error - dnsServiceIP, err = createCustomDnsService(data) - require.NoError(t, err, "Error when creating custom DNS service: %v", err) - } + createCustomDnsPod(t, data) - err := updateAntreaConfigMap(data, dnsServiceIP) - require.NoError(t, err, "Error when updating Antrea configmap with custom dns ip : %v", err) + // set the custom DNS server IP address in Antrea configMap. + setDnsServerAddressInAntrea(t, data, dnsServiceIP) + defer setDnsServerAddressInAntrea(t, data, "") //reset after the test. - if teardown { - t.Logf("removing dns server IP from antrea agent during teardown") - return "" - } - t.Logf("Set DNS server IP in Antrea : %s", dnsServiceIP) - return dnsServiceIP -} + createFqdnPolicyInNamespace(t, data, testFQDN) + createToolboxPod(t, data, dnsServiceIP) -// updateAntreaConfigMap updates the Antrea configuration with the specified DNS server IP. -func updateAntreaConfigMap(data *TestData, dnsServiceIP string) error { - agentChanges := func(config *agentconfig.AgentConfig) { - config.DNSServerOverride = dnsServiceIP + curlTarget := func(podName, containerName, fqdn string) (string, error) { + cmd := []string{"curl", fqdn} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + if err != nil { + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), podName, err, stdout, stderr) + } + return stdout, nil } - return data.mutateAntreaConfigMap(nil, agentChanges, false, true) -} -// createCustomDnsService creates the custom DNS service and returns its ClusterIP as ipv4 address. -func createCustomDnsService(data *TestData) (string, error) { - const dnsPort = 53 - customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, - dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, - false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) - if err != nil { - return "", err - } - return customDnsService.Spec.ClusterIP, nil + assert.EventuallyWithT(t, func(collect *assert.CollectT) { + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFQDN) + assert.NoError(t, err) + t.Logf("response of curl to FQDN - %s", stdout) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFQDN) -} + // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. + t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") + resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) + fqdnIp := resolvedIP.String() + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod : %v", err) + require.Equalf(t, agnHostPodOneIP, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIp, agnHostPodOneIP) + t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIp) -// createDnsConfig generates a DNS configuration for the specified IP address and domain name. -func createDnsConfig(t *testing.T, ipv4Address, domainName string) map[string]string { - const coreFileTemplate = `lfx.test:53 { - errors - log - health - hosts { - {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} - no_reverse - pods verified - ttl 10 - } - loop - reload - }` + // update the IP address mapped to the FQDN in the custom DNS ConfigMap. + t.Logf("New IP to update to DNS | ipAdress : %s", agnHostPodTwoIP) + customDnsConfigMap.Data = createDnsConfig(t, map[string]string{"ipAddress": agnHostPodTwoIP, "domainName": testFQDN}) + require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP : %v", err) + t.Logf("successfully updated dns configMap with new IPs | ipAdress : %s", agnHostPodTwoIP) - generateConfigData := func() (string, error) { - data := struct { - IPv4 string - FQDN string - }{ - IPv4: ipv4Address, - FQDN: domainName, - } + // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server pod, this way + // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. + // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically + require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", + randSeq(8)), "failed to update custom dns pod annotation.") - tmpl, err := template.New("configMapData").Parse(coreFileTemplate) - if err != nil { - return "", err - } - var output bytes.Buffer - err = tmpl.Execute(&output, data) + // finally verify that Curling the previously cached IP fails after DNS update. + assert.Eventually(t, func() bool { + t.Logf("Trying to curl the existing cached IP of the domain - %v", fqdnIp) + stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) if err != nil { - return "", err + t.Logf("The test failed because of error : %+v", err) + } else { + t.Logf("response of curl to cached IP - %+v", stdout) } - return strings.TrimSpace(output.String()), nil - } - - configMapData, err := generateConfigData() - require.NoError(t, err, "error processing configData template for DNS, ", err) - configData := map[string]string{ - "Corefile": configMapData, - } + return assert.Error(t, err) + }, 20*time.Second, 1*time.Second) - return configData } -// createDnsPod creates the CoreDNS pod configured to use the custom DNS ConfigMap. -func createDnsPod(t *testing.T, data *TestData) { - const customDnsImage = "coredns/coredns:1.11.3" - - volume := []corev1.Volume{ - { - Name: "config-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "custom-dns-config", - }, - Items: []corev1.KeyToPath{ - { - Key: "Corefile", - Path: "Corefile", - }, - }, - }, - }, - }, +// setDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. +func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = dnsServiceIP } + err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) + require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) - volumeMount := []corev1.VolumeMount{ - { - Name: "config-volume", - MountPath: "/etc/coredns", - }, + if dnsServiceIP == "" { + t.Logf("removing dns server IP from antrea agent as part of teardown") + } else { + t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) } - require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, customDnsImage). - WithLabels(map[string]string{"app": "custom-dns"}). - WithContainerName("coredns"). - WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). - AddVolume(volume).AddVolumeMount(volumeMount). - WithAnnotations(map[string]string{randomPatchAnnotationKey: randSeq(annotationValueLen)}). - Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) } // createFqdnPolicyInNamespace creates an FQDN policy in the specified namespace. @@ -350,8 +190,7 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string) } // createToolBoxPod creates the toolbox pod with custom DNS settings for test purpose. -func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { - toolBoxLabel := map[string]string{"app": "fqdn-cache-test"} +func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { pod.Spec.DNSPolicy = corev1.DNSNone if pod.Spec.DNSConfig == nil { @@ -360,17 +199,22 @@ func createToolBoxPod(t *testing.T, data *TestData, dnsServiceIP string) { pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} } + //TODO: toolboxPodName ? require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). - WithLabels(toolBoxLabel). + WithLabels(map[string]string{"app": "fqdn-cache-test"}). WithContainerName(toolboxContainerName). WithMutateFunc(mutateSpecForAddingCustomDNS). Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) } -// createHttpAgnhostPod creates an agnHost pod that serves HTTP requests and returns the IP of pod created. -func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabels map[string]string) *PodIPs { - const agnHostPort = 80 +// createHttpAgnHostPod creates an agnHost pod that serves HTTP requests and returns the IP of pod created. +func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { + const ( + agnHostPort = 80 + agnHostPodNamePreFix = "agnhost-" + ) + podName := randName(agnHostPodNamePreFix) args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} ports := []corev1.ContainerPort{ { @@ -383,9 +227,92 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). WithArgs(args). WithPorts(ports). - WithLabels(agnLabels). + WithLabels(map[string]string{"app": "agnhost"}). Create(data)) podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) require.NoError(t, err) return podIPs } + +// createDnsPod creates the CoreDNS pod configured to use the custom DNS ConfigMap. +func createCustomDnsPod(t *testing.T, data *TestData) { + volume := []corev1.Volume{ + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "custom-dns-config", + }, + Items: []corev1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + } + + volumeMount := []corev1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + } + + require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3"). + WithLabels(map[string]string{"app": "custom-dns"}). + WithContainerName("coredns"). + WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). + AddVolume(volume).AddVolumeMount(volumeMount). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) +} + +// createDnsConfig generates a DNS configuration for the specified IP address and domain name. +func createDnsConfig(t *testing.T, hosts map[string]string) map[string]string { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} + no_reverse + pods verified + ttl 10 + } + loop + reload + }` + + generateConfigData := func() (string, error) { + data := struct { + IPv4 string + FQDN string + }{ + IPv4: hosts["ipAddress"], + FQDN: hosts["domainName"], + } + + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + if err != nil { + return "", err + } + var output bytes.Buffer + err = tmpl.Execute(&output, data) + if err != nil { + return "", err + } + return strings.TrimSpace(output.String()), nil + } + + configMapData, err := generateConfigData() + require.NoError(t, err, "error processing configData template for DNS, ", err) + configData := map[string]string{ + "Corefile": configMapData, + } + + return configData +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a5171cd9e1a..029eb4e2d19 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -140,8 +140,6 @@ const ( defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" - randomPatchAnnotationKey = "test.antrea.io/random-value" - annotationValueLen = 8 iperfPort = 5201 iperfSvcPort = 9999 ) From 7cd15c7a9fb3a14ad4595e94032bf5e1400bcffa Mon Sep 17 00:00:00 2001 From: Hemant Date: Sat, 26 Oct 2024 01:25:52 +0530 Subject: [PATCH 15/17] Improved e2e test for FQDN: - Added skipIfNotRequired(t, "mode-irrelevant") and used EventuallyWithT in the final step. - Allowed passing TTL as a parameter for DNS config. - Accomodated other changes as per review. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 90 ++++++++++++++++----------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 0892aa9bd3b..4f57c00e46d 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -38,11 +38,13 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { const ( testFQDN = "fqdn-test-pod.lfx.test" dnsPort = 53 + dnsTTL = 5 ) skipIfAntreaPolicyDisabled(t) skipIfNotIPv4Cluster(t) skipIfIPv6Cluster(t) + skipIfNotRequired(t, "mode-irrelevant") data, err := setupTest(t) if err != nil { @@ -65,59 +67,58 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) - require.NoError(t, err, "Error creating customDNS service: %+v", err) + require.NoError(t, err, "Error creating customDNS service") dnsServiceIP := customDnsService.Spec.ClusterIP // create a ConfigMap for the custom DNS server, mapping IP of agnHost pod 1 to the FQDN. - dnsConfigData := createDnsConfig(t, map[string]string{"ipAddress": agnHostPodOneIP, "domainName": testFQDN}) configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-dns-config", Namespace: data.testNamespace, }, - Data: dnsConfigData, + Data: createDnsConfig(t, map[string]string{agnHostPodOneIP: testFQDN}, dnsTTL), } customDnsConfigMap, err := data.CreateConfigMap(configMap) - require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) + require.NoError(t, err, "failed to create custom dns ConfigMap") - createCustomDnsPod(t, data) + createCustomDnsPod(t, data, configMap.Name) // set the custom DNS server IP address in Antrea configMap. setDnsServerAddressInAntrea(t, data, dnsServiceIP) defer setDnsServerAddressInAntrea(t, data, "") //reset after the test. - createFqdnPolicyInNamespace(t, data, testFQDN) + createFqdnPolicyInNamespace(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") createToolboxPod(t, data, dnsServiceIP) - - curlTarget := func(podName, containerName, fqdn string) (string, error) { - cmd := []string{"curl", fqdn} - stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, podName, containerName, cmd) + curlTarget := testFQDN + curlFQDN := func() (string, error) { + cmd := []string{"curl", curlTarget} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd) if err != nil { return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", - strings.Join(cmd, " "), podName, err, stdout, stderr) + strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr) } return stdout, nil } assert.EventuallyWithT(t, func(collect *assert.CollectT) { - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, testFQDN) + stdout, err := curlFQDN() assert.NoError(t, err) - t.Logf("response of curl to FQDN - %s", stdout) - }, 2*time.Second, 100*time.Millisecond, "trying to curl the fqdn : ", testFQDN) + t.Logf("Response of curl to FQDN - %s", stdout) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the FQDN : ", testFQDN) // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) fqdnIp := resolvedIP.String() - require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod : %v", err) + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod") require.Equalf(t, agnHostPodOneIP, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIp, agnHostPodOneIP) t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIp) // update the IP address mapped to the FQDN in the custom DNS ConfigMap. - t.Logf("New IP to update to DNS | ipAdress : %s", agnHostPodTwoIP) - customDnsConfigMap.Data = createDnsConfig(t, map[string]string{"ipAddress": agnHostPodTwoIP, "domainName": testFQDN}) - require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP : %v", err) - t.Logf("successfully updated dns configMap with new IPs | ipAdress : %s", agnHostPodTwoIP) + t.Logf("Updating host mapping in DNS server config to use new IP : %s", agnHostPodTwoIP) + customDnsConfigMap.Data = createDnsConfig(t, map[string]string{agnHostPodTwoIP: testFQDN}, dnsTTL) + require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP") + t.Logf("Successfully updated dns configMap with new IP : %s", agnHostPodTwoIP) // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server pod, this way // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. @@ -125,18 +126,19 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", randSeq(8)), "failed to update custom dns pod annotation.") + curlTarget = fqdnIp // finally verify that Curling the previously cached IP fails after DNS update. - assert.Eventually(t, func() bool { - t.Logf("Trying to curl the existing cached IP of the domain - %v", fqdnIp) - stdout, err := curlTarget(toolboxPodName, toolboxContainerName, fqdnIp) + assert.EventuallyWithT(t, func(collectT *assert.CollectT) { + t.Logf("Trying to curl the existing cached IP of the domain - %s", fqdnIp) + stdout, err := curlFQDN() if err != nil { t.Logf("The test failed because of error : %+v", err) } else { - t.Logf("response of curl to cached IP - %+v", stdout) + t.Logf("Response of curl to cached IP - %+v", stdout) } - return assert.Error(t, err) - }, 20*time.Second, 1*time.Second) + assert.Error(collectT, err) + }, 20*time.Second, 1*time.Second) } // setDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. @@ -145,24 +147,18 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri config.DNSServerOverride = dnsServiceIP } err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) - require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) + require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap") if dnsServiceIP == "" { - t.Logf("removing dns server IP from antrea agent as part of teardown") + t.Logf("Removing dns server IP from antrea agent as part of teardown") } else { - t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + t.Logf("DNS server value set to %+v in antrea \n", dnsServiceIP) } } // createFqdnPolicyInNamespace creates an FQDN policy in the specified namespace. -func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string) { - const ( - fqdnPolicyName = "test-anp-fqdn" - customDnsLabelValue = "custom-dns" - fqdnPodSelectorLabelValue = "fqdn-cache-test" - ) - +func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDnsLabelValue, fqdnPodSelectorLabelValue string) { podSelectorLabel := map[string]string{ "app": fqdnPodSelectorLabelValue, } @@ -185,7 +181,7 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string) nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") annp, err := data.CreateOrUpdateANNP(builder.Get()) - require.NoError(t, err, "error while deploying antrea policy %+v", err) + require.NoError(t, err, "error while deploying antrea policy") require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) } @@ -199,7 +195,6 @@ func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} } - //TODO: toolboxPodName ? require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). WithLabels(map[string]string{"app": "fqdn-cache-test"}). WithContainerName(toolboxContainerName). @@ -235,14 +230,14 @@ func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { } // createDnsPod creates the CoreDNS pod configured to use the custom DNS ConfigMap. -func createCustomDnsPod(t *testing.T, data *TestData) { +func createCustomDnsPod(t *testing.T, data *TestData, configName string) { volume := []corev1.Volume{ { Name: "config-volume", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: "custom-dns-config", + Name: configName, }, Items: []corev1.KeyToPath{ { @@ -272,28 +267,29 @@ func createCustomDnsPod(t *testing.T, data *TestData) { } // createDnsConfig generates a DNS configuration for the specified IP address and domain name. -func createDnsConfig(t *testing.T, hosts map[string]string) map[string]string { +func createDnsConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { const coreFileTemplate = `lfx.test:53 { errors log health hosts { - {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} + {{ range $ip, $fqdn := .Hosts }}{{ $ip }} {{ $fqdn }}{{ end }} no_reverse pods verified - ttl 10 + ttl {{ .TTL }} } loop reload }` generateConfigData := func() (string, error) { + data := struct { - IPv4 string - FQDN string + Hosts map[string]string + TTL int }{ - IPv4: hosts["ipAddress"], - FQDN: hosts["domainName"], + Hosts: hosts, + TTL: ttl, } tmpl, err := template.New("configMapData").Parse(coreFileTemplate) @@ -309,7 +305,7 @@ func createDnsConfig(t *testing.T, hosts map[string]string) map[string]string { } configMapData, err := generateConfigData() - require.NoError(t, err, "error processing configData template for DNS, ", err) + require.NoError(t, err, "error processing configData template for DNS") configData := map[string]string{ "Corefile": configMapData, } From b7c92c022949cfa02d7014cf1b43d5a24771c730 Mon Sep 17 00:00:00 2001 From: Hemant Date: Sat, 26 Oct 2024 03:28:27 +0530 Subject: [PATCH 16/17] corrected initialisms for FQDN, IP Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 4f57c00e46d..2989a897b11 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -36,7 +36,7 @@ import ( func TestFQDNPolicyWithCachedDNS(t *testing.T) { const ( - testFQDN = "fqdn-test-pod.lfx.test" + testFQDN = "FQDN-test-pod.lfx.test" dnsPort = 53 dnsTTL = 5 ) @@ -54,14 +54,14 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { // create two agnHost pods and get their IPv4 addresses. The IP of these pods will be mapped against the FQDN. podCount := 2 - agnHostPodIps := make([]*PodIPs, podCount) + agnHostPodIPs := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { - agnHostPodIps[i] = createHttpAgnHostPod(t, data) + agnHostPodIPs[i] = createHttpAgnHostPod(t, data) } // get IPv4 addresses of the agnHost pods created. - agnHostPodOneIP, _ := agnHostPodIps[0].AsStrings() - agnHostPodTwoIP, _ := agnHostPodIps[1].AsStrings() + agnHostPodOneIP, _ := agnHostPodIPs[0].AsStrings() + agnHostPodTwoIP, _ := agnHostPodIPs[1].AsStrings() // create customDNS service and get its ClusterIP. customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, @@ -109,10 +109,10 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) - fqdnIp := resolvedIP.String() + fqdnIP := resolvedIP.String() require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod") - require.Equalf(t, agnHostPodOneIP, fqdnIp, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIp, agnHostPodOneIP) - t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIp) + require.Equalf(t, agnHostPodOneIP, fqdnIP, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIP, agnHostPodOneIP) + t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIP) // update the IP address mapped to the FQDN in the custom DNS ConfigMap. t.Logf("Updating host mapping in DNS server config to use new IP : %s", agnHostPodTwoIP) @@ -126,10 +126,10 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", randSeq(8)), "failed to update custom dns pod annotation.") - curlTarget = fqdnIp + curlTarget = fqdnIP // finally verify that Curling the previously cached IP fails after DNS update. assert.EventuallyWithT(t, func(collectT *assert.CollectT) { - t.Logf("Trying to curl the existing cached IP of the domain - %s", fqdnIp) + t.Logf("Trying to curl the existing cached IP of the domain - %s", fqdnIP) stdout, err := curlFQDN() if err != nil { t.Logf("The test failed because of error : %+v", err) @@ -273,7 +273,7 @@ func createDnsConfig(t *testing.T, hosts map[string]string, ttl int) map[string] log health hosts { - {{ range $ip, $fqdn := .Hosts }}{{ $ip }} {{ $fqdn }}{{ end }} + {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }} no_reverse pods verified ttl {{ .TTL }} From 9e76e44c7f322235218d3b75f365dec94f5aa7c9 Mon Sep 17 00:00:00 2001 From: Hemant Date: Sat, 26 Oct 2024 04:31:11 +0530 Subject: [PATCH 17/17] Set reload value for dns config and added an explanatory comment besides other changes Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 50 ++++++++++++++++----------------- test/e2e/framework.go | 3 +- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 2989a897b11..817f7bed4cc 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -36,7 +36,7 @@ import ( func TestFQDNPolicyWithCachedDNS(t *testing.T) { const ( - testFQDN = "FQDN-test-pod.lfx.test" + testFQDN = "fqdn-test-pod.lfx.test" dnsPort = 53 dnsTTL = 5 ) @@ -52,7 +52,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } defer teardownTest(t, data) - // create two agnHost pods and get their IPv4 addresses. The IP of these pods will be mapped against the FQDN. + // create two agnHost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. podCount := 2 agnHostPodIPs := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { @@ -67,10 +67,10 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) - require.NoError(t, err, "Error creating customDNS service") + require.NoError(t, err, "Error creating custom DNS Service") dnsServiceIP := customDnsService.Spec.ClusterIP - // create a ConfigMap for the custom DNS server, mapping IP of agnHost pod 1 to the FQDN. + // create a ConfigMap for the custom DNS server, mapping IP of agnHost Pod 1 to the FQDN. configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-dns-config", @@ -79,7 +79,7 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { Data: createDnsConfig(t, map[string]string{agnHostPodOneIP: testFQDN}, dnsTTL), } customDnsConfigMap, err := data.CreateConfigMap(configMap) - require.NoError(t, err, "failed to create custom dns ConfigMap") + require.NoError(t, err, "failed to create custom DNS ConfigMap") createCustomDnsPod(t, data, configMap.Name) @@ -103,42 +103,42 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { assert.EventuallyWithT(t, func(collect *assert.CollectT) { stdout, err := curlFQDN() assert.NoError(t, err) - t.Logf("Response of curl to FQDN - %s", stdout) - }, 2*time.Second, 100*time.Millisecond, "trying to curl the FQDN : ", testFQDN) + t.Logf("Response of curl to FQDN: %s", stdout) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the FQDN: ", testFQDN) // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. - t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox pod...") + t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox Pod...") resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) fqdnIP := resolvedIP.String() - require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox pod") + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") require.Equalf(t, agnHostPodOneIP, fqdnIP, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIP, agnHostPodOneIP) t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIP) // update the IP address mapped to the FQDN in the custom DNS ConfigMap. - t.Logf("Updating host mapping in DNS server config to use new IP : %s", agnHostPodTwoIP) + t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnHostPodTwoIP) customDnsConfigMap.Data = createDnsConfig(t, map[string]string{agnHostPodTwoIP: testFQDN}, dnsTTL) require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP") - t.Logf("Successfully updated dns configMap with new IP : %s", agnHostPodTwoIP) + t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnHostPodTwoIP) - // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server pod, this way + // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", - randSeq(8)), "failed to update custom dns pod annotation.") + randSeq(8)), "failed to update custom DNS Pod annotation.") curlTarget = fqdnIP // finally verify that Curling the previously cached IP fails after DNS update. + // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. assert.EventuallyWithT(t, func(collectT *assert.CollectT) { - t.Logf("Trying to curl the existing cached IP of the domain - %s", fqdnIP) + t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) stdout, err := curlFQDN() if err != nil { - t.Logf("The test failed because of error : %+v", err) + t.Logf("Curling the cached IP failed") } else { - t.Logf("Response of curl to cached IP - %+v", stdout) + t.Logf("Response of curl to cached IP: %+v", stdout) } assert.Error(collectT, err) - - }, 20*time.Second, 1*time.Second) + }, 10*time.Second, 1*time.Second) } // setDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. @@ -147,12 +147,12 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri config.DNSServerOverride = dnsServiceIP } err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) - require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap") + require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") if dnsServiceIP == "" { - t.Logf("Removing dns server IP from antrea agent as part of teardown") + t.Logf("Removing DNS server IP from antrea agent as part of teardown") } else { - t.Logf("DNS server value set to %+v in antrea \n", dnsServiceIP) + t.Logf("DNS server value set to %s in antrea \n", dnsServiceIP) } } @@ -185,7 +185,7 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) } -// createToolBoxPod creates the toolbox pod with custom DNS settings for test purpose. +// createToolBoxPod creates the toolbox Pod with custom DNS settings for test purpose. func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { pod.Spec.DNSPolicy = corev1.DNSNone @@ -203,7 +203,7 @@ func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) } -// createHttpAgnHostPod creates an agnHost pod that serves HTTP requests and returns the IP of pod created. +// createHttpAgnHostPod creates an agnHost Pod that serves HTTP requests and returns the IP of Pod created. func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { const ( agnHostPort = 80 @@ -229,7 +229,7 @@ func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { return podIPs } -// createDnsPod creates the CoreDNS pod configured to use the custom DNS ConfigMap. +// createDnsPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. func createCustomDnsPod(t *testing.T, data *TestData, configName string) { volume := []corev1.Volume{ { @@ -279,7 +279,7 @@ func createDnsConfig(t *testing.T, hosts map[string]string, ttl int) map[string] ttl {{ .TTL }} } loop - reload + reload 2s }` generateConfigData := func() (string, error) { diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 029eb4e2d19..99cec75fb95 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -3274,8 +3274,7 @@ func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, return err } - _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) - if err != nil { + if _, err := data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}); err != nil { return err }