diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index b6039919ab0..e8fad3d736c 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -15,10 +15,12 @@ package e2e import ( + "bytes" "fmt" "strconv" "strings" "testing" + "text/template" "time" "github.com/stretchr/testify/assert" @@ -92,30 +94,17 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP) - var agnHostPodIps []*PodIPs podCount := 2 + agnHostPodIps := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { - podIp := createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) - agnHostPodIps = append(agnHostPodIps, podIp) - } - //domainMapping holds whether the IP is mapped to Domain or not. - domainMapping := make(map[string]bool) - - // pick an IP to be added in config - var ipForDnsConfig string - for idx, podIp := range agnHostPodIps { - ipStr := podIp.IPv4.String() - domainMapping[ipStr] = false - //pick last IP for config - if idx == len(agnHostPodIps)-1 { - ipForDnsConfig = ipStr - } + agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue}) } - dnsConfigData := createDnsConfig(ipForDnsConfig) + ipv4, ipv6 := extractIPs(agnHostPodIps[0]) + + dnsConfigData := createDnsConfig(t, ipv4, ipv6) customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false) require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err) - domainMapping[ipForDnsConfig] = true createDnsPod(t, data) @@ -149,30 +138,35 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { t.Logf("received ip using dig for test fqdn %+v ", fqdnIp) - var newIP string - for ip, mapped := range domainMapping { - if ip != fqdnIp && mapped == false { - newIP = ip - } + ipv4, ipv6 = extractIPs(agnHostPodIps[1]) + if ipv6 != "" { + t.Logf("New IPs to update to DNS | ipv4 : %v, ipv6 : %+v", ipv4, ipv6) + } else { + t.Logf("New IPs to update to DNS | ipv4 : %v", ipv4) } - t.Logf("New IP to update to DNS %v", newIP) - // Update the custom DNS configMap - UpdatedCustomDNSconfig := createDnsConfig(newIP) + UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6) customDnsConfigMapObject.Data = UpdatedCustomDNSconfig err = data.UpdateConfigMap(customDnsConfigMapObject) require.NoError(t, err, "failed to update configmap with new IP : %v", err) - t.Logf("successfully updated dns configMap with new IP : %+v", newIP) + if ipv6 != "" { + t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v, ipv6 :%+v", ipv4, ipv6) + } else { + t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v", ipv4) + } - require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to update custom dns pod annotation.") + require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to update custom dns pod annotation.") defer setDnsServerAddressInAntrea(t, data, "") assert.Eventually(t, func() bool { t.Logf("trying to curl the existing cached IP of the domain %v", fqdnIp) err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp) + if err != nil { + t.Logf("The test failed because of error %+v", err) + } return assert.Error(t, err) }, eventualWaitTime, 1*time.Second) @@ -185,31 +179,59 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err) - t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + if dnsServiceIP == "" { + t.Logf("removing dns server IP from antrea agent as part of teardown") + } else { + t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP) + } } -func createDnsConfig(ipForConfig string) map[string]string { - config := fmt.Sprintf(`lfx.test:53 { - errors - health - hosts { - %s %s - no_reverse - pods verified - ttl 10 - } - loop - reload -}`, ipForConfig, testFullyQualifiedDomainName) - +func createDnsConfig(t *testing.T, ipv4Address, ipv6Address string) map[string]string { + configMapData, _ := generateCoreFile(t, ipv4Address, ipv6Address) configData := map[string]string{ - "Corefile": config, + "Corefile": configMapData, } return configData } +func generateCoreFile(t *testing.T, ipv4Address, ipv6Address string) (string, error) { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }} + {{ if .IPv6 }}{{ .IPv6 }} {{ $.FQDN }}{{ end }} + no_reverse + pods verified + ttl 10 + } + loop + reload + }` + + data := struct { + IPv4 string + IPv6 string + FQDN string + }{ + IPv4: ipv4Address, + IPv6: ipv6Address, + FQDN: testFullyQualifiedDomainName, + } + + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + require.NoError(t, err, "error while creating template ", err) + + var output bytes.Buffer + err = tmpl.Execute(&output, data) + require.NoError(t, err, "error while executing config map template", err) + + return strings.TrimSpace(output.String()), nil +} + func createDnsPod(t *testing.T, data *TestData) { volume := []corev1.Volume{ { @@ -247,7 +269,7 @@ func createDnsPod(t *testing.T, data *TestData) { require.NoError(t, pb.Create(data)) require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace)) - require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to annotate the custom dns pod.") + require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to annotate the custom dns pod.") } @@ -310,7 +332,19 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel } require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data)) - podIP, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) require.NoError(t, err) - return podIP + return podIPs +} + +func extractIPs(podIPs *PodIPs) (string, string) { + var ipv4, ipv6 string + res := podIPs.AsSlice() + if len(res) > 0 { + ipv4 = res[0].String() + if len(res) == 2 { + ipv6 = res[1].String() + } + } + return ipv4, ipv6 } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index bee8ae3def1..7093ae9eea5 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -141,9 +141,9 @@ const ( statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" randomPatchAnnotationKey = "test.antrea.io/random-value" - - iperfPort = 5201 - iperfSvcPort = 9999 + annotationValueLen = 8 + iperfPort = 5201 + iperfSvcPort = 9999 ) type ClusterNode struct { @@ -3230,17 +3230,14 @@ func (data *TestData) runDNSQuery( useTCP bool, dnsServiceIP string) (net.IP, error) { - digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) + digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) if useTCP { - digCmd += " +tcp" - } - cmd := []string{ - "/bin/sh", - "-c", - digCmd, + digCmdStr += " +tcp" } - fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(cmd, " ")) - stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, cmd) + + digCmd := strings.Fields(digCmdStr) + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(digCmd, " ")) + stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd) if err != nil { return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) } @@ -3253,12 +3250,8 @@ func (data *TestData) runDNSQuery( } } -// patchPodAnnotation Patches an annotation for a given pod in a given namespace. If no annotation is -// provided as a parameter then it generates a random string value for a predefined key and adds it as annotation. -// The patchType used is MergePatchType since we intend to make updates to annotations in pods instead of updating fields in complex k8s -// resources like deployments. +// patchPodAnnotation Patches a pod with given map of keys and values. func (data *TestData) patchPodAnnotation(namespace, podName string, annotation map[string]string) error { - numberOfRunes := 8 annotationPatch := map[string]interface{}{ "metadata": map[string]interface{}{ "annotations": make(map[string]string), @@ -3266,14 +3259,10 @@ func (data *TestData) patchPodAnnotation(namespace, podName string, annotation m } if annotation == nil { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[randomPatchAnnotationKey] = randSeq(numberOfRunes) + return fmt.Errorf("no annotations were provided") } else { for key, value := range annotation { - if key == randomPatchAnnotationKey { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = randSeq(numberOfRunes) - } else { - annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value - } + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value } } @@ -3292,3 +3281,30 @@ func (data *TestData) patchPodAnnotation(namespace, podName string, annotation m log.Infof("Successfully patched pod %s in namespace %s with provided annotation", podName, namespace) return nil } + +// setPodAnnotationToRandomValue Patches a pod by adding an annotation with a specified key and a randomly generated string as the value. +func (data *TestData) setPodAnnotationToRandomValue(namespace, podName, annotationKey string) error { + annotationPatch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": make(map[string]string), + }, + } + + annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = randSeq(annotationValueLen) + + patchData, err := json.Marshal(annotationPatch) + if err != nil { + log.Infof("Error marshalling annotation: %+v", err) + return err + } + + _, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}) + if err != nil { + log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err) + return err + } + + log.Infof("Successfully patched pod %s in namespace %s with a random value annotation", podName, namespace) + return nil + +} diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index 93e3f7c2106..366365783ed 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -103,7 +103,6 @@ type TestNamespaceMeta struct { // GetPodByLabel returns a Pod with the matching Namespace and "pod" label if it's found. // If the pod is not found, GetPodByLabel returns "ErrPodNotFound". func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) { - //TODO: Query why this is hard coded as "pod" pods, err := k.getPodsUncached(ns, "pod", name) if err != nil { return nil, fmt.Errorf("unable to get Pod in Namespace %s with label pod=%s: %w", ns, name, err) @@ -668,20 +667,24 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } -func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string]byte, immutable bool) (*v1.ConfigMap, error) { +func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string][]byte, immutable bool) (*v1.ConfigMap, error) { configMap := &v1.ConfigMap{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, - Immutable: nil, + Immutable: &immutable, Data: configData, - BinaryData: nil, + BinaryData: binaryData, } - configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMap, metav1.CreateOptions{}) - return configMapObject, err + configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + log.Infof("Unable to create configMap : %s", err) + return nil, err + } + return configMapObject, nil } // DeleteService is a convenience function for deleting a Service by Namespace and name.