From be730193ba423b7ba68f586d05318b6bc04bd237 Mon Sep 17 00:00:00 2001 From: Zhao Congqi <52371592+zcq98@users.noreply.github.com> Date: Tue, 5 Dec 2023 14:09:18 +0800 Subject: [PATCH] e2e aap for security group (#3459) Signed-off-by: zcq98 Co-authored-by: zcq98 --- test/e2e/framework/security-group.go | 138 +++++++++++++++++++++++++++ test/e2e/framework/vip.go | 2 +- test/e2e/vip/e2e_test.go | 91 ++++++++++++++++-- 3 files changed, 224 insertions(+), 7 deletions(-) create mode 100644 test/e2e/framework/security-group.go diff --git a/test/e2e/framework/security-group.go b/test/e2e/framework/security-group.go new file mode 100644 index 00000000000..790f18a4ef7 --- /dev/null +++ b/test/e2e/framework/security-group.go @@ -0,0 +1,138 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// SecurityGroupClient is a struct for security-group client. +type SecurityGroupClient struct { + f *Framework + v1.SecurityGroupInterface +} + +func (f *Framework) SecurityGroupClient() *SecurityGroupClient { + return &SecurityGroupClient{ + f: f, + SecurityGroupInterface: f.KubeOVNClientSet.KubeovnV1().SecurityGroups(), + } +} + +func (c *SecurityGroupClient) Get(name string) *apiv1.SecurityGroup { + sg, err := c.SecurityGroupInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return sg.DeepCopy() +} + +// Create creates a new security group according to the framework specifications +func (c *SecurityGroupClient) Create(sg *apiv1.SecurityGroup) *apiv1.SecurityGroup { + sg, err := c.SecurityGroupInterface.Create(context.TODO(), sg, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating security group") + return sg.DeepCopy() +} + +// CreateSync creates a new security group according to the framework specifications, and waits for it to be ready. +func (c *SecurityGroupClient) CreateSync(sg *apiv1.SecurityGroup) *apiv1.SecurityGroup { + sg = c.Create(sg) + ExpectTrue(c.WaitToBeReady(sg.Name, timeout)) + // Get the newest ovn security group after it becomes ready + return c.Get(sg.Name).DeepCopy() +} + +// WaitToBeReady returns whether the security group is ready within timeout. +func (c *SecurityGroupClient) WaitToBeReady(name string, timeout time.Duration) bool { + Logf("Waiting up to %v for security group %s to be ready", timeout, name) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) { + if c.Get(name).Status.PortGroup != "" { + Logf("security group %s is ready ", name) + return true + } + Logf("security group %s is not ready ", name) + } + Logf("security group %s was not ready within %v", name, timeout) + return false +} + +// Patch patches the security group +func (c *SecurityGroupClient) Patch(original, modified *apiv1.SecurityGroup, timeout time.Duration) *apiv1.SecurityGroup { + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedSg *apiv1.SecurityGroup + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + p, err := c.SecurityGroupInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch security group %q", original.Name) + } + patchedSg = p + return true, nil + }) + if err == nil { + return patchedSg.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch security group %s", original.Name) + } + Failf("error occurred while retrying to patch security group %s: %v", original.Name, err) + + return nil +} + +// Delete deletes a security group if the security group exists +func (c *SecurityGroupClient) Delete(name string) { + err := c.SecurityGroupInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete security group %q: %v", name, err) + } +} + +// DeleteSync deletes the security group and waits for the security group to disappear for `timeout`. +// If the security group doesn't disappear before the timeout, it will fail the test. +func (c *SecurityGroupClient) DeleteSync(name string) { + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for security group %q to disappear", name) +} + +// WaitToDisappear waits the given timeout duration for the specified Security Group to disappear. +func (c *SecurityGroupClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.SecurityGroup, error) { + sg, err := c.SecurityGroupInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return sg, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected security group %s to not be found: %w", name, err) + } + return nil +} + +func MakeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup { + sg := &apiv1.SecurityGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: apiv1.SecurityGroupSpec{ + AllowSameGroupTraffic: allowSameGroupTraffic, + IngressRules: ingressRules, + EgressRules: egressRules, + }, + } + return sg +} diff --git a/test/e2e/framework/vip.go b/test/e2e/framework/vip.go index d036097ab43..4cc812faf42 100644 --- a/test/e2e/framework/vip.go +++ b/test/e2e/framework/vip.go @@ -105,7 +105,7 @@ func (c *VipClient) Delete(name string) { // If the ovn vip doesn't disappear before the timeout, it will fail the test. func (c *VipClient) DeleteSync(name string) { c.Delete(name) - gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for ovn eip %q to disappear", name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for ovn vip %q to disappear", name) } // WaitToDisappear waits the given timeout duration for the specified OVN VIP to disappear. diff --git a/test/e2e/vip/e2e_test.go b/test/e2e/vip/e2e_test.go index 20b6ef16860..c9b8663f321 100644 --- a/test/e2e/vip/e2e_test.go +++ b/test/e2e/vip/e2e_test.go @@ -17,15 +17,19 @@ import ( "github.com/onsi/ginkgo/v2" - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" "github.com/kubeovn/kube-ovn/test/e2e/framework" ) -func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *kubeovnv1.Vip { +func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *apiv1.Vip { return framework.MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType) } +func MakeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup { + return framework.MakeSecurityGroup(name, allowSameGroupTraffic, ingressRules, egressRules) +} + var _ = framework.Describe("[group:vip]", func() { f := framework.NewDefaultFramework("vip") @@ -34,9 +38,10 @@ var _ = framework.Describe("[group:vip]", func() { var vpcClient *framework.VpcClient var subnetClient *framework.SubnetClient var vipClient *framework.VipClient - var vpc *kubeovnv1.Vpc - var subnet *kubeovnv1.Subnet + var vpc *apiv1.Vpc + var subnet *apiv1.Subnet var podClient *framework.PodClient + var securityGroupClient *framework.SecurityGroupClient var image, namespaceName, vpcName, subnetName, cidr string // test switch lb vip, which ip is in the vpc subnet cidr @@ -44,7 +49,10 @@ var _ = framework.Describe("[group:vip]", func() { var switchLbVip1Name, switchLbVip2Name string // test allowed address pair vip - var vip1Name, vip2Name, aapPodName1, aapPodName2 string + var vip1Name, vip2Name, aapPodName1, aapPodName2, aapPodName3 string + + // test allowed address pair connectivity in the security group scenario + var securityGroupName string ginkgo.BeforeEach(func() { cs = f.ClientSet @@ -52,6 +60,7 @@ var _ = framework.Describe("[group:vip]", func() { subnetClient = f.SubnetClient() vipClient = f.VipClient() podClient = f.PodClient() + securityGroupClient = f.SecurityGroupClient() namespaceName = f.Namespace.Name cidr = framework.RandomCIDR(f.ClusterIPFamily) @@ -66,6 +75,9 @@ var _ = framework.Describe("[group:vip]", func() { aapPodName1 = "pod1-" + randomSuffix aapPodName2 = "pod2-" + randomSuffix + aapPodName3 = "pod3-" + randomSuffix + + securityGroupName = "sg-" + randomSuffix vpcName = "vpc-" + randomSuffix subnetName = "subnet-" + randomSuffix @@ -96,10 +108,15 @@ var _ = framework.Describe("[group:vip]", func() { podClient.DeleteSync(aapPodName1) ginkgo.By("Deleting pod " + aapPodName2) podClient.DeleteSync(aapPodName2) + ginkgo.By("Deleting pod " + aapPodName3) + podClient.DeleteSync(aapPodName3) ginkgo.By("Deleting subnet " + subnetName) subnetClient.DeleteSync(subnetName) ginkgo.By("Deleting vpc " + vpcName) vpcClient.DeleteSync(vpcName) + // clean security group + ginkgo.By("Deleting security group " + securityGroupName) + securityGroupClient.DeleteSync(securityGroupName) }) framework.ConformanceIt("Test vip", func() { @@ -110,7 +127,7 @@ var _ = framework.Describe("[group:vip]", func() { cmd := []string{"sh", "-c", "sleep infinity"} ginkgo.By("Creating pod1 support allowed address pair using " + vip1Name) aapPod1 := framework.MakeNetAdminPod(namespaceName, aapPodName1, nil, annotations, image, cmd, nil) - _ = podClient.CreateSync(aapPod1) + aapPod1 = podClient.CreateSync(aapPod1) ginkgo.By("Creating pod2 support allowed address pair using " + vip1Name) aapPod2 := framework.MakeNetAdminPod(namespaceName, aapPodName2, nil, annotations, image, cmd, nil) _ = podClient.CreateSync(aapPod2) @@ -168,6 +185,68 @@ var _ = framework.Describe("[group:vip]", func() { stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command) framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout) framework.ExpectNoError(err) + // aapPod1 can not ping aapPod2 vip when ip is deleted + stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, delIP) + framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout) + stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command) + framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout) + framework.ExpectError(err) + ginkgo.By("Creating security group " + securityGroupName) + gatewayV4 := aapPod1.Annotations[util.GatewayAnnotation] + allowAddress := aapPod1.Annotations[util.IPAddressAnnotation] + rules := make([]*apiv1.SgRule, 0, 2) + // gateway should be added for pinger + rules = append(rules, &apiv1.SgRule{ + IPVersion: "ipv4", + Protocol: apiv1.ProtocolALL, + Priority: 1, + RemoteType: apiv1.SgRemoteTypeAddress, + RemoteAddress: gatewayV4, + Policy: apiv1.PolicyAllow, + }) + // aapPod1 should be allowed by aapPod3 for security group aap test + rules = append(rules, &apiv1.SgRule{ + IPVersion: "ipv4", + Protocol: apiv1.ProtocolALL, + Priority: 1, + RemoteType: apiv1.SgRemoteTypeAddress, + RemoteAddress: allowAddress, + Policy: apiv1.PolicyAllow, + }) + sg := MakeSecurityGroup(securityGroupName, true, rules, rules) + _ = securityGroupClient.CreateSync(sg) + ginkgo.By("Creating pod3 support allowed address pair with security group") + annotations[util.PortSecurityAnnotation] = "true" + annotations[fmt.Sprintf(util.SecurityGroupAnnotationTemplate, "ovn")] = securityGroupName + aapPod3 := framework.MakeNetAdminPod(namespaceName, aapPodName3, nil, annotations, image, cmd, nil) + aapPod3 = podClient.CreateSync(aapPod3) + // check if security group working + sgCheck := fmt.Sprintf("ping -W 1 -c 1 %s", aapPod3.Annotations[util.IPAddressAnnotation]) + // aapPod1 can ping aapPod3 with security group + stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, sgCheck) + framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout) + framework.ExpectNoError(err) + // aapPod3 can not ping aapPod3 with security group + stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, sgCheck) + framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout) + framework.ExpectError(err) + ginkgo.By("Checking ovn address_set and lsp port_security") + // address_set should have aap IP + conditions = fmt.Sprintf("name=ovn.sg.%s.associated.v4", strings.ReplaceAll(securityGroupName, "-", ".")) + execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=addresses find address_set " + conditions + output, err = exec.Command("bash", "-c", execCmd).CombinedOutput() + addressSet := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ") + framework.ExpectNoError(err) + framework.ExpectContainElement(addressSet, vip1.Status.V4ip) + // port_security should have aap IP + conditions = fmt.Sprintf("name=%s.%s", aapPodName3, namespaceName) + execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=port_security find logical-switch-port " + conditions + output, err = exec.Command("bash", "-c", execCmd).CombinedOutput() + portSecurity := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ") + framework.ExpectNoError(err) + framework.ExpectContainElement(portSecurity, vip1.Status.V4ip) + // TODO: Checking allow address pair connectivity with security group + // AAP works fine with security group in kind but not working in e2e } else { framework.ExpectNotEqual(vip1.Status.V6ip, vip2.Status.V6ip) }