Skip to content

Commit

Permalink
e2e aap for security group (#3459)
Browse files Browse the repository at this point in the history
Signed-off-by: zcq98 <[email protected]>
Co-authored-by: zcq98 <[email protected]>
  • Loading branch information
2 people authored and bobz965 committed Dec 5, 2023
1 parent 637b781 commit 0a95b1e
Show file tree
Hide file tree
Showing 3 changed files with 224 additions and 7 deletions.
138 changes: 138 additions & 0 deletions test/e2e/framework/security-group.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package framework

import (
"context"
"errors"
"fmt"
"time"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"

"github.com/onsi/gomega"

apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
)

// SecurityGroupClient is a struct for security-group client.
type SecurityGroupClient struct {
f *Framework
v1.SecurityGroupInterface
}

func (f *Framework) SecurityGroupClient() *SecurityGroupClient {
return &SecurityGroupClient{
f: f,
SecurityGroupInterface: f.KubeOVNClientSet.KubeovnV1().SecurityGroups(),
}
}

func (c *SecurityGroupClient) Get(name string) *apiv1.SecurityGroup {
sg, err := c.SecurityGroupInterface.Get(context.TODO(), name, metav1.GetOptions{})
ExpectNoError(err)
return sg.DeepCopy()
}

// Create creates a new security group according to the framework specifications
func (c *SecurityGroupClient) Create(sg *apiv1.SecurityGroup) *apiv1.SecurityGroup {
sg, err := c.SecurityGroupInterface.Create(context.TODO(), sg, metav1.CreateOptions{})
ExpectNoError(err, "Error creating security group")
return sg.DeepCopy()
}

// CreateSync creates a new security group according to the framework specifications, and waits for it to be ready.
func (c *SecurityGroupClient) CreateSync(sg *apiv1.SecurityGroup) *apiv1.SecurityGroup {
sg = c.Create(sg)
ExpectTrue(c.WaitToBeReady(sg.Name, timeout))
// Get the newest ovn security group after it becomes ready
return c.Get(sg.Name).DeepCopy()
}

// WaitToBeReady returns whether the security group is ready within timeout.
func (c *SecurityGroupClient) WaitToBeReady(name string, timeout time.Duration) bool {
Logf("Waiting up to %v for security group %s to be ready", timeout, name)
for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) {
if c.Get(name).Status.PortGroup != "" {
Logf("security group %s is ready ", name)
return true
}
Logf("security group %s is not ready ", name)
}
Logf("security group %s was not ready within %v", name, timeout)
return false
}

// Patch patches the security group
func (c *SecurityGroupClient) Patch(original, modified *apiv1.SecurityGroup, timeout time.Duration) *apiv1.SecurityGroup {
patch, err := util.GenerateMergePatchPayload(original, modified)
ExpectNoError(err)

var patchedSg *apiv1.SecurityGroup
err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
p, err := c.SecurityGroupInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "")
if err != nil {
return handleWaitingAPIError(err, false, "patch security group %q", original.Name)
}
patchedSg = p
return true, nil
})
if err == nil {
return patchedSg.DeepCopy()
}

if errors.Is(err, context.DeadlineExceeded) {
Failf("timed out while retrying to patch security group %s", original.Name)
}
Failf("error occurred while retrying to patch security group %s: %v", original.Name, err)

return nil
}

// Delete deletes a security group if the security group exists
func (c *SecurityGroupClient) Delete(name string) {
err := c.SecurityGroupInterface.Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
Failf("Failed to delete security group %q: %v", name, err)
}
}

// DeleteSync deletes the security group and waits for the security group to disappear for `timeout`.
// If the security group doesn't disappear before the timeout, it will fail the test.
func (c *SecurityGroupClient) DeleteSync(name string) {
c.Delete(name)
gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for security group %q to disappear", name)
}

// WaitToDisappear waits the given timeout duration for the specified Security Group to disappear.
func (c *SecurityGroupClient) WaitToDisappear(name string, _, timeout time.Duration) error {
err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.SecurityGroup, error) {
sg, err := c.SecurityGroupInterface.Get(ctx, name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil, nil
}
return sg, err
})).WithTimeout(timeout).Should(gomega.BeNil())
if err != nil {
return fmt.Errorf("expected security group %s to not be found: %w", name, err)
}
return nil
}

func MakeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup {
sg := &apiv1.SecurityGroup{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: apiv1.SecurityGroupSpec{
AllowSameGroupTraffic: allowSameGroupTraffic,
IngressRules: ingressRules,
EgressRules: egressRules,
},
}
return sg
}
2 changes: 1 addition & 1 deletion test/e2e/framework/vip.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func (c *VipClient) Delete(name string) {
// If the ovn vip doesn't disappear before the timeout, it will fail the test.
func (c *VipClient) DeleteSync(name string) {
c.Delete(name)
gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for ovn eip %q to disappear", name)
gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for ovn vip %q to disappear", name)
}

// WaitToDisappear waits the given timeout duration for the specified OVN VIP to disappear.
Expand Down
91 changes: 85 additions & 6 deletions test/e2e/vip/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,19 @@ import (

"github.com/onsi/ginkgo/v2"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/test/e2e/framework"
)

func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *kubeovnv1.Vip {
func makeOvnVip(namespaceName, name, subnet, v4ip, v6ip, vipType string) *apiv1.Vip {
return framework.MakeVip(namespaceName, name, subnet, v4ip, v6ip, vipType)
}

func MakeSecurityGroup(name string, allowSameGroupTraffic bool, ingressRules, egressRules []*apiv1.SgRule) *apiv1.SecurityGroup {
return framework.MakeSecurityGroup(name, allowSameGroupTraffic, ingressRules, egressRules)
}

var _ = framework.Describe("[group:vip]", func() {
f := framework.NewDefaultFramework("vip")

Expand All @@ -34,24 +38,29 @@ var _ = framework.Describe("[group:vip]", func() {
var vpcClient *framework.VpcClient
var subnetClient *framework.SubnetClient
var vipClient *framework.VipClient
var vpc *kubeovnv1.Vpc
var subnet *kubeovnv1.Subnet
var vpc *apiv1.Vpc
var subnet *apiv1.Subnet
var podClient *framework.PodClient
var securityGroupClient *framework.SecurityGroupClient
var image, namespaceName, vpcName, subnetName, cidr string

// test switch lb vip, which ip is in the vpc subnet cidr
// switch lb vip use gw mac to trigger lb nat flows
var switchLbVip1Name, switchLbVip2Name string

// test allowed address pair vip
var vip1Name, vip2Name, aapPodName1, aapPodName2 string
var vip1Name, vip2Name, aapPodName1, aapPodName2, aapPodName3 string

// test allowed address pair connectivity in the security group scenario
var securityGroupName string

ginkgo.BeforeEach(func() {
cs = f.ClientSet
vpcClient = f.VpcClient()
subnetClient = f.SubnetClient()
vipClient = f.VipClient()
podClient = f.PodClient()
securityGroupClient = f.SecurityGroupClient()
namespaceName = f.Namespace.Name
cidr = framework.RandomCIDR(f.ClusterIPFamily)

Expand All @@ -66,6 +75,9 @@ var _ = framework.Describe("[group:vip]", func() {

aapPodName1 = "pod1-" + randomSuffix
aapPodName2 = "pod2-" + randomSuffix
aapPodName3 = "pod3-" + randomSuffix

securityGroupName = "sg-" + randomSuffix

vpcName = "vpc-" + randomSuffix
subnetName = "subnet-" + randomSuffix
Expand Down Expand Up @@ -96,10 +108,15 @@ var _ = framework.Describe("[group:vip]", func() {
podClient.DeleteSync(aapPodName1)
ginkgo.By("Deleting pod " + aapPodName2)
podClient.DeleteSync(aapPodName2)
ginkgo.By("Deleting pod " + aapPodName3)
podClient.DeleteSync(aapPodName3)
ginkgo.By("Deleting subnet " + subnetName)
subnetClient.DeleteSync(subnetName)
ginkgo.By("Deleting vpc " + vpcName)
vpcClient.DeleteSync(vpcName)
// clean security group
ginkgo.By("Deleting security group " + securityGroupName)
securityGroupClient.DeleteSync(securityGroupName)
})

framework.ConformanceIt("Test vip", func() {
Expand All @@ -108,7 +125,7 @@ var _ = framework.Describe("[group:vip]", func() {
cmd := []string{"sh", "-c", "sleep infinity"}
ginkgo.By("Creating pod1 support allowed address pair using " + vip1Name)
aapPod1 := framework.MakeNetAdminPod(namespaceName, aapPodName1, nil, annotations, image, cmd, nil)
_ = podClient.CreateSync(aapPod1)
aapPod1 = podClient.CreateSync(aapPod1)
ginkgo.By("Creating pod2 support allowed address pair using " + vip1Name)
aapPod2 := framework.MakeNetAdminPod(namespaceName, aapPodName2, nil, annotations, image, cmd, nil)
_ = podClient.CreateSync(aapPod2)
Expand Down Expand Up @@ -166,6 +183,68 @@ var _ = framework.Describe("[group:vip]", func() {
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectNoError(err)
// aapPod1 can not ping aapPod2 vip when ip is deleted
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, delIP)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", delIP, err, stderr, stdout)
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, command)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", command, err, stderr, stdout)
framework.ExpectError(err)
ginkgo.By("Creating security group " + securityGroupName)
gatewayV4 := aapPod1.Annotations[util.GatewayAnnotation]
allowAddress := aapPod1.Annotations[util.IPAddressAnnotation]
rules := make([]*apiv1.SgRule, 0, 2)
// gateway should be added for pinger
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv4",
Protocol: apiv1.ProtocolALL,
Priority: 1,
RemoteType: apiv1.SgRemoteTypeAddress,
RemoteAddress: gatewayV4,
Policy: apiv1.PolicyAllow,
})
// aapPod1 should be allowed by aapPod3 for security group aap test
rules = append(rules, &apiv1.SgRule{
IPVersion: "ipv4",
Protocol: apiv1.ProtocolALL,
Priority: 1,
RemoteType: apiv1.SgRemoteTypeAddress,
RemoteAddress: allowAddress,
Policy: apiv1.PolicyAllow,
})
sg := MakeSecurityGroup(securityGroupName, true, rules, rules)
_ = securityGroupClient.CreateSync(sg)
ginkgo.By("Creating pod3 support allowed address pair with security group")
annotations[util.PortSecurityAnnotation] = "true"
annotations[fmt.Sprintf(util.SecurityGroupAnnotationTemplate, "ovn")] = securityGroupName
aapPod3 := framework.MakeNetAdminPod(namespaceName, aapPodName3, nil, annotations, image, cmd, nil)
aapPod3 = podClient.CreateSync(aapPod3)
// check if security group working
sgCheck := fmt.Sprintf("ping -W 1 -c 1 %s", aapPod3.Annotations[util.IPAddressAnnotation])
// aapPod1 can ping aapPod3 with security group
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName1, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectNoError(err)
// aapPod3 can not ping aapPod3 with security group
stdout, stderr, err = framework.ExecShellInPod(context.Background(), f, namespaceName, aapPodName2, sgCheck)
framework.Logf("exec %s failed err: %v, stderr: %s, stdout: %s", sgCheck, err, stderr, stdout)
framework.ExpectError(err)
ginkgo.By("Checking ovn address_set and lsp port_security")
// address_set should have aap IP
conditions = fmt.Sprintf("name=ovn.sg.%s.associated.v4", strings.ReplaceAll(securityGroupName, "-", "."))
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=addresses find address_set " + conditions
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
addressSet := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(addressSet, vip1.Status.V4ip)
// port_security should have aap IP
conditions = fmt.Sprintf("name=%s.%s", aapPodName3, namespaceName)
execCmd = "kubectl ko nbctl --format=list --data=bare --no-heading --columns=port_security find logical-switch-port " + conditions
output, err = exec.Command("bash", "-c", execCmd).CombinedOutput()
portSecurity := strings.Split(strings.ReplaceAll(string(output), "\n", ""), " ")
framework.ExpectNoError(err)
framework.ExpectContainElement(portSecurity, vip1.Status.V4ip)
// TODO: Checking allow address pair connectivity with security group
// AAP works fine with security group in kind but not working in e2e
} else {
framework.ExpectNotEqual(vip1.Status.V6ip, vip2.Status.V6ip)
}
Expand Down

0 comments on commit 0a95b1e

Please sign in to comment.