From 6774c62d80480ef68ec745908d47ffb7cd8bb12f Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Mon, 6 Jan 2025 10:28:17 +0800 Subject: [PATCH 1/2] add common changes --- pkg/nsx/services/common/policy_tree.go | 527 ++++++++++++++++++++ pkg/nsx/services/common/policy_tree_test.go | 15 + pkg/nsx/services/common/store.go | 54 ++ pkg/nsx/services/common/types.go | 91 ++-- pkg/nsx/services/common/wrap.go | 230 ++++++++- 5 files changed, 860 insertions(+), 57 deletions(-) create mode 100644 pkg/nsx/services/common/policy_tree.go create mode 100644 pkg/nsx/services/common/policy_tree_test.go diff --git a/pkg/nsx/services/common/policy_tree.go b/pkg/nsx/services/common/policy_tree.go new file mode 100644 index 000000000..625221a8a --- /dev/null +++ b/pkg/nsx/services/common/policy_tree.go @@ -0,0 +1,527 @@ +package common + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +type LeafDataWrapper[T any] func(leafData T) (*data.StructValue, error) +type GetPath[T any] func(obj T) *string +type GetId[T any] func(obj T) *string + +func getNSXResourcePath[T any](obj T) *string { + switch v := any(obj).(type) { + case *model.VpcIpAddressAllocation: + return v.Path + case *model.VpcSubnet: + return v.Path + case *model.VpcSubnetPort: + return v.Path + case *model.SubnetConnectionBindingMap: + return v.Path + case *model.Vpc: + return v.Path + case *model.StaticRoutes: + return v.Path + case *model.SecurityPolicy: + return v.Path + case *model.Group: + return v.Path + case *model.Rule: + return v.Path + case *model.Share: + return v.Path + case *model.LBService: + return v.Path + case *model.LBVirtualServer: + return v.Path + case *model.LBPool: + return v.Path + default: + log.Error(nil, "Unknown NSX resource type %v", v) + return nil + } +} + +func getNSXResourceId[T any](obj T) *string { + switch v := any(obj).(type) { + case *model.VpcIpAddressAllocation: + return v.Id + case *model.VpcSubnet: + return v.Id + case *model.VpcSubnetPort: + return v.Id + case *model.SubnetConnectionBindingMap: + return v.Id + case *model.Vpc: + return v.Id + case *model.StaticRoutes: + return v.Id + case *model.SecurityPolicy: + return v.Id + case *model.Group: + return v.Id + case *model.Rule: + return v.Id + case *model.Share: + return v.Id + case *model.LBService: + return v.Id + case *model.LBVirtualServer: + return v.Path + case *model.LBPool: + return v.Id + default: + log.Error(nil, "Unknown NSX resource type %v", v) + return nil + } +} + +func leafWrapper[T any](obj T) (*data.StructValue, error) { + switch v := any(obj).(type) { + case *model.VpcIpAddressAllocation: + return WrapVpcIpAddressAllocation(v) + case *model.VpcSubnet: + return WrapVpcSubnet(v) + case *model.VpcSubnetPort: + return WrapVpcSubnetPort(v) + case *model.SubnetConnectionBindingMap: + return WrapSubnetConnectionBindingMap(v) + case *model.Vpc: + return WrapVPC(v) + case *model.StaticRoutes: + return WrapStaticRoutes(v) + case *model.SecurityPolicy: + return WrapSecurityPolicy(v) + case *model.Group: + return WrapGroup(v) + case *model.Rule: + return WrapRule(v) + case *model.Share: + return WrapShare(v) + case *model.LBService: + return WrapLBService(v) + case *model.LBVirtualServer: + return WrapLBVirtualServer(v) + case *model.LBPool: + return WrapLBPool(v) + default: + log.Error(nil, "Unknown NSX resource type %v", v) + return nil, fmt.Errorf("unsupported NSX resource type %v", v) + } +} + +type PolicyResourceType struct { + ModelKey string + PathKey string +} + +type PolicyResourcePath[T any] []PolicyResourceType + +func (p *PolicyResourcePath[T]) Length() int { + resourceTypes := ([]PolicyResourceType)(*p) + return len(resourceTypes) +} + +func (p *PolicyResourcePath[T]) String() string { + resourceTypes := ([]PolicyResourceType)(*p) + resources := make([]string, len(resourceTypes)) + for i := 0; i < len(resourceTypes); i++ { + resources[i] = resourceTypes[i].ModelKey + } + return strings.Join(resources, "-") +} + +func (p *PolicyResourcePath[T]) getResources() []PolicyResourceType { + return (*p) +} + +func (p *PolicyResourcePath[T]) getChildrenResources() []PolicyResourceType { + resources := p.getResources() + if resources[0] == PolicyResourceInfra { + return resources[1:] + } + return resources +} + +func (p *PolicyResourcePath[T]) getKVPathFormat() []string { + resourceTypes := p.getChildrenResources() + format := make([]string, len(resourceTypes)) + for i := 0; i < len(resourceTypes); i++ { + format[i] = resourceTypes[i].PathKey + } + return format +} + +func (p *PolicyResourcePath[T]) getChildrenModelFormat() []string { + resourceTypes := p.getChildrenResources() + format := make([]string, len(resourceTypes)) + for i := 0; i < len(resourceTypes); i++ { + format[i] = resourceTypes[i].ModelKey + } + return format +} + +func (p *PolicyResourcePath[T]) getRootType() string { + resourceTypes := p.getResources() + if resourceTypes[0] == PolicyResourceOrg { + return ResourceTypeOrgRoot + } + return ResourceTypeInfra +} + +var ( + PolicyResourceInfra = PolicyResourceType{ModelKey: ResourceTypeInfra, PathKey: "infra"} + PolicyResourceOrg = PolicyResourceType{ModelKey: ResourceTypeOrg, PathKey: "orgs"} + PolicyResourceProject = PolicyResourceType{ModelKey: ResourceTypeProject, PathKey: "projects"} + PolicyResourceVpc = PolicyResourceType{ModelKey: ResourceTypeVpc, PathKey: "vpcs"} + PolicyResourceStaticRoutes = PolicyResourceType{ModelKey: ResourceTypeStaticRoutes, PathKey: "static-routes"} + PolicyResourceVpcSubnet = PolicyResourceType{ModelKey: ResourceTypeSubnet, PathKey: "subnets"} + PolicyResourceVpcSubnetPort = PolicyResourceType{ModelKey: ResourceTypeSubnetPort, PathKey: "ports"} + PolicyResourceVpcSubnetConnectionBindingMap = PolicyResourceType{ModelKey: ResourceTypeSubnetConnectionBindingMap, PathKey: "subnet-connection-binding-maps"} + PolicyResourceVpcLBService = PolicyResourceType{ModelKey: ResourceTypeLBService, PathKey: "vpc-lbs"} + PolicyResourceVpcLBPool = PolicyResourceType{ModelKey: ResourceTypeLBPool, PathKey: "vpc-lb-pools"} + PolicyResourceVpcLBVirtualServer = PolicyResourceType{ModelKey: ResourceTypeLBVirtualServer, PathKey: "vpc-lb-virtual-servers"} + PolicyResourceInfraLBService = PolicyResourceType{ModelKey: ResourceTypeLBService, PathKey: "lbs"} + PolicyResourceInfraLBPool = PolicyResourceType{ModelKey: ResourceTypeLBPool, PathKey: "lb-pools"} + PolicyResourceInfraLBVirtualServer = PolicyResourceType{ModelKey: ResourceTypeLBVirtualServer, PathKey: "lb-virtual-servers"} + PolicyResourceVpcIPAddressAllocation = PolicyResourceType{ModelKey: ResourceTypeIPAddressAllocation, PathKey: "ip-address-allocations"} + PolicyResourceDomain = PolicyResourceType{ModelKey: ResourceTypeDomain, PathKey: "domains"} + PolicyResourceShare = PolicyResourceType{ModelKey: ResourceTypeShare, PathKey: "shares"} + PolicyResourceSharedResource = PolicyResourceType{ModelKey: ResourceTypeSharedResource, PathKey: "resources"} + PolicyResourceGroup = PolicyResourceType{ModelKey: ResourceTypeGroup, PathKey: "groups"} + PolicyResourceRule = PolicyResourceType{ModelKey: ResourceTypeRule, PathKey: "rules"} + PolicyResourceSecurityPolicy = PolicyResourceType{ModelKey: ResourceTypeSecurityPolicy, PathKey: "security-policies"} + PolicyResourceTlsCertificate = PolicyResourceType{ModelKey: ResourceTypeTlsCertificate, PathKey: "certificates"} + + PolicyPathVpcSubnet PolicyResourcePath[*model.VpcSubnet] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet} + PolicyPathVpcSubnetConnectionBindingMap PolicyResourcePath[*model.SubnetConnectionBindingMap] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet, PolicyResourceVpcSubnetConnectionBindingMap} + PolicyPathVpcSubnetPort PolicyResourcePath[*model.VpcSubnetPort] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet, PolicyResourceVpcSubnetPort} + PolicyPathVpc PolicyResourcePath[*model.Vpc] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc} + PolicyPathVpcLBPool PolicyResourcePath[*model.LBPool] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBPool} + PolicyPathVpcLBService PolicyResourcePath[*model.LBService] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBService} + PolicyPathVpcLBVirtualServer PolicyResourcePath[*model.LBVirtualServer] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBVirtualServer} + PolicyPathVpcIPAddressAllocation PolicyResourcePath[*model.VpcIpAddressAllocation] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcIPAddressAllocation} + PolicyPathVpcSecurityPolicy PolicyResourcePath[*model.SecurityPolicy] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceSecurityPolicy} + PolicyPathVpcStaticRoutes PolicyResourcePath[*model.StaticRoutes] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceStaticRoutes} + PolicyPathVpcSecurityPolicyRule PolicyResourcePath[*model.Rule] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceSecurityPolicy, PolicyResourceRule} + PolicyPathVpcGroup PolicyResourcePath[*model.Group] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceGroup} + PolicyPathProjectGroup PolicyResourcePath[*model.Group] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceInfra, PolicyResourceDomain, PolicyResourceGroup} + PolicyPathProjectShare PolicyResourcePath[*model.Share] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceInfra, PolicyResourceShare} + PolicyPathInfraGroup PolicyResourcePath[*model.Group] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceDomain, PolicyResourceGroup} + PolicyPathInfraShare PolicyResourcePath[*model.Share] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceShare} + PolicyPathInfraSharedResource PolicyResourcePath[*model.SharedResource] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceShare, PolicyResourceShare, PolicyResourceSharedResource} + PolicyPathInfraCert PolicyResourcePath[*model.TlsCertificate] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceTlsCertificate} + PolicyPathInfraVirtualServer PolicyResourcePath[*model.LBVirtualServer] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBVirtualServer} + PolicyPathInfraLBPool PolicyResourcePath[*model.LBPool] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBPool} + PolicyPathInfraLBService PolicyResourcePath[*model.LBService] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBService} +) + +type hNodeKey struct { + resType string + resID string +} + +func (k *hNodeKey) Equals(other *hNodeKey) bool { + return k.resType == other.resType && k.resID == other.resID +} + +func (k *hNodeKey) String() string { + return fmt.Sprintf("/%s", strings.Join([]string{k.resType, k.resID}, "/")) +} + +type hNode[T any] struct { + key *hNodeKey + leafData *data.StructValue + childNodes map[hNodeKey]*hNode[T] +} + +func (n *hNode[T]) mergeChildNode(node *hNode[T], leafType string) { + if n.childNodes == nil { + n.childNodes = make(map[hNodeKey]*hNode[T]) + } + + if node.key.resType == leafType { + n.childNodes[*node.key] = node + return + } + + cn, found := n.childNodes[*node.key] + if found { + for _, chN := range node.childNodes { + cn.mergeChildNode(chN, leafType) + } + return + } + n.childNodes[*node.key] = node +} + +func (n *hNode[T]) buildTree(rootType, leafType string) ([]*data.StructValue, error) { + if n.key.resType == leafType { + return []*data.StructValue{n.leafData}, nil + } + + children := make([]*data.StructValue, 0) + for _, cn := range n.childNodes { + cnDataValues, err := cn.buildTree(rootType, leafType) + if err != nil { + return nil, err + } + children = append(children, cnDataValues...) + } + + if n.key.resType == rootType { + return children, nil + } + + return wrapChildResourceReference(n.key.resType, n.key.resID, children) +} + +type PolicyTreeBuilder[T any] struct { + leafType string + rootType string + + leafWrapper LeafDataWrapper[T] + pathGetter GetPath[T] + idGetter GetId[T] + + pathFormat []string + modelFormat []string + hasInnerInfra bool +} + +func (b *PolicyTreeBuilder[T]) BuildRootNode(resources []T, parentPath string) *hNode[T] { + rootNode := &hNode[T]{ + key: &hNodeKey{ + resType: b.rootType, + }, + } + leafPathKey := b.pathFormat[len(b.pathFormat)-1] + for _, res := range resources { + var path string + if parentPath != "" { + idValue := b.idGetter(res) + path = fmt.Sprintf("%s/%s/%s", parentPath, leafPathKey, *idValue) + } else { + pathValue := b.pathGetter(res) + if pathValue == nil { + continue + } + path = *pathValue + } + orgNode, err := b.buildHNodeFromResource(path, res) + if err != nil { + log.Error(err, "Failed to build data value for resource, ignore", "Path", path) + continue + } + rootNode.mergeChildNode(orgNode, b.leafType) + } + return rootNode +} + +func (b *PolicyTreeBuilder[T]) buildTree(resources []T, parentPath string) ([]*data.StructValue, error) { + rootNode := b.BuildRootNode(resources, parentPath) + children, err := rootNode.buildTree(b.rootType, b.leafType) + if err != nil { + log.Error(err, "Failed to build data values for multiple resources") + return nil, err + } + return children, nil +} + +func (b *PolicyTreeBuilder[T]) BuildOrgRoot(resources []T, parentPath string) (*model.OrgRoot, error) { + children, err := b.buildTree(resources, parentPath) + if err != nil { + return nil, err + } + + return &model.OrgRoot{ + Children: children, + ResourceType: String(ResourceTypeOrgRoot), + }, nil +} + +func (b *PolicyTreeBuilder[T]) BuildInfra(resources []T, parentPath string) (*model.Infra, error) { + children, err := b.buildTree(resources, parentPath) + if err != nil { + return nil, err + } + + return wrapInfra(children), nil +} + +func (b *PolicyTreeBuilder[T]) buildHNodeFromResource(path string, res T) (*hNode[T], error) { + pathSegments, err := b.parsePathSegments(path) + if err != nil { + return nil, err + } + + dataValue, err := b.leafWrapper(res) + if err != nil { + return nil, err + } + + idx := len(pathSegments) - 1 + nodeCount := len(b.pathFormat) + nodes := make([]*hNode[T], nodeCount) + leafIdx := nodeCount - 1 + nodes[leafIdx] = &hNode[T]{ + key: &hNodeKey{ + resID: pathSegments[idx], + resType: b.modelFormat[leafIdx], + }, + leafData: dataValue, + } + idx -= 2 + + for i := leafIdx - 1; i >= 0; i-- { + child := nodes[i+1] + resType := b.modelFormat[i] + var resID string + if resType != ResourceTypeInfra { + resID = pathSegments[idx] + idx -= 2 + } else { + resID = "" + idx -= 1 + } + node := &hNode[T]{ + key: &hNodeKey{ + resID: resID, + resType: resType, + }, + childNodes: map[hNodeKey]*hNode[T]{ + *child.key: child, + }, + } + nodes[i] = node + } + n := nodes[0] + return n, nil +} + +func (b *PolicyTreeBuilder[T]) parsePathSegments(inputPath string) ([]string, error) { + pathSegments := strings.Split(strings.Trim(inputPath, "/"), "/") + // Remove "infra" in the path since the infra resource's path does not follow the format "/key/value", so we remove + // the first "infra" in the pathSegments. + if b.rootType == ResourceTypeInfra { + pathSegments = pathSegments[1:] + } + + segmentsCount := len(pathSegments) + leafPathKey := b.pathFormat[len(b.pathFormat)-1] + if segmentsCount <= 2 || pathSegments[segmentsCount-2] != leafPathKey { + return nil, fmt.Errorf("invalid input path %s for resource %s", inputPath, b.leafType) + } + if b.hasInnerInfra { + segmentsCount += 1 + } + if segmentsCount != len(b.pathFormat)*2 { + return nil, fmt.Errorf("invalid input path: %s", inputPath) + } + + return pathSegments, nil +} + +func (b *PolicyTreeBuilder[T]) DeleteMultipleResourcesOnNSX(objects []T, nsxClient *nsx.Client) error { + if len(objects) == 0 { + return nil + } + fmt.Println(b.rootType, b.leafType, "count", len(objects)) + enforceRevisionCheckParam := false + if b.rootType == ResourceTypeOrgRoot { + orgRoot, err := b.BuildOrgRoot(objects, "") + if err != nil { + log.Error(err, "Failed to generate OrgRoot with multiple resources", "resourceType", b.leafType) + return err + } + if err = nsxClient.OrgRootClient.Patch(*orgRoot, &enforceRevisionCheckParam); err != nil { + log.Error(err, "Failed to delete multiple resources on NSX with HAPI", "resourceType", b.leafType) + err = util.TransNSXApiError(err) + return err + } + return nil + } + + infraRoot, err := b.BuildInfra(objects, "") + if err != nil { + log.Error(err, "Failed to generate Infra with multiple resources", "resourceType", b.leafType) + return err + } + if err = nsxClient.InfraClient.Patch(*infraRoot, &enforceRevisionCheckParam); err != nil { + log.Error(err, "Failed to delete multiple resources on NSX with HAPI", "resourceType", b.leafType) + err = util.TransNSXApiError(err) + return err + } + + return nil +} + +func PagingNSXResources[T any](resources []T, pageSize int) [][]T { + totalCount := len(resources) + pages := (totalCount + pageSize - 1) / pageSize + pagedResources := make([][]T, 0) + for i := 1; i <= pages; i++ { + start := (i - 1) * pageSize + end := start + pageSize + if end > totalCount { + end = totalCount + } + pagedResources = append(pagedResources, resources[start:end]) + } + return pagedResources +} + +func (p *PolicyResourcePath[T]) NewPolicyTreeBuilder() (*PolicyTreeBuilder[T], error) { + if p.Length() == 0 { + return nil, fmt.Errorf("invalid PolicyResourcePath: %s", p.String()) + } + modelFormat := p.getChildrenModelFormat() + rootType := p.getRootType() + pathFormat := p.getKVPathFormat() + + return &PolicyTreeBuilder[T]{ + pathFormat: pathFormat, + modelFormat: modelFormat, + hasInnerInfra: sets.New[string](pathFormat...).Has(PolicyResourceInfra.PathKey), + rootType: rootType, + leafType: modelFormat[len(modelFormat)-1], + + leafWrapper: leafWrapper[T], + pathGetter: getNSXResourcePath[T], + idGetter: getNSXResourceId[T], + }, nil +} + +func PagingDeleteResources[T any](ctx context.Context, builder *PolicyTreeBuilder[T], objs []T, pageSize int, nsxClient *nsx.Client, delFn func(deletedObjs []T)) error { + if len(objs) == 0 { + return nil + } + var nsxErr error + pagedObjs := PagingNSXResources(objs, pageSize) + for _, partialObjs := range pagedObjs { + select { + case <-ctx.Done(): + return errors.Join(util.TimeoutFailed, ctx.Err()) + default: + delErr := builder.DeleteMultipleResourcesOnNSX(partialObjs, nsxClient) + if delErr == nil { + if delFn != nil { + delFn(partialObjs) + } + continue + } + nsxErr = delErr + } + } + return nsxErr +} diff --git a/pkg/nsx/services/common/policy_tree_test.go b/pkg/nsx/services/common/policy_tree_test.go new file mode 100644 index 000000000..2bbac0dd8 --- /dev/null +++ b/pkg/nsx/services/common/policy_tree_test.go @@ -0,0 +1,15 @@ +package common + +import ( + "github.com/stretchr/testify/assert" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuilder(t *testing.T) { + builder, err := PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() + require.NoError(t, err) + + assert.Equal(t, ResourceTypeOrgRoot, builder.rootType) +} diff --git a/pkg/nsx/services/common/store.go b/pkg/nsx/services/common/store.go index 817c96140..1fd7e8b82 100644 --- a/pkg/nsx/services/common/store.go +++ b/pkg/nsx/services/common/store.go @@ -1,6 +1,7 @@ package common import ( + "errors" "fmt" "net/url" "strconv" @@ -17,6 +18,10 @@ import ( nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) +const ( + IndexByVPCPathFuncKey = "indexedByVPCPath" +) + var pageSize = int64(1000) // Store is the interface for store, it should be implemented by subclass @@ -246,3 +251,52 @@ func formatTagParamTag(paramType, value string) string { valueEscaped := strings.Replace(value, ":", "\\:", -1) return fmt.Sprintf("%s:%s", paramType, valueEscaped) } + +func IndexByVPCFunc(obj interface{}) ([]string, error) { + switch v := obj.(type) { + case *model.Vpc: + return []string{*v.Path}, nil + case *model.VpcSubnet: + return []string{*v.ParentPath}, nil + case *model.VpcSubnetPort: + return getVPCPathFromResourcePath(*v.Path) + case *model.SubnetConnectionBindingMap: + return getVPCPathFromResourcePath(*v.Path) + case *model.VpcIpAddressAllocation: + return []string{*v.ParentPath}, nil + case *model.StaticRoutes: + return []string{*v.ParentPath}, nil + case *model.LBService: + return []string{*v.ParentPath}, nil + case *model.LBVirtualServer: + return []string{*v.ParentPath}, nil + case *model.LBPool: + return []string{*v.ParentPath}, nil + + case *model.SecurityPolicy: + return []string{*v.ParentPath}, nil + case *model.Group: + return []string{*v.ParentPath}, nil + case *model.Rule: + return getVPCPathFromResourcePath(*v.Path) + + /* + Infra resources: + LB related: share/sharedResources/cert/LBAppProfile/LBPersistentProfile/LBMonitorProfile + Security Policy related: Share/Group + Project resources: + Security Policy related: Share/Group + */ + + default: + return []string{}, errors.New("indexFunc doesn't support unknown type") + } +} + +func getVPCPathFromResourcePath(path string) ([]string, error) { + resInfo, err := ParseVPCResourcePath(path) + if err != nil { + return []string{}, err + } + return []string{resInfo.GetVPCPath()}, nil +} diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 6f075260b..2214cad19 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -4,6 +4,7 @@ package common import ( + "fmt" "time" "github.com/openlyinc/pointy" @@ -120,6 +121,8 @@ const ( DstGroupSuffix = "dst" IpSetGroupSuffix = "ipset" ShareSuffix = "share" + + VPCKey = "/orgs/%s/projects/%s/vpcs/%s" ) var ( @@ -129,44 +132,52 @@ var ( ) var ( - ResourceType = "resource_type" - ResourceTypeInfra = "Infra" - ResourceTypeDomain = "Domain" - ResourceTypeSecurityPolicy = "SecurityPolicy" - ResourceTypeNetworkPolicy = "NetworkPolicy" - ResourceTypeGroup = "Group" - ResourceTypeRule = "Rule" - ResourceTypeIPBlock = "IpAddressBlock" - ResourceTypeOrgRoot = "OrgRoot" - ResourceTypeOrg = "Org" - ResourceTypeProject = "Project" - ResourceTypeVpc = "Vpc" - ResourceTypeVpcConnectivityProfile = "VpcConnectivityProfile" - ResourceTypeSubnetPort = "VpcSubnetPort" - ResourceTypeVirtualMachine = "VirtualMachine" - ResourceTypeLBService = "LBService" - ResourceTypeVpcAttachment = "VpcAttachment" - ResourceTypeStaticRoute = "StaticRoutes" - ResourceTypeShare = "Share" - ResourceTypeSharedResource = "SharedResource" - ResourceTypeChildSharedResource = "ChildSharedResource" - ResourceTypeChildShare = "ChildShare" - ResourceTypeChildRule = "ChildRule" - ResourceTypeChildGroup = "ChildGroup" - ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" - ResourceTypeChildVpcAttachment = "ChildVpcAttachment" - ResourceTypeChildResourceReference = "ChildResourceReference" - ResourceTypeTlsCertificate = "TlsCertificate" - ResourceTypeLBHttpProfile = "LBHttpProfile" - ResourceTypeLBFastTcpProfile = "LBFastTcpProfile" - ResourceTypeLBFastUdpProfile = "LBFastUdpProfile" - ResourceTypeLBCookiePersistenceProfile = "LBCookiePersistenceProfile" - ResourceTypeLBSourceIpPersistenceProfile = "LBSourceIpPersistenceProfile" - ResourceTypeLBHttpMonitorProfile = "LBHttpMonitorProfile" - ResourceTypeLBTcpMonitorProfile = "LBTcpMonitorProfile" - ResourceTypeLBVirtualServer = "LBVirtualServer" - ResourceTypeLBPool = "LBPool" - ResourceTypeSubnetConnectionBindingMap = "SubnetConnectionBindingMap" + ResourceType = "resource_type" + ResourceTypeInfra = "Infra" + ResourceTypeDomain = "Domain" + ResourceTypeSecurityPolicy = "SecurityPolicy" + ResourceTypeNetworkPolicy = "NetworkPolicy" + ResourceTypeGroup = "Group" + ResourceTypeRule = "Rule" + ResourceTypeIPBlock = "IpAddressBlock" + ResourceTypeOrgRoot = "OrgRoot" + ResourceTypeOrg = "Org" + ResourceTypeProject = "Project" + ResourceTypeVpc = "Vpc" + ResourceTypeVpcConnectivityProfile = "VpcConnectivityProfile" + ResourceTypeSubnetPort = "VpcSubnetPort" + ResourceTypeVirtualMachine = "VirtualMachine" + ResourceTypeLBService = "LBService" + ResourceTypeVpcAttachment = "VpcAttachment" + ResourceTypeShare = "Share" + ResourceTypeSharedResource = "SharedResource" + ResourceTypeStaticRoutes = "StaticRoutes" + ResourceTypeChildLBPool = "ChildLBPool" + ResourceTypeChildLBService = "ChildLBService" + ResourceTypeChildLBVirtualServer = "ChildLBVirtualServer" + ResourceTypeChildSharedResource = "ChildSharedResource" + ResourceTypeChildShare = "ChildShare" + ResourceTypeChildRule = "ChildRule" + ResourceTypeChildGroup = "ChildGroup" + ResourceTypeChildSecurityPolicy = "ChildSecurityPolicy" + ResourceTypeChildStaticRoutes = "ChildStaticRoutes" + ResourceTypeChildSubnetConnectionBindingMap = "ChildSubnetConnectionBindingMap" + ResourceTypeChildVpcAttachment = "ChildVpcAttachment" + ResourceTypeChildVpcIPAddressAllocation = "ChildVpcIpAddressAllocation" + ResourceTypeChildVpcSubnet = "ChildVpcSubnet" + ResourceTypeChildVpcSubnetPort = "ChildVpcSubnetPort" + ResourceTypeChildResourceReference = "ChildResourceReference" + ResourceTypeTlsCertificate = "TlsCertificate" + ResourceTypeLBHttpProfile = "LBHttpProfile" + ResourceTypeLBFastTcpProfile = "LBFastTcpProfile" + ResourceTypeLBFastUdpProfile = "LBFastUdpProfile" + ResourceTypeLBCookiePersistenceProfile = "LBCookiePersistenceProfile" + ResourceTypeLBSourceIpPersistenceProfile = "LBSourceIpPersistenceProfile" + ResourceTypeLBHttpMonitorProfile = "LBHttpMonitorProfile" + ResourceTypeLBTcpMonitorProfile = "LBTcpMonitorProfile" + ResourceTypeLBVirtualServer = "LBVirtualServer" + ResourceTypeLBPool = "LBPool" + ResourceTypeSubnetConnectionBindingMap = "SubnetConnectionBindingMap" // ResourceTypeClusterControlPlane is used by NSXServiceAccountController ResourceTypeClusterControlPlane = "clustercontrolplane" @@ -215,6 +226,10 @@ type VPCResourceInfo struct { PrivateIpv4Blocks []string } +func (info *VPCResourceInfo) GetVPCPath() string { + return fmt.Sprintf(VPCKey, info.OrgID, info.ProjectID, info.VPCID) +} + type VPCNetworkConfigInfo struct { IsDefault bool Org string diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go index ddc0dfac5..0f3a447a9 100644 --- a/pkg/nsx/services/common/wrap.go +++ b/pkg/nsx/services/common/wrap.go @@ -8,14 +8,7 @@ import ( // WrapInfra TODO(gran) refactor existing code in other package func (service *Service) WrapInfra(children []*data.StructValue) (*model.Infra, error) { - // This is the outermost layer of the hierarchy infra client. - // It doesn't need ID field. - resourceType := ResourceTypeInfra - infraObj := model.Infra{ - Children: children, - ResourceType: &resourceType, - } - return &infraObj, nil + return wrapInfra(children), nil } func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoot, error) { @@ -54,18 +47,11 @@ func wrapChildResourceReference(targetType, id string, children []*data.StructVa } func (service *Service) WrapVPC(vpc *model.Vpc) ([]*data.StructValue, error) { - vpc.ResourceType = pointy.String(ResourceTypeVpc) - childVpc := model.ChildVpc{ - Id: vpc.Id, - MarkedForDelete: vpc.MarkedForDelete, - ResourceType: "ChildVpc", - Vpc: vpc, + dv, err := WrapVPC(vpc) + if err != nil { + return nil, err } - dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) - if len(errs) > 0 { - return nil, errs[0] - } - return []*data.StructValue{dataValue.(*data.StructValue)}, nil + return []*data.StructValue{dv}, nil } func (service *Service) WrapLBS(lbs *model.LBService) ([]*data.StructValue, error) { @@ -97,3 +83,209 @@ func (service *Service) WrapAttachment(attachment *model.VpcAttachment) ([]*data } return []*data.StructValue{dataValue.(*data.StructValue)}, nil } + +func WrapVpcIpAddressAllocation(allocation *model.VpcIpAddressAllocation) (*data.StructValue, error) { + allocation.ResourceType = &ResourceTypeIPAddressAllocation + childAddressAllocation := model.ChildVpcIpAddressAllocation{ + Id: allocation.Id, + MarkedForDelete: allocation.MarkedForDelete, + ResourceType: ResourceTypeChildVpcIPAddressAllocation, + VpcIpAddressAllocation: allocation, + } + dataValue, errors := NewConverter().ConvertToVapi(childAddressAllocation, childAddressAllocation.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapVpcSubnetPort(port *model.VpcSubnetPort) (*data.StructValue, error) { + port.ResourceType = &ResourceTypeSubnetPort + childSubnetPort := model.ChildVpcSubnetPort{ + Id: port.Id, + MarkedForDelete: port.MarkedForDelete, + ResourceType: ResourceTypeChildVpcSubnetPort, + VpcSubnetPort: port, + } + dataValue, errors := NewConverter().ConvertToVapi(childSubnetPort, childSubnetPort.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapSubnetConnectionBindingMap(bindingMap *model.SubnetConnectionBindingMap) (*data.StructValue, error) { + bindingMap.ResourceType = &ResourceTypeSubnetConnectionBindingMap + childBindingMap := model.ChildSubnetConnectionBindingMap{ + Id: bindingMap.Id, + MarkedForDelete: bindingMap.MarkedForDelete, + ResourceType: ResourceTypeChildSubnetConnectionBindingMap, + SubnetConnectionBindingMap: bindingMap, + } + dataValue, errors := NewConverter().ConvertToVapi(childBindingMap, childBindingMap.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapVpcSubnet(subnet *model.VpcSubnet) (*data.StructValue, error) { + subnet.ResourceType = &ResourceTypeSubnet + childSubnet := model.ChildVpcSubnet{ + Id: subnet.Id, + MarkedForDelete: subnet.MarkedForDelete, + ResourceType: ResourceTypeChildVpcSubnet, + VpcSubnet: subnet, + } + dataValue, errors := NewConverter().ConvertToVapi(childSubnet, childSubnet.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapStaticRoutes(route *model.StaticRoutes) (*data.StructValue, error) { + route.ResourceType = &ResourceTypeStaticRoutes + childRoute := model.ChildStaticRoutes{ + Id: route.Id, + MarkedForDelete: route.MarkedForDelete, + ResourceType: ResourceTypeChildStaticRoutes, + StaticRoutes: route, + } + dataValue, errors := NewConverter().ConvertToVapi(childRoute, childRoute.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapSecurityPolicy(sp *model.SecurityPolicy) (*data.StructValue, error) { + sp.ResourceType = &ResourceTypeSecurityPolicy + childPolicy := model.ChildSecurityPolicy{ + Id: sp.Id, + MarkedForDelete: sp.MarkedForDelete, + ResourceType: ResourceTypeChildSecurityPolicy, + SecurityPolicy: sp, + } + dataValue, errors := NewConverter().ConvertToVapi(childPolicy, childPolicy.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapRule(rule *model.Rule) (*data.StructValue, error) { + rule.ResourceType = &ResourceTypeRule + childRule := model.ChildRule{ + Id: rule.Id, + MarkedForDelete: rule.MarkedForDelete, + ResourceType: ResourceTypeChildRule, + Rule: rule, + } + dataValue, errors := NewConverter().ConvertToVapi(childRule, childRule.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapGroup(group *model.Group) (*data.StructValue, error) { + group.ResourceType = &ResourceTypeGroup + childGroup := model.ChildGroup{ + ResourceType: ResourceTypeChildGroup, + Id: group.Id, + MarkedForDelete: group.MarkedForDelete, + Group: group, + } + dataValue, errors := NewConverter().ConvertToVapi(childGroup, childGroup.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapShare(share *model.Share) (*data.StructValue, error) { + share.ResourceType = &ResourceTypeShare + childShare := model.ChildShare{ + ResourceType: ResourceTypeChildShare, + Id: share.Id, + MarkedForDelete: share.MarkedForDelete, + Share: share, + } + dataValue, errors := NewConverter().ConvertToVapi(childShare, childShare.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapLBService(lbService *model.LBService) (*data.StructValue, error) { + lbService.ResourceType = &ResourceTypeLBService + childLBService := model.ChildLBService{ + ResourceType: ResourceTypeChildLBService, + Id: lbService.Id, + MarkedForDelete: lbService.MarkedForDelete, + LbService: lbService, + } + dataValue, errors := NewConverter().ConvertToVapi(childLBService, childLBService.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapLBVirtualServer(lbVS *model.LBVirtualServer) (*data.StructValue, error) { + lbVS.ResourceType = &ResourceTypeLBVirtualServer + childLBVS := model.ChildLBVirtualServer{ + ResourceType: ResourceTypeChildLBVirtualServer, + Id: lbVS.Id, + MarkedForDelete: lbVS.MarkedForDelete, + LbVirtualServer: lbVS, + } + dataValue, errors := NewConverter().ConvertToVapi(childLBVS, childLBVS.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapLBPool(lbPool *model.LBPool) (*data.StructValue, error) { + lbPool.ResourceType = &ResourceTypeLBPool + childLBPool := model.ChildLBPool{ + ResourceType: ResourceTypeChildLBPool, + Id: lbPool.Id, + MarkedForDelete: lbPool.MarkedForDelete, + LbPool: lbPool, + } + dataValue, errors := NewConverter().ConvertToVapi(childLBPool, childLBPool.GetType__()) + if len(errors) > 0 { + return nil, errors[0] + } + return dataValue.(*data.StructValue), nil +} + +func WrapVPC(vpc *model.Vpc) (*data.StructValue, error) { + vpc.ResourceType = pointy.String(ResourceTypeVpc) + childVpc := model.ChildVpc{ + Id: vpc.Id, + MarkedForDelete: vpc.MarkedForDelete, + ResourceType: "ChildVpc", + Vpc: vpc, + } + dataValue, errs := NewConverter().ConvertToVapi(childVpc, childVpc.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return dataValue.(*data.StructValue), nil +} + +func wrapInfra(children []*data.StructValue) *model.Infra { + // This is the outermost layer of the hierarchy infra client. + // It doesn't need ID field. + resourceType := ResourceTypeInfra + infraObj := model.Infra{ + Children: children, + ResourceType: &resourceType, + } + return &infraObj +} From 86367daee49aedb800f773e903b7e75cd9673cd3 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Mon, 6 Jan 2025 10:29:02 +0800 Subject: [PATCH 2/2] Refine the clean up logic --- pkg/clean/clean.go | 83 +- pkg/clean/clean_dlb.go | 90 -- pkg/clean/clean_dlb_infra_test.go | 895 ++++++++++++++++++ pkg/clean/clean_dlb_test.go | 205 ---- pkg/clean/clean_lb_infra.go | 347 +++++++ pkg/clean/clean_test.go | 92 +- pkg/clean/types.go | 271 +++++- pkg/clean/types_test.go | 28 +- pkg/nsx/client.go | 106 +-- pkg/nsx/services/common/policy_tree.go | 30 +- pkg/nsx/services/common/policy_tree_test.go | 564 ++++++++++- pkg/nsx/services/common/store.go | 86 +- pkg/nsx/services/common/types.go | 3 +- pkg/nsx/services/common/wrap.go | 25 +- .../services/ipaddressallocation/cleanup.go | 40 + .../ipaddressallocation.go | 27 +- .../ipaddressallocation_test.go | 110 ++- pkg/nsx/services/ipaddressallocation/store.go | 19 + pkg/nsx/services/securitypolicy/builder.go | 2 +- pkg/nsx/services/securitypolicy/cleanup.go | 174 ++++ .../services/securitypolicy/cleanup_test.go | 206 ++++ pkg/nsx/services/securitypolicy/firewall.go | 98 +- .../services/securitypolicy/firewall_test.go | 199 +--- pkg/nsx/services/securitypolicy/store.go | 24 + pkg/nsx/services/staticroute/cleanup.go | 40 + pkg/nsx/services/staticroute/staticroute.go | 29 +- .../services/staticroute/staticroute_test.go | 31 +- pkg/nsx/services/staticroute/store.go | 19 + pkg/nsx/services/subnet/cleanup.go | 37 + pkg/nsx/services/subnet/store.go | 6 + pkg/nsx/services/subnet/store_test.go | 1 + pkg/nsx/services/subnet/subnet.go | 23 +- pkg/nsx/services/subnet/subnet_test.go | 62 +- .../services/subnetbinding/builder_test.go | 10 - pkg/nsx/services/subnetbinding/cleanup.go | 27 + pkg/nsx/services/subnetbinding/store.go | 28 +- pkg/nsx/services/subnetbinding/store_test.go | 2 + .../services/subnetbinding/subnetbinding.go | 82 +- .../subnetbinding/subnetbinding_test.go | 62 +- pkg/nsx/services/subnetbinding/tree.go | 228 ----- pkg/nsx/services/subnetbinding/tree_test.go | 170 +--- pkg/nsx/services/subnetport/cleanup.go | 28 + pkg/nsx/services/subnetport/store.go | 6 + pkg/nsx/services/subnetport/subnetport.go | 29 +- .../services/subnetport/subnetport_test.go | 23 +- pkg/nsx/services/vpc/cleanup.go | 146 +++ pkg/nsx/services/vpc/cleanup_test.go | 489 ++++++++++ pkg/nsx/services/vpc/store.go | 9 + pkg/nsx/services/vpc/store_test.go | 2 +- pkg/nsx/services/vpc/vpc.go | 420 +------- pkg/nsx/services/vpc/vpc_test.go | 86 +- test/e2e/precreated_vpc_test.go | 7 +- 52 files changed, 3919 insertions(+), 1907 deletions(-) delete mode 100644 pkg/clean/clean_dlb.go create mode 100644 pkg/clean/clean_dlb_infra_test.go delete mode 100644 pkg/clean/clean_dlb_test.go create mode 100644 pkg/clean/clean_lb_infra.go create mode 100644 pkg/nsx/services/ipaddressallocation/cleanup.go create mode 100644 pkg/nsx/services/securitypolicy/cleanup.go create mode 100644 pkg/nsx/services/securitypolicy/cleanup_test.go create mode 100644 pkg/nsx/services/staticroute/cleanup.go create mode 100644 pkg/nsx/services/subnet/cleanup.go create mode 100644 pkg/nsx/services/subnetbinding/cleanup.go delete mode 100644 pkg/nsx/services/subnetbinding/tree.go create mode 100644 pkg/nsx/services/subnetport/cleanup.go create mode 100644 pkg/nsx/services/vpc/cleanup.go create mode 100644 pkg/nsx/services/vpc/cleanup_test.go diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index bca702e7f..dad140bcb 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -6,7 +6,6 @@ package clean import ( "context" "errors" - "fmt" "time" "github.com/vmware-tanzu/nsx-operator/pkg/config" @@ -24,7 +23,6 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" ) var Backoff = wait.Backoff{ @@ -66,18 +64,10 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig, log *logr.Logger, var cleanupService *CleanupService var err error go func() { - cleanupService, err = InitializeCleanupService(cf, nsxClient) + cleanupService, err = InitializeCleanupService(cf, nsxClient, log) errChan <- err }() - retriable := func(err error) bool { - if err != nil && !errors.As(err, &nsxutil.TimeoutFailed) { - log.Info("Retrying to clean up NSX resources", "error", err) - return true - } - return false - } - select { case err := <-errChan: if err != nil { @@ -86,46 +76,27 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig, log *logr.Logger, case <-ctx.Done(): return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) } - if cleanupService.err != nil { - return errors.Join(nsxutil.InitCleanupServiceFailed, cleanupService.err) - } else { - for _, clean := range cleanupService.cleans { - if err := retry.OnError(Backoff, retriable, wrapCleanFunc(ctx, clean)); err != nil { - return errors.Join(nsxutil.CleanupResourceFailed, err) - } - } + + if cleanupService.svcErr != nil { + return errors.Join(nsxutil.InitCleanupServiceFailed, cleanupService.svcErr) } - // delete DLB group -> delete virtual servers -> DLB services -> DLB pools -> persistent profiles for DLB - if err := retry.OnError(retry.DefaultRetry, func(err error) bool { - if err != nil { - log.Info("Retrying to clean up DLB resources", "error", err) - return true - } - return false - }, func() error { - if err := CleanDLB(ctx, nsxClient.Cluster, cf, log); err != nil { - return fmt.Errorf("Failed to clean up specific resource: %w", err) - } - return nil - }); err != nil { - return err + + cleanupService.log = log + + if err := cleanupService.cleanupVPCResources(ctx); err != nil { + return errors.Join(nsxutil.CleanupResourceFailed, err) + } + + if err := cleanupService.cleanupInfraResources(ctx); err != nil { + return errors.Join(nsxutil.CleanupResourceFailed, err) } log.Info("Cleanup NSX resources successfully") return nil } -func wrapCleanFunc(ctx context.Context, clean cleanup) func() error { - return func() error { - if err := clean.Cleanup(ctx); err != nil { - return err - } - return nil - } -} - // InitializeCleanupService initializes all the CR services -func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Client) (*CleanupService, error) { +func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Client, log *logr.Logger) (*CleanupService, error) { cleanupService := NewCleanupService() commonService := common.Service{ @@ -138,42 +109,49 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Clien // Use Fluent Interface to escape error check hell wrapInitializeSubnetService := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return subnet.InitializeSubnetService(service) } } wrapInitializeSecurityPolicy := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { - return securitypolicy.InitializeSecurityPolicy(service, vpcService) + return func() (interface{}, error) { + return securitypolicy.InitializeSecurityPolicy(service, vpcService, true) } } wrapInitializeVPC := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return vpcService, vpcErr } } wrapInitializeStaticRoute := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return sr.InitializeStaticRoute(service, vpcService) } } wrapInitializeSubnetPort := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return subnetport.InitializeSubnetPort(service) } } wrapInitializeIPAddressAllocation := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return ipaddressallocation.InitializeIPAddressAllocation(service, vpcService, true) } } wrapInitializeSubnetBinding := func(service common.Service) cleanupFunc { - return func() (cleanup, error) { + return func() (interface{}, error) { return subnetbinding.InitializeService(service) } } + wrapInitializeLBInfraCleaner := func(service common.Service) cleanupFunc { + return func() (interface{}, error) { + return &LBInfraCleaner{Service: service, log: log}, nil + } + } + + cleanupService.vpcService = vpcService // TODO: initialize other CR services cleanupService = cleanupService. AddCleanupService(wrapInitializeSubnetPort(commonService)). @@ -182,7 +160,8 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Clien AddCleanupService(wrapInitializeSecurityPolicy(commonService)). AddCleanupService(wrapInitializeStaticRoute(commonService)). AddCleanupService(wrapInitializeVPC(commonService)). - AddCleanupService(wrapInitializeIPAddressAllocation(commonService)) + AddCleanupService(wrapInitializeIPAddressAllocation(commonService)). + AddCleanupService(wrapInitializeLBInfraCleaner(commonService)) return cleanupService, nil } diff --git a/pkg/clean/clean_dlb.go b/pkg/clean/clean_dlb.go deleted file mode 100644 index c69639fd3..000000000 --- a/pkg/clean/clean_dlb.go +++ /dev/null @@ -1,90 +0,0 @@ -package clean - -import ( - "context" - "errors" - "fmt" - neturl "net/url" - "strings" - - "github.com/vmware-tanzu/nsx-operator/pkg/config" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx" - nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" - - "github.com/go-logr/logr" -) - -type ( - mapInterface = map[string]interface{} -) - -const TagDLB = "DLB" - -func appendIfNotExist(slice []string, s string) []string { - for _, item := range slice { - if item == s { - return slice - } - } - return append(slice, s) -} - -func httpQueryDLBResources(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { - queryParam := "resource_type:" + resource + - "&tags.scope:ncp\\/cluster" + - "&tags.tag:" + cf.Cluster + - "&tags.scope:ncp\\/created_for" + - "&tags.tag:" + TagDLB - - pairs := strings.Split(queryParam, "&") - var encodedPairs []string - for _, pair := range pairs { - keyValue := strings.Split(pair, ":") - encodedKey := neturl.QueryEscape(keyValue[0]) - encodedValue := neturl.QueryEscape(keyValue[1]) - encodedPairs = append(encodedPairs, fmt.Sprintf("%s:%s", encodedKey, encodedValue)) - } - - encodedQuery := strings.Join(encodedPairs, "%20AND%20") - url := "policy/api/v1/search/query?query=" + encodedQuery - - resp, err := cluster.HttpGet(url) - if err != nil { - return nil, err - } - var resourcePath []string - for _, item := range resp["results"].([]interface{}) { - resourcePath = appendIfNotExist(resourcePath, item.(mapInterface)["path"].(string)) - } - return resourcePath, nil -} - -func CleanDLB(ctx context.Context, cluster *nsx.Cluster, cf *config.NSXOperatorConfig, log *logr.Logger) error { - log.Info("Deleting DLB resources started") - - resources := []string{"Group", "LBVirtualServer", "LBService", "LBPool", "LBCookiePersistenceProfile"} - var allPaths []string - - for _, resource := range resources { - paths, err := httpQueryDLBResources(cluster, cf, resource) - if err != nil { - return err - } - log.Info(resource, "count", len(paths)) - allPaths = append(allPaths, paths...) - } - - log.Info("Deleting DLB resources", "paths", allPaths) - for _, path := range allPaths { - url := "policy/api/v1" + path - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := cluster.HttpDelete(url); err != nil { - return err - } - } - } - return nil -} diff --git a/pkg/clean/clean_dlb_infra_test.go b/pkg/clean/clean_dlb_infra_test.go new file mode 100644 index 000000000..d35b138b7 --- /dev/null +++ b/pkg/clean/clean_dlb_infra_test.go @@ -0,0 +1,895 @@ +package clean + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/logger" + mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/searchclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +var ( + lbVS = &model.LBVirtualServer{ + Id: common.String("infra-vs"), + Path: common.String("/infra/lb-virtual-servers/infra-vs"), + ParentPath: common.String("/infra"), + } + lbPool = &model.LBPool{ + Id: common.String("infra-pool"), + Path: common.String("/infra/lb-pools/infra-pool"), + ParentPath: common.String("/infra"), + } + lbService = &model.LBService{ + Id: common.String("infra-lbs"), + Path: common.String("/infra/lb-services/infra-lbs"), + ParentPath: common.String("/infra"), + } + group = &model.Group{ + Id: common.String("infra-group"), + Path: common.String("/infra/domains/default/groups/infra-group"), + ParentPath: common.String("/infra/domains/default"), + } + share = &model.Share{ + Id: common.String("infra-share"), + Path: common.String("/infra/shares/infra-share"), + ParentPath: common.String("/infra"), + } + cert = &model.TlsCertificate{ + Id: common.String("cert"), + Path: common.String("/infra/certificates/cert"), + ParentPath: common.String("/infra"), + } + appProfile = &model.LBHttpProfile{ + Id: common.String("default-http-lb-app-profile"), + Path: common.String("/infra/lb-app-profiles/default-http-lb-app-profile"), + ParentPath: common.String("/infra"), + ResourceType: common.ResourceTypeLBHttpProfile, + } + monitorProfile = &model.LBTcpMonitorProfile{ + Id: common.String("default-tcp-lb-monitor"), + Path: common.String("/infra/lb-monitor-profiles/default-tcp-lb-monitor"), + ParentPath: common.String("/infra"), + ResourceType: common.ResourceTypeLBTcpMonitorProfile, + } + persistenceProfile = &model.LBSourceIpPersistenceProfile{ + Id: common.String("default-source-ip-lb-persistence-profile"), + Path: common.String("/infra/lb-persistence-profiles/default-source-ip-lb-persistence-profile"), + ParentPath: common.String("/infra"), + ResourceType: common.ResourceTypeLBSourceIpPersistenceProfile, + } +) + +type fakeInfraClient struct{} + +func (f fakeInfraClient) Get(basePathParam *string, filterParam *string, typeFilterParam *string) (model.Infra, error) { + return model.Infra{}, nil +} + +func (f fakeInfraClient) Update(infraParam model.Infra) (model.Infra, error) { + return model.Infra{}, nil +} + +func (f fakeInfraClient) Patch(infraParam model.Infra, enforceRevisionCheckParam *bool) error { + return nil +} + +type fakeLBAppProfileClient struct{} + +func (f fakeLBAppProfileClient) Delete(string, *bool) error { + return nil +} + +func (f fakeLBAppProfileClient) Get(string) (*data.StructValue, error) { + return nil, nil +} + +func (f fakeLBAppProfileClient) List(*string, *bool, *string, *int64, *bool, *string) (model.LBAppProfileListResult, error) { + return model.LBAppProfileListResult{}, nil +} + +func (f fakeLBAppProfileClient) Patch(string, *data.StructValue) error { + return nil +} + +func (f fakeLBAppProfileClient) Update(string, *data.StructValue) (*data.StructValue, error) { + return nil, nil +} + +type fakeLBMonitorProfileClient struct{} + +func (f fakeLBMonitorProfileClient) Delete(string, *bool) error { + return nil +} + +func (f fakeLBMonitorProfileClient) Get(string) (*data.StructValue, error) { + return nil, nil +} + +func (f fakeLBMonitorProfileClient) List(*string, *bool, *string, *int64, *bool, *string) (model.LBMonitorProfileListResult, error) { + return model.LBMonitorProfileListResult{}, nil +} + +func (f fakeLBMonitorProfileClient) Patch(string, *data.StructValue) error { + return nil +} + +func (f fakeLBMonitorProfileClient) Update(string, *data.StructValue) (*data.StructValue, error) { + return nil, nil +} + +type fakeLBPersistenceProfileClient struct{} + +func (f fakeLBPersistenceProfileClient) Delete(string, *bool) error { + return nil +} + +func (f fakeLBPersistenceProfileClient) Get(string) (*data.StructValue, error) { + return nil, nil +} + +func (f fakeLBPersistenceProfileClient) List(*string, *bool, *string, *int64, *bool, *string) (model.LBPersistenceProfileListResult, error) { + return model.LBPersistenceProfileListResult{}, nil +} + +func (f fakeLBPersistenceProfileClient) Patch(string, *data.StructValue) error { + return nil +} + +func (f fakeLBPersistenceProfileClient) Update(string, *data.StructValue) (*data.StructValue, error) { + return nil, nil +} + +func TestCleanupInfraDLBResources(t *testing.T) { + validCtx := context.Background() + invalidCtx, cancelFn := context.WithCancel(validCtx) + cancelFn() + + for _, tc := range []struct { + name string + ctx context.Context + cleanupFn func(svc *LBInfraCleaner) func(ctx context.Context) error + queriedObjects interface{} + mockFn func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches + queryErr error + expErrStr string + }{ + { + name: "success with cleanupInfraDLBVirtualServers", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBVirtualServers + }, + queriedObjects: []*model.LBVirtualServer{lbVS}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBVirtualServer AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraDLBVirtualServers", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBVirtualServers + }, + queriedObjects: []*model.LBVirtualServer{lbVS}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBVirtualServer AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraDLBVirtualServers by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBVirtualServers + }, + queriedObjects: []*model.LBVirtualServer{lbVS}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBVirtualServer AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupInfraDLBPools", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBPools + }, + queriedObjects: []*model.LBPool{lbPool}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBPool AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraDLBPools", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBPools + }, + queriedObjects: []*model.LBPool{lbPool}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBPool AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraDLBPools by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBPools + }, + queriedObjects: []*model.LBPool{lbPool}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBPool AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupInfraDLBServices", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBServices + }, + queriedObjects: []*model.LBService{lbService}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBService AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraDLBServices", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBServices + }, + queriedObjects: []*model.LBService{lbService}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBService AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraDLBServices by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBServices + }, + queriedObjects: []*model.LBService{lbService}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:LBService AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupInfraDLBGroups", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBGroups + }, + queriedObjects: []*model.Group{group}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Group AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraDLBGroups", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBGroups + }, + queriedObjects: []*model.Group{group}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Group AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraDLBGroups by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraDLBGroups + }, + queriedObjects: []*model.Group{group}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Group AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:DLB" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupInfraShares", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraShares + }, + queriedObjects: []*model.Share{share}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Share AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraShares", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraShares + }, + queriedObjects: []*model.Share{share}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Share AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraShares by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraShares + }, + queriedObjects: []*model.Share{share}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:Share AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupInfraCerts", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraCerts + }, + queriedObjects: []*model.TlsCertificate{cert}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:TlsCertificate AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupInfraCerts", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraCerts + }, + queriedObjects: []*model.TlsCertificate{cert}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:TlsCertificate AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupInfraCerts by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupInfraCerts + }, + queriedObjects: []*model.TlsCertificate{cert}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "resource_type:TlsCertificate AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupLBAppProfiles", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBAppProfiles + }, + queriedObjects: []*model.LBHttpProfile{appProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpProfile OR resource_type:LBFastTcpProfile OR resource_type:LBFastUdpProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbAppProfileClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupLBAppProfiles", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBAppProfiles + }, + queriedObjects: []*model.LBHttpProfile{appProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpProfile OR resource_type:LBFastTcpProfile OR resource_type:LBFastUdpProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupLBAppProfiles by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBAppProfiles + }, + queriedObjects: []*model.LBHttpProfile{appProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpProfile OR resource_type:LBFastTcpProfile OR resource_type:LBFastUdpProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbAppProfileClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupLBMonitorProfiles", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbMonitorProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupLBMonitorProfiles", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupLBMonitorProfiles by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbMonitorProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + { + name: "success with cleanupLBMonitorProfiles", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbMonitorProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupLBMonitorProfiles", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupLBMonitorProfiles by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBMonitorProfiles + }, + queriedObjects: []*model.LBTcpMonitorProfile{monitorProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBHttpMonitorProfile OR resource_type:LBTcpMonitorProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbMonitorProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + + { + name: "success with cleanupLBPersistenceProfiles", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBPersistenceProfiles + }, + queriedObjects: []*model.LBSourceIpPersistenceProfile{persistenceProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBCookiePersistenceProfile OR resource_type:LBSourceIpPersistenceProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbPersistenceProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 1, + }}) + return patches + }, + }, + { + name: "timed out with cleanupLBPersistenceProfiles", + ctx: invalidCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBPersistenceProfiles + }, + queriedObjects: []*model.LBSourceIpPersistenceProfile{persistenceProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBCookiePersistenceProfile OR resource_type:LBSourceIpPersistenceProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + return nil + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + { + name: "failed with cleanupLBPersistenceProfiles by NSX error", + ctx: validCtx, + cleanupFn: func(svc *LBInfraCleaner) func(ctx context.Context) error { + return svc.cleanupLBPersistenceProfiles + }, + queriedObjects: []*model.LBSourceIpPersistenceProfile{persistenceProfile}, + mockFn: func(svc *LBInfraCleaner, queryResponse model.SearchResponse, mockQueryClient *mocks.MockQueryClient) *gomonkey.Patches { + query := "(resource_type:LBCookiePersistenceProfile OR resource_type:LBSourceIpPersistenceProfile) AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test" + mockQueryClient.EXPECT().List(query, gomock.Any(), nil, gomock.Any(), nil, nil).Return(queryResponse, nil) + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.LbPersistenceProfilesClient, "Delete", []gomonkey.OutputCell{{ + Values: gomonkey.Params{fmt.Errorf("server error")}, + Times: 1, + }}) + return patches + }, + expErrStr: "server error", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockQueryClient := mocks.NewMockQueryClient(ctrl) + cleaner := prepareCleaner() + cleaner.NSXClient.QueryClient = mockQueryClient + + patches := tc.mockFn(cleaner, generateQueryResponse(t, tc.queriedObjects), mockQueryClient) + if patches != nil { + defer patches.Reset() + } + + err := tc.cleanupFn(cleaner)(tc.ctx) + if tc.expErrStr != "" { + require.EqualError(t, err, tc.expErrStr) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestCleanupInfraResources(t *testing.T) { + for _, tc := range []struct { + name string + mockFn func(cleaner *LBInfraCleaner) *gomonkey.Patches + expErrStr string + }{ + { + name: "success to clean up infra resources", + mockFn: func(cleaner *LBInfraCleaner) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBVirtualServers", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraShares", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBPools", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBServices", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBGroups", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraCerts", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBAppProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBPersistenceProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBMonitorProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + return patches + }, + }, { + name: "failed with LB virtual server clean up", + mockFn: func(cleaner *LBInfraCleaner) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBVirtualServers", func(_ *LBInfraCleaner, ctx context.Context) error { + return fmt.Errorf("failure in cleanupInfraDLBVirtualServers") + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraShares", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBPools", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBServices", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBGroups", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraCerts", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBAppProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBPersistenceProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBMonitorProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + return patches + }, + expErrStr: "failure in cleanupInfraDLBVirtualServers", + }, + { + name: "failed with parallel clean up", + mockFn: func(cleaner *LBInfraCleaner) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBVirtualServers", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraShares", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBPools", func(_ *LBInfraCleaner, ctx context.Context) error { + return fmt.Errorf("failure in cleanupInfraDLBPools") + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBServices", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraDLBGroups", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupInfraCerts", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBAppProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBPersistenceProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + patches.ApplyPrivateMethod(reflect.TypeOf(cleaner), "cleanupLBMonitorProfiles", func(_ *LBInfraCleaner, ctx context.Context) error { + return nil + }) + return patches + }, + expErrStr: "failure in cleanupInfraDLBPools", + }, + } { + t.Run(tc.name, func(t *testing.T) { + cleaner := prepareCleaner() + patches := tc.mockFn(cleaner) + if patches != nil { + defer patches.Reset() + } + + ctx := context.Background() + err := cleaner.CleanupInfraResources(ctx) + if tc.expErrStr != "" { + require.EqualError(t, err, tc.expErrStr) + } else { + require.NoError(t, err) + } + }) + } +} + +func generateQueryResponse(t *testing.T, resources interface{}) model.SearchResponse { + objects := make([]interface{}, 0) + var bindingType bindings.BindingType + switch resources.(type) { + case []*model.LBVirtualServer: + for _, vs := range resources.([]*model.LBVirtualServer) { + objects = append(objects, vs) + } + bindingType = model.LBVirtualServerBindingType() + case []*model.LBPool: + for _, pool := range resources.([]*model.LBPool) { + objects = append(objects, pool) + } + bindingType = model.LBPoolBindingType() + case []*model.LBService: + for _, lbs := range resources.([]*model.LBService) { + objects = append(objects, lbs) + } + bindingType = model.LBServiceBindingType() + case []*model.TlsCertificate: + for _, cert := range resources.([]*model.TlsCertificate) { + objects = append(objects, cert) + } + bindingType = model.TlsCertificateBindingType() + case []*model.Group: + for _, g := range resources.([]*model.Group) { + objects = append(objects, g) + } + bindingType = model.GroupBindingType() + case []*model.Share: + for _, share := range resources.([]*model.Share) { + objects = append(objects, share) + } + bindingType = model.ShareBindingType() + case []*model.LBHttpProfile: + for _, profile := range resources.([]*model.LBHttpProfile) { + objects = append(objects, profile) + } + bindingType = model.LBHttpProfileBindingType() + case []*model.LBTcpMonitorProfile: + for _, profile := range resources.([]*model.LBTcpMonitorProfile) { + objects = append(objects, profile) + } + bindingType = model.LBTcpMonitorProfileBindingType() + case []*model.LBSourceIpPersistenceProfile: + for _, profile := range resources.([]*model.LBSourceIpPersistenceProfile) { + objects = append(objects, profile) + } + bindingType = model.LBSourceIpPersistenceProfileBindingType() + } + return convertToResponse(t, objects, bindingType) +} + +func convertToResponse(t *testing.T, resources []interface{}, bindingType bindings.BindingType) model.SearchResponse { + var results []*data.StructValue + for _, obj := range resources { + vsData, errs := common.NewConverter().ConvertToVapi(obj, bindingType) + require.Equal(t, 0, len(errs)) + results = append(results, vsData.(*data.StructValue)) + } + resultCount := int64(len(results)) + cursor := fmt.Sprintf("%d", resultCount) + return model.SearchResponse{ + Results: results, + Cursor: &cursor, + ResultCount: &resultCount, + } +} + +func prepareCleaner() *LBInfraCleaner { + log := logger.ZapLogger(false, 0) + return &LBInfraCleaner{ + Service: common.Service{ + NSXClient: &nsx.Client{ + InfraClient: &fakeInfraClient{}, + LbAppProfileClient: &fakeLBAppProfileClient{}, + LbMonitorProfilesClient: &fakeLBMonitorProfileClient{}, + LbPersistenceProfilesClient: &fakeLBPersistenceProfileClient{}, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + log: &log, + } +} diff --git a/pkg/clean/clean_dlb_test.go b/pkg/clean/clean_dlb_test.go deleted file mode 100644 index aa416ea5d..000000000 --- a/pkg/clean/clean_dlb_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package clean - -import ( - "context" - "errors" - "reflect" - "testing" - - "github.com/agiledragon/gomonkey/v2" - "github.com/go-logr/logr" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - - "github.com/vmware-tanzu/nsx-operator/pkg/config" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx" -) - -func TestHttpQueryDLBResources_Success(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - resource := "Group" - /* - expectedURL := "policy/api/v1/search/query?query=resource_type%3AGroup%20AND%20tags.scope%3Ancp%5C%2Fcluster%20AND%20tags.tag%3Atest-cluster%20AND%20tags.scope%3Ancp%5C%2Fcreated_for%20AND%20tags.tag%3ADLB" - */ - expectedResponse := map[string]interface{}{ - "results": []interface{}{ - map[string]interface{}{"path": "/test/path/1"}, - map[string]interface{}{"path": "/test/path/2"}, - }, - } - - patches := gomonkey.ApplyMethod(reflect.TypeOf(cluster), "HttpGet", func(cluster *nsx.Cluster, url string) (map[string]interface{}, error) { - return expectedResponse, nil - }) - defer patches.Reset() - - paths, err := httpQueryDLBResources(cluster, cf, resource) - assert.NoError(t, err) - assert.ElementsMatch(t, []string{"/test/path/1", "/test/path/2"}, paths) -} - -func TestHttpQueryDLBResources_Error(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - resource := "Group" - /* - expectedURL := "policy/api/v1/search/query?query=resource_type%3AGroup%20AND%20tags.scope%3Ancp%5C%2Fcluster%20AND%20tags.tag%3Atest-cluster%20AND%20tags.scope%3Ancp%5C%2Fcreated_for%20AND%20tags.tag%3ADLB" - */ - expectedError := errors.New("http error") - - patches := gomonkey.ApplyMethod(reflect.TypeOf(cluster), "HttpGet", func(cluster *nsx.Cluster, url string) (map[string]interface{}, error) { - return nil, expectedError - }) - defer patches.Reset() - - paths, err := httpQueryDLBResources(cluster, cf, resource) - assert.Error(t, err) - assert.Nil(t, paths) -} - -func TestHttpQueryDLBResources_EmptyResponse(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - resource := "Group" - cluster := &nsx.Cluster{} - - expectedResponse := map[string]interface{}{ - "results": []interface{}{}, - } - - patches := gomonkey.ApplyMethod(reflect.TypeOf(cluster), "HttpGet", func(cluster *nsx.Cluster, url string) (map[string]interface{}, error) { - return expectedResponse, nil - }) - defer patches.Reset() - - paths, err := httpQueryDLBResources(cluster, cf, resource) - assert.NoError(t, err) - assert.Empty(t, paths) -} - -func TestCleanDLB_Success(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - log := logr.Discard() - - expectedPaths := []string{"/test/path/1", "/test/path/2"} - patches := gomonkey.ApplyFunc(httpQueryDLBResources, func(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { - return expectedPaths, nil - }) - defer patches.Reset() - - patches.ApplyMethod(reflect.TypeOf(cluster), "HttpDelete", func(cluster *nsx.Cluster, url string) error { - return nil - }) - - ctx := context.Background() - err := CleanDLB(ctx, cluster, cf, &log) - assert.NoError(t, err) -} - -func TestCleanDLB_HttpQueryError(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - log := logr.Discard() - - expectedError := errors.New("http query error") - patches := gomonkey.ApplyFunc(httpQueryDLBResources, func(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { - return nil, expectedError - }) - defer patches.Reset() - - ctx := context.Background() - err := CleanDLB(ctx, cluster, cf, &log) - assert.Error(t, err) - assert.Equal(t, expectedError, err) -} - -func TestCleanDLB_HttpDeleteError(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - log := logr.Discard() - - expectedPaths := []string{"/test/path/1", "/test/path/2"} - patches := gomonkey.ApplyFunc(httpQueryDLBResources, func(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { - return expectedPaths, nil - }) - defer patches.Reset() - - expectedError := errors.New("http delete error") - patches.ApplyMethod(reflect.TypeOf(cluster), "HttpDelete", func(cluster *nsx.Cluster, url string) error { - return expectedError - }) - - ctx := context.Background() - err := CleanDLB(ctx, cluster, cf, &log) - assert.Error(t, err) - assert.Equal(t, expectedError, err) -} - -func TestCleanDLB_ContextDone(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - cluster := &nsx.Cluster{} - cf = &config.NSXOperatorConfig{NsxConfig: &config.NsxConfig{NsxApiManagers: []string{"10.0.0.1"}}, CoeConfig: &config.CoeConfig{Cluster: "test-cluster"}} - log := logr.Discard() - - expectedPaths := []string{"/test/path/1", "/test/path/2"} - patches := gomonkey.ApplyFunc(httpQueryDLBResources, func(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { - return expectedPaths, nil - }) - defer patches.Reset() - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - err := CleanDLB(ctx, cluster, cf, &log) - assert.Error(t, err) - assert.True(t, errors.Is(err, context.Canceled)) -} - -func TestAppendIfNotExist_ItemExists(t *testing.T) { - slice := []string{"test"} - result := appendIfNotExist(slice, "test") - assert.Equal(t, []string{"test"}, result) -} - -func TestAppendIfNotExist_ItemDoesNotExist(t *testing.T) { - slice := []string{"test1"} - result := appendIfNotExist(slice, "test2") - assert.Equal(t, []string{"test1", "test2"}, result) -} - -func TestAppendIfNotExist_MultipleItems(t *testing.T) { - slice := []string{"test1", "test2"} - result := appendIfNotExist(slice, "test3") - assert.Equal(t, []string{"test1", "test2", "test3"}, result) -} - -func TestAppendIfNotExist_DuplicateItems(t *testing.T) { - slice := []string{"test1", "test2", "test1"} - result := appendIfNotExist(slice, "test1") - assert.Equal(t, []string{"test1", "test2", "test1"}, result) -} - -func TestAppendIfNotExist_EmptySlice(t *testing.T) { - slice := []string{} - result := appendIfNotExist(slice, "test") - assert.Equal(t, []string{"test"}, result) -} diff --git a/pkg/clean/clean_lb_infra.go b/pkg/clean/clean_lb_infra.go new file mode 100644 index 000000000..b06dc4a18 --- /dev/null +++ b/pkg/clean/clean_lb_infra.go @@ -0,0 +1,347 @@ +package clean + +import ( + "context" + "errors" + "sync" + + "github.com/go-logr/logr" + "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +var ( + MarkedForDelete = true + forceDelete = true +) + +type LBInfraCleaner struct { + common.Service + log *logr.Logger +} + +func keyFunc(obj interface{}) (string, error) { + switch v := obj.(type) { + case *model.LBVirtualServer: + return *v.Path, nil + case *model.LBService: + return *v.Path, nil + case *model.LBPool: + return *v.Path, nil + case *model.LBAppProfile: + return *v.Path, nil + case *model.LBMonitorProfile: + return *v.Path, nil + case *model.LBPersistenceProfile: + return *v.Path, nil + case *model.Share: + return *v.Path, nil + case *model.TlsCertificate: + return *v.Path, nil + case *model.Group: + return *v.Path, nil + default: + return "", errors.New("keyFunc doesn't support unknown type") + } +} + +// CleanupInfraResources is to clean up the LB related resources created under path /infra, including, +// group/share/cert/LBAppProfile/LBPersistentProfile/dlb virtual servers/dlb services/dlb groups/dlb pools +func (s *LBInfraCleaner) CleanupInfraResources(ctx context.Context) error { + // LB virtual server has dependencies on LB pool, so we can't delete vs and pool in parallel. + if err := s.cleanupInfraDLBVirtualServers(ctx); err != nil { + s.log.Error(err, "Failed to clean up DLB virtual servers") + return err + } + // Share has dependencies on Group, so we can't delete share and groups in parallel. + if err := s.cleanupInfraShares(ctx); err != nil { + s.log.Error(err, "Failed to clean up infra Shares") + return err + } + + parallelCleaners := []func(ctx context.Context) error{ + s.cleanupInfraDLBPools, + s.cleanupInfraDLBServices, + s.cleanupInfraDLBGroups, + s.cleanupInfraCerts, + s.cleanupLBAppProfiles, + s.cleanupLBPersistenceProfiles, + s.cleanupLBMonitorProfiles, + } + + cleanerCount := len(parallelCleaners) + errs := make(chan error, cleanerCount) + defer close(errs) + wg := sync.WaitGroup{} + wg.Add(cleanerCount) + for i := range parallelCleaners { + cleaner := parallelCleaners[i] + go func() { + defer wg.Done() + err := cleaner(ctx) + if err != nil { + errs <- err + } + }() + } + wg.Wait() + if len(errs) > 0 { + return <-errs + } + return nil +} + +func (s *LBInfraCleaner) cleanupInfraShares(ctx context.Context) error { + store, err := s.queryNCPCreatedResources([]string{common.ResourceTypeShare}, model.ShareBindingType(), nil) + if err != nil { + return nil + } + + var sharesSet []*model.Share + for _, obj := range store.List() { + share := obj.(*model.Share) + share.MarkedForDelete = &MarkedForDelete + sharesSet = append(sharesSet, share) + } + + s.log.Info("Cleaning up shares", "Count", len(sharesSet)) + sharesBuilder, _ := common.PolicyPathInfraShare.NewPolicyTreeBuilder() + return sharesBuilder.PagingDeleteResources(ctx, sharesSet, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *LBInfraCleaner) cleanupInfraCerts(ctx context.Context) error { + store, err := s.queryNCPCreatedResources([]string{common.ResourceTypeTlsCertificate}, model.TlsCertificateBindingType(), nil) + if err != nil { + return nil + } + var certsSet []*model.TlsCertificate + for _, obj := range store.List() { + cert := obj.(*model.TlsCertificate) + cert.MarkedForDelete = &MarkedForDelete + certsSet = append(certsSet, cert) + } + + s.log.Info("Cleaning up certificates", "Count", len(certsSet)) + certsBuilder, _ := common.PolicyPathInfraCert.NewPolicyTreeBuilder() + return certsBuilder.PagingDeleteResources(ctx, certsSet, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +type ResourceStore struct { + common.ResourceStore +} + +func (r *ResourceStore) Apply(i interface{}) error { + return nil +} + +func (s *LBInfraCleaner) queryNCPCreatedResources(resourceTypes []string, resourceBindingType bindings.BindingType, additionalQueryFn func(query string) string) (*ResourceStore, error) { + store := &ResourceStore{common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: resourceBindingType, + }} + if err := s.Service.QueryNCPCreatedResources(resourceTypes, store, additionalQueryFn); err != nil { + s.log.Error(err, "Failed to query NCP created resources", "resource types", resourceTypes) + return nil, err + } + return store, nil +} + +func (s *LBInfraCleaner) queryDLBResources(resourceTypes []string, resourceBindingType bindings.BindingType) (*ResourceStore, error) { + return s.queryNCPCreatedResources(resourceTypes, resourceBindingType, func(query string) string { + return common.AddNCPCreatedForTag(query, common.TagValueDLB) + }) +} + +func (s *LBInfraCleaner) cleanupInfraDLBVirtualServers(ctx context.Context) error { + store, err := s.queryDLBResources([]string{common.ResourceTypeLBVirtualServer}, model.LBVirtualServerBindingType()) + if err != nil { + return nil + } + + var vss []*model.LBVirtualServer + for _, obj := range store.List() { + vs := obj.(*model.LBVirtualServer) + vs.MarkedForDelete = &MarkedForDelete + vss = append(vss, vs) + } + + s.log.Info("Cleaning up DLB virtual servers", "Count", len(vss)) + vssBuilder, _ := common.PolicyPathInfraLBVirtualServer.NewPolicyTreeBuilder() + return vssBuilder.PagingDeleteResources(ctx, vss, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *LBInfraCleaner) cleanupInfraDLBPools(ctx context.Context) error { + store, err := s.queryDLBResources([]string{common.ResourceTypeLBPool}, model.LBPoolBindingType()) + if err != nil { + return nil + } + + var pools []*model.LBPool + for _, obj := range store.List() { + pool := obj.(*model.LBPool) + pool.MarkedForDelete = &MarkedForDelete + pools = append(pools, pool) + } + + s.log.Info("Cleaning up DLB pools", "Count", len(pools)) + poolBuilder, _ := common.PolicyPathInfraLBPool.NewPolicyTreeBuilder() + return poolBuilder.PagingDeleteResources(ctx, pools, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *LBInfraCleaner) cleanupInfraDLBServices(ctx context.Context) error { + store, err := s.queryDLBResources([]string{common.ResourceTypeLBService}, model.LBServiceBindingType()) + if err != nil { + return nil + } + + var lbServices []*model.LBService + for _, obj := range store.List() { + svc := obj.(*model.LBService) + svc.MarkedForDelete = &MarkedForDelete + lbServices = append(lbServices, svc) + } + + s.log.Info("Cleaning up DLB services", "Count", len(lbServices)) + lbsBuilder, _ := common.PolicyPathInfraLBService.NewPolicyTreeBuilder() + return lbsBuilder.PagingDeleteResources(ctx, lbServices, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *LBInfraCleaner) cleanupInfraDLBGroups(ctx context.Context) error { + store, err := s.queryDLBResources([]string{common.ResourceTypeGroup}, model.GroupBindingType()) + if err != nil { + return nil + } + + var lbGroups []*model.Group + for _, obj := range store.List() { + grp := obj.(*model.Group) + grp.MarkedForDelete = &MarkedForDelete + lbGroups = append(lbGroups, grp) + } + + s.log.Info("Cleaning up DLB groups", "Count", len(lbGroups)) + groupBuilder, _ := common.PolicyPathInfraGroup.NewPolicyTreeBuilder() + return groupBuilder.PagingDeleteResources(ctx, lbGroups, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *LBInfraCleaner) ListLBAppProfile() []*model.LBAppProfile { + store, err := s.queryNCPCreatedResources([]string{common.ResourceTypeLBHttpProfile, common.ResourceTypeLBFastTcpProfile, common.ResourceTypeLBFastUdpProfile}, model.LBAppProfileBindingType(), nil) + if err != nil { + return nil + } + + lbAppProfiles := store.List() + var lbAppProfilesSet []*model.LBAppProfile + for _, obj := range lbAppProfiles { + appProfile := obj.(*model.LBAppProfile) + lbAppProfilesSet = append(lbAppProfilesSet, appProfile) + } + return lbAppProfilesSet +} + +func (s *LBInfraCleaner) ListLBPersistenceProfile() []*model.LBPersistenceProfile { + store, err := s.queryNCPCreatedResources([]string{common.ResourceTypeLBCookiePersistenceProfile, common.ResourceTypeLBSourceIpPersistenceProfile}, model.LBPersistenceProfileBindingType(), nil) + if err != nil { + return nil + } + lbPersistenceProfiles := store.List() + var lbPersistenceProfilesSet []*model.LBPersistenceProfile + for _, lbPersistenceProfile := range lbPersistenceProfiles { + lbPersistenceProfilesSet = append(lbPersistenceProfilesSet, lbPersistenceProfile.(*model.LBPersistenceProfile)) + } + return lbPersistenceProfilesSet +} + +func (s *LBInfraCleaner) ListLBMonitorProfile() []model.LBMonitorProfile { + store, err := s.queryNCPCreatedResources([]string{common.ResourceTypeLBHttpMonitorProfile, common.ResourceTypeLBTcpMonitorProfile}, model.LBMonitorProfileBindingType(), nil) + if err != nil { + return nil + } + + lbMonitorProfiles := store.List() + var lbMonitorProfilesSet []model.LBMonitorProfile + for _, lbMonitorProfile := range lbMonitorProfiles { + lbMonitorProfilesSet = append(lbMonitorProfilesSet, *lbMonitorProfile.(*model.LBMonitorProfile)) + } + return lbMonitorProfilesSet +} + +func (s *LBInfraCleaner) cleanupLBAppProfiles(ctx context.Context) error { + lbAppProfiles := s.ListLBAppProfile() + s.log.Info("Cleaning up lbAppProfiles", "Count", len(lbAppProfiles)) + var delErr error + for _, lbAppProfile := range lbAppProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + id := *lbAppProfile.Id + if err := s.NSXClient.LbAppProfileClient.Delete(id, &forceDelete); err != nil { + s.log.Error(err, "Failed to deleted NCP created lbAppProfile", "lbAppProfile", id) + delErr = err + continue + } + } + } + + if delErr != nil { + return delErr + } + + s.log.Info("Completed to clean up NCP created lbAppProfiles") + return nil +} + +func (s *LBInfraCleaner) cleanupLBPersistenceProfiles(ctx context.Context) error { + lbPersistenceProfiles := s.ListLBPersistenceProfile() + s.log.Info("Cleaning up lbPersistenceProfiles", "Count", len(lbPersistenceProfiles)) + var delErr error + for _, lbPersistenceProfile := range lbPersistenceProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + id := *lbPersistenceProfile.Id + if err := s.NSXClient.LbPersistenceProfilesClient.Delete(*lbPersistenceProfile.Id, &forceDelete); err != nil { + s.log.Error(err, "Failed to deleted NCP created lbPersistenceProfile", "lbPersistenceProfile", id) + delErr = err + continue + } + } + } + + if delErr != nil { + return delErr + } + + s.log.Info("Completed to clean up NCP created lbPersistenceProfiles") + return nil +} + +func (s *LBInfraCleaner) cleanupLBMonitorProfiles(ctx context.Context) error { + lbMonitorProfiles := s.ListLBMonitorProfile() + s.log.Info("Cleaning up lbMonitorProfiles", "Count", len(lbMonitorProfiles)) + var delErr error + for _, lbMonitorProfile := range lbMonitorProfiles { + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + id := *lbMonitorProfile.Id + if err := s.NSXClient.LbMonitorProfilesClient.Delete(id, &forceDelete); err != nil { + s.log.Error(err, "Failed to deleted NCP created lbMonitorProfile", "lbMonitorProfile", id) + delErr = err + continue + } + } + } + if delErr != nil { + return delErr + } + s.log.Info("Completed to clean up NCP created lbMonitorProfiles") + return nil +} diff --git a/pkg/clean/clean_test.go b/pkg/clean/clean_test.go index 064fc33e8..5d62432c0 100644 --- a/pkg/clean/clean_test.go +++ b/pkg/clean/clean_test.go @@ -10,6 +10,7 @@ import ( "github.com/go-logr/logr" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/sets" "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" @@ -78,7 +79,7 @@ func TestClean_InitError(t *testing.T) { return &nsx.Client{} }) - patches.ApplyFunc(InitializeCleanupService, func(_ *config.NSXOperatorConfig, _ *nsx.Client) (*CleanupService, error) { + patches.ApplyFunc(InitializeCleanupService, func(_ *config.NSXOperatorConfig, _ *nsx.Client, _ *logr.Logger) (*CleanupService, error) { return nil, errors.New("init cleanup service failed") }) @@ -87,7 +88,7 @@ func TestClean_InitError(t *testing.T) { assert.Contains(t, err.Error(), "init cleanup service failed") } -func TestClean_CleanupSucc(t *testing.T) { +func TestClean_Cleanup(t *testing.T) { ctx := context.Background() debug := false @@ -101,57 +102,59 @@ func TestClean_CleanupSucc(t *testing.T) { return &nsx.Client{} }) - cleanupService := &CleanupService{} - clean := &MockCleanup{ - CleanupFunc: func(ctx context.Context) error { - return nil - }, + cleanupService := &CleanupService{ + vpcService: &vpc.VPCService{}, } - cleanupService.cleans = append(cleanupService.cleans, clean) - patches.ApplyFunc(InitializeCleanupService, func(_ *config.NSXOperatorConfig, _ *nsx.Client) (*CleanupService, error) { - return cleanupService, nil + clean := &MockCleanup{} + cleanupService.AddCleanupService(func() (interface{}, error) { + return clean, nil }) - patches.ApplyFunc(CleanDLB, func(ctx context.Context, cluster *nsx.Cluster, cf *config.NSXOperatorConfig, log *logr.Logger) error { + patches.ApplyFunc(InitializeCleanupService, func(_ *config.NSXOperatorConfig, _ *nsx.Client, _ *logr.Logger) (*CleanupService, error) { + return cleanupService, nil + }) + patches.ApplyMethod(reflect.TypeOf(cleanupService.vpcService), "ListAutoCreatedVPCPaths", func(_ *vpc.VPCService) sets.Set[string] { + return sets.New[string]("/orgs/default/projects/p1/vpcs/vpc-1") + }) + patches.ApplyMethod(reflect.TypeOf(cleanupService.vpcService), "DeleteVPC", func(_ *vpc.VPCService, path string) error { return nil }) + err := Clean(ctx, cf, nil, debug, logLevel) assert.Nil(t, err) + assert.True(t, clean.vpcPreCleanupCalled) + assert.True(t, clean.vpcChildrenCleanupCalled) + assert.True(t, clean.infraCleanupCalled) + assert.ElementsMatch(t, []string{"/orgs/default/projects/p1/vpcs/vpc-1", ""}, clean.cleanedVPCs) } type MockCleanup struct { - CleanupFunc func(ctx context.Context) error + CleanupFunc func(ctx context.Context) error + vpcPreCleanupCalled bool + vpcChildrenCleanupCalled bool + infraCleanupCalled bool + + cleanedVPCs []string } func (m *MockCleanup) Cleanup(ctx context.Context) error { return m.CleanupFunc(ctx) } -func TestWrapCleanFunc(t *testing.T) { - // succ case - ctx := context.Background() - clean := &MockCleanup{ - CleanupFunc: func(ctx context.Context) error { - return nil - }, - } - - wrappedFunc := wrapCleanFunc(ctx, clean) - err := wrappedFunc() - assert.NoError(t, err) - - // error case - clean = &MockCleanup{ - CleanupFunc: func(ctx context.Context) error { - return errors.New("cleanup failed") - }, - } +func (m *MockCleanup) CleanupBeforeVPCDeletion(ctx context.Context) error { + m.vpcPreCleanupCalled = true + return nil +} - wrappedFunc = wrapCleanFunc(ctx, clean) - err = wrappedFunc() - assert.Error(t, err) - assert.Equal(t, "cleanup failed", err.Error()) +func (m *MockCleanup) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + m.vpcChildrenCleanupCalled = true + m.cleanedVPCs = append(m.cleanedVPCs, vpcPath) + return nil +} +func (m *MockCleanup) CleanupInfraResources(ctx context.Context) error { + m.infraCleanupCalled = true + return nil } func TestInitializeCleanupService_Success(t *testing.T) { @@ -169,7 +172,7 @@ func TestInitializeCleanupService_Success(t *testing.T) { patches.ApplyFunc(subnet.InitializeSubnetService, func(service common.Service) (*subnet.SubnetService, error) { return &subnet.SubnetService{}, nil }) - patches.ApplyFunc(securitypolicy.InitializeSecurityPolicy, func(service common.Service, vpcService common.VPCServiceProvider) (*securitypolicy.SecurityPolicyService, error) { + patches.ApplyFunc(securitypolicy.InitializeSecurityPolicy, func(service common.Service, vpcService common.VPCServiceProvider, forCleanup bool) (*securitypolicy.SecurityPolicyService, error) { return &securitypolicy.SecurityPolicyService{}, nil }) patches.ApplyFunc(sr.InitializeStaticRoute, func(service common.Service, vpcService common.VPCServiceProvider) (*sr.StaticRouteService, error) { @@ -185,10 +188,12 @@ func TestInitializeCleanupService_Success(t *testing.T) { return &subnetbinding.BindingService{}, nil }) - cleanupService, err := InitializeCleanupService(cf, nsxClient) + cleanupService, err := InitializeCleanupService(cf, nsxClient, nil) assert.NoError(t, err) assert.NotNil(t, cleanupService) - assert.Len(t, cleanupService.cleans, 7) + assert.Len(t, cleanupService.vpcPreCleaners, 3) + assert.Len(t, cleanupService.vpcChildrenCleaners, 5) + assert.Len(t, cleanupService.infraCleaners, 2) } func TestInitializeCleanupService_VPCError(t *testing.T) { @@ -206,7 +211,7 @@ func TestInitializeCleanupService_VPCError(t *testing.T) { patches.ApplyFunc(subnet.InitializeSubnetService, func(service common.Service) (*subnet.SubnetService, error) { return &subnet.SubnetService{}, nil }) - patches.ApplyFunc(securitypolicy.InitializeSecurityPolicy, func(service common.Service, vpcService common.VPCServiceProvider) (*securitypolicy.SecurityPolicyService, error) { + patches.ApplyFunc(securitypolicy.InitializeSecurityPolicy, func(service common.Service, vpcService common.VPCServiceProvider, forCleanup bool) (*securitypolicy.SecurityPolicyService, error) { return &securitypolicy.SecurityPolicyService{}, nil }) patches.ApplyFunc(sr.InitializeStaticRoute, func(service common.Service, vpcService common.VPCServiceProvider) (*sr.StaticRouteService, error) { @@ -222,9 +227,12 @@ func TestInitializeCleanupService_VPCError(t *testing.T) { return &subnetbinding.BindingService{}, nil }) - cleanupService, err := InitializeCleanupService(cf, nsxClient) + cleanupService, err := InitializeCleanupService(cf, nsxClient, nil) assert.NoError(t, err) assert.NotNil(t, cleanupService) - assert.Len(t, cleanupService.cleans, 5) - assert.Equal(t, expectedError, cleanupService.err) + // Note, the services added after VPCService should fail because of the error returned in `InitializeVPC`. + assert.Len(t, cleanupService.vpcChildrenCleaners, 3) + assert.Len(t, cleanupService.vpcPreCleaners, 2) + assert.Len(t, cleanupService.infraCleaners, 1) + assert.Equal(t, expectedError, cleanupService.svcErr) } diff --git a/pkg/clean/types.go b/pkg/clean/types.go index 40c5e43f8..f41592218 100644 --- a/pkg/clean/types.go +++ b/pkg/clean/types.go @@ -1,16 +1,56 @@ package clean -import "context" +import ( + "context" + "errors" + "sync" + "time" -type cleanup interface { - Cleanup(ctx context.Context) error + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + "k8s.io/client-go/util/workqueue" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +const ( + vpcCleanerWorkers = 8 + maxRetries = 12 +) + +type vpcPreCleaner interface { + // CleanupBeforeVPCDeletion is called to clean up the VPC resources which may block the VPC recursive deletion, or + // break the parallel deletion with other resource types, e.g., VpcSubnetPort, SubnetConnectionBindingMap, LBVirtualServer. + // CleanupBeforeVPCDeletion is called before recursively deleting the VPCs in parallel. + CleanupBeforeVPCDeletion(ctx context.Context) error } -type cleanupFunc func() (cleanup, error) +type vpcChildrenCleaner interface { + // CleanupVPCChildResources is called when cleaning up the VPC related resources. For the resources in an auto-created + // VPC, this function is called after the VPC is recursively deleted on NSX, so the providers only needs to clean up + // with the local cache. It uses an empty string for vpcPath for all pre-created VPCs, so the providers should delete + // resources both on NSX and from the local cache. + // CleanupVPCChildResources is called after the VPC with path "vpcPath" is recursively deleted. + CleanupVPCChildResources(ctx context.Context, vpcPath string) error +} + +type infraCleaner interface { + // CleanupInfraResources is to clean up the resources created under path /infra. + CleanupInfraResources(ctx context.Context) error +} + +type cleanupFunc func() (interface{}, error) type CleanupService struct { - cleans []cleanup - err error + log *logr.Logger + + vpcService *vpc.VPCService + vpcPreCleaners []vpcPreCleaner + vpcChildrenCleaners []vpcChildrenCleaner + infraCleaners []infraCleaner + svcErr error } func NewCleanupService() *CleanupService { @@ -18,16 +58,225 @@ func NewCleanupService() *CleanupService { } func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService { - var clean cleanup - if c.err != nil { + if c.svcErr != nil { return c } - clean, c.err = f() - if c.err != nil { + var clean interface{} + clean, c.svcErr = f() + if c.svcErr != nil { return c } - c.cleans = append(c.cleans, clean) + if svc, ok := clean.(vpcPreCleaner); ok { + c.vpcPreCleaners = append(c.vpcPreCleaners, svc) + } + if svc, ok := clean.(vpcChildrenCleaner); ok { + c.vpcChildrenCleaners = append(c.vpcChildrenCleaners, svc) + } + if svc, ok := clean.(infraCleaner); ok { + c.infraCleaners = append(c.infraCleaners, svc) + } + return c } + +func (c *CleanupService) retriable(err error) bool { + if err != nil && !errors.As(err, &nsxutil.TimeoutFailed) { + c.log.Info("Retrying to clean up NSX resources", "error", err) + return true + } + return false +} + +func (c *CleanupService) cleanupBeforeVPCDeletion(ctx context.Context) error { + cleanersCount := len(c.vpcPreCleaners) + if cleanersCount > 0 { + wgForPreVPCCleaners := sync.WaitGroup{} + wgForPreVPCCleaners.Add(cleanersCount) + errorChans := make(chan error, cleanersCount) + for idx := range c.vpcPreCleaners { + cleaner := c.vpcPreCleaners[idx] + go func() { + defer wgForPreVPCCleaners.Done() + err := retry.OnError(Backoff, c.retriable, func() error { + return cleaner.CleanupBeforeVPCDeletion(ctx) + }) + if err != nil { + errorChans <- err + } + return + }() + } + wgForPreVPCCleaners.Wait() + if len(errorChans) > 0 { + err := <-errorChans + return err + } + } + return nil +} + +func (c *CleanupService) cleanupVPCResourcesByVPCPath(ctx context.Context, vpcPath string) error { + c.log.Info("Cleaning VPC resources", "path", vpcPath) + if vpcPath != "" { + if err := c.vpcService.DeleteVPC(vpcPath); err != nil { + c.log.Error(err, "Failed to delete VPC on NSX", "path", vpcPath) + return err + } + c.log.Info("Deleted VPC", "Path", vpcPath) + } + + cleanersCount := len(c.vpcChildrenCleaners) + cleanErrs := make(chan error, len(c.vpcChildrenCleaners)) + defer close(cleanErrs) + + wgForChildrenCleaners := sync.WaitGroup{} + wgForChildrenCleaners.Add(cleanersCount) + for idx := range c.vpcChildrenCleaners { + cleaner := c.vpcChildrenCleaners[idx] + go func() { + defer wgForChildrenCleaners.Done() + err := cleaner.CleanupVPCChildResources(ctx, vpcPath) + if err != nil { + cleanErrs <- err + } + }() + } + wgForChildrenCleaners.Wait() + if len(cleanErrs) > 0 { + return <-cleanErrs + } + return nil +} + +func (c *CleanupService) vpcWorker(ctx context.Context, queue workqueue.TypedRateLimitingInterface[string], potentialVPCs sets.Set[string], completedVPCs sets.Set[string], mu *sync.Mutex, finalErrors chan error) bool { + vpcPath, shutdown := queue.Get() + if shutdown { + return false + } + + // Mark task as done + defer queue.Done(vpcPath) + + err := c.cleanupVPCResourcesByVPCPath(ctx, vpcPath) + if err != nil && queue.NumRequeues(vpcPath) < maxRetries { + queue.AddAfter(vpcPath, 10*time.Second) + return true + } + + defer queue.Forget(vpcPath) + mu.Lock() + defer mu.Unlock() + + if err != nil { + finalErrors <- err + } + + completedVPCs.Insert(vpcPath) + if potentialVPCs.Equal(completedVPCs) { + queue.ShutDown() + } + + return true +} + +func (c *CleanupService) cleanPreCreatedVPCs(ctx context.Context) error { + if err := retry.OnError(Backoff, c.retriable, func() error { + return c.cleanupVPCResourcesByVPCPath(ctx, "") + }); err != nil { + return errors.Join(nsxutil.CleanupResourceFailed, err) + } + return nil +} + +func (c *CleanupService) cleanupAutoCreatedVPCs(ctx context.Context) error { + queue := workqueue.NewTypedRateLimitingQueue[string](workqueue.DefaultTypedControllerRateLimiter[string]()) + defer queue.ShutDown() + + autoCreatedVPCs := c.vpcService.ListAutoCreatedVPCPaths() + if autoCreatedVPCs.Len() == 0 { + return nil + } + + var completedMutex sync.Mutex + completedVPCs := sets.New[string]() + vpcFinalErrors := make(chan error, autoCreatedVPCs.Len()) + defer close(vpcFinalErrors) + + wg := &sync.WaitGroup{} + wg.Add(vpcCleanerWorkers) + for i := 0; i < vpcCleanerWorkers; i++ { + go func() { + defer wg.Done() + for c.vpcWorker(ctx, queue, autoCreatedVPCs, completedVPCs, &completedMutex, vpcFinalErrors) { + } + }() + } + for vpcPath := range autoCreatedVPCs { + queue.Add(vpcPath) + } + + wg.Wait() + + if len(vpcFinalErrors) > 0 { + return <-vpcFinalErrors + } + return nil +} + +// cleanupVPCResources cleans up the VPCs and their children resources created by nsx-operator. +func (c *CleanupService) cleanupVPCResources(ctx context.Context) error { + // Clean up the indirect VPC children resources before deleting the VPCs, otherwise, it may block VPC deletion request + if err := c.cleanupBeforeVPCDeletion(ctx); err != nil { + c.log.Error(err, "Failed to clean up the resources before deleting VPCs") + return err + } + c.log.Info("Completed to clean up the resources before deleting VPCs") + + // Clean up the auto-created VPC and its children resources + if err := c.cleanupAutoCreatedVPCs(ctx); err != nil { + c.log.Error(err, "Failed to clean up the auto created VPCs and their child resources") + return err + } + c.log.Info("Completed to clean up the auto created VPCs and their child resources") + + // Clean up the resources in pre-created VPC. + if err := c.cleanPreCreatedVPCs(ctx); err != nil { + c.log.Error(err, "Failed to clean up the pre-created VPCs' child resources") + return err + } + c.log.Info("Completed to clean up the pre-created VPCs' child resources") + + return nil +} + +func (c *CleanupService) cleanupInfraResources(ctx context.Context) error { + if err := retry.OnError(Backoff, c.retriable, func() error { + cleanersCount := len(c.infraCleaners) + cleanErrs := make([]error, 0) + wgForInfraCleaners := sync.WaitGroup{} + wgForInfraCleaners.Add(cleanersCount) + + for idx := range c.infraCleaners { + cleaner := c.infraCleaners[idx] + go func() { + defer wgForInfraCleaners.Done() + err := cleaner.CleanupInfraResources(ctx) + if err != nil { + cleanErrs = append(cleanErrs, err) + } + }() + } + + wgForInfraCleaners.Wait() + if len(cleanErrs) > 0 { + return cleanErrs[0] + } + + return nil + }); err != nil { + return err + } + return nil +} diff --git a/pkg/clean/types_test.go b/pkg/clean/types_test.go index 678c21cd8..fc973356f 100644 --- a/pkg/clean/types_test.go +++ b/pkg/clean/types_test.go @@ -1,46 +1,42 @@ package clean import ( - "context" "errors" "testing" "github.com/stretchr/testify/assert" ) -type mockCleanup struct{} - -func (m *mockCleanup) Cleanup(ctx context.Context) error { - return nil -} - -func mockCleanupFunc() (cleanup, error) { - return &mockCleanup{}, nil +func mockCleanupFunc() (interface{}, error) { + return &MockCleanup{}, nil } -func mockCleanupFuncWithError() (cleanup, error) { +func mockCleanupFuncWithError() (interface{}, error) { return nil, errors.New("mock error") } func TestNewCleanupService(t *testing.T) { service := NewCleanupService() assert.NotNil(t, service) - assert.Nil(t, service.err) - assert.Empty(t, service.cleans) + assert.Nil(t, service.svcErr) } func TestAddCleanupService_Success(t *testing.T) { service := NewCleanupService() service.AddCleanupService(mockCleanupFunc) - assert.Nil(t, service.err) - assert.Len(t, service.cleans, 1) + assert.Nil(t, service.svcErr) + assert.Len(t, service.vpcPreCleaners, 1) + assert.Len(t, service.vpcChildrenCleaners, 1) + assert.Len(t, service.infraCleaners, 1) } func TestAddCleanupService_Error(t *testing.T) { service := NewCleanupService() service.AddCleanupService(mockCleanupFuncWithError) - assert.NotNil(t, service.err) - assert.Len(t, service.cleans, 0) + assert.NotNil(t, service.svcErr) + assert.Len(t, service.vpcPreCleaners, 0) + assert.Len(t, service.vpcChildrenCleaners, 0) + assert.Len(t, service.infraCleaners, 0) } diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index bb2b981f6..d090f62ab 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -20,7 +20,6 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/domains" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/domains/security_policies" infra_realized "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/realized_state" - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/shares" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/infra/sites/enforcement_points" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/orgs/projects" @@ -96,9 +95,7 @@ type Client struct { ProjectClient orgs.ProjectsClient TransitGatewayClient projects.TransitGatewaysClient TransitGatewayAttachmentClient transit_gateways.AttachmentsClient - CertificateClient infra.CertificatesClient ShareClient infra.SharesClient - SharedResourceClient shares.ResourcesClient LbAppProfileClient infra.LbAppProfilesClient LbPersistenceProfilesClient infra.LbPersistenceProfilesClient LbMonitorProfilesClient infra.LbMonitorProfilesClient @@ -157,55 +154,55 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { c.EnvoyPort = cf.EnvoyPort cluster, _ := NewCluster(c) - queryClient := search.NewQueryClient(restConnector(cluster)) - groupClient := domains.NewGroupsClient(restConnector(cluster)) - securityClient := domains.NewSecurityPoliciesClient(restConnector(cluster)) - ruleClient := security_policies.NewRulesClient(restConnector(cluster)) - infraClient := nsx_policy.NewInfraClient(restConnector(cluster)) - - clusterControlPlanesClient := enforcement_points.NewClusterControlPlanesClient(restConnector(cluster)) - hostTransportNodesClient := enforcement_points.NewHostTransportNodesClient(restConnector(cluster)) - realizedEntitiesClient := infra_realized.NewRealizedEntitiesClient(restConnector(cluster)) - mpQueryClient := mpsearch.NewQueryClient(restConnector(cluster)) - certificatesClient := trust_management.NewCertificatesClient(restConnector(cluster)) - principalIdentitiesClient := trust_management.NewPrincipalIdentitiesClient(restConnector(cluster)) - withCertificateClient := principal_identities.NewWithCertificateClient(restConnector(cluster)) - - orgRootClient := nsx_policy.NewOrgRootClient(restConnector(cluster)) - projectInfraClient := projects.NewInfraClient(restConnector(cluster)) - projectClient := orgs.NewProjectsClient(restConnector(cluster)) - vpcClient := projects.NewVpcsClient(restConnector(cluster)) - vpcConnectivityProfilesClient := projects.NewVpcConnectivityProfilesClient(restConnector(cluster)) - ipBlockClient := project_infra.NewIpBlocksClient(restConnector(cluster)) - staticRouteClient := vpcs.NewStaticRoutesClient(restConnector(cluster)) - natRulesClient := nat.NewNatRulesClient(restConnector(cluster)) - vpcGroupClient := vpcs.NewGroupsClient(restConnector(cluster)) - portClient := subnets.NewPortsClient(restConnectorAllowOverwrite(cluster)) - portStateClient := ports.NewStateClient(restConnector(cluster)) - ipPoolClient := subnets.NewIpPoolsClient(restConnector(cluster)) - ipAllocationClient := ip_pools.NewIpAllocationsClient(restConnector(cluster)) - subnetsClient := vpcs.NewSubnetsClient(restConnector(cluster)) - subnetStatusClient := subnets.NewStatusClient(restConnector(cluster)) - ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(restConnectorAllowOverwrite(cluster)) - vpcLBSClient := vpcs.NewVpcLbsClient(restConnector(cluster)) - vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(restConnector(cluster)) - vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(restConnector(cluster)) - vpcAttachmentClient := vpcs.NewAttachmentsClient(restConnector(cluster)) - - vpcSecurityClient := vpcs.NewSecurityPoliciesClient(restConnector(cluster)) - vpcRuleClient := vpc_sp.NewRulesClient(restConnector(cluster)) - - transitGatewayClient := projects.NewTransitGatewaysClient(restConnector(cluster)) - transitGatewayAttachmentClient := transit_gateways.NewAttachmentsClient(restConnector(cluster)) - - subnetConnectionBindingMapsClient := subnets.NewSubnetConnectionBindingMapsClient(restConnector(cluster)) - - certificateClient := infra.NewCertificatesClient(restConnector(cluster)) - shareClient := infra.NewSharesClient(restConnector(cluster)) - sharedResourceClient := shares.NewResourcesClient(restConnector(cluster)) - lbAppProfileClient := infra.NewLbAppProfilesClient(restConnector(cluster)) - lbPersistenceProfilesClient := infra.NewLbPersistenceProfilesClient(restConnector(cluster)) - lbMonitorProfilesClient := infra.NewLbMonitorProfilesClient(restConnector(cluster)) + connector := restConnector(cluster) + connectorAllowOverwrite := restConnectorAllowOverwrite(cluster) + + queryClient := search.NewQueryClient(connector) + groupClient := domains.NewGroupsClient(connector) + securityClient := domains.NewSecurityPoliciesClient(connector) + ruleClient := security_policies.NewRulesClient(connector) + infraClient := nsx_policy.NewInfraClient(connector) + + clusterControlPlanesClient := enforcement_points.NewClusterControlPlanesClient(connector) + hostTransportNodesClient := enforcement_points.NewHostTransportNodesClient(connector) + realizedEntitiesClient := infra_realized.NewRealizedEntitiesClient(connector) + mpQueryClient := mpsearch.NewQueryClient(connector) + certificatesClient := trust_management.NewCertificatesClient(connector) + principalIdentitiesClient := trust_management.NewPrincipalIdentitiesClient(connector) + withCertificateClient := principal_identities.NewWithCertificateClient(connector) + + lbAppProfileClient := infra.NewLbAppProfilesClient(connector) + lbPersistenceProfilesClient := infra.NewLbPersistenceProfilesClient(connector) + lbMonitorProfilesClient := infra.NewLbMonitorProfilesClient(connector) + + orgRootClient := nsx_policy.NewOrgRootClient(connector) + projectInfraClient := projects.NewInfraClient(connector) + projectClient := orgs.NewProjectsClient(connector) + vpcClient := projects.NewVpcsClient(connector) + vpcConnectivityProfilesClient := projects.NewVpcConnectivityProfilesClient(connector) + ipBlockClient := project_infra.NewIpBlocksClient(connector) + staticRouteClient := vpcs.NewStaticRoutesClient(connector) + natRulesClient := nat.NewNatRulesClient(connector) + vpcGroupClient := vpcs.NewGroupsClient(connector) + portClient := subnets.NewPortsClient(connectorAllowOverwrite) + portStateClient := ports.NewStateClient(connector) + ipPoolClient := subnets.NewIpPoolsClient(connector) + ipAllocationClient := ip_pools.NewIpAllocationsClient(connector) + subnetsClient := vpcs.NewSubnetsClient(connector) + subnetStatusClient := subnets.NewStatusClient(connector) + ipAddressAllocationClient := vpcs.NewIpAddressAllocationsClient(connectorAllowOverwrite) + vpcLBSClient := vpcs.NewVpcLbsClient(connector) + vpcLbVirtualServersClient := vpcs.NewVpcLbVirtualServersClient(connector) + vpcLbPoolsClient := vpcs.NewVpcLbPoolsClient(connector) + vpcAttachmentClient := vpcs.NewAttachmentsClient(connector) + + vpcSecurityClient := vpcs.NewSecurityPoliciesClient(connector) + vpcRuleClient := vpc_sp.NewRulesClient(connector) + + transitGatewayClient := projects.NewTransitGatewaysClient(connector) + transitGatewayAttachmentClient := transit_gateways.NewAttachmentsClient(connector) + + subnetConnectionBindingMapsClient := subnets.NewSubnetConnectionBindingMapsClient(connector) nsxChecker := &NSXHealthChecker{ cluster: cluster, @@ -217,7 +214,7 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { nsxClient := &Client{ NsxConfig: cf, - RestConnector: restConnector(cluster), + RestConnector: connector, QueryClient: queryClient, GroupClient: groupClient, SecurityClient: securityClient, @@ -259,9 +256,6 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { TransitGatewayClient: transitGatewayClient, TransitGatewayAttachmentClient: transitGatewayAttachmentClient, SubnetConnectionBindingMapsClient: subnetConnectionBindingMapsClient, - CertificateClient: certificateClient, - ShareClient: shareClient, - SharedResourceClient: sharedResourceClient, LbAppProfileClient: lbAppProfileClient, LbPersistenceProfilesClient: lbPersistenceProfilesClient, LbMonitorProfilesClient: lbMonitorProfilesClient, diff --git a/pkg/nsx/services/common/policy_tree.go b/pkg/nsx/services/common/policy_tree.go index 625221a8a..410331134 100644 --- a/pkg/nsx/services/common/policy_tree.go +++ b/pkg/nsx/services/common/policy_tree.go @@ -14,6 +14,10 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) +const ( + DefaultHAPIChildrenCount = 500 +) + type LeafDataWrapper[T any] func(leafData T) (*data.StructValue, error) type GetPath[T any] func(obj T) *string type GetId[T any] func(obj T) *string @@ -46,6 +50,8 @@ func getNSXResourcePath[T any](obj T) *string { return v.Path case *model.LBPool: return v.Path + case *model.TlsCertificate: + return v.Path default: log.Error(nil, "Unknown NSX resource type %v", v) return nil @@ -77,9 +83,11 @@ func getNSXResourceId[T any](obj T) *string { case *model.LBService: return v.Id case *model.LBVirtualServer: - return v.Path + return v.Id case *model.LBPool: return v.Id + case *model.TlsCertificate: + return v.Id default: log.Error(nil, "Unknown NSX resource type %v", v) return nil @@ -114,8 +122,10 @@ func leafWrapper[T any](obj T) (*data.StructValue, error) { return WrapLBVirtualServer(v) case *model.LBPool: return WrapLBPool(v) + case *model.TlsCertificate: + return WrapCertificate(v) default: - log.Error(nil, "Unknown NSX resource type %v", v) + log.Error(nil, "Unknown NSX resource type", v) return nil, fmt.Errorf("unsupported NSX resource type %v", v) } } @@ -184,14 +194,14 @@ var ( PolicyResourceOrg = PolicyResourceType{ModelKey: ResourceTypeOrg, PathKey: "orgs"} PolicyResourceProject = PolicyResourceType{ModelKey: ResourceTypeProject, PathKey: "projects"} PolicyResourceVpc = PolicyResourceType{ModelKey: ResourceTypeVpc, PathKey: "vpcs"} - PolicyResourceStaticRoutes = PolicyResourceType{ModelKey: ResourceTypeStaticRoutes, PathKey: "static-routes"} + PolicyResourceStaticRoutes = PolicyResourceType{ModelKey: ResourceTypeStaticRoute, PathKey: "static-routes"} PolicyResourceVpcSubnet = PolicyResourceType{ModelKey: ResourceTypeSubnet, PathKey: "subnets"} PolicyResourceVpcSubnetPort = PolicyResourceType{ModelKey: ResourceTypeSubnetPort, PathKey: "ports"} PolicyResourceVpcSubnetConnectionBindingMap = PolicyResourceType{ModelKey: ResourceTypeSubnetConnectionBindingMap, PathKey: "subnet-connection-binding-maps"} PolicyResourceVpcLBService = PolicyResourceType{ModelKey: ResourceTypeLBService, PathKey: "vpc-lbs"} PolicyResourceVpcLBPool = PolicyResourceType{ModelKey: ResourceTypeLBPool, PathKey: "vpc-lb-pools"} PolicyResourceVpcLBVirtualServer = PolicyResourceType{ModelKey: ResourceTypeLBVirtualServer, PathKey: "vpc-lb-virtual-servers"} - PolicyResourceInfraLBService = PolicyResourceType{ModelKey: ResourceTypeLBService, PathKey: "lbs"} + PolicyResourceInfraLBService = PolicyResourceType{ModelKey: ResourceTypeLBService, PathKey: "lb-services"} PolicyResourceInfraLBPool = PolicyResourceType{ModelKey: ResourceTypeLBPool, PathKey: "lb-pools"} PolicyResourceInfraLBVirtualServer = PolicyResourceType{ModelKey: ResourceTypeLBVirtualServer, PathKey: "lb-virtual-servers"} PolicyResourceVpcIPAddressAllocation = PolicyResourceType{ModelKey: ResourceTypeIPAddressAllocation, PathKey: "ip-address-allocations"} @@ -206,7 +216,6 @@ var ( PolicyPathVpcSubnet PolicyResourcePath[*model.VpcSubnet] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet} PolicyPathVpcSubnetConnectionBindingMap PolicyResourcePath[*model.SubnetConnectionBindingMap] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet, PolicyResourceVpcSubnetConnectionBindingMap} PolicyPathVpcSubnetPort PolicyResourcePath[*model.VpcSubnetPort] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcSubnet, PolicyResourceVpcSubnetPort} - PolicyPathVpc PolicyResourcePath[*model.Vpc] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc} PolicyPathVpcLBPool PolicyResourcePath[*model.LBPool] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBPool} PolicyPathVpcLBService PolicyResourcePath[*model.LBService] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBService} PolicyPathVpcLBVirtualServer PolicyResourcePath[*model.LBVirtualServer] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceVpc, PolicyResourceVpcLBVirtualServer} @@ -219,9 +228,9 @@ var ( PolicyPathProjectShare PolicyResourcePath[*model.Share] = []PolicyResourceType{PolicyResourceOrg, PolicyResourceProject, PolicyResourceInfra, PolicyResourceShare} PolicyPathInfraGroup PolicyResourcePath[*model.Group] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceDomain, PolicyResourceGroup} PolicyPathInfraShare PolicyResourcePath[*model.Share] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceShare} - PolicyPathInfraSharedResource PolicyResourcePath[*model.SharedResource] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceShare, PolicyResourceShare, PolicyResourceSharedResource} + PolicyPathInfraSharedResource PolicyResourcePath[*model.SharedResource] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceShare, PolicyResourceSharedResource} PolicyPathInfraCert PolicyResourcePath[*model.TlsCertificate] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceTlsCertificate} - PolicyPathInfraVirtualServer PolicyResourcePath[*model.LBVirtualServer] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBVirtualServer} + PolicyPathInfraLBVirtualServer PolicyResourcePath[*model.LBVirtualServer] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBVirtualServer} PolicyPathInfraLBPool PolicyResourcePath[*model.LBPool] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBPool} PolicyPathInfraLBService PolicyResourcePath[*model.LBService] = []PolicyResourceType{PolicyResourceInfra, PolicyResourceInfraLBService} ) @@ -283,7 +292,7 @@ func (n *hNode[T]) buildTree(rootType, leafType string) ([]*data.StructValue, er return children, nil } - return wrapChildResourceReference(n.key.resType, n.key.resID, children) + return WrapChildResourceReference(n.key.resType, n.key.resID, children) } type PolicyTreeBuilder[T any] struct { @@ -419,7 +428,7 @@ func (b *PolicyTreeBuilder[T]) parsePathSegments(inputPath string) ([]string, er segmentsCount := len(pathSegments) leafPathKey := b.pathFormat[len(b.pathFormat)-1] - if segmentsCount <= 2 || pathSegments[segmentsCount-2] != leafPathKey { + if segmentsCount < 2 || pathSegments[segmentsCount-2] != leafPathKey { return nil, fmt.Errorf("invalid input path %s for resource %s", inputPath, b.leafType) } if b.hasInnerInfra { @@ -436,7 +445,6 @@ func (b *PolicyTreeBuilder[T]) DeleteMultipleResourcesOnNSX(objects []T, nsxClie if len(objects) == 0 { return nil } - fmt.Println(b.rootType, b.leafType, "count", len(objects)) enforceRevisionCheckParam := false if b.rootType == ResourceTypeOrgRoot { orgRoot, err := b.BuildOrgRoot(objects, "") @@ -502,7 +510,7 @@ func (p *PolicyResourcePath[T]) NewPolicyTreeBuilder() (*PolicyTreeBuilder[T], e }, nil } -func PagingDeleteResources[T any](ctx context.Context, builder *PolicyTreeBuilder[T], objs []T, pageSize int, nsxClient *nsx.Client, delFn func(deletedObjs []T)) error { +func (builder *PolicyTreeBuilder[T]) PagingDeleteResources(ctx context.Context, objs []T, pageSize int, nsxClient *nsx.Client, delFn func(deletedObjs []T)) error { if len(objs) == 0 { return nil } diff --git a/pkg/nsx/services/common/policy_tree_test.go b/pkg/nsx/services/common/policy_tree_test.go index 2bbac0dd8..c1d6b5bd3 100644 --- a/pkg/nsx/services/common/policy_tree_test.go +++ b/pkg/nsx/services/common/policy_tree_test.go @@ -1,15 +1,577 @@ package common import ( - "github.com/stretchr/testify/assert" + "context" + "fmt" + "reflect" "testing" + "time" + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + orgroot_mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" ) +type mockInfraClient struct{} + +func (c *mockInfraClient) Get(*string, *string, *string) (model.Infra, error) { + return model.Infra{}, nil +} + +func (c *mockInfraClient) Patch(model.Infra, *bool) error { + return nil +} + +func (c *mockInfraClient) Update(model.Infra) (model.Infra, error) { + return model.Infra{}, nil +} + func TestBuilder(t *testing.T) { builder, err := PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() require.NoError(t, err) assert.Equal(t, ResourceTypeOrgRoot, builder.rootType) } + +func TestPolicyPathBuilder_DeleteMultipleResourcesOnNSX(t *testing.T) { + testVPCResources(t) + testInfraResources(t) + testProjectInfraResources(t) +} + +func TestPagingDeleteResources(t *testing.T) { + count := 10 + targetSubnets := make([]*model.VpcSubnet, count) + for i := 0; i < count; i++ { + idString := fmt.Sprintf("id-%d", i) + targetSubnets[i] = &model.VpcSubnet{ + Id: String(idString), + Path: String(fmt.Sprintf("/orgs/default/projects/p1/vpcs/vpc1/subnets/%s", idString)), + } + } + + // Verify the happy path. + t.Run("testPagingDeleteResourcesSucceeded", func(t *testing.T) { + testPagingDeleteResourcesSucceeded(t, targetSubnets) + }) + + // Verify the case tha a TimeoutFailed error is returned if context is done. + t.Run("testPagingDeleteResourcesWithContextDone", func(t *testing.T) { + testPagingDeleteResourcesWithContextDone(t, targetSubnets) + }) + + // Verify the case that NSX error is hit when calling HAPI. + t.Run("testPagingDeleteResourcesWithNSXFailure", func(t *testing.T) { + testPagingDeleteResourcesWithNSXFailure(t, targetSubnets) + }) +} + +func testPagingDeleteResourcesSucceeded(t *testing.T, targetSubnets []*model.VpcSubnet) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + builder, err := PolicyPathVpcSubnet.NewPolicyTreeBuilder() + require.NoError(t, err) + + mockRootClient := orgroot_mocks.NewMockOrgRootClient(ctrl) + mockRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) + nsxClient := &nsx.Client{ + OrgRootClient: mockRootClient, + } + + ctx := context.Background() + err = builder.PagingDeleteResources(ctx, targetSubnets, 500, nsxClient, nil) + require.NoError(t, err) +} + +func TestPagingNSXResources(t *testing.T) { + for _, tc := range []struct { + name string + count int + expPages int + }{ + { + name: "no paging on the requests", + count: 50, + expPages: 1, + }, { + name: "paging on the requests", + count: 1501, + expPages: 4, + }, + } { + t.Run(tc.name, func(t *testing.T) { + targetSubnets := make([]*model.VpcSubnet, tc.count) + for i := 0; i < tc.count; i++ { + idString := fmt.Sprintf("id-%d", i) + targetSubnets[i] = &model.VpcSubnet{ + Id: String(idString), + Path: String(fmt.Sprintf("/orgs/default/projects/p1/vpcs/vpc1/subnets/%s", idString)), + } + } + + pagedResources := PagingNSXResources(targetSubnets, 500) + assert.Equal(t, tc.expPages, len(pagedResources)) + + var totalResources []*model.VpcSubnet + for _, pagedSlice := range pagedResources { + totalResources = append(totalResources, pagedSlice...) + } + assert.ElementsMatch(t, targetSubnets, totalResources) + }) + } +} + +func testPagingDeleteResourcesWithContextDone(t *testing.T, targetSubnets []*model.VpcSubnet) { + ctx, cancelFn := context.WithCancel(context.TODO()) + cancelFn() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + nsxClient := &nsx.Client{ + OrgRootClient: orgroot_mocks.NewMockOrgRootClient(ctrl), + } + + builder, err := PolicyPathVpcSubnet.NewPolicyTreeBuilder() + require.NoError(t, err) + + err = builder.PagingDeleteResources(ctx, targetSubnets, 500, nsxClient, nil) + assert.ErrorContains(t, err, "failed because of timeout") +} + +func testPagingDeleteResourcesWithNSXFailure(t *testing.T, targetSubnets []*model.VpcSubnet) { + ctx := context.Background() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockRootClient := orgroot_mocks.NewMockOrgRootClient(ctrl) + mockRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(fmt.Errorf("NSX returned an error")) + nsxClient := &nsx.Client{ + OrgRootClient: mockRootClient, + } + + builder, err := PolicyPathVpcSubnet.NewPolicyTreeBuilder() + require.NoError(t, err) + err = builder.PagingDeleteResources(ctx, targetSubnets, 500, nsxClient, nil) + assert.EqualError(t, err, "NSX returned an error") +} + +func testVPCResources(t *testing.T) { + cases := []struct { + name string + objects any + }{ + { + name: "delete VpcSubnet", + objects: []*model.VpcSubnet{{ + Id: String("subnet1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/subnets/subnet1"), + ResourceType: String(ResourceTypeSubnet), + }}, + }, { + name: "delete VpcSubnetPort", + objects: []*model.VpcSubnetPort{{ + Id: String("port1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/subnets/subnet1/ports/port"), + ResourceType: String(ResourceTypeSubnetPort), + }}, + }, { + name: "delete SubnetConnectionBindingMap", + objects: []*model.SubnetConnectionBindingMap{{ + Id: String("bm1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/subnets/subnet1/subnet-connection-binding-maps/bm1"), + ResourceType: String(ResourceTypeSubnetConnectionBindingMap), + }}, + }, { + name: "delete SecurityPolicy", + objects: []*model.SecurityPolicy{{ + Id: String("sp1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/security-policies/sg1"), + ResourceType: String(ResourceTypeSecurityPolicy), + }}, + }, { + name: "delete SecurityPolicy rule", + objects: []*model.Rule{{ + Id: String("rule1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/security-policies/sg1/rules/rule1"), + Action: String("ALLOW"), + ResourceType: String(ResourceTypeRule), + }}, + }, { + name: "delete VPC group", + objects: []*model.Group{{ + Id: String("group1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/groups/group1"), + ResourceType: String(ResourceTypeGroup), + }}, + }, { + name: "delete VPC IP address allocation", + objects: []*model.VpcIpAddressAllocation{{ + Id: String("allocation1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/ip-address-allocations/allocation1"), + ResourceType: String(ResourceTypeIPAddressAllocation), + }}, + }, { + name: "delete static routes", + objects: []*model.StaticRoutes{{ + Id: String("route1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/static-routes/route1"), + ResourceType: String(ResourceTypeStaticRoute), + }}, + }, { + name: "delete VPC LB service", + objects: []*model.LBService{{ + Id: String("lbs1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/vpc-lbs/lbs1"), + ResourceType: String(ResourceTypeLBService), + }}, + }, { + name: "delete VPC LB pool", + objects: []*model.LBPool{{ + Id: String("pool1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/vpc-lb-pools/pool1"), + ResourceType: String(ResourceTypeLBPool), + }}, + }, { + name: "delete VPC LB virtual server", + objects: []*model.LBVirtualServer{{ + Id: String("vs1"), + Path: String("/orgs/default/projects/p1/vpcs/vpc1/vpc-lb-virtual-servers/vs1"), + ResourceType: String(ResourceTypeLBVirtualServer), + }}, + }, + } + + nsxMockFn := func(ctrl *gomock.Controller, expErr error) (string, *nsx.Client) { + orgRootClient := orgroot_mocks.NewMockOrgRootClient(ctrl) + orgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(expErr) + expErrStr := "" + if expErr != nil { + expErrStr = expErr.Error() + } + return expErrStr, &nsx.Client{ + OrgRootClient: orgRootClient, + } + } + + testVPCResourceDeletion := func(nsxErr error) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + errStr, nsxClient := nsxMockFn(ctrl, nsxErr) + + var err error + switch tc.objects.(type) { + case []*model.VpcSubnet: + res := tc.objects.([]*model.VpcSubnet) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcSubnet, res, nsxClient) + case []*model.VpcSubnetPort: + res := tc.objects.([]*model.VpcSubnetPort) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcSubnetPort, res, nsxClient) + case []*model.SubnetConnectionBindingMap: + res := tc.objects.([]*model.SubnetConnectionBindingMap) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcSubnetConnectionBindingMap, res, nsxClient) + case []*model.SecurityPolicy: + res := tc.objects.([]*model.SecurityPolicy) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcSecurityPolicy, res, nsxClient) + case []*model.Rule: + res := tc.objects.([]*model.Rule) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcSecurityPolicyRule, res, nsxClient) + case []*model.Group: + res := tc.objects.([]*model.Group) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcGroup, res, nsxClient) + case []*model.VpcIpAddressAllocation: + res := tc.objects.([]*model.VpcIpAddressAllocation) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcIPAddressAllocation, res, nsxClient) + case []*model.StaticRoutes: + res := tc.objects.([]*model.StaticRoutes) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcStaticRoutes, res, nsxClient) + case []*model.LBService: + res := tc.objects.([]*model.LBService) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcLBService, res, nsxClient) + case []*model.LBPool: + res := tc.objects.([]*model.LBPool) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcLBPool, res, nsxClient) + case []*model.LBVirtualServer: + res := tc.objects.([]*model.LBVirtualServer) + err = testPolicyPathBuilderDeletion(t, PolicyPathVpcLBVirtualServer, res, nsxClient) + } + + if nsxErr != nil { + assert.EqualError(t, err, errStr) + } else { + assert.NoError(t, err) + } + }) + } + } + + t.Run("testVPCResourceDeletionSucceed", func(t *testing.T) { + testVPCResourceDeletion(nil) + }) + t.Run("testVPCResourceDeletionFailed", func(t *testing.T) { + testVPCResourceDeletion(fmt.Errorf("NSX returns 503")) + }) + +} + +func testInfraResources(t *testing.T) { + cases := []struct { + name string + objects any + }{ + { + name: "delete infra group", + objects: []*model.Group{{ + Id: String("group1"), + Path: String("/infra/domains/default/groups/group1"), + ResourceType: String(ResourceTypeGroup), + }}, + }, { + name: "delete infra share", + objects: []*model.Share{{ + Id: String("share1"), + Path: String("/infra/shares/share1"), + ResourceType: String(ResourceTypeShare), + }}, + }, { + name: "delete infra shared resource", + objects: []*model.SharedResource{{ + Id: String("res1"), + Path: String("/infra/shares/share1/resources/res1"), + ResourceType: String(ResourceTypeSharedResource), + }}, + }, + { + name: "delete infra LB service", + objects: []*model.LBService{{ + Id: String("lbs1"), + Path: String("/infra/lb-services/lbs1"), + ResourceType: String(ResourceTypeLBService), + }}, + }, { + name: "delete infra LB pool", + objects: []*model.LBPool{{ + Id: String("pool1"), + Path: String("/infra/lb-pools/pool1"), + ResourceType: String(ResourceTypeLBPool), + }}, + }, { + name: "delete infra LB virtual server", + objects: []*model.LBVirtualServer{{ + Id: String("vs1"), + Path: String("/infra/lb-virtual-servers/vs1"), + ResourceType: String(ResourceTypeLBVirtualServer), + }}, + }, { + name: "delete infra LB tls certificate", + objects: []*model.TlsCertificate{{ + Id: String("cert1"), + Path: String("/infra/certificates/cert1"), + ResourceType: String(ResourceTypeTlsCertificate), + }}, + }, + } + + nsxMockFn := func(nsxClient *nsx.Client, nsxErr error) (*gomonkey.Patches, string) { + patches := gomonkey.ApplyMethod(reflect.TypeOf(nsxClient.InfraClient), "Patch", func(_ *mockInfraClient, infraParam model.Infra, enforceRevisionCheckParam *bool) error { + return nsxErr + }) + errStr := "" + if nsxErr != nil { + errStr = nsxErr.Error() + } + return patches, errStr + } + + testInfraResourceDeletion := func(nsxErr error) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + nsxClient := &nsx.Client{ + InfraClient: &mockInfraClient{}, + } + patches, expErr := nsxMockFn(nsxClient, nsxErr) + defer patches.Reset() + + var err error + switch tc.objects.(type) { + case []*model.Group: + res := tc.objects.([]*model.Group) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraGroup, res, nsxClient) + case []*model.Share: + res := tc.objects.([]*model.Share) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraShare, res, nsxClient) + case []*model.SharedResource: + res := tc.objects.([]*model.SharedResource) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraSharedResource, res, nsxClient) + case []*model.LBService: + res := tc.objects.([]*model.LBService) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraLBService, res, nsxClient) + case []*model.LBPool: + res := tc.objects.([]*model.LBPool) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraLBPool, res, nsxClient) + case []*model.LBVirtualServer: + res := tc.objects.([]*model.LBVirtualServer) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraLBVirtualServer, res, nsxClient) + case []*model.TlsCertificate: + res := tc.objects.([]*model.TlsCertificate) + err = testPolicyPathBuilderDeletion(t, PolicyPathInfraCert, res, nsxClient) + } + + if nsxErr != nil { + assert.EqualError(t, err, expErr) + } else { + assert.NoError(t, err) + } + }) + } + } + + t.Run("testInfraResourceDeletionSucceeded", func(t *testing.T) { + testInfraResourceDeletion(nil) + }) + t.Run("testInfraResourceDeletionFailed", func(t *testing.T) { + testInfraResourceDeletion(fmt.Errorf("NSX returns 503")) + }) +} + +func testProjectInfraResources(t *testing.T) { + cases := []struct { + name string + objects any + }{ + { + name: "delete project group", + objects: []*model.Group{{ + Id: String("group1"), + Path: String("/orgs/default/projects/p1/infra/domains/default/groups/group1"), + ResourceType: String(ResourceTypeGroup), + }}, + }, { + name: "delete project share", + objects: []*model.Share{{ + Id: String("share1"), + Path: String("/orgs/default/projects/p1/infra/shares/share1"), + ResourceType: String(ResourceTypeShare), + }}, + }, + } + + nsxMockFn := func(ctrl *gomock.Controller, expErr error) (string, *nsx.Client) { + orgRootClient := orgroot_mocks.NewMockOrgRootClient(ctrl) + orgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(expErr) + expErrStr := "" + if expErr != nil { + expErrStr = expErr.Error() + } + return expErrStr, &nsx.Client{ + OrgRootClient: orgRootClient, + } + } + + testProjectResourceDeletion := func(nsxErr error) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + errStr, nsxClient := nsxMockFn(ctrl, nsxErr) + + var err error + switch tc.objects.(type) { + case []*model.Group: + res := tc.objects.([]*model.Group) + err = testPolicyPathBuilderDeletion(t, PolicyPathProjectGroup, res, nsxClient) + case []*model.Share: + res := tc.objects.([]*model.Share) + err = testPolicyPathBuilderDeletion(t, PolicyPathProjectShare, res, nsxClient) + } + + if nsxErr != nil { + assert.EqualError(t, err, errStr) + } else { + assert.NoError(t, err) + } + }) + } + } + t.Run("testProjectResourceDeletionSucceeded", func(t *testing.T) { + testProjectResourceDeletion(nil) + }) + t.Run("testProjectResourceDeletionFailed", func(t *testing.T) { + testProjectResourceDeletion(fmt.Errorf("NSX returns 503")) + }) +} + +func testPolicyPathBuilderDeletion[T any](t *testing.T, resourcePath PolicyResourcePath[T], objects []T, nsxClient *nsx.Client) error { + builder, err := resourcePath.NewPolicyTreeBuilder() + require.Nil(t, err) + return builder.DeleteMultipleResourcesOnNSX(objects, nsxClient) +} + +func TestBuildRootNodePerformance(t *testing.T) { + orgPrefix, orgCount := "org", 1 + projectPrefix, projectCount := "proj", 10 + vpcPrefix, vpcCount := "vpc", 20 + subnetPrefix, subnetCount := "subnet", 10 + bindingPrefix, bindingCount := "binding", 5 + + bindings := make([]*model.SubnetConnectionBindingMap, 0) + for i := 1; i <= orgCount; i++ { + orgID := fmt.Sprintf("%s%d", orgPrefix, i) + for j := 1; j <= projectCount; j++ { + projID := fmt.Sprintf("%s%d", projectPrefix, j) + for k := 1; k <= vpcCount; k++ { + vpcID := fmt.Sprintf("%s%d", vpcPrefix, k) + for l := 1; l <= subnetCount; l++ { + subnetID := fmt.Sprintf("%s%d", subnetPrefix, l) + subnetPath := fmt.Sprintf("/orgs/%s/projects/%s/vpcs/%s/subnets/%s", orgID, projID, vpcID, subnetID) + for m := 0; m <= bindingCount; m++ { + bindingID := fmt.Sprintf("%s%d", bindingPrefix, m) + bindingPath := fmt.Sprintf("%s/subnet-connection-binding-maps/%s", subnetPath, bindingID) + binding := &model.SubnetConnectionBindingMap{ + Id: String(bindingID), + Path: String(bindingPath), + ParentPath: String(subnetPath), + ResourceType: String(ResourceTypeSubnetConnectionBindingMap), + } + bindings = append(bindings, binding) + } + } + } + } + } + + // The total time to build OrgRoot with 10W SubnetConnectionBindngMaps is supposed to less than 10s. + builder, err := PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() + require.NoError(t, err) + start := time.Now() + builder.BuildRootNode(bindings, "") + cost := time.Now().Sub(start) + assert.Truef(t, cost.Seconds() < 3, "It takes %s to build Org root with 10K resources", cost.String()) +} + +func TestParsePathSegments(t *testing.T) { + builder1, err := PolicyPathInfraShare.NewPolicyTreeBuilder() + require.NoError(t, err) + segments, err := builder1.parsePathSegments("/infra/shares/infra-test-share") + require.NoError(t, err) + require.Len(t, segments, 2) + assert.Equal(t, "infra-test-share", segments[len(segments)-1]) + + builder2, err := PolicyPathProjectShare.NewPolicyTreeBuilder() + require.NoError(t, err) + segments2, err := builder2.parsePathSegments("/orgs/default/projects/project1/infra/shares/project-test-share") + require.NoError(t, err) + require.Len(t, segments2, 7) + assert.Equal(t, "project-test-share", segments2[len(segments2)-1]) +} diff --git a/pkg/nsx/services/common/store.go b/pkg/nsx/services/common/store.go index 1fd7e8b82..42e66bb1a 100644 --- a/pkg/nsx/services/common/store.go +++ b/pkg/nsx/services/common/store.go @@ -255,46 +255,86 @@ func formatTagParamTag(paramType, value string) string { func IndexByVPCFunc(obj interface{}) ([]string, error) { switch v := obj.(type) { case *model.Vpc: - return []string{*v.Path}, nil + return getVPCPathFromResourcePath(v.Path) case *model.VpcSubnet: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.VpcSubnetPort: - return getVPCPathFromResourcePath(*v.Path) + return getVPCPathFromResourcePath(v.Path) case *model.SubnetConnectionBindingMap: - return getVPCPathFromResourcePath(*v.Path) + return getVPCPathFromResourcePath(v.Path) case *model.VpcIpAddressAllocation: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.StaticRoutes: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.LBService: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.LBVirtualServer: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.LBPool: - return []string{*v.ParentPath}, nil - + return getVPCPathFromParentPath(v.ParentPath) case *model.SecurityPolicy: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.Group: - return []string{*v.ParentPath}, nil + return getVPCPathFromParentPath(v.ParentPath) case *model.Rule: - return getVPCPathFromResourcePath(*v.Path) - - /* - Infra resources: - LB related: share/sharedResources/cert/LBAppProfile/LBPersistentProfile/LBMonitorProfile - Security Policy related: Share/Group - Project resources: - Security Policy related: Share/Group - */ + return getVPCPathFromResourcePath(v.Path) default: return []string{}, errors.New("indexFunc doesn't support unknown type") } } +func (service *Service) QueryNCPCreatedResources(resourceTypes []string, store Store, additionalQueryFn func(query string) string) error { + resQuery := make([]string, 0) + for _, rt := range resourceTypes { + resQuery = append(resQuery, fmt.Sprintf("%s:%s", ResourceType, rt)) + } + + var query string + if len(resQuery) == 1 { + query = resQuery[0] + } else { + query = fmt.Sprintf("(%s)", strings.Join(resQuery, " OR ")) + } + + query = service.AddNCPClusterTag(query) + if additionalQueryFn != nil { + query = additionalQueryFn(query) + } + count, searchErr := service.SearchResource("", query, store, nil) + if searchErr != nil { + log.Error(searchErr, "Failed to query resources", "query", query) + return searchErr + } + log.V(1).Info("Queried resources", "count", count) + return nil +} + +func (service *Service) AddNCPClusterTag(query string) string { + tagScopeClusterKey := strings.Replace(TagScopeNCPCluster, "/", "\\/", -1) + tagScopeClusterValue := strings.Replace(service.NSXClient.NsxConfig.Cluster, ":", "\\:", -1) + tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) + return query + " AND " + tagParam +} + +func AddNCPCreatedForTag(query string, createdFor string) string { + tagScopeClusterKey := strings.Replace(TagScopeNCPCreateFor, "/", "\\/", -1) + tagScopeClusterValue := strings.Replace(createdFor, ":", "\\:", -1) + tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) + return query + " AND " + tagParam +} -func getVPCPathFromResourcePath(path string) ([]string, error) { - resInfo, err := ParseVPCResourcePath(path) +func getVPCPathFromParentPath(parentPath *string) ([]string, error) { + if parentPath == nil { + return []string{}, errors.New("NSX resource does not set ParentPath field") + } + return []string{*parentPath}, nil +} + +func getVPCPathFromResourcePath(path *string) ([]string, error) { + if path == nil { + return []string{}, errors.New("NSX resource does not set Path field") + } + resInfo, err := ParseVPCResourcePath(*path) if err != nil { return []string{}, err } diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 2214cad19..3592c13c7 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -81,6 +81,7 @@ const ( TagValueShareCreatedForInfra string = "infra" TagValueShareCreatedForProject string = "project" TagValueShareNotCreated string = "notShared" + TagValueDLB string = "DLB" TagValueSLB string = "SLB" AnnotationVPCNetworkConfig string = "nsx.vmware.com/vpc_network_config" AnnotationSharedVPCNamespace string = "nsx.vmware.com/shared_vpc_namespace" @@ -151,7 +152,7 @@ var ( ResourceTypeVpcAttachment = "VpcAttachment" ResourceTypeShare = "Share" ResourceTypeSharedResource = "SharedResource" - ResourceTypeStaticRoutes = "StaticRoutes" + ResourceTypeStaticRoute = "StaticRoutes" ResourceTypeChildLBPool = "ChildLBPool" ResourceTypeChildLBService = "ChildLBService" ResourceTypeChildLBVirtualServer = "ChildLBVirtualServer" diff --git a/pkg/nsx/services/common/wrap.go b/pkg/nsx/services/common/wrap.go index 0f3a447a9..3c87289af 100644 --- a/pkg/nsx/services/common/wrap.go +++ b/pkg/nsx/services/common/wrap.go @@ -22,15 +22,15 @@ func (service *Service) WrapOrgRoot(children []*data.StructValue) (*model.OrgRoo func (service *Service) WrapOrg(org string, children []*data.StructValue) ([]*data.StructValue, error) { targetType := ResourceTypeOrg - return wrapChildResourceReference(targetType, org, children) + return WrapChildResourceReference(targetType, org, children) } func (service *Service) WrapProject(nsxtProject string, children []*data.StructValue) ([]*data.StructValue, error) { targetType := ResourceTypeProject - return wrapChildResourceReference(targetType, nsxtProject, children) + return WrapChildResourceReference(targetType, nsxtProject, children) } -func wrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) { +func WrapChildResourceReference(targetType, id string, children []*data.StructValue) ([]*data.StructValue, error) { resourceType := ResourceTypeChildResourceReference childProject := model.ChildResourceReference{ Id: &id, @@ -145,7 +145,7 @@ func WrapVpcSubnet(subnet *model.VpcSubnet) (*data.StructValue, error) { } func WrapStaticRoutes(route *model.StaticRoutes) (*data.StructValue, error) { - route.ResourceType = &ResourceTypeStaticRoutes + route.ResourceType = &ResourceTypeStaticRoute childRoute := model.ChildStaticRoutes{ Id: route.Id, MarkedForDelete: route.MarkedForDelete, @@ -265,7 +265,7 @@ func WrapLBPool(lbPool *model.LBPool) (*data.StructValue, error) { } func WrapVPC(vpc *model.Vpc) (*data.StructValue, error) { - vpc.ResourceType = pointy.String(ResourceTypeVpc) + vpc.ResourceType = &ResourceTypeVpc childVpc := model.ChildVpc{ Id: vpc.Id, MarkedForDelete: vpc.MarkedForDelete, @@ -279,6 +279,21 @@ func WrapVPC(vpc *model.Vpc) (*data.StructValue, error) { return dataValue.(*data.StructValue), nil } +func WrapCertificate(cert *model.TlsCertificate) (*data.StructValue, error) { + cert.ResourceType = &ResourceTypeTlsCertificate + childCert := model.ChildTlsCertificate{ + Id: cert.Id, + MarkedForDelete: cert.MarkedForDelete, + ResourceType: "ChildTlsCertificate", + TlsCertificate: cert, + } + dataValue, errs := NewConverter().ConvertToVapi(childCert, childCert.GetType__()) + if len(errs) > 0 { + return nil, errs[0] + } + return dataValue.(*data.StructValue), nil +} + func wrapInfra(children []*data.StructValue) *model.Infra { // This is the outermost layer of the hierarchy infra client. // It doesn't need ID field. diff --git a/pkg/nsx/services/ipaddressallocation/cleanup.go b/pkg/nsx/services/ipaddressallocation/cleanup.go new file mode 100644 index 000000000..21ff1eec5 --- /dev/null +++ b/pkg/nsx/services/ipaddressallocation/cleanup.go @@ -0,0 +1,40 @@ +package ipaddressallocation + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +// CleanupVPCChildResources is deleting all the NSX VpcIPAddressAllocations in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with an auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached +// VpcIPAddressAllocations on NSX and in local cache. +func (service *IPAddressAllocationService) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + allocations, err := service.ipAddressAllocationStore.GetByVPCPath(vpcPath) + if err != nil { + log.Error(err, "Failed to list VpcIPAddressAllocations under the VPC", "path", vpcPath) + } + if len(allocations) == 0 { + return nil + } + // Delete resources from the store and return. + service.ipAddressAllocationStore.DeleteMultipleObjects(allocations) + return nil + } + + allocations := make([]*model.VpcIpAddressAllocation, 0) + // Mark the resources for delete. + for _, obj := range service.ipAddressAllocationStore.List() { + allocation := obj.(*model.VpcIpAddressAllocation) + allocation.MarkedForDelete = &MarkedForDelete + allocations = append(allocations, allocation) + } + + return service.builder.PagingDeleteResources(ctx, allocations, common.DefaultHAPIChildrenCount, service.NSXClient, func(deletedObjs []*model.VpcIpAddressAllocation) { + service.ipAddressAllocationStore.DeleteMultipleObjects(deletedObjs) + }) +} diff --git a/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go index e8469ac48..3f1241915 100644 --- a/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go +++ b/pkg/nsx/services/ipaddressallocation/ipaddressallocation.go @@ -27,17 +27,23 @@ type IPAddressAllocationService struct { common.Service ipAddressAllocationStore *IPAddressAllocationStore VPCService common.VPCServiceProvider + builder *common.PolicyTreeBuilder[*model.VpcIpAddressAllocation] } func InitializeIPAddressAllocation(service common.Service, vpcService common.VPCServiceProvider, includeNCP bool) (*IPAddressAllocationService, error) { + builder, _ := common.PolicyPathVpcIPAddressAllocation.NewPolicyTreeBuilder() + wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) - ipAddressAllocationService := &IPAddressAllocationService{Service: service, VPCService: vpcService} + ipAddressAllocationService := &IPAddressAllocationService{Service: service, VPCService: vpcService, builder: builder} ipAddressAllocationService.ipAddressAllocationStore = &IPAddressAllocationStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{common.TagScopeIPAddressAllocationCRUID: indexFunc}), + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeIPAddressAllocationCRUID: indexFunc, + common.IndexByVPCPathFuncKey: common.IndexByVPCFunc, + }), BindingType: model.VpcIpAddressAllocationBindingType(), }} @@ -270,20 +276,3 @@ func (service *IPAddressAllocationService) GetIPAddressAllocationUID(nsxIPAddres } return "" } - -func (service *IPAddressAllocationService) Cleanup(ctx context.Context) error { - keys := service.ListIPAddressAllocationKeys() - log.Info("Cleaning up ipaddressallocation", "count", len(keys)) - for _, key := range keys { - select { - case <-ctx.Done(): - return util.TimeoutFailed - default: - err := service.DeleteIPAddressAllocation(key) - if err != nil { - return err - } - } - } - return nil -} diff --git a/pkg/nsx/services/ipaddressallocation/ipaddressallocation_test.go b/pkg/nsx/services/ipaddressallocation/ipaddressallocation_test.go index 9be1c5dd2..cde4852a0 100644 --- a/pkg/nsx/services/ipaddressallocation/ipaddressallocation_test.go +++ b/pkg/nsx/services/ipaddressallocation/ipaddressallocation_test.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/ipaddressallocation" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" @@ -90,12 +91,15 @@ func TestIPAddressAllocationService_DeleteIPAddressAllocation(t *testing.T) { defer mockController.Finish() var tc *bindings.TypeConverter + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), + ParentPath: String(vpcPath), } var j interface{} = m return j, nil @@ -115,8 +119,8 @@ func TestIPAddressAllocationService_DeleteIPAddressAllocation(t *testing.T) { } id := util.GenerateIDByObject(srObj) tags := util.BuildBasicTags(service.NSXConfig.Cluster, srObj, "") - path := "/orgs/default/projects/project-1/vpcs/vpc-1" - sr1 := &model.VpcIpAddressAllocation{Id: &id, Path: &path, Tags: tags} + path := fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id) + sr1 := &model.VpcIpAddressAllocation{Id: &id, Path: &path, Tags: tags, ParentPath: &vpcPath} // no record found mockVPCIPAddressAllocationclient.EXPECT().Delete(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Times(0) @@ -137,13 +141,16 @@ func TestIPAddressAllocationService_CreateorUpdateIPAddressAllocation(t *testing service, mockController, mockVPCIPAddressallocationclient := createIPAddressAllocationService(t) defer mockController.Finish() + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), } var j interface{} = m return j, nil @@ -167,6 +174,8 @@ func TestIPAddressAllocationService_CreateorUpdateIPAddressAllocation(t *testing m := model.VpcIpAddressAllocation{ Id: &mId, Tags: []model.Tag{{Tag: &tag, Scope: &scope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), AllocationIps: &cidr, } mockVPCIPAddressallocationclient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(m, nil).Times(2) @@ -192,18 +201,22 @@ func TestIPAddressAllocationService_CreateorUpdateIPAddressAllocation(t *testing } func TestIPAddressAllocationService_Cleanup(t *testing.T) { - service, mockController, mockVPCIPAddressAllocationclient := createIPAddressAllocationService(t) + service, mockController, _ := createIPAddressAllocationService(t) defer mockController.Finish() + mockOrgRootClient := mock_org_root.NewMockOrgRootClient(mockController) + service.NSXClient.OrgRootClient = mockOrgRootClient + + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" - path := "/orgs/default/projects/project-1/vpcs/vpc-1" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - Path: &path, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), + ParentPath: &vpcPath, } var j interface{} = m return j, nil @@ -215,11 +228,11 @@ func TestIPAddressAllocationService_Cleanup(t *testing.T) { assert.NoError(t, err) // Set up expectations - mockVPCIPAddressAllocationclient.EXPECT().Delete("default", "project-1", "vpc-1", "test_id").Return(nil).Times(1) + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) // Call Cleanup ctx := context.Background() - err = returnService.Cleanup(ctx) + err = returnService.CleanupVPCChildResources(ctx, "") // Assert assert.NoError(t, err) @@ -231,7 +244,7 @@ func TestIPAddressAllocationService_Cleanup(t *testing.T) { returnService, err = InitializeIPAddressAllocation(service.Service, vpcService, false) assert.NoError(t, err) - err = returnService.Cleanup(cancelledCtx) + err = returnService.CleanupVPCChildResources(cancelledCtx, "") assert.Error(t, err) } @@ -239,15 +252,16 @@ func TestIPAddressAllocationService_ListIPAddressAllocationID(t *testing.T) { service, mockController, _ := createIPAddressAllocationService(t) defer mockController.Finish() + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" - path := "/orgs/default/projects/project-1/vpcs/vpc-1" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - Path: &path, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), } var j interface{} = m return j, nil @@ -274,10 +288,9 @@ func TestIPAddressAllocationService_ListIPAddressAllocationID(t *testing.T) { id1 := util.GenerateIDByObject(ipa1) id2 := util.GenerateIDByObject(ipa2) - path := "/orgs/default/projects/project-1/vpcs/vpc-1" - sr1 := &model.VpcIpAddressAllocation{Id: &id1, Path: &path, Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa1, "")} - sr2 := &model.VpcIpAddressAllocation{Id: &id2, Path: &path, Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa2, "")} + sr1 := &model.VpcIpAddressAllocation{Id: &id1, ParentPath: &vpcPath, Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id1)), Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa1, "")} + sr2 := &model.VpcIpAddressAllocation{Id: &id2, ParentPath: &vpcPath, Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id2)), Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa2, "")} returnService.ipAddressAllocationStore.Add(sr1) returnService.ipAddressAllocationStore.Add(sr2) @@ -293,15 +306,16 @@ func TestIPAddressAllocationService_ListIPAddressAllocationKeys(t *testing.T) { service, mockController, _ := createIPAddressAllocationService(t) defer mockController.Finish() + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" - path := "/orgs/default/projects/project-1/vpcs/vpc-1" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - Path: &path, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), } var j interface{} = m return j, nil @@ -328,10 +342,9 @@ func TestIPAddressAllocationService_ListIPAddressAllocationKeys(t *testing.T) { id1 := util.GenerateIDByObject(ipa1) id2 := util.GenerateIDByObject(ipa2) - path := "/orgs/default/projects/project-1/vpcs/vpc-1" - sr1 := &model.VpcIpAddressAllocation{Id: &id1, Path: &path, Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa1, "")} - sr2 := &model.VpcIpAddressAllocation{Id: &id2, Path: &path, Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa2, "")} + sr1 := &model.VpcIpAddressAllocation{Id: &id1, ParentPath: &vpcPath, Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id1)), Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa1, "")} + sr2 := &model.VpcIpAddressAllocation{Id: &id2, ParentPath: &vpcPath, Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id2)), Tags: util.BuildBasicTags(service.NSXConfig.Cluster, ipa2, "")} returnService.ipAddressAllocationStore.Add(sr1) returnService.ipAddressAllocationStore.Add(sr2) @@ -347,14 +360,15 @@ func TestIPAddressAllocationService_CreateOrUpdateIPAddressAllocation_Errors(t * service, mockController, _ := createIPAddressAllocationService(t) defer mockController.Finish() + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId := "test_id" - path := "/orgs/default/projects/project-1/vpcs/vpc-1" m := model.VpcIpAddressAllocation{ - Id: &mId, - Path: &path, + Id: &mId, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), } var j interface{} = m return j, nil @@ -417,13 +431,16 @@ func TestIPAddressAllocationService_DeleteIPAddressAllocation_Errors(t *testing. service, mockController, mockVPCIPAddressAllocationclient := createIPAddressAllocationService(t) defer mockController.Finish() + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + Id: &mId, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("/orgs/default/projects/project-1/vpcs/vpc-1/ip-address-allocations/%s", mId)), + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, } var j interface{} = m return j, nil @@ -443,8 +460,7 @@ func TestIPAddressAllocationService_DeleteIPAddressAllocation_Errors(t *testing. } id := util.GenerateIDByObject(srObj) tags := util.BuildBasicTags(service.NSXConfig.Cluster, srObj, "") - path := "/orgs/default/projects/project-1/vpcs/vpc-1" - sr1 := &model.VpcIpAddressAllocation{Id: &id, Path: &path, Tags: tags} + sr1 := &model.VpcIpAddressAllocation{Id: &id, ParentPath: &vpcPath, Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, id)), Tags: tags} returnservice.ipAddressAllocationStore.Add(sr1) @@ -473,14 +489,15 @@ func TestIPAddressAllocationService_Cleanup_Error(t *testing.T) { service, mockController, _ := createIPAddressAllocationService(t) defer mockController.Finish() var tc *bindings.TypeConverter + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" patchConvertToGolang := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { mId, mTag, mScope := "test_id", "test_tag", "test_scope" - path := "/orgs/default/projects/project-1/vpcs/vpc-1" m := model.VpcIpAddressAllocation{ - Id: &mId, - Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, - Path: &path, + Id: &mId, + Tags: []model.Tag{{Tag: &mTag, Scope: &mScope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/ip-address-allocations/%s", vpcPath, mId)), } var j interface{} = m return j, nil @@ -489,23 +506,22 @@ func TestIPAddressAllocationService_Cleanup_Error(t *testing.T) { vpcService := &vpc.VPCService{} returnservice, _ := InitializeIPAddressAllocation(service.Service, vpcService, false) + mockOrgRootClient := mock_org_root.NewMockOrgRootClient(mockController) + returnservice.NSXClient.OrgRootClient = mockOrgRootClient // Add a test IPAddressAllocation to the store testIPA := &model.VpcIpAddressAllocation{ - Id: String("test-id"), - Path: String("/test/path"), + Id: String("test-id"), + Path: String("/test/path"), + ParentPath: String(vpcPath), } returnservice.ipAddressAllocationStore.Add(testIPA) // Test case: DeleteIPAddressAllocation error - patchDeleteIPAddressAllocation := gomonkey.ApplyMethod(reflect.TypeOf(returnservice), "DeleteIPAddressAllocation", - func(_ *IPAddressAllocationService, _ interface{}) error { - return fmt.Errorf("delete error") - }) - defer patchDeleteIPAddressAllocation.Reset() + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(fmt.Errorf("delete error")) ctx := context.Background() - err := returnservice.Cleanup(ctx) + err := returnservice.CleanupVPCChildResources(ctx, "") assert.Error(t, err) assert.Contains(t, err.Error(), "delete error") } diff --git a/pkg/nsx/services/ipaddressallocation/store.go b/pkg/nsx/services/ipaddressallocation/store.go index e0efdcdac..2bee7f5c0 100644 --- a/pkg/nsx/services/ipaddressallocation/store.go +++ b/pkg/nsx/services/ipaddressallocation/store.go @@ -88,3 +88,22 @@ func (ipAddressAllocationStore *IPAddressAllocationStore) GetByIndex(uid types.U } return nsxIPAddressAllocation, nil } + +func (ipAddressAllocationStore *IPAddressAllocationStore) GetByVPCPath(vpcPath string) ([]*model.VpcIpAddressAllocation, error) { + objs, err := ipAddressAllocationStore.ResourceStore.ByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if err != nil { + return nil, err + } + allocations := make([]*model.VpcIpAddressAllocation, len(objs)) + for i, obj := range objs { + allocation := obj.(*model.VpcIpAddressAllocation) + allocations[i] = allocation + } + return allocations, nil +} + +func (ipAddressAllocationStore *IPAddressAllocationStore) DeleteMultipleObjects(allocations []*model.VpcIpAddressAllocation) { + for _, allocation := range allocations { + ipAddressAllocationStore.Delete(allocation) + } +} diff --git a/pkg/nsx/services/securitypolicy/builder.go b/pkg/nsx/services/securitypolicy/builder.go index 754294994..6e7649f36 100644 --- a/pkg/nsx/services/securitypolicy/builder.go +++ b/pkg/nsx/services/securitypolicy/builder.go @@ -1829,7 +1829,7 @@ func (service *SecurityPolicyService) buildSharedWith(vpcInfo *common.VPCResourc return &sharedWith } if projectGroupShared { - sharedWithPath := fmt.Sprintf("/orgs/%s/projects/%s/vpcs/%s", vpcInfo.OrgID, vpcInfo.ProjectID, vpcInfo.VPCID) + sharedWithPath := vpcInfo.GetVPCPath() sharedWith = append(sharedWith, sharedWithPath) return &sharedWith } diff --git a/pkg/nsx/services/securitypolicy/cleanup.go b/pkg/nsx/services/securitypolicy/cleanup.go new file mode 100644 index 000000000..0fa6ad8f3 --- /dev/null +++ b/pkg/nsx/services/securitypolicy/cleanup.go @@ -0,0 +1,174 @@ +package securitypolicy + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +// CleanupVPCChildResources is called when cleaning up the VPC related resources. For the resources in an auto-created +// VPC, this function is called after the VPC is deleted on NSX, so the provider only needs to clean up with the local +// cache. For the resources in a pre-created VPC, this function is called to delete resources on NSX and in the local cache. +func (service *SecurityPolicyService) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + if err := service.cleanupRulesByVPC(ctx, vpcPath); err != nil { + log.Error(err, "Failed to clean up Rule by VPC", "VPC", vpcPath) + return err + } + + if err := service.cleanupSecurityPoliciesByVPC(ctx, vpcPath); err != nil { + log.Error(err, "Failed to clean up SecurityPolicy by VPC", "VPC", vpcPath) + return err + } + if err := service.cleanupGroupsByVPC(ctx, vpcPath); err != nil { + log.Error(err, "Failed to clean up Group by VPC", "VPC", vpcPath) + return err + } + return nil +} + +// cleanupSecurityPoliciesByVPC is deleting all the NSX security policies in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached seurity policies +// on NSX and in local cache. +func (service *SecurityPolicyService) cleanupSecurityPoliciesByVPC(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + securityPolicies := service.securityPolicyStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if len(securityPolicies) == 0 { + return nil + } + // Delete resources from the store and return. + service.securityPolicyStore.DeleteMultipleObjects(securityPolicies) + return nil + } + + securityPolicies := make([]*model.SecurityPolicy, 0) + // Mark the resources for delete. + for _, obj := range service.securityPolicyStore.List() { + sp := obj.(*model.SecurityPolicy) + sp.MarkedForDelete = &MarkedForDelete + securityPolicies = append(securityPolicies, sp) + } + + return service.securityPolicyBuilder.PagingDeleteResources(ctx, securityPolicies, common.DefaultHAPIChildrenCount, service.NSXClient, func(deletedObjs []*model.SecurityPolicy) { + service.securityPolicyStore.DeleteMultipleObjects(deletedObjs) + }) +} + +// cleanupRulesByVPC is deleting all the NSX rules in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached rules +// on NSX and in local cache. +func (service *SecurityPolicyService) cleanupRulesByVPC(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + rules := service.ruleStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if len(rules) == 0 { + return nil + } + // Delete resources from the store and return. + service.ruleStore.DeleteMultipleObjects(rules) + return nil + } + + rules := make([]*model.Rule, 0) + // Mark the resources for delete. + for _, obj := range service.ruleStore.List() { + rule := obj.(*model.Rule) + rule.MarkedForDelete = &MarkedForDelete + rules = append(rules, rule) + } + + return service.ruleBuilder.PagingDeleteResources(ctx, rules, common.DefaultHAPIChildrenCount, service.NSXClient, func(deletedObjs []*model.Rule) { + service.ruleStore.DeleteMultipleObjects(deletedObjs) + }) +} + +// cleanupRulesByVPC is deleting all the NSX groups in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached groups +// on NSX and in local cache. +func (service *SecurityPolicyService) cleanupGroupsByVPC(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + groups := service.groupStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if len(groups) == 0 { + return nil + } + // Delete resources from the store and return. + service.groupStore.DeleteMultipleObjects(groups) + return nil + } + + return cleanGroups(ctx, service.groupStore, service.groupBuilder, service.NSXClient) +} + +// CleanupInfraResources is to clean up the resources created by SecurityPolicyService under path /infra. +func (service *SecurityPolicyService) CleanupInfraResources(ctx context.Context) error { + for _, config := range []struct { + store *ShareStore + builder *common.PolicyTreeBuilder[*model.Share] + }{ + { + store: service.projectShareStore, + builder: service.projectShareBuilder, + }, { + store: service.infraShareStore, + builder: service.infraShareBuilder, + }, + } { + if err := cleanShares(ctx, config.store, config.builder, service.NSXClient); err != nil { + return err + } + } + for _, config := range []struct { + store *GroupStore + builder *common.PolicyTreeBuilder[*model.Group] + }{ + { + store: service.projectGroupStore, + builder: service.projectGroupBuilder, + }, { + store: service.infraGroupStore, + builder: service.infraGroupBuilder, + }, + } { + if err := cleanGroups(ctx, config.store, config.builder, service.NSXClient); err != nil { + return err + } + } + return nil +} + +func cleanShares(ctx context.Context, store *ShareStore, builder *common.PolicyTreeBuilder[*model.Share], nsxClient *nsx.Client) error { + cachedObjs := store.List() + if len(cachedObjs) == 0 { + return nil + } + log.Info("Cleaning up Shares", "Count", len(cachedObjs)) + cachedShares := make([]*model.Share, 0) + for _, obj := range cachedObjs { + share := obj.(*model.Share) + share.MarkedForDelete = &MarkedForDelete + cachedShares = append(cachedShares, share) + } + + return builder.PagingDeleteResources(ctx, cachedShares, common.DefaultHAPIChildrenCount, nsxClient, func(deletedObjs []*model.Share) { + store.DeleteMultipleObjects(deletedObjs) + }) +} + +func cleanGroups(ctx context.Context, store *GroupStore, builder *common.PolicyTreeBuilder[*model.Group], nsxClient *nsx.Client) error { + cachedObjs := store.List() + log.Info("Cleaning up Groups", "Count", len(cachedObjs)) + + cachedGroups := make([]*model.Group, 0) + for _, obj := range cachedObjs { + group := obj.(*model.Group) + group.MarkedForDelete = &MarkedForDelete + cachedGroups = append(cachedGroups, group) + } + return builder.PagingDeleteResources(ctx, cachedGroups, common.DefaultHAPIChildrenCount, nsxClient, func(deletedObjs []*model.Group) { + store.DeleteMultipleObjects(deletedObjs) + }) +} diff --git a/pkg/nsx/services/securitypolicy/cleanup_test.go b/pkg/nsx/services/securitypolicy/cleanup_test.go new file mode 100644 index 000000000..e5d93a9d4 --- /dev/null +++ b/pkg/nsx/services/securitypolicy/cleanup_test.go @@ -0,0 +1,206 @@ +package securitypolicy + +import ( + "context" + "fmt" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +var ( + projectPath = "/orgs/default/projects/project-1" + infraShareId = "infra-share" + projectShareId = "proj-share" + infraGroupId = "infra-group" + projectGroupId = "proj-group" + vpcPath = fmt.Sprintf("%s/vpcs/vpc-1", projectPath) + vpcRuleId = "rule0" + vpcGroupId = "vpc-group" + vpcSecurityPolicyId = "security-policy" + + infraResourceTags = []model.Tag{ + { + Scope: String(common.TagScopeSecurityPolicyCRUID), + Tag: String("test-security-policy-cr-id"), + }, { + Scope: String(common.TagScopeNetworkPolicyUID), + Tag: String("test-network-policy-id"), + }, + } +) + +func TestCleanupInfraResources(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + orgRootClient := mock_org_root.NewMockOrgRootClient(ctrl) + svc := prepareServiceForCleanup(orgRootClient) + + infraShare := &model.Share{ + Id: String(infraShareId), + Path: String(fmt.Sprintf("/infra/shares/%s", infraShareId)), + ParentPath: String("/infra"), + Tags: infraResourceTags, + } + svc.infraShareStore.Add(infraShare) + + projectShare := &model.Share{ + Id: String(projectShareId), + Path: String(fmt.Sprintf("%s/shares/%s", projectPath, infraShareId)), + ParentPath: String(projectPath), + Tags: infraResourceTags, + } + svc.projectShareStore.Add(projectShare) + + infraGroup := &model.Group{ + Id: String(infraGroupId), + Path: String(fmt.Sprintf("/infra/domains/default/groups/%s", infraGroupId)), + ParentPath: String("/infra/domains/default"), + Tags: infraResourceTags, + } + svc.infraGroupStore.Add(infraGroup) + + projectGroup := &model.Group{ + Id: String(projectGroupId), + Path: String(fmt.Sprintf("%s/infra/domains/default/groups/%s", projectPath, projectGroupId)), + ParentPath: String(fmt.Sprintf("%s/infra/domains/default", projectPath)), + Tags: infraResourceTags, + } + svc.projectGroupStore.Add(projectGroup) + + assert.Equal(t, 1, len(svc.infraShareStore.List())) + assert.Equal(t, 1, len(svc.infraGroupStore.List())) + assert.Equal(t, 1, len(svc.projectGroupStore.List())) + assert.Equal(t, 1, len(svc.projectShareStore.List())) + + patches := gomonkey.ApplyMethodSeq(svc.NSXClient.InfraClient, "Patch", []gomonkey.OutputCell{{ + Values: gomonkey.Params{nil}, + Times: 2, + }}) + defer patches.Reset() + orgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).Times(2) + + ctx := context.Background() + err := svc.CleanupInfraResources(ctx) + require.NoError(t, err) + assert.Equal(t, 0, len(svc.infraShareStore.List())) + assert.Equal(t, 0, len(svc.infraGroupStore.List())) + assert.Equal(t, 0, len(svc.projectGroupStore.List())) + assert.Equal(t, 0, len(svc.projectShareStore.List())) +} + +func TestCleanupVPCChildResources(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + securityPolicyPath := fmt.Sprintf("%s/security-policies/%s", vpcPath, vpcSecurityPolicyId) + securityPolicy := &model.SecurityPolicy{ + Id: String(vpcSecurityPolicyId), + Path: String(securityPolicyPath), + ParentPath: String(vpcPath), + Tags: infraResourceTags, + } + vpcRule := &model.Rule{ + Id: String(vpcRuleId), + Path: String(fmt.Sprintf("%s/rules/%s", securityPolicyPath, vpcRuleId)), + ParentPath: String(securityPolicyPath), + Tags: infraResourceTags, + } + vpcGroup := &model.Group{ + Id: String(vpcGroupId), + Path: String(fmt.Sprintf("%s/security-policies/%s", vpcPath, vpcGroupId)), + ParentPath: String(vpcPath), + Tags: infraResourceTags, + } + + for _, tc := range []struct { + name string + mockFn func() *mock_org_root.MockOrgRootClient + vpcPath string + }{ + { + name: "clean up with a given VPC path", + mockFn: func() *mock_org_root.MockOrgRootClient { + return mock_org_root.NewMockOrgRootClient(ctrl) + }, + vpcPath: vpcPath, + }, { + name: "clean up with all resources", + mockFn: func() *mock_org_root.MockOrgRootClient { + client := mock_org_root.NewMockOrgRootClient(ctrl) + client.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).Times(3) + return client + }, + vpcPath: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + orgRootClient := tc.mockFn() + svc := prepareServiceForCleanup(orgRootClient) + svc.ruleStore.Add(vpcRule) + rulesBeforeCleanup := svc.ruleStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 1, len(rulesBeforeCleanup)) + + svc.securityPolicyStore.Add(securityPolicy) + securityPoliciesBeforeCleanup := svc.securityPolicyStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 1, len(securityPoliciesBeforeCleanup)) + + svc.groupStore.Add(vpcGroup) + groupsBeforeCleanup := svc.groupStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 1, len(groupsBeforeCleanup)) + + ctx := context.Background() + err := svc.CleanupVPCChildResources(ctx, tc.vpcPath) + require.NoError(t, err) + if tc.vpcPath != "" { + rulesAfterCleanup := svc.ruleStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 0, len(rulesAfterCleanup)) + + securityPoliciesAfterCleanup := svc.securityPolicyStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 0, len(securityPoliciesAfterCleanup)) + + groupsAfterCleanup := svc.groupStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + assert.Equal(t, 0, len(groupsAfterCleanup)) + } else { + assert.Equal(t, 0, len(svc.ruleStore.List())) + assert.Equal(t, 0, len(svc.securityPolicyStore.List())) + assert.Equal(t, 0, len(svc.groupStore.List())) + } + }) + } +} + +func prepareServiceForCleanup(orgRootClient *mock_org_root.MockOrgRootClient) *SecurityPolicyService { + svc := &SecurityPolicyService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + OrgRootClient: orgRootClient, + InfraClient: &fakeInfraClient{}, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + } + svc.setUpStore(common.TagScopeSecurityPolicyUID, true) + svc.securityPolicyBuilder, _ = common.PolicyPathVpcSecurityPolicy.NewPolicyTreeBuilder() + svc.ruleBuilder, _ = common.PolicyPathVpcSecurityPolicyRule.NewPolicyTreeBuilder() + svc.groupBuilder, _ = common.PolicyPathVpcGroup.NewPolicyTreeBuilder() + svc.infraShareBuilder, _ = common.PolicyPathInfraShare.NewPolicyTreeBuilder() + svc.projectShareBuilder, _ = common.PolicyPathProjectShare.NewPolicyTreeBuilder() + svc.projectGroupBuilder, _ = common.PolicyPathProjectGroup.NewPolicyTreeBuilder() + svc.infraGroupBuilder, _ = common.PolicyPathInfraGroup.NewPolicyTreeBuilder() + return svc +} diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 4e3e868ad..d2be2a856 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -4,7 +4,6 @@ package securitypolicy import ( - "context" "errors" "fmt" "os" @@ -48,6 +47,14 @@ type SecurityPolicyService struct { projectGroupStore *GroupStore projectShareStore *ShareStore vpcService common.VPCServiceProvider + + securityPolicyBuilder *common.PolicyTreeBuilder[*model.SecurityPolicy] + ruleBuilder *common.PolicyTreeBuilder[*model.Rule] + groupBuilder *common.PolicyTreeBuilder[*model.Group] + infraGroupBuilder *common.PolicyTreeBuilder[*model.Group] + projectGroupBuilder *common.PolicyTreeBuilder[*model.Group] + infraShareBuilder *common.PolicyTreeBuilder[*model.Share] + projectShareBuilder *common.PolicyTreeBuilder[*model.Share] } type GroupShare struct { @@ -67,7 +74,7 @@ func GetSecurityService(service common.Service, vpcService common.VPCServiceProv defer lock.Unlock() if securityService == nil { var err error - if securityService, err = InitializeSecurityPolicy(service, vpcService); err != nil { + if securityService, err = InitializeSecurityPolicy(service, vpcService, false); err != nil { log.Error(err, "Failed to initialize SecurityPolicy service") os.Exit(1) } @@ -77,21 +84,33 @@ func GetSecurityService(service common.Service, vpcService common.VPCServiceProv } // InitializeSecurityPolicy sync NSX resources -func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServiceProvider) (*SecurityPolicyService, error) { +func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServiceProvider, forCleanUp bool) (*SecurityPolicyService, error) { wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) wg.Add(7) - securityPolicyService := &SecurityPolicyService{Service: service} + securityPolicyService := &SecurityPolicyService{ + Service: service, + } + + if forCleanUp { + securityPolicyService.securityPolicyBuilder, _ = common.PolicyPathVpcSecurityPolicy.NewPolicyTreeBuilder() + securityPolicyService.ruleBuilder, _ = common.PolicyPathVpcSecurityPolicyRule.NewPolicyTreeBuilder() + securityPolicyService.groupBuilder, _ = common.PolicyPathVpcGroup.NewPolicyTreeBuilder() + securityPolicyService.infraShareBuilder, _ = common.PolicyPathInfraShare.NewPolicyTreeBuilder() + securityPolicyService.projectShareBuilder, _ = common.PolicyPathProjectShare.NewPolicyTreeBuilder() + securityPolicyService.projectGroupBuilder, _ = common.PolicyPathProjectGroup.NewPolicyTreeBuilder() + securityPolicyService.infraGroupBuilder, _ = common.PolicyPathInfraGroup.NewPolicyTreeBuilder() + } if IsVPCEnabled(securityPolicyService) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID } indexScope := common.TagValueScopeSecurityPolicyUID - securityPolicyService.setUpStore(indexScope) + securityPolicyService.setUpStore(indexScope, forCleanUp) securityPolicyService.vpcService = vpcService infraShareTag := []model.Tag{ @@ -142,29 +161,35 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi return securityPolicyService, nil } -func (s *SecurityPolicyService) setUpStore(indexScope string) { +func (s *SecurityPolicyService) setUpStore(indexScope string, indexWithVPCPath bool) { + vpcResourceIndexWrapper := func(indexers cache.Indexers) cache.Indexers { + indexers[indexScope] = indexBySecurityPolicyUID + indexers[common.TagScopeNetworkPolicyUID] = indexByNetworkPolicyUID + // Note: we can't use indexer `common.IndexByVPCPathFuncKey` with group/rule stores by default because the + // caller may not use the object read from NSX to apply on the store which is possibly not set with path or + // the parent path. But for cleanup logic, indexWithVPCPath is always set true and the store is re-built from + // the NSX resources but not from nsx-operator local calculation. + if indexWithVPCPath { + indexers[common.IndexByVPCPathFuncKey] = common.IndexByVPCFunc + } + return indexers + } + s.securityPolicyStore = &SecurityPolicyStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer( - keyFunc, cache.Indexers{ - indexScope: indexBySecurityPolicyUID, - common.TagScopeNetworkPolicyUID: indexByNetworkPolicyUID, - common.TagScopeNamespace: indexBySecurityPolicyNamespace, - }), + keyFunc, vpcResourceIndexWrapper(cache.Indexers{ + common.TagScopeNamespace: indexBySecurityPolicyNamespace, + })), BindingType: model.SecurityPolicyBindingType(), }} s.groupStore = &GroupStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ - indexScope: indexBySecurityPolicyUID, - common.TagScopeNetworkPolicyUID: indexByNetworkPolicyUID, - common.TagScopeRuleID: indexGroupFunc, - }), + Indexer: cache.NewIndexer(keyFunc, vpcResourceIndexWrapper(cache.Indexers{ + common.TagScopeRuleID: indexGroupFunc, + })), BindingType: model.GroupBindingType(), }} s.ruleStore = &RuleStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ - indexScope: indexBySecurityPolicyUID, - common.TagScopeNetworkPolicyUID: indexByNetworkPolicyUID, - }), + Indexer: cache.NewIndexer(keyFunc, vpcResourceIndexWrapper(cache.Indexers{})), BindingType: model.RuleBindingType(), }} s.infraGroupStore = &GroupStore{ResourceStore: common.ResourceStore{ @@ -1133,39 +1158,6 @@ func (service *SecurityPolicyService) ListNetworkPolicyByName(ns, name string) [ return result } -func (service *SecurityPolicyService) Cleanup(ctx context.Context) error { - // Delete all the security policies in store - uids := service.ListSecurityPolicyID() - log.Info("Cleaning up security policies created for SecurityPolicy CR", "count", len(uids)) - for uid := range uids { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSecurityPolicy(types.UID(uid), false, true, common.ResourceTypeSecurityPolicy) - if err != nil { - return err - } - } - } - - // Delete all the security policies created for network policy in store - uids = service.ListNetworkPolicyID() - log.Info("Cleaning up security policies created for network policy", "count", len(uids)) - for uid := range uids { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSecurityPolicy(types.UID(uid), false, true, common.ResourceTypeNetworkPolicy) - if err != nil { - return err - } - } - } - return nil -} - func (service *SecurityPolicyService) gcInfraSharesGroups(sp types.UID, indexScope string) error { var err error var infraResource *model.Infra diff --git a/pkg/nsx/services/securitypolicy/firewall_test.go b/pkg/nsx/services/securitypolicy/firewall_test.go index 8e3c9eb2b..a9ad9fdfe 100644 --- a/pkg/nsx/services/securitypolicy/firewall_test.go +++ b/pkg/nsx/services/securitypolicy/firewall_test.go @@ -4,7 +4,6 @@ package securitypolicy import ( - "context" "fmt" "reflect" "strings" @@ -542,7 +541,7 @@ func Test_InitializeSecurityPolicy(t *testing.T) { }) defer patch.Reset() - _, err := InitializeSecurityPolicy(commonService, vpcService) + _, err := InitializeSecurityPolicy(commonService, vpcService, true) if err != nil { t.Error(err) } @@ -552,7 +551,7 @@ func Test_ListSecurityPolicyID(t *testing.T) { service := &SecurityPolicyService{ Service: common.Service{NSXClient: nil}, } - service.setUpStore(common.TagValueScopeSecurityPolicyUID) + service.setUpStore(common.TagValueScopeSecurityPolicyUID, false) group := model.Group{} scope := common.TagValueScopeSecurityPolicyUID @@ -594,6 +593,7 @@ func Test_ListSecurityPolicyID(t *testing.T) { share.Id = &id3 share.UniqueId = &uuid3 share.Tags = []model.Tag{{Scope: &scope, Tag: &id3}} + share.Path = String(fmt.Sprintf("/orgs/default/projects/p1/infra/shares/%s", id3)) err = service.projectShareStore.Add(&share) if err != nil { t.Fatalf("Failed to add share to store: %v", err) @@ -605,6 +605,7 @@ func Test_ListSecurityPolicyID(t *testing.T) { share1.Id = &id4 share1.UniqueId = &uuid4 share1.Tags = []model.Tag{{Scope: &scope, Tag: &id4}} + share1.Path = String(fmt.Sprintf("/infra/shares/%s", id4)) err = service.infraShareStore.Add(&share1) if err != nil { t.Fatalf("Failed to add share to store: %v", err) @@ -642,7 +643,6 @@ func Test_createOrUpdateGroups(t *testing.T) { VPCInfo[0].OrgID = "default" VPCInfo[0].ProjectID = "projectQuality" VPCInfo[0].VPCID = "vpc1" - mId, mTag, mScope := "spA_uidA_scope", "uidA", tagScopeSecurityPolicyUID markDelete := true @@ -701,7 +701,7 @@ func Test_createOrUpdateGroups(t *testing.T) { mockVPCService := mock.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) defer patches.Reset() @@ -1160,7 +1160,7 @@ func Test_DeleteVPCSecurityPolicy(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.inputPolicy)) assert.NoError(t, fakeService.ruleStore.Apply(&tt.inputPolicy.Rules)) @@ -1329,7 +1329,7 @@ func Test_DeleteVPCSecurityPolicyForNetworkPolicy(t *testing.T) { mockVPCService := mock.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) defer patches.Reset() @@ -1519,7 +1519,7 @@ func Test_deleteSecurityPolicy(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = false - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.inputPolicy)) assert.NoError(t, fakeService.ruleStore.Apply(&tt.inputPolicy.Rules)) @@ -1714,7 +1714,7 @@ func Test_deleteVPCSecurityPolicy(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.inputPolicy)) assert.NoError(t, fakeService.ruleStore.Apply(&tt.inputPolicy.Rules)) @@ -1918,7 +1918,7 @@ func Test_deleteVPCSecurityPolicyInDefaultProject(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.inputPolicy)) assert.NoError(t, fakeService.ruleStore.Apply(&tt.inputPolicy.Rules)) @@ -2021,7 +2021,7 @@ func Test_CreateOrUpdateSecurityPolicy(t *testing.T) { mockVPCService := mock.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) patches.ApplyMethodSeq(fakeService.NSXClient.VPCSecurityClient, "Get", []gomonkey.OutputCell{{ @@ -2133,7 +2133,7 @@ func Test_CreateOrUpdateSecurityPolicyFromNetworkPolicy(t *testing.T) { mockVPCService := mock.MockVPCServiceProvider{} fakeService.vpcService = &mockVPCService - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) patches.ApplyMethodSeq(fakeService.NSXClient.VPCSecurityClient, "Get", []gomonkey.OutputCell{ @@ -2286,7 +2286,7 @@ func Test_createOrUpdateSecurityPolicy(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyCRName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyCRUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) patches.ApplyMethodSeq(fakeService.NSXClient.SecurityClient, "Get", []gomonkey.OutputCell{{ @@ -2448,7 +2448,7 @@ func Test_createOrUpdateVPCSecurityPolicy(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) patches.ApplyMethodSeq(fakeService.NSXClient.VPCSecurityClient, "Get", []gomonkey.OutputCell{{ @@ -2620,7 +2620,7 @@ func Test_createOrUpdateVPCSecurityPolicyInDefaultProject(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) patches.ApplyMethodSeq(fakeService.NSXClient.VPCSecurityClient, "Get", []gomonkey.OutputCell{{ @@ -2701,7 +2701,7 @@ func Test_GetFinalSecurityPolicyResourceForT1(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyCRName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyCRUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) defer patches.Reset() @@ -2833,7 +2833,7 @@ func Test_GetFinalSecurityPolicyResourceForVPC(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) defer patches.Reset() @@ -3119,7 +3119,7 @@ func Test_GetFinalSecurityPolicyResourceFromNetworkPolicy(t *testing.T) { common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) var finalAllowSecurityPolicy *model.SecurityPolicy var finalIsolationSecurityPolicy *model.SecurityPolicy var finalGroups []model.Group @@ -3163,7 +3163,7 @@ func Test_ListSecurityPolicyByName(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) sp1 := &model.SecurityPolicy{ DisplayName: &spName, @@ -3205,7 +3205,7 @@ func Test_ListNetworkPolicyByName(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) sp1 := &model.SecurityPolicy{ DisplayName: &spName, @@ -3243,163 +3243,6 @@ func Test_ListNetworkPolicyByName(t *testing.T) { assert.Len(t, result, 0) } -func Test_Cleanup(t *testing.T) { - spPath := "/orgs/default/projects/projectQuality/vpcs/vpc1" - - tests := []struct { - name string - prepareFunc func(*testing.T, *SecurityPolicyService) *gomonkey.Patches - inputPolicy *model.SecurityPolicy - wantErr bool - wantSecurityPolicyStoreCount int - }{ - { - name: "success Cleanup", - prepareFunc: func(t *testing.T, s *SecurityPolicyService) *gomonkey.Patches { - patches := gomonkey.ApplyMethodSeq(s.NSXClient.OrgRootClient, "Patch", []gomonkey.OutputCell{{ - Values: gomonkey.Params{nil}, - Times: 1, - }}) - return patches - }, - inputPolicy: &model.SecurityPolicy{ - DisplayName: &spName, - Id: common.String("spA_uidA"), - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_scope"}, - SequenceNumber: &seq0, - Rules: []model.Rule{}, - Tags: vpcBasicTags, - Path: &spPath, - }, - wantErr: false, - wantSecurityPolicyStoreCount: 0, - }, - { - name: "error Cleanup", - prepareFunc: func(t *testing.T, s *SecurityPolicyService) *gomonkey.Patches { - patches := gomonkey.ApplyMethodSeq(s.NSXClient.OrgRootClient, "Patch", []gomonkey.OutputCell{{ - Values: gomonkey.Params{nil}, - Times: 1, - }}) - return patches - }, - inputPolicy: &model.SecurityPolicy{ - DisplayName: &spName, - Id: common.String("spA_uidA"), - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/spA_uidA_scope"}, - SequenceNumber: &seq0, - Rules: []model.Rule{}, - Tags: vpcBasicTags, - Path: &spPath, - }, - wantErr: true, - wantSecurityPolicyStoreCount: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName - common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - - fakeService := fakeSecurityPolicyService() - fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) - - assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.inputPolicy)) - - patches := tt.prepareFunc(t, fakeService) - defer patches.Reset() - ctx := context.Background() - - if tt.name == "error Cleanup" { - ctx, cancel := context.WithCancel(ctx) - cancel() - if err := fakeService.Cleanup(ctx); (err != nil) != tt.wantErr { - t.Errorf("Cleanup error = %v, wantErr %v", err, tt.wantErr) - } - } - - if tt.name == "success Cleanup" { - if err := fakeService.Cleanup(ctx); (err != nil) != tt.wantErr { - t.Errorf("Cleanup error = %v, wantErr %v", err, tt.wantErr) - } - } - - assert.Equal(t, tt.wantSecurityPolicyStoreCount, len(fakeService.securityPolicyStore.ListKeys())) - }) - } -} - -func Test_Cleanup_ForNetworkPolicy(t *testing.T) { - spPath := "/orgs/default/projects/projectQuality/vpcs/vpc1" - - tests := []struct { - name string - prepareFunc func(*testing.T, *SecurityPolicyService) *gomonkey.Patches - expAllowPolicy *model.SecurityPolicy - expIsolationPolicy *model.SecurityPolicy - wantErr bool - wantSecurityPolicyStoreCount int - }{ - { - name: "success Cleanup for NetworkPolicy", - prepareFunc: func(t *testing.T, s *SecurityPolicyService) *gomonkey.Patches { - patches := gomonkey.ApplyMethodSeq(s.NSXClient.OrgRootClient, "Patch", []gomonkey.OutputCell{{ - Values: gomonkey.Params{nil}, - Times: 2, - }}) - return patches - }, - expAllowPolicy: &model.SecurityPolicy{ - DisplayName: common.String("np-app-access"), - Id: common.String("np-app-access_uidNP_allow"), - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/np-app-access_uidNP_allow_scope"}, - SequenceNumber: Int64(int64(common.PriorityNetworkPolicyAllowRule)), - Rules: []model.Rule{}, - Tags: npAllowBasicTags, - Path: &spPath, - }, - expIsolationPolicy: &model.SecurityPolicy{ - DisplayName: common.String("np-app-access"), - Id: common.String("np-app-access_uidNP_isolation"), - Scope: []string{"/orgs/default/projects/projectQuality/vpcs/vpc1/groups/np-app-access_uidNP_isolation_scope"}, - SequenceNumber: Int64(int64(common.PriorityNetworkPolicyIsolationRule)), - Rules: []model.Rule{}, - Tags: npIsolationBasicTags, - Path: &spPath, - }, - wantErr: false, - wantSecurityPolicyStoreCount: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - common.TagValueScopeSecurityPolicyName = common.TagScopeSecurityPolicyName - common.TagValueScopeSecurityPolicyUID = common.TagScopeSecurityPolicyUID - - fakeService := fakeSecurityPolicyService() - fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) - - assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.expAllowPolicy)) - assert.NoError(t, fakeService.securityPolicyStore.Apply(tt.expIsolationPolicy)) - assert.Equal(t, 2, len(fakeService.securityPolicyStore.ListKeys())) - - patches := tt.prepareFunc(t, fakeService) - defer patches.Reset() - ctx := context.Background() - - if err := fakeService.Cleanup(ctx); (err != nil) != tt.wantErr { - t.Errorf("Cleanup error = %v, wantErr %v", err, tt.wantErr) - } - - assert.Equal(t, tt.wantSecurityPolicyStoreCount, len(fakeService.securityPolicyStore.ListKeys())) - }) - } -} - func Test_gcInfraSharesGroups(t *testing.T) { markNoDelete := false @@ -3495,7 +3338,7 @@ func Test_gcInfraSharesGroups(t *testing.T) { fakeService := fakeSecurityPolicyService() fakeService.NSXConfig.EnableVPCNetwork = true - fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID) + fakeService.setUpStore(common.TagValueScopeSecurityPolicyUID, false) patches := tt.prepareFunc(t, fakeService) defer patches.Reset() diff --git a/pkg/nsx/services/securitypolicy/store.go b/pkg/nsx/services/securitypolicy/store.go index 59724e83c..6949e2d6a 100644 --- a/pkg/nsx/services/securitypolicy/store.go +++ b/pkg/nsx/services/securitypolicy/store.go @@ -157,6 +157,12 @@ func (securityPolicyStore *SecurityPolicyStore) GetByIndex(key string, value str return securityPolicies } +func (securityPolicyStore *SecurityPolicyStore) DeleteMultipleObjects(securityPolicies []*model.SecurityPolicy) { + for _, securityPolicy := range securityPolicies { + securityPolicyStore.Delete(securityPolicy) + } +} + func (ruleStore *RuleStore) Apply(i interface{}) error { rules := i.(*[]model.Rule) for _, rule := range *rules { @@ -187,6 +193,12 @@ func (ruleStore *RuleStore) GetByIndex(key string, value string) []*model.Rule { return rules } +func (ruleStore *RuleStore) DeleteMultipleObjects(rules []*model.Rule) { + for _, rule := range rules { + ruleStore.Delete(rule) + } +} + func (groupStore *GroupStore) Apply(i interface{}) error { gs := i.(*[]model.Group) for _, group := range *gs { @@ -217,6 +229,12 @@ func (groupStore *GroupStore) GetByIndex(key string, value string) []*model.Grou return groups } +func (groupStore *GroupStore) DeleteMultipleObjects(groups []*model.Group) { + for _, group := range groups { + groupStore.Delete(group) + } +} + func (shareStore *ShareStore) Apply(i interface{}) error { shares := i.(*[]model.Share) for _, share := range *shares { @@ -246,3 +264,9 @@ func (shareStore *ShareStore) GetByIndex(key string, value string) []*model.Shar } return shares } + +func (shareStore *ShareStore) DeleteMultipleObjects(shares []*model.Share) { + for _, share := range shares { + shareStore.Delete(share) + } +} diff --git a/pkg/nsx/services/staticroute/cleanup.go b/pkg/nsx/services/staticroute/cleanup.go new file mode 100644 index 000000000..d21bc2b9c --- /dev/null +++ b/pkg/nsx/services/staticroute/cleanup.go @@ -0,0 +1,40 @@ +package staticroute + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +// CleanupVPCChildResources is deleting all the NSX StaticRoutes in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with an auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached StaticRoutes +// on NSX and in local cache. +func (service *StaticRouteService) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + routes, err := service.StaticRouteStore.GetByVPCPath(vpcPath) + if err != nil { + log.Error(err, "Failed to list StaticRoutes under the VPC", "path", vpcPath) + } + if len(routes) == 0 { + return nil + } + // Delete resources from the store and return. + service.StaticRouteStore.DeleteMultipleObjects(routes) + return nil + } + + routes := make([]*model.StaticRoutes, 0) + MarkedForDelete := true + // Mark the resources for delete. + for _, obj := range service.StaticRouteStore.List() { + route := obj.(*model.StaticRoutes) + route.MarkedForDelete = &MarkedForDelete + routes = append(routes, route) + } + return service.builder.PagingDeleteResources(ctx, routes, common.DefaultHAPIChildrenCount, service.NSXClient, func(deletedObjs []*model.StaticRoutes) { + service.StaticRouteStore.DeleteMultipleObjects(deletedObjs) + }) +} diff --git a/pkg/nsx/services/staticroute/staticroute.go b/pkg/nsx/services/staticroute/staticroute.go index 7144761ad..5487f6dc3 100644 --- a/pkg/nsx/services/staticroute/staticroute.go +++ b/pkg/nsx/services/staticroute/staticroute.go @@ -1,10 +1,7 @@ package staticroute import ( - "context" - "errors" "fmt" - "strings" "sync" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" @@ -21,6 +18,7 @@ type StaticRouteService struct { common.Service StaticRouteStore *StaticRouteStore VPCService common.VPCServiceProvider + builder *common.PolicyTreeBuilder[*model.StaticRoutes] } var ( @@ -30,16 +28,19 @@ var ( // InitializeStaticRoute sync NSX resources func InitializeStaticRoute(commonService common.Service, vpcService common.VPCServiceProvider) (*StaticRouteService, error) { + builder, _ := common.PolicyPathVpcStaticRoutes.NewPolicyTreeBuilder() + wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) wg.Add(1) - staticRouteService := &StaticRouteService{Service: commonService} + staticRouteService := &StaticRouteService{Service: commonService, builder: builder} staticRouteStore := &StaticRouteStore{} staticRouteStore.Indexer = cache.NewIndexer(keyFunc, cache.Indexers{ common.TagScopeStaticRouteCRUID: indexFunc, common.TagScopeNamespace: indexStaticRouteNamespace, + common.IndexByVPCPathFuncKey: common.IndexByVPCFunc, }) staticRouteStore.BindingType = model.StaticRoutesBindingType() staticRouteService.StaticRouteStore = staticRouteStore @@ -169,23 +170,3 @@ func (service *StaticRouteService) ListStaticRoute() []*model.StaticRoutes { } return staticRouteSet } - -func (service *StaticRouteService) Cleanup(ctx context.Context) error { - staticRouteSet := service.ListStaticRoute() - log.Info("Cleanup staticroute", "count", len(staticRouteSet)) - for _, staticRoute := range staticRouteSet { - path := strings.Split(*staticRoute.Path, "/") - log.Info("Deleting staticroute", "staticroute path", *staticRoute.Path) - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteStaticRouteByPath(path[2], path[4], path[6], *staticRoute.Id) - if err != nil { - log.Error(err, "Delete staticroute failed", "staticroute id", *staticRoute.Id) - return err - } - } - } - return nil -} diff --git a/pkg/nsx/services/staticroute/staticroute_test.go b/pkg/nsx/services/staticroute/staticroute_test.go index bcfabd749..fce8ac9d2 100644 --- a/pkg/nsx/services/staticroute/staticroute_test.go +++ b/pkg/nsx/services/staticroute/staticroute_test.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/staticrouteclient" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" @@ -144,8 +145,9 @@ func TestStaticRouteService_DeleteStaticRoute(t *testing.T) { }, } id := util.GenerateIDByObject(srObj) - path := "/orgs/default/projects/project-1/vpcs/vpc-1" - sr1 := &model.StaticRoutes{Id: &id, Path: &path} + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" + path := fmt.Sprintf("%s/static-routes/%s", vpcPath, id) + sr1 := &model.StaticRoutes{Id: &id, Path: &path, ParentPath: &vpcPath} // no record found mockStaticRouteclient.EXPECT().Delete(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Times(0) @@ -165,7 +167,7 @@ func TestStaticRouteService_DeleteStaticRoute(t *testing.T) { func TestStaticRouteService_CreateorUpdateStaticRoute(t *testing.T) { service, mockController, mockStaticRouteclient := createService(t) defer mockController.Finish() - + vpcPath := "/orgs/default/projects/project-1/vpcs/vpc-1" var tc *bindings.TypeConverter patches2 := gomonkey.ApplyMethod(reflect.TypeOf(tc), "ConvertToGolang", func(_ *bindings.TypeConverter, d data.DataValue, b bindings.BindingType) (interface{}, []error) { @@ -193,8 +195,10 @@ func TestStaticRouteService_CreateorUpdateStaticRoute(t *testing.T) { scope := common.TagScopeStaticRouteCRUID tag := "test_tag" m := model.StaticRoutes{ - Id: &mId, - Tags: []model.Tag{{Tag: &tag, Scope: &scope}}, + Id: &mId, + Tags: []model.Tag{{Tag: &tag, Scope: &scope}}, + ParentPath: &vpcPath, + Path: String(fmt.Sprintf("%s/static-routes/%s", vpcPath, mId)), } mockStaticRouteclient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(m, nil).Times(2) patches := gomonkey.ApplyMethod(reflect.TypeOf(returnservice.VPCService), "ListVPCInfo", func(_ common.VPCServiceProvider, ns string) []common.VPCResourceInfo { @@ -323,6 +327,9 @@ func TestListStaticRoute(t *testing.T) { func TestStaticRouteService_Cleanup(t *testing.T) { service, mockController, mockStaticRouteclient := createService(t) defer mockController.Finish() + builder, _ := common.PolicyPathVpcStaticRoutes.NewPolicyTreeBuilder() + service.builder = builder + mockOrgRootClient := mock_org_root.NewMockOrgRootClient(mockController) ctx := context.Background() @@ -339,14 +346,14 @@ func TestStaticRouteService_Cleanup(t *testing.T) { } t.Run("Successful cleanup", func(t *testing.T) { + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) service.StaticRouteStore.Add(staticRoute1) service.StaticRouteStore.Add(staticRoute2) mockStaticRouteclient = mocks.NewMockStaticRoutesClient(mockController) service.NSXClient.StaticRouteClient = mockStaticRouteclient - mockStaticRouteclient.EXPECT().Delete("org1", "project1", "vpc1", "staticroute1").Return(nil).Times(1) - mockStaticRouteclient.EXPECT().Delete("org2", "project2", "vpc2", "staticroute2").Return(nil).Times(1) + service.NSXClient.OrgRootClient = mockOrgRootClient - err := service.Cleanup(ctx) + err := service.CleanupVPCChildResources(ctx, "") assert.NoError(t, err) }) @@ -355,19 +362,19 @@ func TestStaticRouteService_Cleanup(t *testing.T) { ctx, cancel := context.WithCancel(ctx) cancel() - err := service.Cleanup(ctx) + err := service.CleanupVPCChildResources(ctx, "") assert.Error(t, err) assert.Contains(t, err.Error(), "context canceled") }) t.Run("Delete static route error", func(t *testing.T) { + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(fmt.Errorf("delete error")) + service.StaticRouteStore.Add(staticRoute1) mockStaticRouteclient = mocks.NewMockStaticRoutesClient(mockController) service.NSXClient.StaticRouteClient = mockStaticRouteclient - mockStaticRouteclient.EXPECT().Delete("org1", "project1", "vpc1", "staticroute1").Return(fmt.Errorf("delete error")).Times(1) - - err := service.Cleanup(ctx) + err := service.CleanupVPCChildResources(ctx, "") assert.Error(t, err) assert.Contains(t, err.Error(), "delete error") }) diff --git a/pkg/nsx/services/staticroute/store.go b/pkg/nsx/services/staticroute/store.go index 1d64ce2a2..8c07470f2 100644 --- a/pkg/nsx/services/staticroute/store.go +++ b/pkg/nsx/services/staticroute/store.go @@ -68,3 +68,22 @@ func (StaticRouteStore *StaticRouteStore) GetByKey(key string) *model.StaticRout } return nil } + +func (StaticRouteStore *StaticRouteStore) GetByVPCPath(vpcPath string) ([]*model.StaticRoutes, error) { + objs, err := StaticRouteStore.ResourceStore.ByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if err != nil { + return nil, err + } + routes := make([]*model.StaticRoutes, len(objs)) + for i, obj := range objs { + route := obj.(*model.StaticRoutes) + routes[i] = route + } + return routes, nil +} + +func (StaticRouteStore *StaticRouteStore) DeleteMultipleObjects(routes []*model.StaticRoutes) { + for _, route := range routes { + StaticRouteStore.Delete(route) + } +} diff --git a/pkg/nsx/services/subnet/cleanup.go b/pkg/nsx/services/subnet/cleanup.go new file mode 100644 index 000000000..e24174e88 --- /dev/null +++ b/pkg/nsx/services/subnet/cleanup.go @@ -0,0 +1,37 @@ +package subnet + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +// CleanupVPCChildResources is deleting all the NSX VpcSubnets in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with an auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached VpcSubnets +// on NSX and in local cache. +func (service *SubnetService) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + subnets := service.SubnetStore.GetByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if len(subnets) == 0 { + return nil + } + // Delete resources from the store and return. + service.SubnetStore.DeleteMultipleObjects(subnets) + return nil + } + + subnets := make([]*model.VpcSubnet, 0) + // Mark the resources for delete. + for _, obj := range service.SubnetStore.List() { + subnet := obj.(*model.VpcSubnet) + subnet.MarkedForDelete = &MarkedForDelete + subnets = append(subnets, subnet) + } + + return service.builder.PagingDeleteResources(ctx, subnets, common.DefaultHAPIChildrenCount, service.NSXClient, func(deletedObjs []*model.VpcSubnet) { + service.SubnetStore.DeleteMultipleObjects(deletedObjs) + }) +} diff --git a/pkg/nsx/services/subnet/store.go b/pkg/nsx/services/subnet/store.go index a8ee9d230..cde4fb470 100644 --- a/pkg/nsx/services/subnet/store.go +++ b/pkg/nsx/services/subnet/store.go @@ -107,3 +107,9 @@ func (subnetStore *SubnetStore) GetByKey(key string) *model.VpcSubnet { subnet := obj.(*model.VpcSubnet) return subnet } + +func (subnetStore *SubnetStore) DeleteMultipleObjects(subnets []*model.VpcSubnet) { + for _, subnet := range subnets { + subnetStore.Delete(subnet) + } +} diff --git a/pkg/nsx/services/subnet/store_test.go b/pkg/nsx/services/subnet/store_test.go index 556051567..97e7ce19a 100644 --- a/pkg/nsx/services/subnet/store_test.go +++ b/pkg/nsx/services/subnet/store_test.go @@ -32,6 +32,7 @@ func (qIface *fakeQueryClient) List(_ string, _ *string, _ *string, _ *int64, _ "resource_type": data.NewStringValue("VpcSubnet"), "id": data.NewStringValue("subnet1"), "path": data.NewStringValue("/orgs/default/projects/default/vpcs/vpc2/subnets/subnet2"), + "parent_path": data.NewStringValue("/orgs/default/projects/default/vpcs/vpc2"), })}, Cursor: &cursor, ResultCount: &resultCount, }, nil diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index 04df49b1c..1c2a9a7bd 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -36,6 +36,7 @@ var ( type SubnetService struct { common.Service SubnetStore *SubnetStore + builder *common.PolicyTreeBuilder[*model.VpcSubnet] } // SubnetParameters stores parameters to CRUD Subnet object @@ -47,6 +48,8 @@ type SubnetParameters struct { // InitializeSubnetService initialize Subnet service. func InitializeSubnetService(service common.Service) (*SubnetService, error) { + builder, _ := common.PolicyPathVpcSubnet.NewPolicyTreeBuilder() + wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) @@ -59,10 +62,12 @@ func InitializeSubnetService(service common.Service) (*SubnetService, error) { common.TagScopeSubnetSetCRUID: subnetSetIndexFunc, common.TagScopeVMNamespace: subnetIndexVMNamespaceFunc, common.TagScopeNamespace: subnetIndexNamespaceFunc, + common.IndexByVPCPathFuncKey: common.IndexByVPCFunc, }), BindingType: model.VpcSubnetBindingType(), }, }, + builder: builder, } wg.Add(1) @@ -135,6 +140,7 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet // For Subnets, it's important to reuse the already created NSXSubnet. // For SubnetSets, since the ID includes a random value, the created NSX Subnet needs to be deleted and recreated. + fmt.Println("Checking who is nil", "realizeService", realizeService, "nsxSubnet path", nsxSubnet.Path) if err = realizeService.CheckRealizeState(util.NSXTRealizeRetry, *nsxSubnet.Path); err != nil { log.Error(err, "Failed to check subnet realization state", "ID", *nsxSubnet.Id) // Delete the subnet if realization check fails, avoiding creating duplicate subnets continuously. @@ -329,23 +335,6 @@ func (service *SubnetService) ListAllSubnet() []*model.VpcSubnet { return allNSXSubnets } -func (service *SubnetService) Cleanup(ctx context.Context) error { - allNSXSubnets := service.ListAllSubnet() - log.Info("Cleaning up Subnet", "Count", len(allNSXSubnets)) - for _, nsxSubnet := range allNSXSubnets { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSubnet(*nsxSubnet) - if err != nil { - return err - } - } - } - return nil -} - func (service *SubnetService) GetSubnetsByIndex(key, value string) []*model.VpcSubnet { return service.SubnetStore.GetByIndex(key, value) } diff --git a/pkg/nsx/services/subnet/subnet_test.go b/pkg/nsx/services/subnet/subnet_test.go index 3ad8186dd..804b7bd76 100644 --- a/pkg/nsx/services/subnet/subnet_test.go +++ b/pkg/nsx/services/subnet/subnet_test.go @@ -3,6 +3,7 @@ package subnet import ( "context" "errors" + "fmt" "reflect" "testing" @@ -25,6 +26,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" @@ -104,17 +106,6 @@ func TestGenerateSubnetNSTags(t *testing.T) { assert.Equal(t, "test-ns", *tagsSet[1].Tag) } -type fakeOrgRootClient struct { -} - -func (f fakeOrgRootClient) Get(basePathParam *string, filterParam *string, typeFilterParam *string) (model.OrgRoot, error) { - return model.OrgRoot{}, nil -} - -func (f fakeOrgRootClient) Patch(orgRootParam model.OrgRoot, enforceRevisionCheckParam *bool) error { - return nil -} - type fakeSubnetsClient struct { } @@ -187,19 +178,15 @@ func TestInitializeSubnetService(t *testing.T) { Spec: v1alpha1.SubnetSpec{}, Status: v1alpha1.SubnetStatus{}, } - vpcResourceInfo := &common.VPCResourceInfo{ - OrgID: "", - ProjectID: "", - VPCID: "", - ID: "", - ParentID: "", - PrivateIpv4Blocks: nil, - } nsxSubnetID := util.GenerateIDByObject(subnet) basicTags := util.BuildBasicTags(clusterName, subnet, "") - fakeSubnetPath := "/orgs/default/projects/nsx_operator_e2e_test/vpcs/subnet-e2e_8f36f7fc-90cd-4e65-a816-daf3ecd6a0f9/subnets/" + nsxSubnetID + fakeVPCPath := "/orgs/default/projects/nsx_operator_e2e_test/vpcs/subnet-e2e_8f36f7fc-90cd-4e65-a816-daf3ecd6a0f9" + fakeSubnetPath := fmt.Sprintf("%s/subnet-e2e_8f36f7fc-90cd-4e65-a816-daf3ecd6a0f9/subnets/%s", fakeVPCPath, nsxSubnetID) + + vpcResourceInfo, _ := common.ParseVPCResourcePath(fakeVPCPath) + var mockOrgRootClient *mock_org_root.MockOrgRootClient testCases := []struct { name string prepareFunc func() *gomonkey.Patches @@ -214,12 +201,14 @@ func TestInitializeSubnetService(t *testing.T) { name: "Subnet does not exist", existingSubnetCR: subnet, expectAllSubnetNum: 0, - existingVPCInfo: vpcResourceInfo, + existingVPCInfo: &vpcResourceInfo, prepareFunc: func() *gomonkey.Patches { - var fakeVpcSubnet = model.VpcSubnet{Path: &fakeSubnetPath, Id: &nsxSubnetID, Tags: basicTags} + var fakeVpcSubnet = model.VpcSubnet{Path: &fakeSubnetPath, Id: &nsxSubnetID, Tags: basicTags, ParentPath: &fakeVPCPath} patches := gomonkey.ApplyMethod(reflect.TypeOf(&fakeSubnetsClient{}), "Get", func(_ *fakeSubnetsClient, orgIdParam string, projectIdParam string, vpcIdParam string, subnetIdParam string) (model.VpcSubnet, error) { return fakeVpcSubnet, nil }) + // OrgRootClient.Patch is called twice, one is to create the VpcSubnet, and the other is for cleanup. + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).Times(2) return patches }, subnetCRTags: []model.Tag{}, @@ -229,7 +218,7 @@ func TestInitializeSubnetService(t *testing.T) { { name: "Subnet exists and not change", existingSubnetCR: subnet, - existingVPCInfo: vpcResourceInfo, + existingVPCInfo: &vpcResourceInfo, prepareFunc: func() *gomonkey.Patches { patches := gomonkey.ApplyMethod(reflect.TypeOf(&fakeQueryClient{}), "List", func(_ *fakeQueryClient, _ string, _ *string, _ *string, _ *int64, _ *bool, _ *string) (model.SearchResponse, error) { cursor := "1" @@ -247,12 +236,15 @@ func TestInitializeSubnetService(t *testing.T) { "id": data.NewStringValue(nsxSubnetID), "display_name": data.NewStringValue(subnetName), "path": data.NewStringValue(fakeSubnetPath), + "parent_path": data.NewStringValue(fakeVPCPath), "tags": tags, "subnet_dhcp_config": dhcpConfig, })}, Cursor: &cursor, ResultCount: &resultCount, }, nil }) + // OrgRootClient.Patch is called only once for cleanup. + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).Times(1) return patches }, expectAllSubnetNum: 1, @@ -262,7 +254,7 @@ func TestInitializeSubnetService(t *testing.T) { { name: "Subnet exists and changed", existingSubnetCR: subnet, - existingVPCInfo: vpcResourceInfo, + existingVPCInfo: &vpcResourceInfo, prepareFunc: func() *gomonkey.Patches { patches := gomonkey.ApplyMethod(reflect.TypeOf(&fakeQueryClient{}), "List", func(_ *fakeQueryClient, _ string, _ *string, _ *string, _ *int64, _ *bool, _ *string) (model.SearchResponse, error) { cursor := "1" @@ -279,15 +271,18 @@ func TestInitializeSubnetService(t *testing.T) { "id": data.NewStringValue(nsxSubnetID), "display_name": data.NewStringValue(subnetName), "path": data.NewStringValue(fakeSubnetPath), + "parent_path": data.NewStringValue(fakeVPCPath), "tags": tags, })}, Cursor: &cursor, ResultCount: &resultCount, }, nil }) - var fakeVpcSubnet = model.VpcSubnet{Path: &fakeSubnetPath, Id: &nsxSubnetID, Tags: basicTags} + var fakeVpcSubnet = model.VpcSubnet{Path: &fakeSubnetPath, Id: &nsxSubnetID, Tags: basicTags, ParentPath: &fakeVPCPath} patches.ApplyMethod(reflect.TypeOf(&fakeSubnetsClient{}), "Get", func(_ *fakeSubnetsClient, orgIdParam string, projectIdParam string, vpcIdParam string, subnetIdParam string) (model.VpcSubnet, error) { return fakeVpcSubnet, nil }) + // OrgRootClient.Patch is called twice, one is to update the VpcSubnet, and the other is for cleanup. + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).Times(2) return patches }, expectAllSubnetNum: 1, @@ -298,6 +293,10 @@ func TestInitializeSubnetService(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOrgRootClient = mock_org_root.NewMockOrgRootClient(ctrl) + newScheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(newScheme)) utilruntime.Must(v1alpha1.AddToScheme(newScheme)) @@ -305,7 +304,7 @@ func TestInitializeSubnetService(t *testing.T) { commonService := common.Service{ Client: fakeClient, NSXClient: &nsx.Client{ - OrgRootClient: &fakeOrgRootClient{}, + OrgRootClient: mockOrgRootClient, SubnetsClient: &fakeSubnetsClient{}, QueryClient: &fakeQueryClient{}, RealizedEntitiesClient: &fakeRealizedEntitiesClient{}, @@ -321,8 +320,9 @@ func TestInitializeSubnetService(t *testing.T) { }, }, } + var patches *gomonkey.Patches if tc.prepareFunc != nil { - patches := tc.prepareFunc() + patches = tc.prepareFunc() defer patches.Reset() } @@ -351,7 +351,7 @@ func TestInitializeSubnetService(t *testing.T) { assert.NoError(t, err) assert.Equal(t, nsxSubnetID, *getByPath.Id) - err = service.Cleanup(context.TODO()) + err = service.CleanupVPCChildResources(context.TODO(), "") assert.NoError(t, err) assert.Equal(t, 0, len(service.ListAllSubnet())) @@ -436,11 +436,13 @@ func TestSubnetService_createOrUpdateSubnet(t *testing.T) { mockCtl := gomock.NewController(t) k8sClient := mock_client.NewMockClient(mockCtl) defer mockCtl.Finish() + + fakeOrgRootClient := mock_org_root.NewMockOrgRootClient(mockCtl) service := &SubnetService{ Service: common.Service{ Client: k8sClient, NSXClient: &nsx.Client{ - OrgRootClient: &fakeOrgRootClient{}, + OrgRootClient: fakeOrgRootClient, SubnetsClient: &fakeSubnetsClient{}, SubnetStatusClient: &fakeSubnetStatusClient{}, }, @@ -479,6 +481,7 @@ func TestSubnetService_createOrUpdateSubnet(t *testing.T) { { name: "Update Subnet with RealizedState and deletion error", prepareFunc: func() *gomonkey.Patches { + fakeOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) patches := gomonkey.ApplyFunc((*realizestate.RealizeStateService).CheckRealizeState, func(_ *realizestate.RealizeStateService, _ wait.Backoff, _ string) error { return realizestate.NewRealizeStateError("mocked realized error") @@ -497,6 +500,7 @@ func TestSubnetService_createOrUpdateSubnet(t *testing.T) { { name: "Create Subnet for SubnetSet Success", prepareFunc: func() *gomonkey.Patches { + fakeOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) patches := gomonkey.ApplyFunc((*realizestate.RealizeStateService).CheckRealizeState, func(_ *realizestate.RealizeStateService, _ wait.Backoff, _ string) error { return nil diff --git a/pkg/nsx/services/subnetbinding/builder_test.go b/pkg/nsx/services/subnetbinding/builder_test.go index 8023b27de..9bb676736 100644 --- a/pkg/nsx/services/subnetbinding/builder_test.go +++ b/pkg/nsx/services/subnetbinding/builder_test.go @@ -111,16 +111,6 @@ func TestBuildSubnetConnectionBindingMapCR(t *testing.T) { assert.Equal(t, expCR, cr) } -func genSubnetConnectionBindingMap(bmID, displayName, subnetPath, parentPath string, vlanTag int64) *model.SubnetConnectionBindingMap { - return &model.SubnetConnectionBindingMap{ - Id: String(bmID), - DisplayName: String(displayName), - SubnetPath: String(subnetPath), - VlanTrafficTag: Int64(vlanTag), - ParentPath: String(parentPath), - } -} - func mockService() *BindingService { return &BindingService{ Service: common.Service{ diff --git a/pkg/nsx/services/subnetbinding/cleanup.go b/pkg/nsx/services/subnetbinding/cleanup.go new file mode 100644 index 000000000..4dd098050 --- /dev/null +++ b/pkg/nsx/services/subnetbinding/cleanup.go @@ -0,0 +1,27 @@ +package subnetbinding + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func (s *BindingService) CleanupBeforeVPCDeletion(ctx context.Context) error { + allNSXBindings := s.BindingStore.List() + log.Info("Cleaning up SubnetConnectionBindingMaps", "Count", len(allNSXBindings)) + if len(allNSXBindings) == 0 { + return nil + } + + finalBindingMaps := make([]*model.SubnetConnectionBindingMap, len(allNSXBindings)) + for i, obj := range allNSXBindings { + binding, _ := obj.(*model.SubnetConnectionBindingMap) + binding.MarkedForDelete = &markedForDelete + finalBindingMaps[i] = binding + } + return s.builder.PagingDeleteResources(ctx, finalBindingMaps, servicecommon.DefaultHAPIChildrenCount, s.NSXClient, func(deletedObjs []*model.SubnetConnectionBindingMap) { + s.BindingStore.DeleteMultipleObjects(deletedObjs) + }) +} diff --git a/pkg/nsx/services/subnetbinding/store.go b/pkg/nsx/services/subnetbinding/store.go index 57948e4fa..b1a5538a5 100644 --- a/pkg/nsx/services/subnetbinding/store.go +++ b/pkg/nsx/services/subnetbinding/store.go @@ -79,6 +79,25 @@ func (s *BindingStore) getBindingsByBindingMapCRName(bindingName string, binding return s.GetByIndex(bindingMapCRNameIndexKey, nn.String()) } +func (s *BindingStore) DeleteMultipleObjects(bindingMaps []*model.SubnetConnectionBindingMap) { + for _, bindingMap := range bindingMaps { + s.Delete(bindingMap) + } +} + +func (s *BindingStore) GetByVPCPath(vpcPath string) ([]*model.SubnetConnectionBindingMap, error) { + objs, err := s.ResourceStore.ByIndex(common.IndexByVPCPathFuncKey, vpcPath) + if err != nil { + return nil, err + } + bindingMaps := make([]*model.SubnetConnectionBindingMap, len(objs)) + for i, obj := range objs { + bindingMap := obj.(*model.SubnetConnectionBindingMap) + bindingMaps[i] = bindingMap + } + return bindingMaps, nil +} + func keyFunc(obj interface{}) (string, error) { switch v := obj.(type) { case *model.SubnetConnectionBindingMap: @@ -151,10 +170,11 @@ func SetupStore() *BindingStore { return &BindingStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer( keyFunc, cache.Indexers{ - bindingMapCRUIDIndexKey: bindingMapCRUIDIndexFunc, - bindingMapCRNameIndexKey: bindingMapCRNameIndexFunc, - childSubnetIndexKey: childSubnetIndexFunc, - parentSubnetIndexKey: parentSubnetIndexFunc, + bindingMapCRUIDIndexKey: bindingMapCRUIDIndexFunc, + bindingMapCRNameIndexKey: bindingMapCRNameIndexFunc, + childSubnetIndexKey: childSubnetIndexFunc, + parentSubnetIndexKey: parentSubnetIndexFunc, + common.IndexByVPCPathFuncKey: common.IndexByVPCFunc, }), BindingType: model.SubnetConnectionBindingMapBindingType(), }} diff --git a/pkg/nsx/services/subnetbinding/store_test.go b/pkg/nsx/services/subnetbinding/store_test.go index fcb152c21..ed4281d69 100644 --- a/pkg/nsx/services/subnetbinding/store_test.go +++ b/pkg/nsx/services/subnetbinding/store_test.go @@ -1,6 +1,7 @@ package subnetbinding import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -45,6 +46,7 @@ var ( DisplayName: String("binding1"), SubnetPath: String(parentSubnetPath1), ParentPath: childSubnet.Path, + Path: String(fmt.Sprintf("%s/subnet-connection-binding-maps/incomplete", *childSubnet.Path)), VlanTrafficTag: Int64(201), Tags: []model.Tag{ { diff --git a/pkg/nsx/services/subnetbinding/subnetbinding.go b/pkg/nsx/services/subnetbinding/subnetbinding.go index 6e392e45a..f3fb2365e 100644 --- a/pkg/nsx/services/subnetbinding/subnetbinding.go +++ b/pkg/nsx/services/subnetbinding/subnetbinding.go @@ -10,25 +10,25 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" -) - -const ( - hAPIPageSize = 1000 + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( log = &logger.Log ResourceTypeSubnetConnectionBindingMap = servicecommon.ResourceTypeSubnetConnectionBindingMap enforceRevisionCheckParam = false + markedForDelete = true ) type BindingService struct { + builder *servicecommon.PolicyTreeBuilder[*model.SubnetConnectionBindingMap] servicecommon.Service BindingStore *BindingStore } // InitializeService initializes SubnetConnectionBindingMap service. func InitializeService(service servicecommon.Service) (*BindingService, error) { + builder, _ := servicecommon.PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() wg := sync.WaitGroup{} fatalErrors := make(chan error, 1) defer close(fatalErrors) @@ -36,6 +36,7 @@ func InitializeService(service servicecommon.Service) (*BindingService, error) { bindingService := &BindingService{ Service: service, BindingStore: SetupStore(), + builder: builder, } wg.Add(1) @@ -163,7 +164,46 @@ func (s *BindingService) ListSubnetConnectionBindingMapCRUIDsInStore() sets.Set[ // Apply sync bindingMaps on NSX and save into the store if succeeded to realize. func (s *BindingService) Apply(subnetPath string, bindingMaps []*model.SubnetConnectionBindingMap) error { - return s.hUpdateSubnetConnectionBindingMaps(subnetPath, bindingMaps) + vpcInfo, err := servicecommon.ParseVPCResourcePath(subnetPath) + if err != nil { + return err + } + subnetID := vpcInfo.ID + orgRoot, err := s.builder.BuildOrgRoot(bindingMaps, subnetPath) + if err != nil { + return err + } + + if err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &enforceRevisionCheckParam); err != nil { + log.Error(err, "Failed to patch SubnetConnectionBindingMaps on NSX", "orgID", vpcInfo.OrgID, "projectID", vpcInfo.ProjectID, "vpcID", vpcInfo.VPCID, "subnetID", subnetID, "subnetConnectionBindingMaps", bindingMaps) + err = nsxutil.TransNSXApiError(err) + return err + } + + // Get SubnetConnectionBindingMaps from NSX after patch operation as NSX renders several fields like `path`/`parent_path`. + subnetBindingListResult, err := s.NSXClient.SubnetConnectionBindingMapsClient.List(vpcInfo.OrgID, vpcInfo.ProjectID, vpcInfo.VPCID, subnetID, nil, nil, nil, nil, nil, nil) + if err != nil { + log.Error(err, "Failed to list SubnetConnectionBindingMaps from NSX under subnet", "orgID", vpcInfo.OrgID, "projectID", vpcInfo.ProjectID, "vpcID", vpcInfo.VPCID, "subnetID", subnetID, "subnetConnectionBindingMaps", bindingMaps) + err = nsxutil.TransNSXApiError(err) + return err + } + + nsxBindingMaps := make(map[string]model.SubnetConnectionBindingMap) + for _, bm := range subnetBindingListResult.Results { + nsxBindingMaps[*bm.Id] = bm + } + + for i := range bindingMaps { + bm := bindingMaps[i] + if bm.MarkedForDelete != nil && *bm.MarkedForDelete { + s.BindingStore.Apply(bm) + } else { + nsxBindingMap := nsxBindingMaps[*bm.Id] + s.BindingStore.Apply(&nsxBindingMap) + } + } + + return nil } // deleteSubnetConnectionBindingMaps uses HAPI call to delete multiple SubnetConnectionBindingMaps on NSX in one @@ -176,18 +216,14 @@ func (s *BindingService) deleteSubnetConnectionBindingMaps(bindingMaps []*model. log.Info("No existing SubnetConnectionBindingMaps found in the store") return nil } - pages := (bindingMapsCount + hAPIPageSize - 1) / hAPIPageSize - for i := 1; i <= pages; i++ { - start := (i - 1) * hAPIPageSize - end := start + hAPIPageSize - if end > bindingMapsCount { - end = bindingMapsCount - } - if err := s.hDeleteSubnetConnectionBindingMap(bindingMaps[start:end]); err != nil { - return err - } + markForDelete := true + for _, bm := range bindingMaps { + bm.MarkedForDelete = &markForDelete } - return nil + + return s.builder.PagingDeleteResources(context.TODO(), bindingMaps, servicecommon.DefaultHAPIChildrenCount, s.NSXClient, func(deletedObjs []*model.SubnetConnectionBindingMap) { + s.BindingStore.DeleteMultipleObjects(deletedObjs) + }) } func (s *BindingService) DeleteMultiSubnetConnectionBindingMapsByCRs(bindingCRs sets.Set[string]) error { @@ -195,7 +231,7 @@ func (s *BindingService) DeleteMultiSubnetConnectionBindingMapsByCRs(bindingCRs return nil } finalBindingMaps := make([]*model.SubnetConnectionBindingMap, 0) - for _, crID := range bindingCRs.UnsortedList() { + for crID := range bindingCRs { bms := s.BindingStore.getBindingsByBindingMapCRUID(crID) finalBindingMaps = append(finalBindingMaps, bms...) } @@ -214,18 +250,6 @@ func (s *BindingService) GetSubnetConnectionBindingMapCRName(bindingMap *model.S return "" } -func (s *BindingService) Cleanup(ctx context.Context) error { - allNSXBindings := s.BindingStore.List() - log.Info("Cleaning up SubnetConnectionBindingMaps", "Count", len(allNSXBindings)) - finalBindingMaps := make([]*model.SubnetConnectionBindingMap, len(allNSXBindings)) - for i, obj := range allNSXBindings { - binding, _ := obj.(*model.SubnetConnectionBindingMap) - finalBindingMaps[i] = binding - } - - return s.deleteSubnetConnectionBindingMaps(finalBindingMaps) -} - func bindingMapsToMap(bindingMaps []*model.SubnetConnectionBindingMap) map[string]*model.SubnetConnectionBindingMap { bmMap := make(map[string]*model.SubnetConnectionBindingMap) for _, bm := range bindingMaps { diff --git a/pkg/nsx/services/subnetbinding/subnetbinding_test.go b/pkg/nsx/services/subnetbinding/subnetbinding_test.go index 1271187ee..b6f6dc9c3 100644 --- a/pkg/nsx/services/subnetbinding/subnetbinding_test.go +++ b/pkg/nsx/services/subnetbinding/subnetbinding_test.go @@ -3,10 +3,8 @@ package subnetbinding import ( "context" "fmt" - "reflect" "testing" - "github.com/agiledragon/gomonkey/v2" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -122,6 +120,7 @@ func TestGetSubnetConnectionBindingMapCRsBySubnet(t *testing.T) { require.Equal(t, 1, len(bindingMaps)) bm := bindingMaps[0] bm.ParentPath = childSubnet.Path + bm.Path = String(fmt.Sprintf("%s/subnet-connection-binding-maps/%s", *bm.ParentPath, *bm.Id)) svc.BindingStore.Apply(bm) gotBMs1 = svc.GetSubnetConnectionBindingMapCRsBySubnet(parentSubnet1) @@ -149,8 +148,10 @@ func TestListSubnetConnectionBindingMapCRUIDsInStore(t *testing.T) { // Case: success bm := svc.buildSubnetBindings(binding1, []*model.VpcSubnet{parentSubnet1})[0] bm.ParentPath = String(childSubnetPath1) + bm.Path = String(fmt.Sprintf("%s/subnet-connection-binding-maps/%s", *bm.ParentPath, *bm.Id)) bm2 := svc.buildSubnetBindings(binding2, []*model.VpcSubnet{parentSubnet2})[0] bm2.ParentPath = String(childSubnetPath2) + bm2.Path = String(fmt.Sprintf("%s/subnet-connection-binding-maps/%s", *bm2.ParentPath, *bm2.Id)) svc.BindingStore.Apply(bm) svc.BindingStore.Apply(bm2) crIDs = svc.ListSubnetConnectionBindingMapCRUIDsInStore() @@ -234,6 +235,7 @@ func TestCreateOrUpdateSubnetConnectionBindingMap(t *testing.T) { mockSubnetBindingClient := bindingmap_mocks.NewMockSubnetConnectionBindingMapsClient(ctrl) oriBM2 := *bindingMap2 + oriBM2.Path = String(fmt.Sprintf("%s/subnet-connection-binding-maps/%s", *childSubnet.Path, *oriBM2.Id)) oriBM2.VlanTrafficTag = Int64(200) expAddBM := *bindingMap2 @@ -336,6 +338,7 @@ func TestCreateOrUpdateSubnetConnectionBindingMap(t *testing.T) { }, BindingStore: SetupStore(), } + svc.builder, _ = common.PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() for _, bm := range tc.existingBindingMaps { svc.BindingStore.Add(bm) } @@ -405,6 +408,8 @@ func TestDeleteMultiSubnetConnectionBindingMapsByCRs(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + builder, err := common.PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() + require.NoError(t, err) svc := &BindingService{ Service: common.Service{ NSXClient: &nsx.Client{ @@ -417,6 +422,7 @@ func TestDeleteMultiSubnetConnectionBindingMapsByCRs(t *testing.T) { }, }, BindingStore: SetupStore(), + builder: builder, } svc.BindingStore.Add(createdBM1) svc.BindingStore.Add(createdBM2) @@ -425,7 +431,7 @@ func TestDeleteMultiSubnetConnectionBindingMapsByCRs(t *testing.T) { tc.prepareFunc() } - err := svc.DeleteMultiSubnetConnectionBindingMapsByCRs(sets.New[string](tc.bindingCRIDs...)) + err = svc.DeleteMultiSubnetConnectionBindingMapsByCRs(sets.New[string](tc.bindingCRIDs...)) if tc.expErr != "" { require.EqualError(t, err, tc.expErr) } else { @@ -468,14 +474,14 @@ func TestDeleteSubnetConnectionBindingMaps(t *testing.T) { deleteFn: func(svc *BindingService) error { mockOrgRootClient.EXPECT().Patch(gomock.Any(), &enforceRevisionCheckParam).Return(nil) ctx := context.Background() - return svc.Cleanup(ctx) + return svc.CleanupBeforeVPCDeletion(ctx) }, expErr: "", expBindingMapsInStore: []*model.SubnetConnectionBindingMap{}, }, } { t.Run(tc.name, func(t *testing.T) { - + builder, _ := common.PolicyPathVpcSubnetConnectionBindingMap.NewPolicyTreeBuilder() svc := &BindingService{ Service: common.Service{ NSXClient: &nsx.Client{ @@ -488,6 +494,7 @@ func TestDeleteSubnetConnectionBindingMaps(t *testing.T) { }, }, BindingStore: SetupStore(), + builder: builder, } svc.BindingStore.Add(createdBM1) err := tc.deleteFn(svc) @@ -501,48 +508,3 @@ func TestDeleteSubnetConnectionBindingMaps(t *testing.T) { }) } } - -func TestDeleteSubnetConnectionBindingMaps_WithPaging(t *testing.T) { - for _, tc := range []struct { - name string - bindingCount int - expPages int - }{ - { - name: "no paging on the requests", - bindingCount: 50, - expPages: 1, - }, { - name: "paging on the requests", - bindingCount: 1501, - expPages: 2, - }, - } { - t.Run(tc.name, func(t *testing.T) { - actPages := 0 - targetBindingMaps := make([]*model.SubnetConnectionBindingMap, tc.bindingCount) - expDeletedBindings := make([]string, tc.bindingCount) - for i := 0; i < tc.bindingCount; i++ { - idString := fmt.Sprintf("id-%d", i) - expDeletedBindings[i] = idString - targetBindingMaps[i] = &model.SubnetConnectionBindingMap{ - Id: common.String(idString), - } - } - actDeletedBindings := make([]string, 0) - svc := &BindingService{} - patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(svc), "hDeleteSubnetConnectionBindingMap", func(_ *BindingService, bindingMaps []*model.SubnetConnectionBindingMap) error { - for _, bm := range bindingMaps { - actDeletedBindings = append(actDeletedBindings, *bm.Id) - } - actPages += 1 - return nil - }) - defer patches.Reset() - err := svc.deleteSubnetConnectionBindingMaps(targetBindingMaps) - require.NoError(t, err) - assert.ElementsMatch(t, expDeletedBindings, actDeletedBindings) - assert.Equal(t, tc.expPages, actPages) - }) - } -} diff --git a/pkg/nsx/services/subnetbinding/tree.go b/pkg/nsx/services/subnetbinding/tree.go deleted file mode 100644 index e123bce5d..000000000 --- a/pkg/nsx/services/subnetbinding/tree.go +++ /dev/null @@ -1,228 +0,0 @@ -package subnetbinding - -import ( - "github.com/vmware/vsphere-automation-sdk-go/runtime/data" - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" - - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" -) - -var leafType = "SubnetConnectionBindingMap" - -type hNode struct { - resourceType string - resourceID string - bindingMap *model.SubnetConnectionBindingMap - childNodes []*hNode -} - -// TODO: Refine the struct of hNode to avoid "linear search" when merging nodes in case it has performance -// issue if the number of resources is huge. -func (n *hNode) mergeChildNode(node *hNode) { - if node.resourceType == leafType { - n.childNodes = append(n.childNodes, node) - return - } - - for _, cn := range n.childNodes { - if cn.resourceType == node.resourceType && cn.resourceID == node.resourceID { - for _, chN := range node.childNodes { - cn.mergeChildNode(chN) - } - return - } - } - n.childNodes = append(n.childNodes, node) -} - -func (n *hNode) buildTree() ([]*data.StructValue, error) { - if n.resourceType == leafType { - dataValue, err := wrapSubnetBindingMap(n.bindingMap) - if err != nil { - return nil, err - } - return []*data.StructValue{dataValue}, nil - } - - children := make([]*data.StructValue, 0) - for _, cn := range n.childNodes { - cnDataValues, err := cn.buildTree() - if err != nil { - return nil, err - } - children = append(children, cnDataValues...) - } - if n.resourceType == "OrgRoot" { - return children, nil - } - - return wrapChildResourceReference(n.resourceType, n.resourceID, children) -} - -func buildHNodeFromSubnetConnectionBindingMap(subnetPath string, bindingMap *model.SubnetConnectionBindingMap) (*hNode, error) { - vpcInfo, err := common.ParseVPCResourcePath(subnetPath) - if err != nil { - return nil, err - } - return &hNode{ - resourceType: "Org", - resourceID: vpcInfo.OrgID, - childNodes: []*hNode{ - { - resourceType: "Project", - resourceID: vpcInfo.ProjectID, - childNodes: []*hNode{ - { - resourceID: vpcInfo.VPCID, - resourceType: "Vpc", - childNodes: []*hNode{ - { - resourceID: vpcInfo.ID, - resourceType: "VpcSubnet", - childNodes: []*hNode{ - { - resourceID: *bindingMap.Id, - resourceType: leafType, - bindingMap: bindingMap, - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} - -func buildRootNode(bindingMaps []*model.SubnetConnectionBindingMap, subnetPath string) *hNode { - rootNode := &hNode{ - resourceType: "OrgRoot", - } - - for _, bm := range bindingMaps { - parentPath := subnetPath - if parentPath == "" { - parentPath = *bm.ParentPath - } - orgNode, err := buildHNodeFromSubnetConnectionBindingMap(parentPath, bm) - if err != nil { - log.Error(err, "Failed to build data value for SubnetConnectionBindingMap, ignore", "bindingMap", *bm.Path) - continue - } - rootNode.mergeChildNode(orgNode) - } - return rootNode -} - -func buildOrgRootBySubnetConnectionBindingMaps(bindingMaps []*model.SubnetConnectionBindingMap, subnetPath string) (*model.OrgRoot, error) { - rootNode := buildRootNode(bindingMaps, subnetPath) - - children, err := rootNode.buildTree() - if err != nil { - log.Error(err, "Failed to build data values for multiple SubnetConnectionBindingMaps") - return nil, err - } - - return &model.OrgRoot{ - Children: children, - ResourceType: String("OrgRoot"), - }, nil -} - -func wrapChildResourceReference(targetType, resID string, children []*data.StructValue) ([]*data.StructValue, error) { - childRes := model.ChildResourceReference{ - Id: &resID, - ResourceType: "ChildResourceReference", - TargetType: &targetType, - Children: children, - } - dataValue, errors := common.NewConverter().ConvertToVapi(childRes, model.ChildResourceReferenceBindingType()) - if len(errors) > 0 { - return nil, errors[0] - } - return []*data.StructValue{dataValue.(*data.StructValue)}, nil -} - -func wrapSubnetBindingMap(bindingMap *model.SubnetConnectionBindingMap) (*data.StructValue, error) { - bindingMap.ResourceType = &common.ResourceTypeSubnetConnectionBindingMap - childBindingMap := model.ChildSubnetConnectionBindingMap{ - Id: bindingMap.Id, - MarkedForDelete: bindingMap.MarkedForDelete, - ResourceType: "ChildSubnetConnectionBindingMap", - SubnetConnectionBindingMap: bindingMap, - } - dataValue, errors := common.NewConverter().ConvertToVapi(childBindingMap, model.ChildSubnetConnectionBindingMapBindingType()) - if len(errors) > 0 { - return nil, errors[0] - } - return dataValue.(*data.StructValue), nil -} - -func (s *BindingService) hUpdateSubnetConnectionBindingMaps(subnetPath string, bindingMaps []*model.SubnetConnectionBindingMap) error { - vpcInfo, err := common.ParseVPCResourcePath(subnetPath) - if err != nil { - return err - } - subnetID := vpcInfo.ID - orgRoot, err := buildOrgRootBySubnetConnectionBindingMaps(bindingMaps, subnetPath) - if err != nil { - return err - } - - if err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &enforceRevisionCheckParam); err != nil { - log.Error(err, "Failed to patch SubnetConnectionBindingMaps on NSX", "orgID", vpcInfo.OrgID, "projectID", vpcInfo.ProjectID, "vpcID", vpcInfo.VPCID, "subnetID", subnetID, "subnetConnectionBindingMaps", bindingMaps) - err = nsxutil.TransNSXApiError(err) - return err - } - - // Get SubnetConnectionBindingMaps from NSX after patch operation as NSX renders several fields like `path`/`parent_path`. - subnetBindingListResult, err := s.NSXClient.SubnetConnectionBindingMapsClient.List(vpcInfo.OrgID, vpcInfo.ProjectID, vpcInfo.VPCID, subnetID, nil, nil, nil, nil, nil, nil) - if err != nil { - log.Error(err, "Failed to list SubnetConnectionBindingMaps from NSX under subnet", "orgID", vpcInfo.OrgID, "projectID", vpcInfo.ProjectID, "vpcID", vpcInfo.VPCID, "subnetID", subnetID, "subnetConnectionBindingMaps", bindingMaps) - err = nsxutil.TransNSXApiError(err) - return err - } - - nsxBindingMaps := make(map[string]model.SubnetConnectionBindingMap) - for _, bm := range subnetBindingListResult.Results { - nsxBindingMaps[*bm.Id] = bm - } - - for i := range bindingMaps { - bm := bindingMaps[i] - if bm.MarkedForDelete != nil && *bm.MarkedForDelete { - s.BindingStore.Apply(bm) - } else { - nsxBindingMap := nsxBindingMaps[*bm.Id] - s.BindingStore.Apply(&nsxBindingMap) - } - } - - return nil -} - -func (s *BindingService) hDeleteSubnetConnectionBindingMap(bindingMaps []*model.SubnetConnectionBindingMap) error { - markForDelete := true - for _, bm := range bindingMaps { - bm.MarkedForDelete = &markForDelete - } - - orgRoot, err := buildOrgRootBySubnetConnectionBindingMaps(bindingMaps, "") - if err != nil { - return err - } - - if err = s.NSXClient.OrgRootClient.Patch(*orgRoot, &enforceRevisionCheckParam); err != nil { - log.Error(err, "Failed to delete multiple SubnetConnectionBindingMaps on NSX with HAPI") - err = nsxutil.TransNSXApiError(err) - return err - } - - // Remove SubnetConnectionBindingMap from local store. - for _, bm := range bindingMaps { - s.BindingStore.Apply(bm) - } - return nil -} diff --git a/pkg/nsx/services/subnetbinding/tree_test.go b/pkg/nsx/services/subnetbinding/tree_test.go index 6c8ac16eb..25bffa89f 100644 --- a/pkg/nsx/services/subnetbinding/tree_test.go +++ b/pkg/nsx/services/subnetbinding/tree_test.go @@ -3,11 +3,7 @@ package subnetbinding import ( "fmt" "strings" - "testing" - "time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/vmware/vsphere-automation-sdk-go/runtime/data" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" @@ -167,123 +163,6 @@ func stringEquals(s1, s2 *string) bool { return *s1 == *s2 } -func TestBuildHNodeFromSubnetConnectionBindingMap(t *testing.T) { - for _, tc := range []struct { - name string - subnetBindings []*model.SubnetConnectionBindingMap - expOrgRootConfig map[string]map[string]map[string]map[string][]string - }{ - { - name: "bindings under same subnets", - subnetBindings: []*model.SubnetConnectionBindingMap{ - genSubnetConnectionBindingMap(bm1ID, "binding1", parentSubnetPath1, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet1", 201), - genSubnetConnectionBindingMap(bm2ID, "binding2", parentSubnetPath2, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet1", 202), - }, - expOrgRootConfig: map[string]map[string]map[string]map[string][]string{ - "default": { - "default": { - "vpc1": { - "subnet1": []string{bm1ID, bm2ID}, - }, - }, - }, - }, - }, - { - name: "bindings under different subnets", - subnetBindings: []*model.SubnetConnectionBindingMap{ - genSubnetConnectionBindingMap(bm1ID, "binding1", parentSubnetPath1, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet1", 201), - genSubnetConnectionBindingMap(bm2ID, "binding2", parentSubnetPath2, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet2", 202), - }, - expOrgRootConfig: map[string]map[string]map[string]map[string][]string{ - "default": { - "default": { - "vpc1": { - "subnet1": []string{bm1ID}, - "subnet2": []string{bm2ID}, - }, - }, - }, - }, - }, { - name: "bindings under different VPCs", - subnetBindings: []*model.SubnetConnectionBindingMap{ - genSubnetConnectionBindingMap(bm1ID, "binding1", parentSubnetPath1, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet1", 201), - genSubnetConnectionBindingMap(bm2ID, "binding2", parentSubnetPath2, "/orgs/default/projects/default/vpcs/vpc2/subnets/subnet1", 202), - }, - expOrgRootConfig: map[string]map[string]map[string]map[string][]string{ - "default": { - "default": { - "vpc1": { - "subnet1": []string{bm1ID}, - }, - "vpc2": { - "subnet1": []string{bm2ID}, - }, - }, - }, - }, - }, { - name: "bindings under different projects", - subnetBindings: []*model.SubnetConnectionBindingMap{ - genSubnetConnectionBindingMap(bm1ID, "binding1", parentSubnetPath1, "/orgs/default/projects/default/vpcs/vpc1/subnets/subnet1", 201), - genSubnetConnectionBindingMap(bm2ID, "binding2", parentSubnetPath2, "/orgs/default/projects/project1/vpcs/vpc2/subnets/subnet1", 202), - }, - expOrgRootConfig: map[string]map[string]map[string]map[string][]string{ - "default": { - "default": { - "vpc1": { - "subnet1": []string{bm1ID}, - }, - }, - "project1": { - "vpc2": { - "subnet1": []string{bm2ID}, - }, - }, - }, - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - bmMappings := make(map[string]*model.SubnetConnectionBindingMap) - for _, bm := range tc.subnetBindings { - bmMappings[*bm.Id] = bm - } - orgRootConfig := convertToOrgConfig(tc.expOrgRootConfig, bmMappings) - expOrgRoot, err := wrapOrgRoot(orgRootConfig) - require.Nil(t, err) - - orgRoot, err := buildOrgRootBySubnetConnectionBindingMaps(tc.subnetBindings, "") - require.NoError(t, err) - - expRoot := orgRootMatcher{expOrgRoot} - assert.True(t, expRoot.matches(orgRoot)) - }) - } -} - -func convertToOrgConfig(testCfg map[string]map[string]map[string]map[string][]string, subnetBindings map[string]*model.SubnetConnectionBindingMap) map[string]map[string]map[string]map[string][]*model.SubnetConnectionBindingMap { - out := make(map[string]map[string]map[string]map[string][]*model.SubnetConnectionBindingMap) - for k1, v1 := range testCfg { - out[k1] = make(map[string]map[string]map[string][]*model.SubnetConnectionBindingMap) - for k2, v2 := range v1 { - out[k1][k2] = make(map[string]map[string][]*model.SubnetConnectionBindingMap) - for k3, v3 := range v2 { - out[k1][k2][k3] = make(map[string][]*model.SubnetConnectionBindingMap) - for k4, v4 := range v3 { - bms := make([]*model.SubnetConnectionBindingMap, len(v4)) - for i, bmID := range v4 { - bms[i] = subnetBindings[bmID] - } - out[k1][k2][k3][k4] = bms - } - } - } - } - return out -} - func wrapOrgRoot(orgConfigs map[string]map[string]map[string]map[string][]*model.SubnetConnectionBindingMap) (*model.OrgRoot, error) { // This is the outermost layer of the hierarchy SubnetConnectionBindingMaps. // It doesn't need ID field. @@ -312,7 +191,7 @@ func wrapOrg(orgID string, orgConfig map[string]map[string]map[string][]*model.S } children = append(children, child...) } - return wrapChildResourceReference("Org", orgID, children) + return common.WrapChildResourceReference("Org", orgID, children) } func wrapProject(projectID string, projectConfig map[string]map[string][]*model.SubnetConnectionBindingMap) ([]*data.StructValue, error) { @@ -324,7 +203,7 @@ func wrapProject(projectID string, projectConfig map[string]map[string][]*model. } children = append(children, child...) } - return wrapChildResourceReference("Project", projectID, children) + return common.WrapChildResourceReference("Project", projectID, children) } func wrapVPC(vpcID string, vpcConfig map[string][]*model.SubnetConnectionBindingMap) ([]*data.StructValue, error) { @@ -336,7 +215,7 @@ func wrapVPC(vpcID string, vpcConfig map[string][]*model.SubnetConnectionBinding } children = append(children, child...) } - return wrapChildResourceReference("Vpc", vpcID, children) + return common.WrapChildResourceReference("Vpc", vpcID, children) } func wrapSubnet(subnetId string, bindingMaps []*model.SubnetConnectionBindingMap) ([]*data.StructValue, error) { @@ -344,13 +223,13 @@ func wrapSubnet(subnetId string, bindingMaps []*model.SubnetConnectionBindingMap if err != nil { return nil, err } - return wrapChildResourceReference("VpcSubnet", subnetId, children) + return common.WrapChildResourceReference("VpcSubnet", subnetId, children) } func wrapSubnetBindingMaps(bindingMaps []*model.SubnetConnectionBindingMap) ([]*data.StructValue, error) { dataValues := make([]*data.StructValue, 0) for _, bindingMap := range bindingMaps { - dataValue, err := wrapSubnetBindingMap(bindingMap) + dataValue, err := common.WrapSubnetConnectionBindingMap(bindingMap) if err != nil { return nil, err } @@ -358,42 +237,3 @@ func wrapSubnetBindingMaps(bindingMaps []*model.SubnetConnectionBindingMap) ([]* } return dataValues, nil } - -func TestBuildRootNodePerformance(t *testing.T) { - orgPrefix, orgCount := "org", 1 - projectPrefix, projectCount := "proj", 10 - vpcPrefix, vpcCount := "vpc", 20 - subnetPrefix, subnetCount := "subnet", 100 - bindingPrefix, bindingCount := "binding", 5 - - bindings := make([]*model.SubnetConnectionBindingMap, 0) - for i := 1; i <= orgCount; i++ { - orgID := fmt.Sprintf("%s%d", orgPrefix, i) - for j := 1; j <= projectCount; j++ { - projID := fmt.Sprintf("%s%d", projectPrefix, j) - for k := 1; k <= vpcCount; k++ { - vpcID := fmt.Sprintf("%s%d", vpcPrefix, k) - for l := 1; l <= subnetCount; l++ { - subnetID := fmt.Sprintf("%s%d", subnetPrefix, l) - subnetPath := fmt.Sprintf("/orgs/%s/projects/%s/vpcs/%s/subnets/%s", orgID, projID, vpcID, subnetID) - for m := 0; m <= bindingCount; m++ { - bindingID := fmt.Sprintf("%s%d", bindingPrefix, m) - bindingPath := fmt.Sprintf("%s/subnet-connection-binding-maps/%s", subnetPath, bindingID) - binding := &model.SubnetConnectionBindingMap{ - Id: common.String(bindingID), - Path: common.String(bindingPath), - ParentPath: common.String(subnetPath), - ResourceType: common.String(leafType), - } - bindings = append(bindings, binding) - } - } - } - } - } - - // The total time to build OrgRoot with 10W SubnetConnectionBindngMaps is supposed to less than 10s. - start := time.Now() - buildRootNode(bindings, "") - assert.True(t, time.Now().Sub(start).Seconds() < 5) -} diff --git a/pkg/nsx/services/subnetport/cleanup.go b/pkg/nsx/services/subnetport/cleanup.go new file mode 100644 index 000000000..eb83a5b77 --- /dev/null +++ b/pkg/nsx/services/subnetport/cleanup.go @@ -0,0 +1,28 @@ +package subnetport + +import ( + "context" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func (service *SubnetPortService) CleanupBeforeVPCDeletion(ctx context.Context) error { + objs := service.SubnetPortStore.List() + log.Info("Cleaning up VpcSubnetPorts", "Count", len(objs)) + if len(objs) == 0 { + return nil + } + + // Mark the resources for delete. + ports := make([]*model.VpcSubnetPort, len(objs)) + for i, obj := range objs { + port := obj.(*model.VpcSubnetPort) + port.MarkedForDelete = &MarkedForDelete + ports[i] = port + } + return service.builder.PagingDeleteResources(ctx, ports, common.DefaultHAPIChildrenCount, service.NSXClient, func(delObjs []*model.VpcSubnetPort) { + service.SubnetPortStore.DeleteMultiplePorts(delObjs) + }) +} diff --git a/pkg/nsx/services/subnetport/store.go b/pkg/nsx/services/subnetport/store.go index 30f6177a4..21d3980ac 100644 --- a/pkg/nsx/services/subnetport/store.go +++ b/pkg/nsx/services/subnetport/store.go @@ -140,3 +140,9 @@ func (subnetPortStore *SubnetPortStore) GetByIndex(key string, value string) []* } return subnetPorts } + +func (subnetPortStore *SubnetPortStore) DeleteMultiplePorts(allocations []*model.VpcSubnetPort) { + for _, allocation := range allocations { + subnetPortStore.Delete(allocation) + } +} diff --git a/pkg/nsx/services/subnetport/subnetport.go b/pkg/nsx/services/subnetport/subnetport.go index b3270e1f5..3b447dd57 100644 --- a/pkg/nsx/services/subnetport/subnetport.go +++ b/pkg/nsx/services/subnetport/subnetport.go @@ -32,17 +32,20 @@ var ( type SubnetPortService struct { servicecommon.Service SubnetPortStore *SubnetPortStore + builder *servicecommon.PolicyTreeBuilder[*model.VpcSubnetPort] } // InitializeSubnetPort sync NSX resources. func InitializeSubnetPort(service servicecommon.Service) (*SubnetPortService, error) { + builder, _ := servicecommon.PolicyPathVpcSubnetPort.NewPolicyTreeBuilder() + wg := sync.WaitGroup{} wgDone := make(chan bool) fatalErrors := make(chan error) wg.Add(1) - subnetPortService := &SubnetPortService{Service: service} + subnetPortService := &SubnetPortService{Service: service, builder: builder} subnetPortService.SubnetPortStore = &SubnetPortStore{ ResourceStore: servicecommon.ResourceStore{ @@ -54,10 +57,10 @@ func InitializeSubnetPort(service servicecommon.Service) (*SubnetPortService, er servicecommon.TagScopeVMNamespace: subnetPortIndexNamespace, servicecommon.TagScopeNamespace: subnetPortIndexPodNamespace, servicecommon.IndexKeySubnetID: subnetPortIndexBySubnetID, + servicecommon.IndexByVPCPathFuncKey: servicecommon.IndexByVPCFunc, }), BindingType: model.VpcSubnetPortBindingType(), - }, - } + }} go subnetPortService.InitializeResourceStore(&wg, fatalErrors, ResourceTypeSubnetPort, nil, subnetPortService.SubnetPortStore) @@ -346,26 +349,6 @@ func (service *SubnetPortService) ListSubnetPortByPodName(ns string, name string return result } -func (service *SubnetPortService) Cleanup(ctx context.Context) error { - subnetPorts := service.SubnetPortStore.List() - log.Info("cleanup subnetports", "count", len(subnetPorts)) - for _, subnetPort := range subnetPorts { - subnetPortID := *subnetPort.(*model.VpcSubnetPort).Id - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - err := service.DeleteSubnetPortById(subnetPortID) - if err != nil { - log.Error(err, "cleanup subnetport failed", "subnetPortID", subnetPortID) - return err - } - - } - } - return nil -} - // AllocatePortFromSubnet checks the number of SubnetPorts on the Subnet. // If the Subnet has capacity for the new SubnetPorts, it will increase // the number of SubnetPort under creation and return true. diff --git a/pkg/nsx/services/subnetport/subnetport_test.go b/pkg/nsx/services/subnetport/subnetport_test.go index bdd42ea23..cdc1b79f3 100644 --- a/pkg/nsx/services/subnetport/subnetport_test.go +++ b/pkg/nsx/services/subnetport/subnetport_test.go @@ -19,6 +19,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" mock_client "github.com/vmware-tanzu/nsx-operator/pkg/mock/controller-runtime/client" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/realizestate" @@ -153,6 +154,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { mockCtl := gomock.NewController(t) k8sClient := mock_client.NewMockClient(mockCtl) defer mockCtl.Finish() + orgRootClient := mock_org_root.NewMockOrgRootClient(mockCtl) commonService := common.Service{ Client: k8sClient, NSXClient: &nsx.Client{ @@ -160,6 +162,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { PortClient: &fakePortClient{}, RealizedEntitiesClient: &fakeRealizedEntitiesClient{}, PortStateClient: &fakePortStateClient{}, + OrgRootClient: orgRootClient, NsxConfig: &config.NSXOperatorConfig{ CoeConfig: &config.CoeConfig{ Cluster: "k8scl-one:test", @@ -172,6 +175,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { }, }, } + builder, _ := common.PolicyPathVpcSubnetPort.NewPolicyTreeBuilder() service := &SubnetPortService{ Service: commonService, SubnetPortStore: &SubnetPortStore{ResourceStore: common.ResourceStore{ @@ -184,6 +188,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { }), BindingType: model.VpcSubnetPortBindingType(), }}, + builder: builder, } subnetPortCR := &v1alpha1.SubnetPort{ @@ -220,6 +225,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { namespaceCR.UID = "ns1" return nil }) + orgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() return nil }, wantErr: false, @@ -233,6 +239,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { return nil }) service.SubnetPortStore.Add(&nsxSubnetPort) + orgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() return nil }, wantErr: false, @@ -298,7 +305,7 @@ func TestSubnetPortService_CreateOrUpdateSubnetPort(t *testing.T) { if (err != nil) != tt.wantErr { t.Errorf("CreateOrUpdateSubnetPort() error = %v, wantErr %v", err, tt.wantErr) } - err = service.Cleanup(context.TODO()) + err = service.CleanupBeforeVPCDeletion(context.TODO()) assert.Nil(t, err) }) } @@ -581,15 +588,23 @@ func TestSubnetPortService_GetPortsOfSubnet(t *testing.T) { } func TestSubnetPortService_Cleanup(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockOrgRootClient := mock_org_root.NewMockOrgRootClient(ctrl) + mockOrgRootClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) + port := model.VpcSubnetPort{ Id: &subnetPortId1, Path: &subnetPortPath1, ParentPath: &subnetPath, } + builder, _ := common.PolicyPathVpcSubnetPort.NewPolicyTreeBuilder() service := &SubnetPortService{ Service: common.Service{ NSXClient: &nsx.Client{ - PortClient: &fakePortClient{}, + PortClient: &fakePortClient{}, + OrgRootClient: mockOrgRootClient, }, }, SubnetPortStore: &SubnetPortStore{ResourceStore: common.ResourceStore{ @@ -600,9 +615,11 @@ func TestSubnetPortService_Cleanup(t *testing.T) { }), BindingType: model.VpcSubnetPortBindingType(), }}, + builder: builder, } + service.SubnetPortStore.Add(&port) - err := service.Cleanup(context.TODO()) + err := service.CleanupBeforeVPCDeletion(context.TODO()) assert.Nil(t, err) assert.Nil(t, service.SubnetPortStore.GetByKey(*port.Id)) } diff --git a/pkg/nsx/services/vpc/cleanup.go b/pkg/nsx/services/vpc/cleanup.go new file mode 100644 index 000000000..d98de8f00 --- /dev/null +++ b/pkg/nsx/services/vpc/cleanup.go @@ -0,0 +1,146 @@ +package vpc + +import ( + "context" + "strings" + + "github.com/vmware/vsphere-automation-sdk-go/runtime/bindings" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +func (s *VPCService) ListAutoCreatedVPCPaths() sets.Set[string] { + vpcPaths := sets.New[string]() + for _, obj := range s.VpcStore.List() { + vpc := obj.(*model.Vpc) + vpcPaths.Insert(*vpc.Path) + } + return vpcPaths +} + +func (s *VPCService) CleanupBeforeVPCDeletion(ctx context.Context) error { + // LB Virtual Servers are removed before deleting VPC, otherwise it may block the parallel deletion with other + // VPC children e.g., VpcIPAddressAllocations. + if err := s.cleanupSLBVirtualServers(ctx); err != nil { + log.Error(err, "Failed to clean up SLB VirtualServers") + return err + } + return nil +} + +// CleanupVPCChildResources is deleting all the VPC LB pools in the given vpcPath on NSX and/or in local cache. +// If vpcPath is not empty, the function is called with auto-created VPC case, so it only deletes in the local cache for +// the NSX resources are already removed when VPC is deleted recursively. Otherwise, it should delete all cached groups +// on NSX and in local cache. +func (s *VPCService) CleanupVPCChildResources(ctx context.Context, vpcPath string) error { + if err := s.cleanupLBPools(ctx, vpcPath); err != nil { + log.Error(err, "Failed to clean up LB Pool") + return err + } + return nil +} + +func (s *VPCService) cleanupSLBVirtualServers(ctx context.Context) error { + lbVSs, err := s.getStaleSLBVirtualServers() + if err != nil { + return err + } + log.Info("Cleaning up SLB virtual servers ", "Count", len(lbVSs)) + lbVSBuilder, _ := common.PolicyPathVpcLBVirtualServer.NewPolicyTreeBuilder() + return lbVSBuilder.PagingDeleteResources(ctx, lbVSs, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *VPCService) getStaleSLBPools() ([]*model.LBPool, error) { + objs, err := s.getStaleSLBResources(common.ResourceTypeLBPool, model.LBPoolBindingType()) + if err != nil { + return nil, err + } + var lbPools []*model.LBPool + for _, obj := range objs { + lbPools = append(lbPools, obj.(*model.LBPool)) + } + return lbPools, nil +} + +func (s *VPCService) getStaleSLBVirtualServers() ([]*model.LBVirtualServer, error) { + objs, err := s.getStaleSLBResources(common.ResourceTypeLBVirtualServer, model.LBVirtualServerBindingType()) + if err != nil { + return nil, err + } + var lbVSs []*model.LBVirtualServer + for _, obj := range objs { + lbVSs = append(lbVSs, obj.(*model.LBVirtualServer)) + } + return lbVSs, nil +} + +func (s *VPCService) getStaleSLBResources(resourceType string, resourceBindingType bindings.BindingType) ([]interface{}, error) { + store, err := s.querySLBResources(resourceType, resourceBindingType) + if err != nil { + return nil, err + } + + autoCreatedVPCPaths := s.ListAutoCreatedVPCPaths() + + isSLBResourceValidToDelete := func(obj interface{}) bool { + var parentPath *string + switch obj.(type) { + case *model.LBVirtualServer: + lbVS := obj.(*model.LBVirtualServer) + parentPath = lbVS.ParentPath + lbVS.MarkedForDelete = &MarkedForDelete + case *model.LBPool: + lbPool := obj.(*model.LBPool) + parentPath = lbPool.ParentPath + lbPool.MarkedForDelete = &MarkedForDelete + default: + return false + } + if parentPath == nil { + return false + } + return strings.HasPrefix(*parentPath, "/orgs") && !autoCreatedVPCPaths.Has(*parentPath) + } + + var slbObjects []interface{} + for _, obj := range store.List() { + if !isSLBResourceValidToDelete(obj) { + continue + } + slbObjects = append(slbObjects, obj) + } + return slbObjects, nil +} + +func (s *VPCService) cleanupLBPools(ctx context.Context, vpcPath string) error { + if vpcPath != "" { + // Return directly with the case for auto-created VPC by which vpcPath is not empty, since the LB Pool is + // automatically removed when deleting the VPC recursively. + return nil + } + + lbPools, err := s.getStaleSLBPools() + if err != nil { + return err + } + log.Info("Cleaning up SLB pools ", "Count", len(lbPools)) + lbPoolBuilder, _ := common.PolicyPathVpcLBPool.NewPolicyTreeBuilder() + return lbPoolBuilder.PagingDeleteResources(ctx, lbPools, common.DefaultHAPIChildrenCount, s.NSXClient, nil) +} + +func (s *VPCService) querySLBResources(resourceType string, resourceBindingType bindings.BindingType) (*ResourceStore, error) { + store := &ResourceStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), + BindingType: resourceBindingType, + }} + if err := s.Service.QueryNCPCreatedResources([]string{resourceType}, store, func(query string) string { + return common.AddNCPCreatedForTag(query, common.TagValueSLB) + }); err != nil { + log.Error(err, "Failed to query SLB resources", "resource type", resourceType) + return store, err + } + return store, nil +} diff --git a/pkg/nsx/services/vpc/cleanup_test.go b/pkg/nsx/services/vpc/cleanup_test.go new file mode 100644 index 000000000..f50e0ca7f --- /dev/null +++ b/pkg/nsx/services/vpc/cleanup_test.go @@ -0,0 +1,489 @@ +package vpc + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vmware/vsphere-automation-sdk-go/runtime/data" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/cache" + + "github.com/vmware-tanzu/nsx-operator/pkg/config" + mock_org_root "github.com/vmware-tanzu/nsx-operator/pkg/mock/orgrootclient" + mocks "github.com/vmware-tanzu/nsx-operator/pkg/mock/searchclient" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" +) + +var ( + projectPath = fmt.Sprintf("/orgs/default/projects/project-1") + vpcPath = fmt.Sprintf("%s/vpcs/vpc-1", projectPath) + infraVSId = "infra-lb-vs" + vpcVSId = "vpc-lb-vs" + infraPoolId = "infra-lb-pool" + vpcPoolId = "vpc-lb-pool" + autoVpcID = "auto-vpc1" + autoVpcPath = fmt.Sprintf("%s/vpcs/%s", projectPath, autoVpcID) +) + +func TestListAutoCreatedVPCPaths(t *testing.T) { + vpcService := &VPCService{ + VpcStore: &VPCStore{ResourceStore: common.ResourceStore{ + Indexer: cache.NewIndexer(keyFunc, cache.Indexers{ + common.TagScopeNamespaceUID: vpcIndexNamespaceIDFunc, + common.TagScopeNamespace: vpcIndexNamespaceNameFunc, + }), + BindingType: model.VpcBindingType(), + }}, + } + + vpc1 := &model.Vpc{ + Id: common.String(autoVpcID), + Path: common.String(autoVpcPath), + Tags: []model.Tag{ + { + Scope: common.String(common.TagScopeNamespaceUID), + Tag: common.String("namespace1-uid"), + }, + { + Scope: common.String(common.TagScopeNamespace), + Tag: common.String("namespace1"), + }, + { + Scope: common.String(common.TagScopeVPCManagedBy), + Tag: common.String(common.AutoCreatedVPCTagValue), + }, + }, + } + err := vpcService.VpcStore.Add(vpc1) + require.NoError(t, err) + vpcPathSet := vpcService.ListAutoCreatedVPCPaths() + require.Equal(t, 1, vpcPathSet.Len()) + assert.Equal(t, *vpc1.Path, vpcPathSet.UnsortedList()[0]) +} + +func TestGetStaleSLBVirtualServers(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockQueryClient := mocks.NewMockQueryClient(ctrl) + vpcService := &VPCService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + QueryClient: mockQueryClient, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + VpcStore: &VPCStore{}, + } + + patches := gomonkey.ApplyMethod(reflect.TypeOf(vpcService), "ListAutoCreatedVPCPaths", func(*VPCService) sets.Set[string] { + return sets.New[string](autoVpcPath) + }) + defer patches.Reset() + + results := make([]*data.StructValue, 0) + for _, vs := range prepareLBVirtualServers() { + vsData, errs := NewConverter().ConvertToVapi(vs, model.LBVirtualServerBindingType()) + require.Equal(t, 0, len(errs)) + results = append(results, vsData.(*data.StructValue)) + } + + // Test with happy path. + expQueryParam := "resource_type:LBVirtualServer AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:SLB" + resultCount := int64(len(results)) + cursor := fmt.Sprintf("%d", resultCount) + expResponse := model.SearchResponse{ + Results: results, + Cursor: &cursor, + ResultCount: &resultCount, + } + for _, tc := range []struct { + name string + nsxErr error + expErrStr string + }{ + { + name: "success to search on NSX", + nsxErr: nil, + }, { + name: "failed to query on NSX", + nsxErr: fmt.Errorf("connection issue"), + expErrStr: "connection issue", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mockQueryClient.EXPECT().List(expQueryParam, gomock.Any(), nil, gomock.Any(), nil, nil).Return(expResponse, tc.nsxErr) + vss, err := vpcService.getStaleSLBVirtualServers() + if tc.nsxErr != nil { + require.EqualError(t, err, tc.expErrStr) + } else { + require.NoError(t, err) + assert.Equal(t, 1, len(vss)) + vs := vss[0] + assert.Equal(t, vpcVSId, *vs.Id) + } + }) + } + +} + +func TestCleanupBeforeVPCDeletion(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockOrgClient := mock_org_root.NewMockOrgRootClient(ctrl) + vpcService := &VPCService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + OrgRootClient: mockOrgClient, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + VpcStore: &VPCStore{}, + } + + validCtx := context.Background() + canceledCtx, cancelFn := context.WithCancel(validCtx) + cancelFn() + + for _, tc := range []struct { + name string + ctx context.Context + mockFn func(s *VPCService) *gomonkey.Patches + expErrStr string + }{ + { + name: "success with no SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBVirtualServers", func(_ *VPCService) ([]*model.LBVirtualServer, error) { + return nil, nil + }) + return patches + }, + }, { + name: "success to cleanup SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBVirtualServers", func(_ *VPCService) ([]*model.LBVirtualServer, error) { + return []*model.LBVirtualServer{ + { + Id: common.String(vpcVSId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-virtual-servers/%s", vpcPath, vpcVSId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + mockOrgClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) + return patches + }, + }, { + name: "failed to query SLB on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBVirtualServers", func(_ *VPCService) ([]*model.LBVirtualServer, error) { + return nil, fmt.Errorf("failed to query SLB") + }) + return patches + }, + expErrStr: "failed to query SLB", + }, { + name: "failed to clean up SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBVirtualServers", func(_ *VPCService) ([]*model.LBVirtualServer, error) { + return []*model.LBVirtualServer{ + { + Id: common.String(vpcVSId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-virtual-servers/%s", vpcPath, vpcVSId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + mockOrgClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(fmt.Errorf("issues to clean up resources")) + return patches + }, + expErrStr: "issues to clean up resources", + }, { + name: "failed to clean up SLB with canceled context", + ctx: canceledCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBVirtualServers", func(_ *VPCService) ([]*model.LBVirtualServer, error) { + return []*model.LBVirtualServer{ + { + Id: common.String(vpcVSId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-virtual-servers/%s", vpcPath, vpcVSId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + return patches + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + } { + t.Run(tc.name, func(t *testing.T) { + patches := tc.mockFn(vpcService) + defer patches.Reset() + + err := vpcService.CleanupBeforeVPCDeletion(tc.ctx) + if tc.expErrStr == "" { + require.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expErrStr) + } + }) + } + +} + +func TestGetStaleSLBPools(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockQueryClient := mocks.NewMockQueryClient(ctrl) + vpcService := &VPCService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + QueryClient: mockQueryClient, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + VpcStore: &VPCStore{}, + } + + patches := gomonkey.ApplyMethod(reflect.TypeOf(vpcService), "ListAutoCreatedVPCPaths", func(*VPCService) sets.Set[string] { + return sets.New[string](autoVpcPath) + }) + defer patches.Reset() + + results := make([]*data.StructValue, 0) + for _, vs := range prepareLBPools() { + lbData, errs := NewConverter().ConvertToVapi(vs, model.LBPoolBindingType()) + require.Equal(t, 0, len(errs)) + results = append(results, lbData.(*data.StructValue)) + } + + // Test with happy path. + expQueryParam := "resource_type:LBPool AND tags.scope:ncp\\/cluster AND tags.tag:k8scl-one\\:test AND tags.scope:ncp\\/created_for AND tags.tag:SLB" + resultCount := int64(len(results)) + cursor := fmt.Sprintf("%d", resultCount) + expResponse := model.SearchResponse{ + Results: results, + Cursor: &cursor, + ResultCount: &resultCount, + } + for _, tc := range []struct { + name string + nsxErr error + expErrStr string + }{ + { + name: "success to search on NSX", + nsxErr: nil, + }, { + name: "failed to query on NSX", + nsxErr: fmt.Errorf("connection issue"), + expErrStr: "connection issue", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mockQueryClient.EXPECT().List(expQueryParam, gomock.Any(), nil, gomock.Any(), nil, nil).Return(expResponse, tc.nsxErr) + pools, err := vpcService.getStaleSLBPools() + if tc.nsxErr != nil { + require.EqualError(t, err, tc.expErrStr) + } else { + require.NoError(t, err) + assert.Equal(t, 1, len(pools)) + pool := pools[0] + assert.Equal(t, vpcPoolId, *pool.Id) + } + }) + } + +} + +func TestCleanupVPCChildResources(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockOrgClient := mock_org_root.NewMockOrgRootClient(ctrl) + vpcService := &VPCService{ + Service: common.Service{ + NSXClient: &nsx.Client{ + OrgRootClient: mockOrgClient, + NsxConfig: &config.NSXOperatorConfig{ + CoeConfig: &config.CoeConfig{ + Cluster: "k8scl-one:test", + }, + }, + }, + }, + VpcStore: &VPCStore{}, + } + + validCtx := context.Background() + canceledCtx, cancelFn := context.WithCancel(validCtx) + cancelFn() + + for _, tc := range []struct { + name string + ctx context.Context + mockFn func(s *VPCService) *gomonkey.Patches + vpcPath string + expErrStr string + }{ + { + name: "success with auto-created VPC", + ctx: validCtx, + vpcPath: autoVpcPath, + mockFn: func(s *VPCService) *gomonkey.Patches { + return nil + }, + }, + { + name: "success with no SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBPools", func(_ *VPCService) ([]*model.LBPool, error) { + return nil, nil + }) + return patches + }, + }, { + name: "success to cleanup SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBPools", func(_ *VPCService) ([]*model.LBPool, error) { + return []*model.LBPool{ + { + Id: common.String(vpcPoolId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-pools/%s", vpcPath, vpcPoolId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + mockOrgClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(nil) + return patches + }, + }, { + name: "failed to query SLB on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBPools", func(_ *VPCService) ([]*model.LBPool, error) { + return nil, fmt.Errorf("failed to query SLB") + }) + return patches + }, + expErrStr: "failed to query SLB", + }, { + name: "failed to clean up SLB found on NSX", + ctx: validCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBPools", func(_ *VPCService) ([]*model.LBPool, error) { + return []*model.LBPool{ + { + Id: common.String(vpcPoolId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-pools/%s", vpcPath, vpcPoolId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + mockOrgClient.EXPECT().Patch(gomock.Any(), gomock.Any()).Return(fmt.Errorf("issues to clean up resources")) + return patches + }, + expErrStr: "issues to clean up resources", + }, { + name: "failed to clean up SLB with canceled context", + ctx: canceledCtx, + mockFn: func(s *VPCService) *gomonkey.Patches { + patches := gomonkey.ApplyPrivateMethod(reflect.TypeOf(vpcService), "getStaleSLBPools", func(_ *VPCService) ([]*model.LBPool, error) { + return []*model.LBPool{ + { + Id: common.String(vpcPoolId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-pools/%s", vpcPath, vpcPoolId)), + ParentPath: common.String(vpcPath), + }, + }, nil + }) + return patches + }, + expErrStr: "failed because of timeout\ncontext canceled", + }, + } { + t.Run(tc.name, func(t *testing.T) { + patches := tc.mockFn(vpcService) + if patches != nil { + defer patches.Reset() + } + + err := vpcService.CleanupVPCChildResources(tc.ctx, tc.vpcPath) + if tc.expErrStr == "" { + require.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expErrStr) + } + }) + } + +} + +func prepareLBVirtualServers() []*model.LBVirtualServer { + return []*model.LBVirtualServer{ + { + Id: common.String(infraVSId), + Path: common.String(fmt.Sprintf("/infra/lb-virtual-servers/%s", infraVSId)), + ParentPath: common.String("/infra"), + }, + { + Id: common.String(vpcVSId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-virtual-servers/%s", vpcPath, vpcVSId)), + ParentPath: common.String(vpcPath), + }, + { + Id: common.String("autovpc-vs"), + Path: common.String(fmt.Sprintf("%s/vpc-lb-virtual-servers/autovpc-vs", autoVpcPath)), + ParentPath: common.String(autoVpcPath), + }, + } +} + +func prepareLBPools() []*model.LBPool { + return []*model.LBPool{ + { + Id: common.String(infraPoolId), + Path: common.String(fmt.Sprintf("/infra/lb-pools/%s", infraPoolId)), + ParentPath: common.String("/infra"), + }, + { + Id: common.String(vpcPoolId), + Path: common.String(fmt.Sprintf("%s/vpc-lb-pools/%s", vpcPath, vpcPoolId)), + ParentPath: common.String(vpcPath), + }, + { + Id: common.String("autovpc-pool"), + Path: common.String(fmt.Sprintf("%s/vpc-lb-pools/autovpc-pool", autoVpcPath)), + ParentPath: common.String(autoVpcPath), + }, + } +} diff --git a/pkg/nsx/services/vpc/store.go b/pkg/nsx/services/vpc/store.go index 682bf28ce..8dc26ba8a 100644 --- a/pkg/nsx/services/vpc/store.go +++ b/pkg/nsx/services/vpc/store.go @@ -148,3 +148,12 @@ func (ls *LBSStore) GetByKey(vpcID string) *model.LBService { } return nil } + +func (ls *LBSStore) GetByIndex(key string, value string) []*model.LBService { + lbss := make([]*model.LBService, 0) + objs := ls.ResourceStore.GetByIndex(key, value) + for _, lbs := range objs { + lbss = append(lbss, lbs.(*model.LBService)) + } + return lbss +} diff --git a/pkg/nsx/services/vpc/store_test.go b/pkg/nsx/services/vpc/store_test.go index 09f110bd3..c305301c4 100644 --- a/pkg/nsx/services/vpc/store_test.go +++ b/pkg/nsx/services/vpc/store_test.go @@ -268,7 +268,7 @@ func TestVPCStore_CRUDResource_List(t *testing.T) { func Test_keyFunc(t *testing.T) { id := "test_id" - vpcPath := fmt.Sprintf(VPCKey, "fake-org", "fake-project", "fake-vpc-id") + vpcPath := fmt.Sprintf(common.VPCKey, "fake-org", "fake-project", "fake-vpc-id") type args struct { obj interface{} } diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 5ef880519..1e44aabbf 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -26,7 +26,6 @@ import ( type LBProvider string const ( - VPCKey = "/orgs/%s/projects/%s/vpcs/%s" albEndpointPath = "policy/api/v1/infra/sites/default/enforcement-points/alb-endpoint" NSXLB = LBProvider("nsx-lb") AVILB = LBProvider("avi") @@ -141,7 +140,7 @@ func (s *VPCService) ValidateNetworkConfig(nc common.VPCNetworkConfigInfo) bool func InitializeVPC(service common.Service) (*VPCService, error) { wg := sync.WaitGroup{} wgDone := make(chan bool) - fatalErrors := make(chan error) + fatalErrors := make(chan error, 2) VPCService := &VPCService{Service: service} VPCService.VpcStore = &VPCStore{ResourceStore: common.ResourceStore{ @@ -151,6 +150,7 @@ func InitializeVPC(service common.Service) (*VPCService, error) { }), BindingType: model.VpcBindingType(), }} + VPCService.LbsStore = &LBSStore{ResourceStore: common.ResourceStore{ Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), BindingType: model.LBServiceBindingType(), @@ -243,290 +243,6 @@ func (s *VPCService) DeleteVPC(path string) error { return nil } -func (s *VPCService) addClusterTag(query string) string { - tagScopeClusterKey := strings.Replace(common.TagScopeNCPCluster, "/", "\\/", -1) - tagScopeClusterValue := strings.Replace(s.NSXClient.NsxConfig.Cluster, ":", "\\:", -1) - tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) - return query + " AND " + tagParam -} - -func (s *VPCService) addNCPCreatedForTag(query string) string { - tagScopeClusterKey := strings.Replace(common.TagScopeNCPCreateFor, "/", "\\/", -1) - tagScopeClusterValue := strings.Replace(common.TagValueSLB, ":", "\\:", -1) - tagParam := fmt.Sprintf("tags.scope:%s AND tags.tag:%s", tagScopeClusterKey, tagScopeClusterValue) - return query + " AND " + tagParam -} - -func (s *VPCService) ListCert() []model.TlsCertificate { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.TlsCertificateBindingType(), - }} - query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeTlsCertificate) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query certificate", "query", query) - } else { - log.V(1).Info("Query certificate", "count", count) - } - certs := store.List() - var certsSet []model.TlsCertificate - for _, cert := range certs { - certsSet = append(certsSet, *cert.(*model.TlsCertificate)) - } - return certsSet -} - -func (s *VPCService) DeleteCert(id string) error { - certClient := s.NSXClient.CertificateClient - if err := certClient.Delete(id); err != nil { - return err - } - log.Info("Successfully deleted NCP created certificate", "certificate", id) - return nil -} - -func (s *VPCService) ListShare() []model.Share { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.ShareBindingType(), - }} - query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeShare) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query share", "query", query) - } else { - log.V(1).Info("Query share", "count", count) - } - shares := store.List() - var sharesSet []model.Share - for _, cert := range shares { - sharesSet = append(sharesSet, *cert.(*model.Share)) - } - return sharesSet -} - -func (s *VPCService) DeleteShare(shareId string) error { - shareClient := s.NSXClient.ShareClient - if err := shareClient.Delete(shareId); err != nil { - return err - } - log.Info("Successfully deleted NCP created share", "share", shareId) - return nil -} - -func (s *VPCService) ListSharedResource() []model.SharedResource { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.SharedResourceBindingType(), - }} - query := fmt.Sprintf("%s:%s", common.ResourceType, common.ResourceTypeSharedResource) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query sharedResource", "query", query) - } else { - log.V(1).Info("Query sharedResource", "count", count) - } - sharedResources := store.List() - var sharedResourcesSet []model.SharedResource - for _, sharedResource := range sharedResources { - sharedResourcesSet = append(sharedResourcesSet, *sharedResource.(*model.SharedResource)) - } - return sharedResourcesSet -} - -func (s *VPCService) DeleteSharedResource(shareId, id string) error { - sharedResourceClient := s.NSXClient.SharedResourceClient - if err := sharedResourceClient.Delete(shareId, id); err != nil { - return err - } - log.Info("Successfully deleted NCP created sharedResource", "shareId", shareId, "sharedResource", id) - return nil -} - -func (s *VPCService) ListLBAppProfile() []model.LBAppProfile { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBAppProfileBindingType(), - }} - query := fmt.Sprintf("(%s:%s OR %s:%s OR %s:%s)", - common.ResourceType, common.ResourceTypeLBHttpProfile, - common.ResourceType, common.ResourceTypeLBFastTcpProfile, - common.ResourceType, common.ResourceTypeLBFastUdpProfile) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query LBAppProfile", "query", query) - } else { - log.V(1).Info("Query LBAppProfile", "Count", count) - } - lbAppProfiles := store.List() - var lbAppProfilesSet []model.LBAppProfile - for _, lbAppProfile := range lbAppProfiles { - lbAppProfilesSet = append(lbAppProfilesSet, *lbAppProfile.(*model.LBAppProfile)) - } - return lbAppProfilesSet -} - -func (s *VPCService) DeleteLBAppProfile(id string) error { - lbAppProfileClient := s.NSXClient.LbAppProfileClient - boolValue := false - if err := lbAppProfileClient.Delete(id, &boolValue); err != nil { - return err - } - log.Info("Successfully deleted NCP created lbAppProfile", "lbAppProfile", id) - return nil -} - -func (s *VPCService) ListLBPersistenceProfile() []model.LBPersistenceProfile { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBPersistenceProfileBindingType(), - }} - query := fmt.Sprintf("(%s:%s OR %s:%s)", - common.ResourceType, common.ResourceTypeLBCookiePersistenceProfile, - common.ResourceType, common.ResourceTypeLBSourceIpPersistenceProfile) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query LBPersistenceProfile", "query", query) - } else { - log.V(1).Info("Query LBPersistenceProfile", "count", count) - } - lbPersistenceProfiles := store.List() - var lbPersistenceProfilesSet []model.LBPersistenceProfile - for _, lbPersistenceProfile := range lbPersistenceProfiles { - lbPersistenceProfilesSet = append(lbPersistenceProfilesSet, *lbPersistenceProfile.(*model.LBPersistenceProfile)) - } - return lbPersistenceProfilesSet -} - -func (s *VPCService) DeleteLBPersistenceProfile(id string) error { - lbPersistenceProfilesClient := s.NSXClient.LbPersistenceProfilesClient - boolValue := false - if err := lbPersistenceProfilesClient.Delete(id, &boolValue); err != nil { - return err - } - log.Info("Successfully deleted NCP created lbPersistenceProfile", "lbPersistenceProfile", id) - return nil -} - -func (s *VPCService) ListLBMonitorProfile() []model.LBMonitorProfile { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBMonitorProfileBindingType(), - }} - query := fmt.Sprintf("(%s:%s OR %s:%s)", - common.ResourceType, common.ResourceTypeLBHttpMonitorProfile, - common.ResourceType, common.ResourceTypeLBTcpMonitorProfile) - query = s.addClusterTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query LBMonitorProfile", "query", query) - } else { - log.V(1).Info("Query LBMonitorProfile", "count", count) - } - lbMonitorProfiles := store.List() - var lbMonitorProfilesSet []model.LBMonitorProfile - for _, lbMonitorProfile := range lbMonitorProfiles { - lbMonitorProfilesSet = append(lbMonitorProfilesSet, *lbMonitorProfile.(*model.LBMonitorProfile)) - } - return lbMonitorProfilesSet -} - -func (s *VPCService) DeleteLBMonitorProfile(id string) error { - lbMonitorProfilesClient := s.NSXClient.LbMonitorProfilesClient - boolValue := false - //nolint:staticcheck // SA1019 ignore this! - if err := lbMonitorProfilesClient.Delete(id, &boolValue); err != nil { - return err - } - log.Info("Successfully deleted NCP created lbMonitorProfile", "lbMonitorProfile", id) - return nil -} - -func (s *VPCService) ListLBVirtualServer() []model.LBVirtualServer { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBVirtualServerBindingType(), - }} - query := fmt.Sprintf("(%s:%s)", - common.ResourceType, common.ResourceTypeLBVirtualServer) - query = s.addClusterTag(query) - query = s.addNCPCreatedForTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query LBVirtualServer", "query", query) - } else { - log.V(1).Info("Query LBVirtualServer", "count", count) - } - lbVirtualServers := store.List() - var lbVirtualServersSet []model.LBVirtualServer - for _, lbVirtualServer := range lbVirtualServers { - lbVirtualServersSet = append(lbVirtualServersSet, *lbVirtualServer.(*model.LBVirtualServer)) - } - return lbVirtualServersSet -} - -func (s *VPCService) DeleteLBVirtualServer(path string) error { - lbVirtualServersClient := s.NSXClient.VpcLbVirtualServersClient - boolValue := false - paths := strings.Split(path, "/") - - if len(paths) < common.VPCLbResourcePathMinSegments { - // skip virtual server under infra - log.Info("Failed to parse virtual server path", "path", path) - return nil - } - if err := lbVirtualServersClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil { - return err - } - log.Info("Successfully deleted NCP created lbVirtualServer", "lbVirtualServer", path) - return nil -} - -func (s *VPCService) ListLBPool() []model.LBPool { - store := &ResourceStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBPoolBindingType(), - }} - query := fmt.Sprintf("(%s:%s)", - common.ResourceType, common.ResourceTypeLBPool) - query = s.addClusterTag(query) - query = s.addNCPCreatedForTag(query) - count, searchErr := s.SearchResource("", query, store, nil) - if searchErr != nil { - log.Error(searchErr, "Failed to query LBPool", "query", query) - } else { - log.V(1).Info("Query LBPool", "Count", count) - } - lbPools := store.List() - var lbPoolsSet []model.LBPool - for _, lbPool := range lbPools { - lbPoolsSet = append(lbPoolsSet, *lbPool.(*model.LBPool)) - } - return lbPoolsSet -} - -func (s *VPCService) DeleteLBPool(path string) error { - lbPoolsClient := s.NSXClient.VpcLbPoolsClient - boolValue := false - paths := strings.Split(path, "/") - if len(paths) < 8 { - // skip lb pool under infra - log.Info("Failed to parse lb pool path", "path", path) - return nil - } - if err := lbPoolsClient.Delete(paths[2], paths[4], paths[6], paths[8], &boolValue); err != nil { - return err - } - log.Info("Successfully deleted NCP created lbPool", "lbPool", path) - return nil -} - func (s *VPCService) IsSharedVPCNamespaceByNS(ctx context.Context, ns string) (bool, error) { _, sharedNamespaceObj, err := s.resolveSharedVPCNamespace(ctx, ns) if err != nil { @@ -781,7 +497,7 @@ func (s *VPCService) CreateOrUpdateVPC(ctx context.Context, obj *v1alpha1.Networ var createdLBS *model.LBService if lbProvider == NSXLB { lbsSize := s.NSXConfig.NsxConfig.GetNSXLBSize() - vpcPath := fmt.Sprintf(VPCKey, nc.Org, nc.NSXProject, nc.Name) + vpcPath := fmt.Sprintf(common.VPCKey, nc.Org, nc.NSXProject, nc.Name) var relaxScaleValidation *bool if s.NSXConfig.NsxConfig.RelaxNSXLBScaleValication { relaxScaleValidation = common.Bool(true) @@ -979,136 +695,6 @@ func (s *VPCService) ValidateGatewayConnectionStatus(nc *common.VPCNetworkConfig return true, "", nil } -func (s *VPCService) Cleanup(ctx context.Context) error { - vpcs := s.ListVPC() - log.Info("Cleaning up VPCs", "Count", len(vpcs)) - for _, vpc := range vpcs { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - // first clean avi subnet ports, or else vpc delete will fail - if err := CleanAviSubnetPorts(ctx, s.NSXClient.Cluster, *vpc.Path); err != nil { - return err - } - - if err := s.DeleteVPC(*vpc.Path); err != nil { - return err - } - } - } - - // Delete NCP created resources (share/sharedResources/cert/LBAppProfile/LBPersistentProfile - sharedResources := s.ListSharedResource() - log.Info("Cleaning up sharedResources", "Count", len(sharedResources)) - for _, sharedResource := range sharedResources { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - parentPath := strings.Split(*sharedResource.ParentPath, "/") - shareId := parentPath[len(parentPath)-1] - if err := s.DeleteSharedResource(shareId, *sharedResource.Id); err != nil { - return err - } - } - } - - shares := s.ListShare() - log.Info("Cleaning up shares", "Count", len(shares)) - for _, share := range shares { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteShare(*share.Id); err != nil { - return err - } - } - } - - certs := s.ListCert() - log.Info("Cleaning up certificates", "Count", len(certs)) - for _, cert := range certs { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteCert(*cert.Id); err != nil { - return err - } - } - } - - lbAppProfiles := s.ListLBAppProfile() - log.Info("Cleaning up lbAppProfiles", "Count", len(lbAppProfiles)) - for _, lbAppProfile := range lbAppProfiles { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteLBAppProfile(*lbAppProfile.Id); err != nil { - return err - } - } - } - - lbPersistenceProfiles := s.ListLBPersistenceProfile() - log.Info("Cleaning up lbPersistenceProfiles", "Count", len(lbPersistenceProfiles)) - for _, lbPersistenceProfile := range lbPersistenceProfiles { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteLBPersistenceProfile(*lbPersistenceProfile.Id); err != nil { - return err - } - } - } - - lbMonitorProfiles := s.ListLBMonitorProfile() - log.Info("Cleaning up lbMonitorProfiles", "Count", len(lbMonitorProfiles)) - for _, lbMonitorProfile := range lbMonitorProfiles { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteLBMonitorProfile(*lbMonitorProfile.Id); err != nil { - return err - } - } - } - - // Clean vs/lb pool created for pre-created vpc - lbVirtualServers := s.ListLBVirtualServer() - log.Info("Cleaning up lbVirtualServers", "Count", len(lbVirtualServers)) - for _, lbVirtualServer := range lbVirtualServers { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteLBVirtualServer(*lbVirtualServer.Path); err != nil { - return err - } - } - } - - lbPools := s.ListLBPool() - log.Info("Cleaning up lbPools", "Count", len(lbPools)) - for _, lbPool := range lbPools { - select { - case <-ctx.Done(): - return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) - default: - if err := s.DeleteLBPool(*lbPool.Path); err != nil { - return err - } - } - } - // We don't clean client_ssl_profile as client_ssl_profile is not created by ncp or nsx-operator - return nil -} - func (s *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { var VPCInfoList []common.VPCResourceInfo nc := s.GetVPCNetworkConfigByNamespace(ns) diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index 350e416ec..4e03cde97 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -7,14 +7,6 @@ import ( "reflect" "strings" "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/agiledragon/gomonkey/v2" "github.com/golang/mock/gomock" @@ -24,9 +16,14 @@ import ( "github.com/vmware/vsphere-automation-sdk-go/runtime/data" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/vmware-tanzu/nsx-operator/pkg/apis/vpc/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/config" @@ -1650,77 +1647,6 @@ func TestListNamespacesWithPreCreatedVPCs(t *testing.T) { } } -func TestVPCService_Cleanup(t *testing.T) { - vpcCacheIndexer := cache.NewIndexer(keyFunc, cache.Indexers{}) - resourceStore := common.ResourceStore{ - Indexer: vpcCacheIndexer, - BindingType: model.VpcBindingType(), - } - - mockCtrl := gomock.NewController(t) - mockVpcclient := mocks.NewMockVpcsClient(mockCtrl) - - vpcService := &VPCService{ - Service: common.Service{ - NSXConfig: &config.NSXOperatorConfig{ - NsxConfig: &config.NsxConfig{ - UseAVILoadBalancer: false, - }, - }, - NSXClient: &nsx.Client{ - Cluster: &nsx.Cluster{}, - VPCClient: mockVpcclient, - NsxConfig: &config.NSXOperatorConfig{ - DefaultConfig: nil, - CoeConfig: &config.CoeConfig{ - Cluster: "", - EnableVPCNetwork: false, - }, - NsxConfig: nil, - K8sConfig: nil, - VCConfig: nil, - HAConfig: nil, - LibMode: false, - }, - }, - }, - - LbsStore: &LBSStore{ResourceStore: common.ResourceStore{ - Indexer: cache.NewIndexer(keyFunc, cache.Indexers{}), - BindingType: model.LBServiceBindingType(), - }}, - VpcStore: &VPCStore{resourceStore}, - } - - fakeVPCID := "fakeID" - vpcPath := "/orgs/default/projects/default/vpcs/vpc2" - err := vpcService.VpcStore.Add(&model.Vpc{Id: &fakeVPCID, Path: &vpcPath}) - assert.NoError(t, err) - - mockCtx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - patches := gomonkey.ApplyFunc(httpGetAviPortsPaths, func(cluster *nsx.Cluster, vpcPath string) (sets.Set[string], error) { - return nil, nil - }) - patches.ApplyMethodSeq(reflect.TypeOf(vpcService.NSXClient.VPCClient), "Delete", []gomonkey.OutputCell{{ - Values: gomonkey.Params{ - nil, - }, - Times: 1, - }}) - - patches.ApplyMethod(reflect.TypeOf(&common.Service{}), "SearchResource", func(_ *common.Service, _ string, _ string, store common.Store, _ common.Filter) (uint64, error) { - return 0, nil - }) - - defer patches.Reset() - - err = vpcService.Cleanup(mockCtx) - - assert.NoError(t, err) -} - func createFakeVPCService(t *testing.T, objs []client.Object) *VPCService { newScheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(newScheme)) diff --git a/test/e2e/precreated_vpc_test.go b/test/e2e/precreated_vpc_test.go index 4c46cb378..f86566062 100644 --- a/test/e2e/precreated_vpc_test.go +++ b/test/e2e/precreated_vpc_test.go @@ -38,7 +38,6 @@ const ( var ( projectPathFormat = "/orgs/%s/projects/%s" - vpcPathFormat = "/orgs/%s/projects/%s/vpcs/%s" defaultConnectivityProfileFormat = "/orgs/%s/projects/%s/vpc-connectivity-profiles/default" ) @@ -47,7 +46,7 @@ func TestPreCreatedVPC(t *testing.T) { nsName := "test-prevpc" projectPath := fmt.Sprintf(projectPathFormat, orgID, projectID) profilePath := fmt.Sprintf(defaultConnectivityProfileFormat, orgID, projectID) - preCreatedVPCPath := fmt.Sprintf(vpcPathFormat, orgID, projectID, vpcID) + preCreatedVPCPath := fmt.Sprintf(common.VPCKey, orgID, projectID, vpcID) log.Info("Created VPC on NSX", "path", preCreatedVPCPath) defer func() { log.Info("Deleting the created VPC from NSX", "path", preCreatedVPCPath) @@ -258,7 +257,7 @@ func (data *TestData) createVPC(orgID, projectID, vpcID string, privateIPs []str PrivateIps: privateIPs, ResourceType: common.String(common.ResourceTypeVpc), } - vpcPath := fmt.Sprintf(vpcPathFormat, orgID, projectID, vpcID) + vpcPath := fmt.Sprintf(common.VPCKey, orgID, projectID, vpcID) var lbsPath string var createdLBS *model.LBService if !useNSXLB { @@ -311,7 +310,7 @@ func (data *TestData) createVPC(orgID, projectID, vpcID string, privateIPs []str log.Error(pollErr, "Failed to realize VPC and related resources within 2m") data.nsxClient.VPCClient.Delete(orgID, projectID, vpcID, common.Bool(true)) if err := data.nsxClient.VPCClient.Delete(orgID, projectID, vpcID, common.Bool(true)); err != nil { - log.Error(err, "Failed to recursively delete NSX VPC", "path", fmt.Sprintf("/orgs/%s/projects/%s/vpcs/%s", orgID, projectID, vpcID)) + log.Error(err, "Failed to recursively delete NSX VPC", "path", vpcPath) } return pollErr }