From e61f080e144a14debcb696aed31a8c6364772e4c Mon Sep 17 00:00:00 2001 From: KMAnju-2021 Date: Thu, 12 Dec 2024 00:32:51 +0530 Subject: [PATCH] Delete secondaryNetwork OVS ports correctly after an Agent restart Signed-off-by: KMAnju-2021 --- cmd/antrea-agent/agent.go | 24 +-- pkg/agent/secondarynetwork/init.go | 4 +- .../secondarynetwork/podwatch/controller.go | 161 ++++++++++++++---- .../podwatch/controller_test.go | 2 +- 4 files changed, 148 insertions(+), 43 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index dfd740ed2e0..062ce30ceb0 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -680,6 +680,18 @@ func run(o *Options) error { return err } + var secondaryNetworkController *secondarynetwork.Controller + if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) { + secondaryNetworkController, err = secondarynetwork.NewController( + o.config.ClientConnection, o.config.KubeAPIServerOverride, + k8sClient, localPodInformer.Get(), + podUpdateChannel, ifaceStore, + &o.config.SecondaryNetwork, ovsdbConnection) + if err != nil { + return fmt.Errorf("failed to create secondary network controller: %w", err) + } + } + var flowExporter *exporter.FlowExporter if enableFlowExporter { podStore := podstore.NewPodStore(localPodInformer.Get()) @@ -760,18 +772,6 @@ func run(o *Options) error { go ipamController.Run(stopCh) } - var secondaryNetworkController *secondarynetwork.Controller - if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) { - secondaryNetworkController, err = secondarynetwork.NewController( - o.config.ClientConnection, o.config.KubeAPIServerOverride, - k8sClient, localPodInformer.Get(), - podUpdateChannel, - &o.config.SecondaryNetwork, ovsdbConnection) - if err != nil { - return fmt.Errorf("failed to create secondary network controller: %w", err) - } - } - var bgpController *bgp.Controller if features.DefaultFeatureGate.Enabled(features.BGPPolicy) { bgpPolicyInformer := crdInformerFactory.Crd().V1alpha1().BGPPolicies() diff --git a/pkg/agent/secondarynetwork/init.go b/pkg/agent/secondarynetwork/init.go index 721027f1a13..e3ad278a161 100644 --- a/pkg/agent/secondarynetwork/init.go +++ b/pkg/agent/secondarynetwork/init.go @@ -24,6 +24,7 @@ import ( componentbaseconfig "k8s.io/component-base/config" "k8s.io/klog/v2" + "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/secondarynetwork/podwatch" agentconfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/ovs/ovsconfig" @@ -47,6 +48,7 @@ func NewController( k8sClient clientset.Interface, podInformer cache.SharedIndexInformer, podUpdateSubscriber channel.Subscriber, + pIfaceStore interfacestore.InterfaceStore, secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB, ) (*Controller, error) { ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb) @@ -65,7 +67,7 @@ func NewController( // k8s.v1.cni.cncf.io/networks Annotation defined. podWatchController, err := podwatch.NewPodController( k8sClient, netAttachDefClient, podInformer, - podUpdateSubscriber, ovsBridgeClient) + podUpdateSubscriber, pIfaceStore, ovsBridgeClient) if err != nil { return nil, err } diff --git a/pkg/agent/secondarynetwork/podwatch/controller.go b/pkg/agent/secondarynetwork/podwatch/controller.go index db9c576e614..e61b974ba93 100644 --- a/pkg/agent/secondarynetwork/podwatch/controller.go +++ b/pkg/agent/secondarynetwork/podwatch/controller.go @@ -102,6 +102,7 @@ func NewPodController( netAttachDefClient netdefclient.K8sCniCncfIoV1Interface, podInformer cache.SharedIndexInformer, podUpdateSubscriber channel.Subscriber, + pIfaceStore interfacestore.InterfaceStore, ovsBridgeClient ovsconfig.OVSBridgeClient, ) (*PodController, error) { ifaceStore := interfacestore.NewInterfaceStore() @@ -133,6 +134,16 @@ func NewPodController( }, resyncPeriod, ) + + err = pc.initializeSecondaryInterfaceStore() + if err != nil { + klog.ErrorS(err, "Failed to initialize the secondary bridge interface store") + } + + if err := pc.reconcileSecondaryInterfaces(pIfaceStore); err != nil { + klog.ErrorS(err, "Failed to restore the cniCache") + } + // podUpdateSubscriber can be nil with test code. if podUpdateSubscriber != nil { // Subscribe Pod CNI add/del events. @@ -223,40 +234,58 @@ func (pc *PodController) handleAddUpdatePod(pod *corev1.Pod, podCNIInfo *podCNII return pc.configurePodSecondaryNetwork(pod, networklist, podCNIInfo) } -func (pc *PodController) removeInterfaces(interfaces []*interfacestore.InterfaceConfig) error { - var savedErr error - for _, interfaceConfig := range interfaces { - podName := interfaceConfig.PodName - podNamespace := interfaceConfig.PodNamespace - klog.V(1).InfoS("Deleting secondary interface", - "Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev) +// deleteInterfaceAndReleaseResources to delete interface and release resources +func (pc *PodController) deleteInterfaceAndReleaseResources(iface *interfacestore.InterfaceConfig) error { + podName := iface.PodName + podNamespace := iface.PodNamespace + klog.V(1).InfoS("Deleting secondary interface", + "Pod", klog.KRef(podNamespace, podName), "interface", iface.IFDev) + + var err error + // Since only VLAN and SR-IOV interfaces are supported by now, we judge the + // interface type by checking interfaceConfig.OVSPortConfig is set or not. + if iface.OVSPortConfig != nil { + err = pc.interfaceConfigurator.DeleteVLANSecondaryInterface(iface) + } else { + err = pc.deleteSriovSecondaryInterface(iface) + } - var err error - // Since only VLAN and SR-IOV interfaces are supported by now, we judge the - // interface type by checking interfaceConfig.OVSPortConfig is set or not. - if interfaceConfig.OVSPortConfig != nil { - err = pc.interfaceConfigurator.DeleteVLANSecondaryInterface(interfaceConfig) - } else { - err = pc.deleteSriovSecondaryInterface(interfaceConfig) - } + if err != nil { + return err + } + + // Release IPAM allocation + podOwner := &crdv1b1.PodOwner{ + Name: iface.PodName, + Namespace: iface.PodNamespace, + ContainerID: iface.ContainerID, + IFName: iface.IFDev, + } + + return pc.ipamAllocator.SecondaryNetworkRelease(podOwner) +} + +func (pc *PodController) removeInterfaces(interfaces []*interfacestore.InterfaceConfig, ifaceToRemove *interfacestore.InterfaceConfig) error { + var savedErr error + // If ifaceToRemove is provided, directly remove the associated interface's resources + if ifaceToRemove != nil { + err := pc.deleteInterfaceAndReleaseResources(ifaceToRemove) if err != nil { - klog.ErrorS(err, "Error when deleting secondary interface", - "Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev) + klog.ErrorS(err, "Error when deleting interface and releasing resources", + "Pod", klog.KRef(ifaceToRemove.PodNamespace, ifaceToRemove.PodName), "interface", ifaceToRemove.IFDev) savedErr = err - continue } - - podOwner := &crdv1b1.PodOwner{ - Name: interfaceConfig.PodName, - Namespace: interfaceConfig.PodNamespace, - ContainerID: interfaceConfig.ContainerID, - IFName: interfaceConfig.IFDev} - if err = pc.ipamAllocator.SecondaryNetworkRelease(podOwner); err != nil { - klog.ErrorS(err, "Error when releasing IPAM allocation", - "Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev) - savedErr = err + } else { + for _, iface := range interfaces { + err := pc.deleteInterfaceAndReleaseResources(iface) + if err != nil { + klog.ErrorS(err, "Error when deleting interface and releasing resources", + "Pod", klog.KRef(iface.PodNamespace, iface.PodName), "interface", iface.IFDev) + savedErr = err + } } } + return savedErr } @@ -288,9 +317,11 @@ func (pc *PodController) syncPod(key string) error { // not deleted yet. First delete them before processing the new Pod's // secondary networks. storedInterfaces[0].ContainerID != cniInfo.containerID { - if err := pc.removeInterfaces(storedInterfaces); err != nil { + if err := pc.removeInterfaces(storedInterfaces, nil); err != nil { return err } + } else { + return nil } } @@ -502,3 +533,75 @@ func checkForPodSecondaryNetworkAttachement(pod *corev1.Pod) (string, bool) { return netObj, false } } + +// initializeSecondaryInterfaceStore restore secondary interfacestore when agent restart. +func (pc *PodController) initializeSecondaryInterfaceStore() error { + ovsPorts, err := pc.ovsBridgeClient.GetPortList() + if err != nil { + klog.ErrorS(err, "Failed to list OVS ports for the secondary bridge", "bridgeName", pc.ovsBridgeClient.GetBridgeName()) + return err + } + + ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts)) + for index := range ovsPorts { + port := &ovsPorts[index] + ovsPort := &interfacestore.OVSPortConfig{ + PortUUID: port.UUID, + OFPort: port.OFPort, + } + + interfaceType, ok := port.ExternalIDs[interfacestore.AntreaInterfaceTypeKey] + if !ok { + klog.InfoS("Interface type is not set for the secondary bridge", "interfaceName", port.Name) + continue + } + + var intf *interfacestore.InterfaceConfig + switch interfaceType { + case interfacestore.AntreaContainer: + intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort) + default: + klog.InfoS("Unknown Antrea interface type for the secondary bridge", "type", interfaceType) + } + + if intf != nil { + ifaceList = append(ifaceList, intf) + } + + } + + pc.interfaceStore.Initialize(ifaceList) + klog.InfoS("Successfully initialized the secondary bridge interface store") + + return nil +} + +// reconcileSecondaryInterfaces restore cniCache when agent restart using primary interfacestore. +func (pc *PodController) reconcileSecondaryInterfaces(pIfaceStore interfacestore.InterfaceStore) error { + knownInterfaces := pIfaceStore.GetInterfacesByType(interfacestore.ContainerInterface) + for _, containerConfig := range knownInterfaces { + config := containerConfig.ContainerInterfaceConfig + podKey := podKeyGet(config.PodName, config.PodNamespace) + + pc.cniCache.Store(podKey, &podCNIInfo{ + containerID: config.ContainerID, + }) + pc.queue.Add(podKey) + } + + // secondaryInterfaces is the list of interfaces currently in the secondary local cache and delete ports not in the CNI cache. + secondaryInterfaces := pc.interfaceStore.GetInterfacesByType(interfacestore.ContainerInterface) + for _, containerConfig := range secondaryInterfaces { + containerID := containerConfig.ContainerID + _, exists := pIfaceStore.GetContainerInterface(containerID) + if !exists || containerConfig.OFPort == -1 { + pc.interfaceStore.DeleteInterface(containerConfig) + err := pc.removeInterfaces(secondaryInterfaces, containerConfig) + + if err != nil { + return err + } + } + } + return nil +} \ No newline at end of file diff --git a/pkg/agent/secondarynetwork/podwatch/controller_test.go b/pkg/agent/secondarynetwork/podwatch/controller_test.go index b65bb02aff0..2d0d8857cc4 100644 --- a/pkg/agent/secondarynetwork/podwatch/controller_test.go +++ b/pkg/agent/secondarynetwork/podwatch/controller_test.go @@ -205,7 +205,7 @@ func TestPodControllerRun(t *testing.T) { client, netdefclient, informerFactory.Core().V1().Pods().Informer(), - nil, nil) + nil, nil, nil) podController.interfaceConfigurator = interfaceConfigurator podController.ipamAllocator = mockIPAM cniCache := &podController.cniCache