Skip to content

Commit

Permalink
Delete secondaryNetwork OVS ports correctly after an Agent restart
Browse files Browse the repository at this point in the history
Signed-off-by: KMAnju-2021 <[email protected]>
  • Loading branch information
KMAnju-2021 committed Jan 20, 2025
1 parent 5ee28ec commit d0a9c32
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 43 deletions.
24 changes: 12 additions & 12 deletions cmd/antrea-agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,18 @@ func run(o *Options) error {
return err
}

var secondaryNetworkController *secondarynetwork.Controller
if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
secondaryNetworkController, err = secondarynetwork.NewController(
o.config.ClientConnection, o.config.KubeAPIServerOverride,
k8sClient, localPodInformer.Get(),
podUpdateChannel, ifaceStore,
&o.config.SecondaryNetwork, ovsdbConnection)
if err != nil {
return fmt.Errorf("failed to create secondary network controller: %w", err)
}
}

var flowExporter *exporter.FlowExporter
if enableFlowExporter {
podStore := podstore.NewPodStore(localPodInformer.Get())
Expand Down Expand Up @@ -760,18 +772,6 @@ func run(o *Options) error {
go ipamController.Run(stopCh)
}

var secondaryNetworkController *secondarynetwork.Controller
if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
secondaryNetworkController, err = secondarynetwork.NewController(
o.config.ClientConnection, o.config.KubeAPIServerOverride,
k8sClient, localPodInformer.Get(),
podUpdateChannel,
&o.config.SecondaryNetwork, ovsdbConnection)
if err != nil {
return fmt.Errorf("failed to create secondary network controller: %w", err)
}
}

var bgpController *bgp.Controller
if features.DefaultFeatureGate.Enabled(features.BGPPolicy) {
bgpPolicyInformer := crdInformerFactory.Crd().V1alpha1().BGPPolicies()
Expand Down
4 changes: 3 additions & 1 deletion pkg/agent/secondarynetwork/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
componentbaseconfig "k8s.io/component-base/config"
"k8s.io/klog/v2"

"antrea.io/antrea/pkg/agent/interfacestore"
"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
agentconfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/ovs/ovsconfig"
Expand All @@ -47,6 +48,7 @@ func NewController(
k8sClient clientset.Interface,
podInformer cache.SharedIndexInformer,
podUpdateSubscriber channel.Subscriber,
pIfaceStore interfacestore.InterfaceStore,
secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB,
) (*Controller, error) {
ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
Expand All @@ -65,7 +67,7 @@ func NewController(
// k8s.v1.cni.cncf.io/networks Annotation defined.
podWatchController, err := podwatch.NewPodController(
k8sClient, netAttachDefClient, podInformer,
podUpdateSubscriber, ovsBridgeClient)
podUpdateSubscriber, pIfaceStore, ovsBridgeClient)
if err != nil {
return nil, err
}
Expand Down
161 changes: 132 additions & 29 deletions pkg/agent/secondarynetwork/podwatch/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ func NewPodController(
netAttachDefClient netdefclient.K8sCniCncfIoV1Interface,
podInformer cache.SharedIndexInformer,
podUpdateSubscriber channel.Subscriber,
pIfaceStore interfacestore.InterfaceStore,
ovsBridgeClient ovsconfig.OVSBridgeClient,
) (*PodController, error) {
ifaceStore := interfacestore.NewInterfaceStore()
Expand Down Expand Up @@ -133,6 +134,16 @@ func NewPodController(
},
resyncPeriod,
)

err = pc.initializeSecondaryInterfaceStore()
if err != nil {
klog.ErrorS(err, "Failed to initialize the secondary bridge interface store")
}

if err := pc.reconcileSecondaryInterfaces(pIfaceStore); err != nil {
klog.ErrorS(err, "Failed to restore the cniCache")
}

// podUpdateSubscriber can be nil with test code.
if podUpdateSubscriber != nil {
// Subscribe Pod CNI add/del events.
Expand Down Expand Up @@ -223,40 +234,58 @@ func (pc *PodController) handleAddUpdatePod(pod *corev1.Pod, podCNIInfo *podCNII
return pc.configurePodSecondaryNetwork(pod, networklist, podCNIInfo)
}

func (pc *PodController) removeInterfaces(interfaces []*interfacestore.InterfaceConfig) error {
var savedErr error
for _, interfaceConfig := range interfaces {
podName := interfaceConfig.PodName
podNamespace := interfaceConfig.PodNamespace
klog.V(1).InfoS("Deleting secondary interface",
"Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev)
// deleteInterfaceAndReleaseResources to delete interface and release resources
func (pc *PodController) deleteInterfaceAndReleaseResources(iface *interfacestore.InterfaceConfig) error {
podName := iface.PodName
podNamespace := iface.PodNamespace
klog.V(1).InfoS("Deleting secondary interface",
"Pod", klog.KRef(podNamespace, podName), "interface", iface.IFDev)

var err error
// Since only VLAN and SR-IOV interfaces are supported by now, we judge the
// interface type by checking interfaceConfig.OVSPortConfig is set or not.
if iface.OVSPortConfig != nil {
err = pc.interfaceConfigurator.DeleteVLANSecondaryInterface(iface)
} else {
err = pc.deleteSriovSecondaryInterface(iface)
}

var err error
// Since only VLAN and SR-IOV interfaces are supported by now, we judge the
// interface type by checking interfaceConfig.OVSPortConfig is set or not.
if interfaceConfig.OVSPortConfig != nil {
err = pc.interfaceConfigurator.DeleteVLANSecondaryInterface(interfaceConfig)
} else {
err = pc.deleteSriovSecondaryInterface(interfaceConfig)
}
if err != nil {
return err
}

// Release IPAM allocation
podOwner := &crdv1b1.PodOwner{
Name: iface.PodName,
Namespace: iface.PodNamespace,
ContainerID: iface.ContainerID,
IFName: iface.IFDev,
}

return pc.ipamAllocator.SecondaryNetworkRelease(podOwner)
}

func (pc *PodController) removeInterfaces(interfaces []*interfacestore.InterfaceConfig, ifaceToRemove *interfacestore.InterfaceConfig) error {
var savedErr error
// If ifaceToRemove is provided, directly remove the associated interface's resources
if ifaceToRemove != nil {
err := pc.deleteInterfaceAndReleaseResources(ifaceToRemove)
if err != nil {
klog.ErrorS(err, "Error when deleting secondary interface",
"Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev)
klog.ErrorS(err, "Error when deleting interface and releasing resources",
"Pod", klog.KRef(ifaceToRemove.PodNamespace, ifaceToRemove.PodName), "interface", ifaceToRemove.IFDev)
savedErr = err
continue
}

podOwner := &crdv1b1.PodOwner{
Name: interfaceConfig.PodName,
Namespace: interfaceConfig.PodNamespace,
ContainerID: interfaceConfig.ContainerID,
IFName: interfaceConfig.IFDev}
if err = pc.ipamAllocator.SecondaryNetworkRelease(podOwner); err != nil {
klog.ErrorS(err, "Error when releasing IPAM allocation",
"Pod", klog.KRef(podNamespace, podName), "interface", interfaceConfig.IFDev)
savedErr = err
} else {
for _, iface := range interfaces {
err := pc.deleteInterfaceAndReleaseResources(iface)
if err != nil {
klog.ErrorS(err, "Error when deleting interface and releasing resources",
"Pod", klog.KRef(iface.PodNamespace, iface.PodName), "interface", iface.IFDev)
savedErr = err
}
}
}

return savedErr
}

Expand Down Expand Up @@ -288,9 +317,11 @@ func (pc *PodController) syncPod(key string) error {
// not deleted yet. First delete them before processing the new Pod's
// secondary networks.
storedInterfaces[0].ContainerID != cniInfo.containerID {
if err := pc.removeInterfaces(storedInterfaces); err != nil {
if err := pc.removeInterfaces(storedInterfaces, nil); err != nil {
return err
}
} else {
return nil
}
}

Expand Down Expand Up @@ -502,3 +533,75 @@ func checkForPodSecondaryNetworkAttachement(pod *corev1.Pod) (string, bool) {
return netObj, false
}
}

// initializeSecondaryInterfaceStore restore secondary interfacestore when agent restart.
func (pc *PodController) initializeSecondaryInterfaceStore() error {
ovsPorts, err := pc.ovsBridgeClient.GetPortList()
if err != nil {
klog.ErrorS(err, "Failed to list OVS ports for the secondary bridge", "bridgeName", pc.ovsBridgeClient.GetBridgeName())
return err
}

ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts))
for index := range ovsPorts {
port := &ovsPorts[index]
ovsPort := &interfacestore.OVSPortConfig{
PortUUID: port.UUID,
OFPort: port.OFPort,
}

interfaceType, ok := port.ExternalIDs[interfacestore.AntreaInterfaceTypeKey]
if !ok {
klog.InfoS("Interface type is not set for the secondary bridge", "interfaceName", port.Name)
continue
}

var intf *interfacestore.InterfaceConfig
switch interfaceType {
case interfacestore.AntreaContainer:
intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort)
default:
klog.InfoS("Unknown Antrea interface type for the secondary bridge", "type", interfaceType)
}

if intf != nil {
ifaceList = append(ifaceList, intf)
}

}

pc.interfaceStore.Initialize(ifaceList)
klog.InfoS("Successfully initialized the secondary bridge interface store")

return nil
}

// reconcileSecondaryInterfaces restore cniCache when agent restart using primary interfacestore.
func (pc *PodController) reconcileSecondaryInterfaces(pIfaceStore interfacestore.InterfaceStore) error {
knownInterfaces := pIfaceStore.GetInterfacesByType(interfacestore.ContainerInterface)
for _, containerConfig := range knownInterfaces {
config := containerConfig.ContainerInterfaceConfig
podKey := podKeyGet(config.PodName, config.PodNamespace)

pc.cniCache.Store(podKey, &podCNIInfo{
containerID: config.ContainerID,
})
pc.queue.Add(podKey)
}

// secondaryInterfaces is the list of interfaces currently in the secondary local cache and delete ports not in the CNI cache.
secondaryInterfaces := pc.interfaceStore.GetInterfacesByType(interfacestore.ContainerInterface)
for _, containerConfig := range secondaryInterfaces {
containerID := containerConfig.ContainerID
_, exists := pIfaceStore.GetContainerInterface(containerID)
if !exists || containerConfig.OFPort == -1 {
pc.interfaceStore.DeleteInterface(containerConfig)
err := pc.removeInterfaces(secondaryInterfaces, containerConfig)

if err != nil {
return err
}
}
}
return nil
}
2 changes: 1 addition & 1 deletion pkg/agent/secondarynetwork/podwatch/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func TestPodControllerRun(t *testing.T) {
client,
netdefclient,
informerFactory.Core().V1().Pods().Informer(),
nil, nil)
nil, nil, nil)
podController.interfaceConfigurator = interfaceConfigurator
podController.ipamAllocator = mockIPAM
cniCache := &podController.cniCache
Expand Down

0 comments on commit d0a9c32

Please sign in to comment.