make vpc_nat_gateway redo eip fip snat dnat iptables rules after k8s …

…cluster reboot (kubeovn#3267) * make vpc_nat_gateway judge redo snat dnat iptables rules after k8s cluster reboot Signed-off-by: bobz965 <[email protected]> --------- Signed-off-by: bobz965 <[email protected]> Co-authored-by: xiongww <[email protected]> Co-authored-by: bobz965 <[email protected]>
pandaoknight · Mar 19, 2024 · d36801b · d36801b
1 parent 4c19787
commit d36801b
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 4 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v1.12.9
+v1.12.8.1
diff --git a/go.mod b/go.mod
@@ -286,7 +286,7 @@ replace (
 	github.com/mdlayher/arp => github.com/kubeovn/arp v0.0.0-20230101053045-8a0772d9c34c
 	github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20221107163225-3335a34a1d24
 	github.com/ovn-org/libovsdb => github.com/kubeovn/libovsdb v0.0.0-20230517064328-9d5a1383643f
-	github.com/vishvananda/netlink => github.com/kubeovn/netlink v0.0.0-20230322092337-960188369daf
+	github.com/vishvananda/netlink => github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f
 	k8s.io/api => k8s.io/api v0.27.10
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.27.10
 	k8s.io/apimachinery => k8s.io/apimachinery v0.27.10

diff --git a/go.sum b/go.sum
@@ -1364,8 +1364,8 @@ github.com/kubeovn/kubevirt-client-go v0.0.0-20230517062539-8dd832f39ec5 h1:3tyg
 github.com/kubeovn/kubevirt-client-go v0.0.0-20230517062539-8dd832f39ec5/go.mod h1:GhMrao/zkhyDt9T4Azt3/2zxztchdSecAumECdFi+XQ=
 github.com/kubeovn/libovsdb v0.0.0-20230517064328-9d5a1383643f h1:HDjnbJZN+2T3XH7usjtO2+PYDA2fyrLGYjypEA/87pM=
 github.com/kubeovn/libovsdb v0.0.0-20230517064328-9d5a1383643f/go.mod h1:NHoQwGSKygdpFb8y7HBS6b1HP4EtJ14zzLrnd/A1fmY=
-github.com/kubeovn/netlink v0.0.0-20230322092337-960188369daf h1:inZiuUjcQaX0O0Sdki38TWzCl0+wJty+vaQKEr47by8=
-github.com/kubeovn/netlink v0.0.0-20230322092337-960188369daf/go.mod h1:p3BbJwJMQKnFy+IfKc5stjSCxpLN5d6R3MFOM3TQitw=
+github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f h1:3hH6U+CRilak3SxAX9YykAXxxAY25GTEJANLlJNE2jU=
+github.com/kubeovn/netlink v0.0.0-20240218024530-d3ada5dae96f/go.mod h1:KjTlcXwJZNXDSeBgPMWF8yKVqYrIP1cpe5HfyfEi4Ls=
 github.com/kubeovn/ovsdb v0.0.0-20221213053943-9372db56919f h1:nm0ZlQesCje/A5D0LyWfaSUM8/0ro9PVpwd8hVbLBeM=
 github.com/kubeovn/ovsdb v0.0.0-20221213053943-9372db56919f/go.mod h1:LAd0qoeAAm/QyZcpxN2BnpndM2/dhZt+/kokPvcxKcE=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA=
@@ -2261,6 +2261,7 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

diff --git a/pkg/controller/config.go b/pkg/controller/config.go
@@ -150,6 +150,7 @@ func ParseFlags() (*Configuration, error) {
 		argLsDnatModDlDst          = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch")
 		argLsCtSkipDstLportIPs     = pflag.Bool("ls-ct-skip-dst-lport-ips", true, "Skip conntrack for direct traffic between lports")
 		argPodNicType              = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type")
+		argPodDefaultFipType       = pflag.String("pod-default-fip-type", "iptables", "The type of fip bind to pod automatically: iptables")
 		argEnableLb                = pflag.Bool("enable-lb", true, "Enable load balancer")
 		argEnableNP                = pflag.Bool("enable-np", true, "Enable network policy support")
 		argEnableEipSnat           = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT")

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -1097,6 +1097,9 @@ func (c *Controller) initResourceOnce() {
 		util.LogFatalAndExit(err, "failed to sync crd vpc nat gateways")
 	}
 
+	if err := c.initVpcNatGw(); err != nil {
+		util.LogFatalAndExit(err, "failed to initialize vpc nat gateways")
+	}
 	if c.config.EnableLb {
 		if err := c.initVpcDNSConfig(); err != nil {
 			util.LogFatalAndExit(err, "failed to initialize vpc-dns")

diff --git a/pkg/controller/vpc_nat_gateway.go b/pkg/controller/vpc_nat_gateway.go
@@ -1076,3 +1076,34 @@ func (c *Controller) execNatGwQoSInPod(
 	}
 	return nil
 }
+
+func (c *Controller) initVpcNatGw() error {
+	if vpcNatEnabled != "true" {
+		err := fmt.Errorf("iptables nat gw not enable")
+		klog.Warning(err)
+		return nil
+	}
+	klog.Infof("init all vpc nat gateways")
+	gws, err := c.vpcNatGatewayLister.List(labels.Everything())
+	if err != nil {
+		err = fmt.Errorf("failed to get vpc nat gw list, %v", err)
+		klog.Error(err)
+		return err
+	}
+	for _, gw := range gws {
+		pod, err := c.getNatGwPod(gw.Name)
+		if err != nil {
+			// the nat gw maybe deleted
+			err := fmt.Errorf("failed to get nat gw %s pod: %v", gw.Name, err)
+			klog.Error(err)
+			continue
+		}
+		if vpcGwName, isVpcNatGw := pod.Annotations[util.VpcNatGatewayAnnotation]; isVpcNatGw {
+			if _, hasInit := pod.Annotations[util.VpcNatGatewayInitAnnotation]; hasInit {
+				return nil
+			}
+			c.initVpcNatGatewayQueue.Add(vpcGwName)
+		}
+	}
+	return nil
+}
diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -367,6 +368,18 @@ func (c *Controller) handleUpdateIptablesEip(key string) error {
 		cachedEip.Status.Redo != "" &&
 		cachedEip.Status.IP != "" &&
 		cachedEip.DeletionTimestamp.IsZero() {
+		gwPod, err := c.getNatGwPod(cachedEip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with eip redo time. if redo time before gw pod started. redo again
+		eipRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedEip.Status.Redo, time.Local)
+		if cachedEip.Status.Ready && cachedEip.Status.IP != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: eipRedo}) {
+			// already ok
+			klog.V(3).Infof("eip %s already ok", key)
+			return nil
+		}
 		eipV4Cidr, err := c.getEipV4Cidr(cachedEip.Status.IP, externalNetwork)
 		if err != nil {
 			klog.Errorf("failed to get eip or v4Cidr, %v", err)

diff --git a/pkg/controller/vpc_nat_gw_nat.go b/pkg/controller/vpc_nat_gw_nat.go
@@ -643,6 +643,18 @@ func (c *Controller) handleUpdateIptablesFip(key string) error {
 		cachedFip.Status.V4ip != "" &&
 		cachedFip.DeletionTimestamp.IsZero() {
 		klog.V(3).Infof("reapply fip '%s' in pod ", key)
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with fip redo time. if fip redo time before gw pod started. should redo again
+		fipRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedFip.Status.Redo, time.Local)
+		if cachedFip.Status.Ready && cachedFip.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: fipRedo}) {
+			// already ok
+			klog.V(3).Infof("fip %s already ok", key)
+			return nil
+		}
 		if err = c.createFipInPod(eip.Spec.NatGwDp, cachedFip.Status.V4ip, cachedFip.Spec.InternalIP); err != nil {
 			klog.Errorf("failed to create fip, %v", err)
 			return err
@@ -814,6 +826,18 @@ func (c *Controller) handleUpdateIptablesDnatRule(key string) error {
 		cachedDnat.Status.V4ip != "" &&
 		cachedDnat.DeletionTimestamp.IsZero() {
 		klog.V(3).Infof("reapply dnat in pod for %s", key)
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with dnat redo time. if redo time before gw pod started. redo again
+		dnatRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedDnat.Status.Redo, time.Local)
+		if cachedDnat.Status.Ready && cachedDnat.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: dnatRedo}) {
+			// already ok
+			klog.V(3).Infof("dnat %s already ok", key)
+			return nil
+		}
 		if err = c.createDnatInPod(eip.Spec.NatGwDp, cachedDnat.Spec.Protocol,
 			cachedDnat.Status.V4ip, cachedDnat.Spec.InternalIP,
 			cachedDnat.Spec.ExternalPort, cachedDnat.Spec.InternalPort); err != nil {
@@ -986,6 +1010,18 @@ func (c *Controller) handleUpdateIptablesSnatRule(key string) error {
 		cachedSnat.Status.Redo != "" &&
 		cachedSnat.Status.V4ip != "" &&
 		cachedSnat.DeletionTimestamp.IsZero() {
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with snat redo time. if redo time before gw pod started. redo again
+		snatRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedSnat.Status.Redo, time.Local)
+		if cachedSnat.Status.Ready && cachedSnat.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: snatRedo}) {
+			// already ok
+			klog.V(3).Infof("snat %s already ok", key)
+			return nil
+		}
 		if err = c.createSnatInPod(cachedSnat.Status.NatGwDp, cachedSnat.Status.V4ip, v4CidrSpec); err != nil {
 			klog.Errorf("failed to create new snat, %v", err)
 			return err