make vpc_nat_gateway redo eip fip snat dnat iptables rules after k8s …

…cluster reboot (#3267) * make vpc_nat_gateway judge redo snat dnat iptables rules after k8s cluster reboot Signed-off-by: bobz965 <[email protected]> --------- Signed-off-by: bobz965 <[email protected]> Co-authored-by: xiongww <[email protected]> Co-authored-by: bobz965 <[email protected]>
kubeovn · Dec 5, 2023 · f7ff76b · f7ff76b
1 parent 8628642
commit f7ff76b
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 1 deletion.
diff --git a/pkg/controller/config.go b/pkg/controller/config.go
@@ -150,7 +150,7 @@ func ParseFlags() (*Configuration, error) {
 		argDefaultVlanID           = pflag.Int("default-vlan-id", 1, "The default vlan id")
 		argLsDnatModDlDst          = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch")
 		argPodNicType              = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type")
-		argPodDefaultFipType       = pflag.String("pod-default-fip-type", "", "The type of fip bind to pod automatically: iptables")
+		argPodDefaultFipType       = pflag.String("pod-default-fip-type", "iptables", "The type of fip bind to pod automatically: iptables")
 		argEnableLb                = pflag.Bool("enable-lb", true, "Enable load balancer")
 		argEnableNP                = pflag.Bool("enable-np", true, "Enable network policy support")
 		argEnableEipSnat           = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT")

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -1156,6 +1156,9 @@ func (c *Controller) initResourceOnce() {
 		}
 	}
 
+	if err := c.initVpcNatGw(); err != nil {
+		util.LogFatalAndExit(err, "failed to initialize vpc nat gateways")
+	}
 	if c.config.EnableLb {
 		if err := c.initVpcDNSConfig(); err != nil {
 			util.LogFatalAndExit(err, "failed to initialize vpc-dns")

diff --git a/pkg/controller/vpc_nat_gateway.go b/pkg/controller/vpc_nat_gateway.go
@@ -1092,3 +1092,34 @@ func (c *Controller) execNatGwQoSInPod(
 	}
 	return nil
 }
+
+func (c *Controller) initVpcNatGw() error {
+	if vpcNatEnabled != "true" {
+		err := fmt.Errorf("iptables nat gw not enable")
+		klog.Warning(err)
+		return nil
+	}
+	klog.Infof("init all vpc nat gateways")
+	gws, err := c.vpcNatGatewayLister.List(labels.Everything())
+	if err != nil {
+		err = fmt.Errorf("failed to get vpc nat gw list, %v", err)
+		klog.Error(err)
+		return err
+	}
+	for _, gw := range gws {
+		pod, err := c.getNatGwPod(gw.Name)
+		if err != nil {
+			// the nat gw maybe deleted
+			err := fmt.Errorf("failed to get nat gw %s pod: %v", gw.Name, err)
+			klog.Error(err)
+			continue
+		}
+		if vpcGwName, isVpcNatGw := pod.Annotations[util.VpcNatGatewayAnnotation]; isVpcNatGw {
+			if _, hasInit := pod.Annotations[util.VpcNatGatewayInitAnnotation]; hasInit {
+				return nil
+			}
+			c.initVpcNatGatewayQueue.Add(vpcGwName)
+		}
+	}
+	return nil
+}
diff --git a/pkg/controller/vpc_nat_gw_eip.go b/pkg/controller/vpc_nat_gw_eip.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -366,6 +367,18 @@ func (c *Controller) handleUpdateIptablesEip(key string) error {
 		cachedEip.Status.Redo != "" &&
 		cachedEip.Status.IP != "" &&
 		cachedEip.DeletionTimestamp.IsZero() {
+		gwPod, err := c.getNatGwPod(cachedEip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with eip redo time. if redo time before gw pod started. redo again
+		eipRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedEip.Status.Redo, time.Local)
+		if cachedEip.Status.Ready && cachedEip.Status.IP != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: eipRedo}) {
+			// already ok
+			klog.V(3).Infof("eip %s already ok", key)
+			return nil
+		}
 		eipV4Cidr, err := c.getEipV4Cidr(cachedEip.Status.IP, externalNetwork)
 		if err != nil {
 			klog.Errorf("failed to get eip or v4Cidr, %v", err)

diff --git a/pkg/controller/vpc_nat_gw_nat.go b/pkg/controller/vpc_nat_gw_nat.go
@@ -643,6 +643,18 @@ func (c *Controller) handleUpdateIptablesFip(key string) error {
 		cachedFip.Status.V4ip != "" &&
 		cachedFip.DeletionTimestamp.IsZero() {
 		klog.V(3).Infof("reapply fip '%s' in pod ", key)
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with fip redo time. if fip redo time before gw pod started. should redo again
+		fipRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedFip.Status.Redo, time.Local)
+		if cachedFip.Status.Ready && cachedFip.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: fipRedo}) {
+			// already ok
+			klog.V(3).Infof("fip %s already ok", key)
+			return nil
+		}
 		if err = c.createFipInPod(eip.Spec.NatGwDp, cachedFip.Status.V4ip, cachedFip.Spec.InternalIP); err != nil {
 			klog.Errorf("failed to create fip, %v", err)
 			return err
@@ -814,6 +826,18 @@ func (c *Controller) handleUpdateIptablesDnatRule(key string) error {
 		cachedDnat.Status.V4ip != "" &&
 		cachedDnat.DeletionTimestamp.IsZero() {
 		klog.V(3).Infof("reapply dnat in pod for %s", key)
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with dnat redo time. if redo time before gw pod started. redo again
+		dnatRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedDnat.Status.Redo, time.Local)
+		if cachedDnat.Status.Ready && cachedDnat.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: dnatRedo}) {
+			// already ok
+			klog.V(3).Infof("dnat %s already ok", key)
+			return nil
+		}
 		if err = c.createDnatInPod(eip.Spec.NatGwDp, cachedDnat.Spec.Protocol,
 			cachedDnat.Status.V4ip, cachedDnat.Spec.InternalIP,
 			cachedDnat.Spec.ExternalPort, cachedDnat.Spec.InternalPort); err != nil {
@@ -986,6 +1010,18 @@ func (c *Controller) handleUpdateIptablesSnatRule(key string) error {
 		cachedSnat.Status.Redo != "" &&
 		cachedSnat.Status.V4ip != "" &&
 		cachedSnat.DeletionTimestamp.IsZero() {
+		gwPod, err := c.getNatGwPod(eip.Spec.NatGwDp)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		// compare gw pod started time with snat redo time. if redo time before gw pod started. redo again
+		snatRedo, _ := time.ParseInLocation("2006-01-02T15:04:05", cachedSnat.Status.Redo, time.Local)
+		if cachedSnat.Status.Ready && cachedSnat.Status.V4ip != "" && gwPod.Status.ContainerStatuses[0].State.Running.StartedAt.Before(&metav1.Time{Time: snatRedo}) {
+			// already ok
+			klog.V(3).Infof("snat %s already ok", key)
+			return nil
+		}
 		if err = c.createSnatInPod(cachedSnat.Status.NatGwDp, cachedSnat.Status.V4ip, v4CidrSpec); err != nil {
 			klog.Errorf("failed to create new snat, %v", err)
 			return err