control: bind routing policy to dae netns

daeuniverse · Feb 27, 2024 · 9ad68bc · 9ad68bc
1 parent 16dfabc
commit 9ad68bc
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 137 deletions.
diff --git a/control/control_plane.go b/control/control_plane.go
@@ -470,10 +470,6 @@ func NewControlPlane(
 	}
 	go dnsUpstream.InitUpstreams()
 
-	if err = GetDaeNetns().With(core.setupRoutingPolicy); err != nil {
-		return nil, err
-	}
-
 	close(plane.ready)
 	return plane, nil
 }

diff --git a/control/control_plane_core.go b/control/control_plane_core.go
@@ -9,7 +9,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"net/netip"
 	"os"
 	"regexp"
@@ -192,138 +191,6 @@ func (c *controlPlaneCore) delQdisc(ifname string) error {
 	return nil
 }
 
-func (c *controlPlaneCore) setupRoutingPolicy() (err error) {
-	/// Insert ip rule / ip route.
-	var table = 2023 + c.flip
-
-	/** ip table
-	ip route add local default dev lo table 2023
-	ip -6 route add local default dev lo table 2023
-	*/
-	routes := []netlink.Route{{
-		Scope:     unix.RT_SCOPE_HOST,
-		LinkIndex: consts.LoopbackIfIndex,
-		Dst: &net.IPNet{
-			IP:   []byte{0, 0, 0, 0},
-			Mask: net.CIDRMask(0, 32),
-		},
-		Table: table,
-		Type:  unix.RTN_LOCAL,
-	}, {
-		Scope:     unix.RT_SCOPE_HOST,
-		LinkIndex: consts.LoopbackIfIndex,
-		Dst: &net.IPNet{
-			IP:   []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
-			Mask: net.CIDRMask(0, 128),
-		},
-		Table: table,
-		Type:  unix.RTN_LOCAL,
-	}}
-	var routeBadIpv6 bool
-	cleanRoutes := func() error {
-		var errs error
-		for _, route := range routes {
-			if e := netlink.RouteDel(&route); e != nil {
-				if len(route.Dst.IP) == net.IPv6len && routeBadIpv6 {
-					// Not clean for bad ipv6.
-					continue
-				}
-				if errs != nil {
-					errs = fmt.Errorf("%w; %v", errs, e)
-				} else {
-					errs = e
-				}
-			}
-		}
-		if errs != nil {
-			c.log.Debugf("IpRouteDel: %w\n", errs)
-		}
-		return nil
-	}
-tryRouteAddAgain:
-	for _, route := range routes {
-		if err = netlink.RouteAdd(&route); err != nil {
-			if os.IsExist(err) {
-				_ = cleanRoutes()
-				goto tryRouteAddAgain
-			}
-			if len(route.Dst.IP) == net.IPv6len {
-				// ipv6
-				c.log.Warnln("IpRouteAdd: Bad IPv6 support. Perhaps your machine disabled IPv6.")
-				routeBadIpv6 = true
-				continue
-			}
-			return fmt.Errorf("IpRouteAdd: %w", err)
-		}
-	}
-	c.deferFuncs = append(c.deferFuncs, cleanRoutes)
-
-	/** ip rule
-	ip rule add fwmark 0x8000000/0x8000000 table 2023
-	ip -6 rule add fwmark 0x8000000/0x8000000 table 2023
-	*/
-	rules := []netlink.Rule{{
-		SuppressIfgroup:   -1,
-		SuppressPrefixlen: -1,
-		Priority:          -1,
-		Goto:              -1,
-		Flow:              -1,
-		Family:            unix.AF_INET,
-		Table:             table,
-		Mark:              int(consts.TproxyMark),
-		Mask:              int(consts.TproxyMark),
-	}, {
-		SuppressIfgroup:   -1,
-		SuppressPrefixlen: -1,
-		Priority:          -1,
-		Goto:              -1,
-		Flow:              -1,
-		Family:            unix.AF_INET6,
-		Table:             table,
-		Mark:              int(consts.TproxyMark),
-		Mask:              int(consts.TproxyMark),
-	}}
-	var ruleBadIpv6 bool
-	cleanRules := func() error {
-		var errs error
-		for _, rule := range rules {
-			if rule.Family == unix.AF_INET6 && ruleBadIpv6 {
-				// Not clean for bad ipv6.
-				continue
-			}
-			if e := netlink.RuleDel(&rule); e != nil {
-				if errs != nil {
-					errs = fmt.Errorf("%w; %v", errs, e)
-				} else {
-					errs = e
-				}
-			}
-		}
-		if errs != nil {
-			c.log.Debugf("IpRuleDel: %w\n", errs)
-		}
-		return nil
-	}
-tryRuleAddAgain:
-	for _, rule := range rules {
-		if err = netlink.RuleAdd(&rule); err != nil {
-			if os.IsExist(err) {
-				_ = cleanRules()
-				goto tryRuleAddAgain
-			}
-			if rule.Family == unix.AF_INET6 {
-				// ipv6
-				c.log.Warnln("IpRuleAdd: Bad IPv6 support. Perhaps your machine disabled IPv6 (need CONFIG_IPV6_MULTIPLE_TABLES).")
-				ruleBadIpv6 = true
-				continue
-			}
-			return fmt.Errorf("IpRuleAdd: %w", err)
-		}
-	}
-	c.deferFuncs = append(c.deferFuncs, cleanRules)
-	return nil
-}
-
 func (c *controlPlaneCore) addLinkCb(_ifname string, rtmType uint16, cb func()) error {
 	ch := make(chan netlink.LinkUpdate)
 	done := make(chan struct{})

diff --git a/control/netns_utils.go b/control/netns_utils.go
@@ -9,6 +9,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/daeuniverse/dae/common/consts"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
@@ -127,9 +128,93 @@ func (ns *DaeNetns) setup() (err error) {
 	if err = ns.setupIPv6Datapath(); err != nil {
 		return
 	}
+	if err = ns.setupRoutingPolicy(); err != nil {
+		return
+	}
 	return
 }
 
+func (ns *DaeNetns) setupRoutingPolicy() (err error) {
+	if err = netns.Set(ns.daeNs); err != nil {
+		return fmt.Errorf("failed to switch to daens: %v", err)
+	}
+	defer netns.Set(ns.hostNs)
+
+	/// Insert ip rule / ip route.
+	var table = 2023
+
+	/** ip table
+	ip route add local default dev lo table 2023
+	ip -6 route add local default dev lo table 2023
+	*/
+	routes := []netlink.Route{{
+		Scope:     unix.RT_SCOPE_HOST,
+		LinkIndex: consts.LoopbackIfIndex,
+		Dst: &net.IPNet{
+			IP:   []byte{0, 0, 0, 0},
+			Mask: net.CIDRMask(0, 32),
+		},
+		Table: table,
+		Type:  unix.RTN_LOCAL,
+	}, {
+		Scope:     unix.RT_SCOPE_HOST,
+		LinkIndex: consts.LoopbackIfIndex,
+		Dst: &net.IPNet{
+			IP:   []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+			Mask: net.CIDRMask(0, 128),
+		},
+		Table: table,
+		Type:  unix.RTN_LOCAL,
+	}}
+	for _, route := range routes {
+		if err = netlink.RouteAdd(&route); err != nil {
+			if len(route.Dst.IP) == net.IPv6len {
+				// ipv6
+				ns.log.Warnln("IpRouteAdd: Bad IPv6 support. Perhaps your machine disabled IPv6.")
+				continue
+			}
+			return fmt.Errorf("IpRouteAdd: %w", err)
+		}
+	}
+
+	/** ip rule
+	ip rule add fwmark 0x8000000/0x8000000 table 2023
+	ip -6 rule add fwmark 0x8000000/0x8000000 table 2023
+	*/
+	rules := []netlink.Rule{{
+		SuppressIfgroup:   -1,
+		SuppressPrefixlen: -1,
+		Priority:          -1,
+		Goto:              -1,
+		Flow:              -1,
+		Family:            unix.AF_INET,
+		Table:             table,
+		Mark:              int(consts.TproxyMark),
+		Mask:              int(consts.TproxyMark),
+	}, {
+		SuppressIfgroup:   -1,
+		SuppressPrefixlen: -1,
+		Priority:          -1,
+		Goto:              -1,
+		Flow:              -1,
+		Family:            unix.AF_INET6,
+		Table:             table,
+		Mark:              int(consts.TproxyMark),
+		Mask:              int(consts.TproxyMark),
+	}}
+
+	for _, rule := range rules {
+		if err = netlink.RuleAdd(&rule); err != nil {
+			if rule.Family == unix.AF_INET6 {
+				// ipv6
+				ns.log.Warnln("IpRuleAdd: Bad IPv6 support. Perhaps your machine disabled IPv6 (need CONFIG_IPV6_MULTIPLE_TABLES).")
+				continue
+			}
+			return fmt.Errorf("IpRuleAdd: %w", err)
+		}
+	}
+	return nil
+}
 func (ns *DaeNetns) setupVeth() (err error) {
 	// ip l a dae0 type veth peer name dae0peer
 	DeleteLink(HostVethName)