diff --git a/control/control_plane.go b/control/control_plane.go index 9ca7364ff..8f1fdf7a3 100644 --- a/control/control_plane.go +++ b/control/control_plane.go @@ -470,10 +470,6 @@ func NewControlPlane( } go dnsUpstream.InitUpstreams() - if err = GetDaeNetns().With(core.setupRoutingPolicy); err != nil { - return nil, err - } - close(plane.ready) return plane, nil } diff --git a/control/control_plane_core.go b/control/control_plane_core.go index bd6ff6d74..0c2176f94 100644 --- a/control/control_plane_core.go +++ b/control/control_plane_core.go @@ -9,7 +9,6 @@ import ( "context" "errors" "fmt" - "net" "net/netip" "os" "regexp" @@ -192,138 +191,6 @@ func (c *controlPlaneCore) delQdisc(ifname string) error { return nil } -func (c *controlPlaneCore) setupRoutingPolicy() (err error) { - /// Insert ip rule / ip route. - var table = 2023 + c.flip - - /** ip table - ip route add local default dev lo table 2023 - ip -6 route add local default dev lo table 2023 - */ - routes := []netlink.Route{{ - Scope: unix.RT_SCOPE_HOST, - LinkIndex: consts.LoopbackIfIndex, - Dst: &net.IPNet{ - IP: []byte{0, 0, 0, 0}, - Mask: net.CIDRMask(0, 32), - }, - Table: table, - Type: unix.RTN_LOCAL, - }, { - Scope: unix.RT_SCOPE_HOST, - LinkIndex: consts.LoopbackIfIndex, - Dst: &net.IPNet{ - IP: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - Mask: net.CIDRMask(0, 128), - }, - Table: table, - Type: unix.RTN_LOCAL, - }} - var routeBadIpv6 bool - cleanRoutes := func() error { - var errs error - for _, route := range routes { - if e := netlink.RouteDel(&route); e != nil { - if len(route.Dst.IP) == net.IPv6len && routeBadIpv6 { - // Not clean for bad ipv6. - continue - } - if errs != nil { - errs = fmt.Errorf("%w; %v", errs, e) - } else { - errs = e - } - } - } - if errs != nil { - c.log.Debugf("IpRouteDel: %w\n", errs) - } - return nil - } -tryRouteAddAgain: - for _, route := range routes { - if err = netlink.RouteAdd(&route); err != nil { - if os.IsExist(err) { - _ = cleanRoutes() - goto tryRouteAddAgain - } - if len(route.Dst.IP) == net.IPv6len { - // ipv6 - c.log.Warnln("IpRouteAdd: Bad IPv6 support. Perhaps your machine disabled IPv6.") - routeBadIpv6 = true - continue - } - return fmt.Errorf("IpRouteAdd: %w", err) - } - } - c.deferFuncs = append(c.deferFuncs, cleanRoutes) - - /** ip rule - ip rule add fwmark 0x8000000/0x8000000 table 2023 - ip -6 rule add fwmark 0x8000000/0x8000000 table 2023 - */ - rules := []netlink.Rule{{ - SuppressIfgroup: -1, - SuppressPrefixlen: -1, - Priority: -1, - Goto: -1, - Flow: -1, - Family: unix.AF_INET, - Table: table, - Mark: int(consts.TproxyMark), - Mask: int(consts.TproxyMark), - }, { - SuppressIfgroup: -1, - SuppressPrefixlen: -1, - Priority: -1, - Goto: -1, - Flow: -1, - Family: unix.AF_INET6, - Table: table, - Mark: int(consts.TproxyMark), - Mask: int(consts.TproxyMark), - }} - var ruleBadIpv6 bool - cleanRules := func() error { - var errs error - for _, rule := range rules { - if rule.Family == unix.AF_INET6 && ruleBadIpv6 { - // Not clean for bad ipv6. - continue - } - if e := netlink.RuleDel(&rule); e != nil { - if errs != nil { - errs = fmt.Errorf("%w; %v", errs, e) - } else { - errs = e - } - } - } - if errs != nil { - c.log.Debugf("IpRuleDel: %w\n", errs) - } - return nil - } -tryRuleAddAgain: - for _, rule := range rules { - if err = netlink.RuleAdd(&rule); err != nil { - if os.IsExist(err) { - _ = cleanRules() - goto tryRuleAddAgain - } - if rule.Family == unix.AF_INET6 { - // ipv6 - c.log.Warnln("IpRuleAdd: Bad IPv6 support. Perhaps your machine disabled IPv6 (need CONFIG_IPV6_MULTIPLE_TABLES).") - ruleBadIpv6 = true - continue - } - return fmt.Errorf("IpRuleAdd: %w", err) - } - } - c.deferFuncs = append(c.deferFuncs, cleanRules) - return nil -} - func (c *controlPlaneCore) addLinkCb(_ifname string, rtmType uint16, cb func()) error { ch := make(chan netlink.LinkUpdate) done := make(chan struct{}) diff --git a/control/netns_utils.go b/control/netns_utils.go index bd219d52e..a32c983af 100644 --- a/control/netns_utils.go +++ b/control/netns_utils.go @@ -9,6 +9,7 @@ import ( "sync" "sync/atomic" + "github.com/daeuniverse/dae/common/consts" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" @@ -127,9 +128,93 @@ func (ns *DaeNetns) setup() (err error) { if err = ns.setupIPv6Datapath(); err != nil { return } + if err = ns.setupRoutingPolicy(); err != nil { + return + } return } +func (ns *DaeNetns) setupRoutingPolicy() (err error) { + if err = netns.Set(ns.daeNs); err != nil { + return fmt.Errorf("failed to switch to daens: %v", err) + } + defer netns.Set(ns.hostNs) + + /// Insert ip rule / ip route. + var table = 2023 + + /** ip table + ip route add local default dev lo table 2023 + ip -6 route add local default dev lo table 2023 + */ + routes := []netlink.Route{{ + Scope: unix.RT_SCOPE_HOST, + LinkIndex: consts.LoopbackIfIndex, + Dst: &net.IPNet{ + IP: []byte{0, 0, 0, 0}, + Mask: net.CIDRMask(0, 32), + }, + Table: table, + Type: unix.RTN_LOCAL, + }, { + Scope: unix.RT_SCOPE_HOST, + LinkIndex: consts.LoopbackIfIndex, + Dst: &net.IPNet{ + IP: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + Mask: net.CIDRMask(0, 128), + }, + Table: table, + Type: unix.RTN_LOCAL, + }} + for _, route := range routes { + if err = netlink.RouteAdd(&route); err != nil { + if len(route.Dst.IP) == net.IPv6len { + // ipv6 + ns.log.Warnln("IpRouteAdd: Bad IPv6 support. Perhaps your machine disabled IPv6.") + continue + } + return fmt.Errorf("IpRouteAdd: %w", err) + } + } + + /** ip rule + ip rule add fwmark 0x8000000/0x8000000 table 2023 + ip -6 rule add fwmark 0x8000000/0x8000000 table 2023 + */ + rules := []netlink.Rule{{ + SuppressIfgroup: -1, + SuppressPrefixlen: -1, + Priority: -1, + Goto: -1, + Flow: -1, + Family: unix.AF_INET, + Table: table, + Mark: int(consts.TproxyMark), + Mask: int(consts.TproxyMark), + }, { + SuppressIfgroup: -1, + SuppressPrefixlen: -1, + Priority: -1, + Goto: -1, + Flow: -1, + Family: unix.AF_INET6, + Table: table, + Mark: int(consts.TproxyMark), + Mask: int(consts.TproxyMark), + }} + + for _, rule := range rules { + if err = netlink.RuleAdd(&rule); err != nil { + if rule.Family == unix.AF_INET6 { + // ipv6 + ns.log.Warnln("IpRuleAdd: Bad IPv6 support. Perhaps your machine disabled IPv6 (need CONFIG_IPV6_MULTIPLE_TABLES).") + continue + } + return fmt.Errorf("IpRuleAdd: %w", err) + } + } + return nil +} func (ns *DaeNetns) setupVeth() (err error) { // ip l a dae0 type veth peer name dae0peer DeleteLink(HostVethName)