diff --git a/docs/troubleshooting/ovnkube-trace.md b/docs/troubleshooting/ovnkube-trace.md index e04cca6e0b0..9da480c8214 100644 --- a/docs/troubleshooting/ovnkube-trace.md +++ b/docs/troubleshooting/ovnkube-trace.md @@ -8,7 +8,7 @@ A tool to trace packet simulations for arbitrary UDP or TCP traffic between poin Given the command-line arguments, ovnkube-trace would inspect the cluster to determine the addresses (MAC and IP) of the source and destination and perform `ovn-trace`, `ovs-appctl ofproto/trace`, and `ovn-detrace` from/to both directions. ``` -Usage of _output/go/bin/ovnkube-trace: +Usage of /tmp/go-build1564673416/b001/exe/ovnkube-trace: -addr-family string Address family (ip4 or ip6) to be used for tracing (default "ip4") -dst string @@ -19,6 +19,8 @@ Usage of _output/go/bin/ovnkube-trace: k8s namespace of dest pod (default "default") -dst-port string dst-port: destination port (default "80") + -dump-udn-vrf-table-ids + Dump the VRF table ID per node for all the user defined networks -kubeconfig string absolute path to the kubeconfig file -loglevel string diff --git a/docs/troubleshooting/udn.md b/docs/troubleshooting/udn.md new file mode 100644 index 00000000000..c8e83d7fa20 --- /dev/null +++ b/docs/troubleshooting/udn.md @@ -0,0 +1,27 @@ +# User Defined Networks + +To debug UDN the ovnkube-trace can dump multiple elements of the topology to +make easier to match to what network they belong. + +## Local gateway VRF table ID numbers for networks + +Following command will dump the VRF table IDs from a local gateway system for +UDNs. + +```bash +ovnkube-trace -dump-udn-vrf-table-ids +``` + +The output will be tableIDs indexed by node and network name +```json +{ + "ovn-control-plane": { + "net-blue": 1, + "net-red": 2 + }, + "ovn-worker": { + "net-blue": 3, + "net-red": 4 + } +} +``` diff --git a/go-controller/cmd/ovnkube-trace/ovnkube-trace.go b/go-controller/cmd/ovnkube-trace/ovnkube-trace.go index dd602f3cdfa..5fdf28c2aee 100644 --- a/go-controller/cmd/ovnkube-trace/ovnkube-trace.go +++ b/go-controller/cmd/ovnkube-trace/ovnkube-trace.go @@ -1179,49 +1179,13 @@ func main() { udp := flag.Bool("udp", false, "use udp transport protocol") addressFamily := flag.String("addr-family", ip4, "Address family (ip4 or ip6) to be used for tracing") skipOvnDetrace := flag.Bool("skip-detrace", false, "skip ovn-detrace command") + dumpVRFTableIDs := flag.Bool("dump-udn-vrf-table-ids", false, "Dump the VRF table ID per node for all the user defined networks") loglevel := flag.String("loglevel", "0", "loglevel: klog level") flag.Parse() // Set the application's log level. setLogLevel(*loglevel) - // Verify CLI flags. - if *srcPodName == "" { - klog.Exitf("Usage: source pod must be specified") - } - if !*tcp && !*udp { - klog.Exitf("Usage: either tcp or udp must be specified") - } - if *udp && *tcp { - klog.Exitf("Usage: Both tcp and udp cannot be specified at the same time") - } - if *tcp { - protocol = "tcp" - } - if *udp { - if *dstSvcName != "" { - klog.Exitf("Usage: udp option is not compatible with destination service trace") - } - protocol = "udp" - } - targetOptions := 0 - if *dstPodName != "" { - targetOptions++ - } - if *dstSvcName != "" { - targetOptions++ - } - if *dstIP != "" { - targetOptions++ - parsedDstIP = net.ParseIP(*dstIP) - if parsedDstIP == nil { - klog.Exitf("Usage: cannot parse IP address provided in -dst-ip") - } - } - if targetOptions != 1 { - klog.Exitf("Usage: exactly one of -dst, -service or -dst-ip must be set") - } - // Get the ClientConfig. // This might work better? https://godoc.org/sigs.k8s.io/controller-runtime/pkg/client/config // When supplied the kubeconfig supplied via cli takes precedence @@ -1258,8 +1222,55 @@ func main() { if err != nil { klog.Exitf(" Unexpected error: %v", err) } + klog.V(5).Infof("OVN Kubernetes namespace is %s", ovnNamespace) + if *dumpVRFTableIDs { + nodesVRFTableIDs, err := findUserDefinedNetworkVRFTableIDs(coreclient, restconfig, ovnNamespace) + if err != nil { + klog.Exitf("Failed dumping VRF table IDs: %s", err) + } + fmt.Println(string(nodesVRFTableIDs)) + return + } + + // Verify CLI flags. + if *srcPodName == "" { + klog.Exitf("Usage: source pod must be specified") + } + if !*tcp && !*udp { + klog.Exitf("Usage: either tcp or udp must be specified") + } + if *udp && *tcp { + klog.Exitf("Usage: Both tcp and udp cannot be specified at the same time") + } + if *tcp { + protocol = "tcp" + } + if *udp { + if *dstSvcName != "" { + klog.Exitf("Usage: udp option is not compatible with destination service trace") + } + protocol = "udp" + } + targetOptions := 0 + if *dstPodName != "" { + targetOptions++ + } + if *dstSvcName != "" { + targetOptions++ + } + if *dstIP != "" { + targetOptions++ + parsedDstIP = net.ParseIP(*dstIP) + if parsedDstIP == nil { + klog.Exitf("Usage: cannot parse IP address provided in -dst-ip") + } + } + if targetOptions != 1 { + klog.Exitf("Usage: exactly one of -dst, -service or -dst-ip must be set") + } + // Show some information about the nodes in this cluster - only if log level 5 or higher. if lvl, err := strconv.Atoi(*loglevel); err == nil && lvl >= 5 { displayNodeInfo(coreclient) diff --git a/go-controller/cmd/ovnkube-trace/udn.go b/go-controller/cmd/ovnkube-trace/udn.go new file mode 100644 index 00000000000..16d0d9d07ce --- /dev/null +++ b/go-controller/cmd/ovnkube-trace/udn.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + types "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + util "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +func findUserDefinedNetworkVRFTableIDs(coreclient *corev1client.CoreV1Client, restconfig *rest.Config, ovnNamespace string) (string, error) { + nodeList, err := coreclient.Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return "", err + } + nodesTableIDs := map[string]map[string]uint{} + for _, node := range nodeList.Items { + networks, err := findNetworks(&node) + if err != nil { + return "", err + } + networksTableIDs := map[string]uint{} + for _, networkName := range networks { + tableID, err := findUserDefinedNetworkVRFTableID(coreclient, restconfig, &node, ovnNamespace, networkName) + if err != nil { + return "", err + } + networksTableIDs[networkName] = tableID + } + nodesTableIDs[node.Name] = networksTableIDs + } + nodesTableIDsJSON, err := json.Marshal(&nodesTableIDs) + if err != nil { + return "", err + } + return string(nodesTableIDsJSON), nil +} + +func findUserDefinedNetworkVRFTableID(coreclient *corev1client.CoreV1Client, restconfig *rest.Config, node *corev1.Node, ovnNamespace, networkName string) (uint, error) { + ovnKubePodName, err := getOvnKubePodOnNode(coreclient, ovnNamespace, node.Name) + if err != nil { + return 0, err + } + + mgmtPortLinkName := util.GetSecondaryNetworkPrefix(networkName) + types.K8sMgmtIntfName + ipLinkCmd := "ip -j link show dev " + mgmtPortLinkName + stdout, stderr, err := execInPod(coreclient, restconfig, ovnNamespace, ovnKubePodName, ovnKubeNodePodContainers[1], ipLinkCmd, "") + if err != nil { + return 0, fmt.Errorf("%s: %s: %w", stdout, stderr, err) + } + links := []struct { + Index uint `json:"ifindex"` + }{} + if err := json.Unmarshal([]byte(stdout), &links); err != nil { + return 0, err + } + if len(links) < 1 { + return 0, fmt.Errorf("link '%s' not found", mgmtPortLinkName) + } + return uint(util.CalculateRouteTableID(int(links[0].Index))), nil +} + +func findNetworks(node *corev1.Node) ([]string, error) { + annotation, ok := node.Annotations["k8s.ovn.org/network-ids"] + if !ok { + return []string{}, nil + } + networkIDs := make(map[string]json.RawMessage) + if err := json.Unmarshal([]byte(annotation), &networkIDs); err != nil { + return nil, err + } + networks := []string{} + for networkName := range networkIDs { + networks = append(networks, networkName) + } + return networks, nil +} diff --git a/go-controller/pkg/node/controllers/egressip/egressip.go b/go-controller/pkg/node/controllers/egressip/egressip.go index 1b3685cb3e2..ecd34d584ea 100644 --- a/go-controller/pkg/node/controllers/egressip/egressip.go +++ b/go-controller/pkg/node/controllers/egressip/egressip.go @@ -628,7 +628,7 @@ func generateRoutesForLink(link netlink.Link, isV6 bool) ([]netlink.Route, error return nil, fmt.Errorf("failed to get routes for link %s: %v", link.Attrs().Name, err) } linkRoutes = ensureAtLeastOneDefaultRoute(linkRoutes, link.Attrs().Index, isV6) - overwriteRoutesTableID(linkRoutes, getRouteTableID(link.Attrs().Index)) + overwriteRoutesTableID(linkRoutes, util.CalculateRouteTableID(link.Attrs().Index)) return linkRoutes, nil } @@ -877,7 +877,7 @@ func (c *Controller) repairNode() error { assignedAddrStrToAddrs[addressStr] = address } } - filter, mask := filterRouteByLinkTable(linkIdx, getRouteTableID(linkIdx)) + filter, mask := filterRouteByLinkTable(linkIdx, util.CalculateRouteTableID(linkIdx)) existingRoutes, err := util.GetNetLinkOps().RouteListFiltered(netlink.FAMILY_ALL, filter, mask) if err != nil { return fmt.Errorf("unable to get route list using filter (%s): %v", filter.String(), err) @@ -1320,10 +1320,6 @@ func overwriteRoutesTableID(routes []netlink.Route, tableID int) { } } -func getRouteTableID(ifIndex int) int { - return ifIndex + routingTableIDStart -} - func findLinkOnSameNetworkAsIP(ip net.IP, v4, v6 bool) (bool, netlink.Link, error) { found, link, err := findLinkOnSameNetworkAsIPUsingLPM(ip, v4, v6) if err != nil { @@ -1437,7 +1433,7 @@ func getNetlinkAddress(addr *net.IPNet, ifindex int) *netlink.Addr { // from the links 'ifindex' func generateIPRule(srcIP net.IP, isIPv6 bool, ifIndex int) netlink.Rule { r := *netlink.NewRule() - r.Table = getRouteTableID(ifIndex) + r.Table = util.CalculateRouteTableID(ifIndex) r.Priority = rulePriority var ipFullMask string if isIPv6 { diff --git a/go-controller/pkg/node/controllers/egressip/egressip_test.go b/go-controller/pkg/node/controllers/egressip/egressip_test.go index 0a564ab938e..bab7a45f95d 100644 --- a/go-controller/pkg/node/controllers/egressip/egressip_test.go +++ b/go-controller/pkg/node/controllers/egressip/egressip_test.go @@ -475,7 +475,7 @@ var _ = table.DescribeTable("EgressIP selectors", if err != nil { return err } - filter, mask := filterRouteByLinkTable(link.Attrs().Index, getRouteTableID(link.Attrs().Index)) + filter, mask := filterRouteByLinkTable(link.Attrs().Index, util.CalculateRouteTableID(link.Attrs().Index)) existingRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, mask) if err != nil { return fmt.Errorf("failed to list routes for link %q: %v", expectedEIPConfig.inf, err) @@ -617,7 +617,7 @@ var _ = table.DescribeTable("EgressIP selectors", if err != nil { return err } - filter, mask := filterRouteByLinkTable(link.Attrs().Index, getRouteTableID(link.Attrs().Index)) + filter, mask := filterRouteByLinkTable(link.Attrs().Index, util.CalculateRouteTableID(link.Attrs().Index)) existingRoutes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, filter, mask) if err != nil { return err @@ -704,7 +704,7 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod1IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -729,7 +729,7 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod1IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -754,7 +754,7 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod1IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -779,12 +779,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod1IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod2IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -812,12 +812,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod1IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod2IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -843,12 +843,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod1IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod2IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -876,12 +876,12 @@ var _ = table.DescribeTable("EgressIP selectors", []testPodConfig{ {pod1Name, getIPTableMasqRule(pod1IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod1IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod2IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -909,12 +909,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod1IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv4CIDR, dummyLink1Name, egressIP1IPV4), - getRule(pod2IPv4, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -928,12 +928,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod3Name, getIPTableMasqRule(pod3IPv4CIDR, dummyLink2Name, egressIP2IPV4), - getRule(pod3IPv4, getRouteTableID(getLinkIndex(dummyLink2Name))), + getRule(pod3IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink2Name))), }, { pod4Name, getIPTableMasqRule(pod4IPv4CIDR, dummyLink2Name, egressIP2IPV4), - getRule(pod4IPv4, getRouteTableID(getLinkIndex(dummyLink2Name))), + getRule(pod4IPv4, util.CalculateRouteTableID(getLinkIndex(dummyLink2Name))), }, }, }, @@ -964,12 +964,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod1Name, getIPTableMasqRule(pod1IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod1IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod1IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, { pod2Name, getIPTableMasqRule(pod2IPv6CIDRCompressed, dummyLink1Name, egressIP1IPV6Compressed), - getRule(pod2IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink1Name))), + getRule(pod2IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink1Name))), }, }, }, @@ -984,12 +984,12 @@ var _ = table.DescribeTable("EgressIP selectors", { pod3Name, getIPTableMasqRule(pod3IPv6CIDRCompressed, dummyLink2Name, egressIP2IPV6Compressed), - getRule(pod3IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink2Name))), + getRule(pod3IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink2Name))), }, { pod4Name, getIPTableMasqRule(pod4IPv6CIDRCompressed, dummyLink2Name, egressIP2IPV6Compressed), - getRule(pod4IPv6Compressed, getRouteTableID(getLinkIndex(dummyLink2Name))), + getRule(pod4IPv6Compressed, util.CalculateRouteTableID(getLinkIndex(dummyLink2Name))), }, }, }, @@ -1415,7 +1415,7 @@ func getRoutesForLinkFromTable(testNS ns.NetNS, linkName string) ([]netlink.Rout if err != nil { return err } - filterRoute, filterMask := filterRouteByTable(link.Attrs().Index, getRouteTableID(link.Attrs().Index)) + filterRoute, filterMask := filterRouteByTable(link.Attrs().Index, util.CalculateRouteTableID(link.Attrs().Index)) routes, err = netlink.RouteListFiltered(netlink.FAMILY_ALL, filterRoute, filterMask) if err != nil { return err @@ -1540,12 +1540,12 @@ func isIPTableRuleArgIPV6(ruleArgs []string) bool { func getDefaultIPv4Route(linkIndex int) netlink.Route { // dst is nil because netlink represents a default route as nil - return netlink.Route{LinkIndex: linkIndex, Table: getRouteTableID(linkIndex), Dst: defaultV4AnyCIDR} + return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV4AnyCIDR} } func getDefaultIPv6Route(linkIndex int) netlink.Route { // dst is nil because netlink represents a default route as nil - return netlink.Route{LinkIndex: linkIndex, Table: getRouteTableID(linkIndex), Dst: defaultV6AnyCIDR} + return netlink.Route{LinkIndex: linkIndex, Table: util.CalculateRouteTableID(linkIndex), Dst: defaultV6AnyCIDR} } func getDstRoute(linkIndex int, dst string) netlink.Route { @@ -1553,7 +1553,7 @@ func getDstRoute(linkIndex int, dst string) netlink.Route { if err != nil { panic(err.Error()) } - return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Table: getRouteTableID(linkIndex)} + return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Table: util.CalculateRouteTableID(linkIndex)} } func getDstWithSrcRoute(linkIndex int, dst, src string) netlink.Route { @@ -1565,11 +1565,11 @@ func getDstWithSrcRoute(linkIndex int, dst, src string) netlink.Route { if len(ip) == 0 { panic("invalid src IP") } - return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Src: ip, Table: getRouteTableID(linkIndex)} + return netlink.Route{LinkIndex: linkIndex, Dst: dstIPNet, Src: ip, Table: util.CalculateRouteTableID(linkIndex)} } func getLinkLocalRoute(linkIndex int) netlink.Route { - return netlink.Route{LinkIndex: linkIndex, Dst: linkLocalCIDR, Table: getRouteTableID(linkIndex)} + return netlink.Route{LinkIndex: linkIndex, Dst: linkLocalCIDR, Table: util.CalculateRouteTableID(linkIndex)} } func getNetlinkAddr(ip, netmask string) *netlink.Addr { diff --git a/go-controller/pkg/util/net.go b/go-controller/pkg/util/net.go index 9a81eb6d98a..b13e3da96a7 100644 --- a/go-controller/pkg/util/net.go +++ b/go-controller/pkg/util/net.go @@ -13,6 +13,10 @@ import ( utilnet "k8s.io/utils/net" ) +const ( + routingTableIDStart = 1000 +) + var ErrorNoIP = errors.New("no IP available") // GetOVSPortMACAddress returns the MAC address of a given OVS port @@ -324,3 +328,9 @@ func IPNetsIPToStringSlice(ips []*net.IPNet) []string { } return ipAddrs } + +// CalculateRouteTableID will calculate route table ID based on the network +// interface index +func CalculateRouteTableID(ifIndex int) int { + return ifIndex + routingTableIDStart +}