From 3bc3d8fb9bedc7e3984ac925f157ac24790ce386 Mon Sep 17 00:00:00 2001 From: "bingshen.wbs" Date: Mon, 22 Apr 2024 20:47:35 +0800 Subject: [PATCH] rdma: support rdma metrics: resource and statistic Signed-off-by: bingshen.wbs --- nl/rdma_link_linux.go | 58 +++++++----- rdma_link_linux.go | 216 +++++++++++++++++++++++++++++++++++++++++- rdma_link_test.go | 27 ++++++ 3 files changed, 276 insertions(+), 25 deletions(-) diff --git a/nl/rdma_link_linux.go b/nl/rdma_link_linux.go index ce43ee15..02448362 100644 --- a/nl/rdma_link_linux.go +++ b/nl/rdma_link_linux.go @@ -9,31 +9,41 @@ const ( ) const ( - RDMA_NLDEV_CMD_GET = 1 - RDMA_NLDEV_CMD_SET = 2 - RDMA_NLDEV_CMD_NEWLINK = 3 - RDMA_NLDEV_CMD_DELLINK = 4 - RDMA_NLDEV_CMD_SYS_GET = 6 - RDMA_NLDEV_CMD_SYS_SET = 7 + RDMA_NLDEV_CMD_GET = 1 + RDMA_NLDEV_CMD_SET = 2 + RDMA_NLDEV_CMD_NEWLINK = 3 + RDMA_NLDEV_CMD_DELLINK = 4 + RDMA_NLDEV_CMD_SYS_GET = 6 + RDMA_NLDEV_CMD_SYS_SET = 7 + RDMA_NLDEV_CMD_RES_GET = 9 + RDMA_NLDEV_CMD_STAT_GET = 17 ) const ( - RDMA_NLDEV_ATTR_DEV_INDEX = 1 - RDMA_NLDEV_ATTR_DEV_NAME = 2 - RDMA_NLDEV_ATTR_PORT_INDEX = 3 - RDMA_NLDEV_ATTR_CAP_FLAGS = 4 - RDMA_NLDEV_ATTR_FW_VERSION = 5 - RDMA_NLDEV_ATTR_NODE_GUID = 6 - RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7 - RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8 - RDMA_NLDEV_ATTR_LID = 9 - RDMA_NLDEV_ATTR_SM_LID = 10 - RDMA_NLDEV_ATTR_LMC = 11 - RDMA_NLDEV_ATTR_PORT_STATE = 12 - RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13 - RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14 - RDMA_NLDEV_ATTR_NDEV_NAME = 51 - RDMA_NLDEV_ATTR_LINK_TYPE = 65 - RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66 - RDMA_NLDEV_NET_NS_FD = 68 + RDMA_NLDEV_ATTR_DEV_INDEX = 1 + RDMA_NLDEV_ATTR_DEV_NAME = 2 + RDMA_NLDEV_ATTR_PORT_INDEX = 3 + RDMA_NLDEV_ATTR_CAP_FLAGS = 4 + RDMA_NLDEV_ATTR_FW_VERSION = 5 + RDMA_NLDEV_ATTR_NODE_GUID = 6 + RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7 + RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8 + RDMA_NLDEV_ATTR_LID = 9 + RDMA_NLDEV_ATTR_SM_LID = 10 + RDMA_NLDEV_ATTR_LMC = 11 + RDMA_NLDEV_ATTR_PORT_STATE = 12 + RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13 + RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14 + RDMA_NLDEV_ATTR_RES_SUMMARY = 15 + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY = 16 + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME = 17 + RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR = 18 + RDMA_NLDEV_ATTR_NDEV_NAME = 51 + RDMA_NLDEV_ATTR_LINK_TYPE = 65 + RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66 + RDMA_NLDEV_NET_NS_FD = 68 + RDMA_NLDEV_ATTR_STAT_HWCOUNTERS = 80 + RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY = 81 + RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME = 82 + RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE = 83 ) diff --git a/rdma_link_linux.go b/rdma_link_linux.go index 036399db..04e60cc1 100644 --- a/rdma_link_linux.go +++ b/rdma_link_linux.go @@ -17,6 +17,7 @@ type RdmaLinkAttrs struct { FirmwareVersion string NodeGuid string SysImageGuid string + NumPorts uint32 } // Link represents a rdma device from netlink. @@ -68,6 +69,11 @@ func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) { r := bytes.NewReader(value) binary.Read(r, nl.NativeEndian(), &sysGuid) link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid) + case nl.RDMA_NLDEV_ATTR_PORT_INDEX: + var availablePort uint32 + r := bytes.NewReader(value) + binary.Read(r, nl.NativeEndian(), &availablePort) + link.Attrs.NumPorts = availablePort } if (len % 4) != 0 { // Skip pad bytes @@ -304,7 +310,6 @@ func (h *Handle) RdmaLinkDel(name string) error { _, err = req.Execute(unix.NETLINK_RDMA, 0) return err } - // RdmaLinkAdd adds an rdma link for the specified type to the network device. // Similar to: rdma link add NAME type TYPE netdev NETDEV // NAME - specifies the new name of the rdma link to add @@ -329,3 +334,212 @@ func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) er _, err := req.Execute(unix.NETLINK_RDMA, 0) return err } + +// RdmaResource represents a rdma device resource tracking summaries +type RdmaResource struct { + Index uint32 + Name string + RdmaResourceSummaryEntries map[string]uint64 +} + +// RdmaResourceList list rdma resource tracking information +// Returns all rdma devices resource tracking summary on success or returns error +// otherwise. +// Equivalent to: `rdma resource' +func RdmaResourceList() ([]*RdmaResource, error) { + return pkgHandle.RdmaResourceList() +} + +// RdmaResourceList list rdma resource tracking information +// Returns all rdma devices resource tracking summary on success or returns error +// otherwise. +// Equivalent to: `rdma resource' +func (h *Handle) RdmaResourceList() ([]*RdmaResource, error) { + proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_RES_GET) + req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP) + + msgs, err := req.Execute(unix.NETLINK_RDMA, 0) + if err != nil { + return nil, err + } + if len(msgs) == 0 { + return nil, fmt.Errorf("No valid response from kernel") + } + var rdmaResources []*RdmaResource + for _, msg := range msgs { + res, err := executeOneGetRdmaResourceList(msg) + if err != nil { + return nil, err + } + rdmaResources = append(rdmaResources, res) + } + return rdmaResources, nil +} + +func parseRdmaCounters(counterType uint16, data []byte) (map[string]uint64, error) { + var counterKeyType, counterValueType uint16 + switch counterType { + case nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY: + counterKeyType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME + counterValueType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR + case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY: + counterKeyType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME + counterValueType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE + default: + return nil, fmt.Errorf("Invalid counter type: %d", counterType) + } + counters := make(map[string]uint64) + reader := bytes.NewReader(data) + + for reader.Len() >= 4 { + _, attrType, _, value := parseNfAttrTLV(reader) + if attrType != counterType { + return nil, fmt.Errorf("Invalid resource summary entry type; %d", attrType) + } + + summaryReader := bytes.NewReader(value) + for summaryReader.Len() >= 4 { + _, attrType, len, value := parseNfAttrTLV(summaryReader) + if attrType != counterKeyType { + return nil, fmt.Errorf("Invalid resource summary entry name type; %d", attrType) + } + name := string(value[0 : len-1]) + // Skip pad bytes + if (len % 4) != 0 { + summaryReader.Seek(int64(4-(len%4)), seekCurrent) + } + _, attrType, len, value = parseNfAttrTLV(summaryReader) + if attrType != counterValueType { + return nil, fmt.Errorf("Invalid resource summary entry value type; %d", attrType) + } + counters[name] = native.Uint64(value) + } + } + return counters, nil +} + +func executeOneGetRdmaResourceList(data []byte) (*RdmaResource, error) { + var res RdmaResource + reader := bytes.NewReader(data) + for reader.Len() >= 4 { + _, attrType, len, value := parseNfAttrTLV(reader) + + switch attrType { + case nl.RDMA_NLDEV_ATTR_DEV_INDEX: + var Index uint32 + r := bytes.NewReader(value) + binary.Read(r, nl.NativeEndian(), &Index) + res.Index = Index + case nl.RDMA_NLDEV_ATTR_DEV_NAME: + res.Name = string(value[0 : len-1]) + case nl.RDMA_NLDEV_ATTR_RES_SUMMARY: + var err error + res.RdmaResourceSummaryEntries, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY, value) + if err != nil { + return nil, err + } + } + if (len % 4) != 0 { + // Skip pad bytes + reader.Seek(int64(4-(len%4)), seekCurrent) + } + } + return &res, nil +} + +// RdmaPortStatistic represents a rdma port statistic counter +type RdmaPortStatistic struct { + PortIndex uint32 + Statistics map[string]uint64 +} + +// RdmaDeviceStatistic represents a rdma device statistic counter +type RdmaDeviceStatistic struct { + RdmaPortStatistics []*RdmaPortStatistic +} + +// RdmaStatistic get rdma device statistic counters +// Returns rdma device statistic counters on success or returns error +// otherwise. +// Equivalent to: `rdma statistic show link [DEV]' +func RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) { + return pkgHandle.RdmaStatistic(link) +} + +// RdmaStatistic get rdma device statistic counters +// Returns rdma device statistic counters on success or returns error +// otherwise. +// Equivalent to: `rdma statistic show link [DEV]' +func (h *Handle) RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) { + rdmaLinkStatistic := make([]*RdmaPortStatistic, 0) + for portIndex := uint32(1); portIndex <= link.Attrs.NumPorts; portIndex++ { + portStatistic, err := h.RdmaPortStatisticList(link, portIndex) + if err != nil { + return nil, err + } + rdmaLinkStatistic = append(rdmaLinkStatistic, portStatistic) + } + return &RdmaDeviceStatistic{RdmaPortStatistics: rdmaLinkStatistic}, nil +} + +// RdmaPortStatisticList get rdma device port statistic counters +// Returns rdma device port statistic counters on success or returns error +// otherwise. +// Equivalent to: `rdma statistic show link [DEV/PORT]' +func RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) { + return pkgHandle.RdmaPortStatisticList(link, port) +} + +// RdmaPortStatisticList get rdma device port statistic counters +// Returns rdma device port statistic counters on success or returns error +// otherwise. +// Equivalent to: `rdma statistic show link [DEV/PORT]' +func (h *Handle) RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) { + proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_STAT_GET) + req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_REQUEST) + b := make([]byte, 4) + native.PutUint32(b, link.Attrs.Index) + data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b) + req.AddData(data) + + b = make([]byte, 4) + native.PutUint32(b, port) + data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_PORT_INDEX, b) + req.AddData(data) + + msgs, err := req.Execute(unix.NETLINK_RDMA, 0) + if err != nil { + return nil, err + } + if len(msgs) != 1 { + return nil, fmt.Errorf("No valid response from kernel") + } + return executeOneGetRdmaPortStatistics(msgs[0]) +} + +func executeOneGetRdmaPortStatistics(data []byte) (*RdmaPortStatistic, error) { + var stat RdmaPortStatistic + reader := bytes.NewReader(data) + for reader.Len() >= 4 { + _, attrType, len, value := parseNfAttrTLV(reader) + + switch attrType { + case nl.RDMA_NLDEV_ATTR_PORT_INDEX: + var Index uint32 + r := bytes.NewReader(value) + binary.Read(r, nl.NativeEndian(), &Index) + stat.PortIndex = Index + case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTERS: + var err error + stat.Statistics, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY, value) + if err != nil { + return nil, err + } + } + if (len % 4) != 0 { + // Skip pad bytes + reader.Seek(int64(4-(len%4)), seekCurrent) + } + } + return &stat, nil +} diff --git a/rdma_link_test.go b/rdma_link_test.go index 241b89e8..6c337758 100644 --- a/rdma_link_test.go +++ b/rdma_link_test.go @@ -205,3 +205,30 @@ func TestRdmaLinkAddAndDel(t *testing.T) { checkPresence(linkName, false) } + +func TestRdmaLinkMetrics(t *testing.T) { + minKernelRequired(t, 5, 1) + setupRdmaKModule(t, "rdma_rxe") + if err := RdmaLinkAdd(t.Name(), "rxe", "lo"); err != nil { + t.Fatal(err) + } + link, err := RdmaLinkByName(t.Name()) + if err != nil { + t.Fatal(err) + } + defer RdmaLinkDel(t.Name()) + resources, err := RdmaResourceList() + if err != nil { + t.Fatal(err) + } + for _, resource := range resources { + t.Logf("resource: %+v", resource) + } + stats, err := RdmaStatistic(link) + if err != nil { + t.Fatal(err) + } + for _, stat := range stats.RdmaPortStatistics { + t.Logf("stat: %+v", stat) + } +}