Skip to content

Commit

Permalink
rdma: support rdma metrics: resource and statistic
Browse files Browse the repository at this point in the history 
Signed-off-by: bingshen.wbs <[email protected]>
  • Loading branch information
@BSWANG
BSWANG committed Apr 23, 2024
1 parent 578e95c commit 3bc3d8f
Show file tree
Hide file tree
Showing 3 changed files with 276 additions and 25 deletions.
58 changes: 34 additions & 24 deletions nl/rdma_link_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,41 @@ const (
)

const (
RDMA_NLDEV_CMD_GET = 1
RDMA_NLDEV_CMD_SET = 2
RDMA_NLDEV_CMD_NEWLINK = 3
RDMA_NLDEV_CMD_DELLINK = 4
RDMA_NLDEV_CMD_SYS_GET = 6
RDMA_NLDEV_CMD_SYS_SET = 7
RDMA_NLDEV_CMD_GET = 1
RDMA_NLDEV_CMD_SET = 2
RDMA_NLDEV_CMD_NEWLINK = 3
RDMA_NLDEV_CMD_DELLINK = 4
RDMA_NLDEV_CMD_SYS_GET = 6
RDMA_NLDEV_CMD_SYS_SET = 7
RDMA_NLDEV_CMD_RES_GET = 9
RDMA_NLDEV_CMD_STAT_GET = 17
)

const (
RDMA_NLDEV_ATTR_DEV_INDEX = 1
RDMA_NLDEV_ATTR_DEV_NAME = 2
RDMA_NLDEV_ATTR_PORT_INDEX = 3
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
RDMA_NLDEV_ATTR_FW_VERSION = 5
RDMA_NLDEV_ATTR_NODE_GUID = 6
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
RDMA_NLDEV_ATTR_LID = 9
RDMA_NLDEV_ATTR_SM_LID = 10
RDMA_NLDEV_ATTR_LMC = 11
RDMA_NLDEV_ATTR_PORT_STATE = 12
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
RDMA_NLDEV_ATTR_NDEV_NAME = 51
RDMA_NLDEV_ATTR_LINK_TYPE = 65
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
RDMA_NLDEV_NET_NS_FD = 68
RDMA_NLDEV_ATTR_DEV_INDEX = 1
RDMA_NLDEV_ATTR_DEV_NAME = 2
RDMA_NLDEV_ATTR_PORT_INDEX = 3
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
RDMA_NLDEV_ATTR_FW_VERSION = 5
RDMA_NLDEV_ATTR_NODE_GUID = 6
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
RDMA_NLDEV_ATTR_LID = 9
RDMA_NLDEV_ATTR_SM_LID = 10
RDMA_NLDEV_ATTR_LMC = 11
RDMA_NLDEV_ATTR_PORT_STATE = 12
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
RDMA_NLDEV_ATTR_RES_SUMMARY = 15
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY = 16
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME = 17
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR = 18
RDMA_NLDEV_ATTR_NDEV_NAME = 51
RDMA_NLDEV_ATTR_LINK_TYPE = 65
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
RDMA_NLDEV_NET_NS_FD = 68
RDMA_NLDEV_ATTR_STAT_HWCOUNTERS = 80
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY = 81
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME = 82
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE = 83
)
216 changes: 215 additions & 1 deletion rdma_link_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type RdmaLinkAttrs struct {
FirmwareVersion string
NodeGuid string
SysImageGuid string
NumPorts uint32
}

// Link represents a rdma device from netlink.
Expand Down Expand Up @@ -68,6 +69,11 @@ func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) {
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &sysGuid)
link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid)
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
var availablePort uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &availablePort)
link.Attrs.NumPorts = availablePort
}
if (len % 4) != 0 {
// Skip pad bytes
Expand Down Expand Up @@ -304,7 +310,6 @@ func (h *Handle) RdmaLinkDel(name string) error {
_, err = req.Execute(unix.NETLINK_RDMA, 0)
return err
}

// RdmaLinkAdd adds an rdma link for the specified type to the network device.
// Similar to: rdma link add NAME type TYPE netdev NETDEV
// NAME - specifies the new name of the rdma link to add
Expand All @@ -329,3 +334,212 @@ func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) er
_, err := req.Execute(unix.NETLINK_RDMA, 0)
return err
}

// RdmaResource represents a rdma device resource tracking summaries
type RdmaResource struct {
Index uint32
Name string
RdmaResourceSummaryEntries map[string]uint64
}

// RdmaResourceList list rdma resource tracking information
// Returns all rdma devices resource tracking summary on success or returns error
// otherwise.
// Equivalent to: `rdma resource'
func RdmaResourceList() ([]*RdmaResource, error) {
return pkgHandle.RdmaResourceList()
}

// RdmaResourceList list rdma resource tracking information
// Returns all rdma devices resource tracking summary on success or returns error
// otherwise.
// Equivalent to: `rdma resource'
func (h *Handle) RdmaResourceList() ([]*RdmaResource, error) {
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_RES_GET)
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP)

msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
if err != nil {
return nil, err
}
if len(msgs) == 0 {
return nil, fmt.Errorf("No valid response from kernel")
}
var rdmaResources []*RdmaResource
for _, msg := range msgs {
res, err := executeOneGetRdmaResourceList(msg)
if err != nil {
return nil, err
}
rdmaResources = append(rdmaResources, res)
}
return rdmaResources, nil
}

func parseRdmaCounters(counterType uint16, data []byte) (map[string]uint64, error) {
var counterKeyType, counterValueType uint16
switch counterType {
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY:
counterKeyType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME
counterValueType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY:
counterKeyType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME
counterValueType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE
default:
return nil, fmt.Errorf("Invalid counter type: %d", counterType)
}
counters := make(map[string]uint64)
reader := bytes.NewReader(data)

for reader.Len() >= 4 {
_, attrType, _, value := parseNfAttrTLV(reader)
if attrType != counterType {
return nil, fmt.Errorf("Invalid resource summary entry type; %d", attrType)
}

summaryReader := bytes.NewReader(value)
for summaryReader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(summaryReader)
if attrType != counterKeyType {
return nil, fmt.Errorf("Invalid resource summary entry name type; %d", attrType)
}
name := string(value[0 : len-1])
// Skip pad bytes
if (len % 4) != 0 {
summaryReader.Seek(int64(4-(len%4)), seekCurrent)
}
_, attrType, len, value = parseNfAttrTLV(summaryReader)
if attrType != counterValueType {
return nil, fmt.Errorf("Invalid resource summary entry value type; %d", attrType)
}
counters[name] = native.Uint64(value)
}
}
return counters, nil
}

func executeOneGetRdmaResourceList(data []byte) (*RdmaResource, error) {
var res RdmaResource
reader := bytes.NewReader(data)
for reader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(reader)

switch attrType {
case nl.RDMA_NLDEV_ATTR_DEV_INDEX:
var Index uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &Index)
res.Index = Index
case nl.RDMA_NLDEV_ATTR_DEV_NAME:
res.Name = string(value[0 : len-1])
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY:
var err error
res.RdmaResourceSummaryEntries, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY, value)
if err != nil {
return nil, err
}
}
if (len % 4) != 0 {
// Skip pad bytes
reader.Seek(int64(4-(len%4)), seekCurrent)
}
}
return &res, nil
}

// RdmaPortStatistic represents a rdma port statistic counter
type RdmaPortStatistic struct {
PortIndex uint32
Statistics map[string]uint64
}

// RdmaDeviceStatistic represents a rdma device statistic counter
type RdmaDeviceStatistic struct {
RdmaPortStatistics []*RdmaPortStatistic
}

// RdmaStatistic get rdma device statistic counters
// Returns rdma device statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV]'
func RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
return pkgHandle.RdmaStatistic(link)
}

// RdmaStatistic get rdma device statistic counters
// Returns rdma device statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV]'
func (h *Handle) RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
rdmaLinkStatistic := make([]*RdmaPortStatistic, 0)
for portIndex := uint32(1); portIndex <= link.Attrs.NumPorts; portIndex++ {
portStatistic, err := h.RdmaPortStatisticList(link, portIndex)
if err != nil {
return nil, err
}
rdmaLinkStatistic = append(rdmaLinkStatistic, portStatistic)
}
return &RdmaDeviceStatistic{RdmaPortStatistics: rdmaLinkStatistic}, nil
}

// RdmaPortStatisticList get rdma device port statistic counters
// Returns rdma device port statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV/PORT]'
func RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
return pkgHandle.RdmaPortStatisticList(link, port)
}

// RdmaPortStatisticList get rdma device port statistic counters
// Returns rdma device port statistic counters on success or returns error
// otherwise.
// Equivalent to: `rdma statistic show link [DEV/PORT]'
func (h *Handle) RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_STAT_GET)
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_REQUEST)
b := make([]byte, 4)
native.PutUint32(b, link.Attrs.Index)
data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b)
req.AddData(data)

b = make([]byte, 4)
native.PutUint32(b, port)
data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_PORT_INDEX, b)
req.AddData(data)

msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
if err != nil {
return nil, err
}
if len(msgs) != 1 {
return nil, fmt.Errorf("No valid response from kernel")
}
return executeOneGetRdmaPortStatistics(msgs[0])
}

func executeOneGetRdmaPortStatistics(data []byte) (*RdmaPortStatistic, error) {
var stat RdmaPortStatistic
reader := bytes.NewReader(data)
for reader.Len() >= 4 {
_, attrType, len, value := parseNfAttrTLV(reader)

switch attrType {
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
var Index uint32
r := bytes.NewReader(value)
binary.Read(r, nl.NativeEndian(), &Index)
stat.PortIndex = Index
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTERS:
var err error
stat.Statistics, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY, value)
if err != nil {
return nil, err
}
}
if (len % 4) != 0 {
// Skip pad bytes
reader.Seek(int64(4-(len%4)), seekCurrent)
}
}
return &stat, nil
}
27 changes: 27 additions & 0 deletions rdma_link_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,30 @@ func TestRdmaLinkAddAndDel(t *testing.T) {

checkPresence(linkName, false)
}

func TestRdmaLinkMetrics(t *testing.T) {
minKernelRequired(t, 5, 1)
setupRdmaKModule(t, "rdma_rxe")
if err := RdmaLinkAdd(t.Name(), "rxe", "lo"); err != nil {
t.Fatal(err)
}
link, err := RdmaLinkByName(t.Name())
if err != nil {
t.Fatal(err)
}
defer RdmaLinkDel(t.Name())
resources, err := RdmaResourceList()
if err != nil {
t.Fatal(err)
}
for _, resource := range resources {
t.Logf("resource: %+v", resource)
}
stats, err := RdmaStatistic(link)
if err != nil {
t.Fatal(err)
}
for _, stat := range stats.RdmaPortStatistics {
t.Logf("stat: %+v", stat)
}
}

0 comments on commit 3bc3d8f

Please sign in to comment.