metrics: support gpu memory usage based ratio, when process utilizati…

…on is unavailable

Signed-off-by: Huamin Chen <[email protected]>

pkg/collector/stats/stats.go

@@ -75,7 +75,7 @@ func NewStats() *Stats {
 		m.ResourceUsage[metricName] = types.NewUInt64StatCollection()
 	}
 
-	if gpu.IsGPUCollectionSupported() {
+	if config.EnabledGPU {
 		m.ResourceUsage[config.GPUSMUtilization] = types.NewUInt64StatCollection()
 		m.ResourceUsage[config.GPUMemUtilization] = types.NewUInt64StatCollection()
 	}

pkg/config/config.go

@@ -455,3 +455,7 @@ func IsCgroupMetricsEnabled() bool {
 func IsIRQCounterMetricsEnabled() bool {
 	return ExposeIRQCounterMetrics
 }
+
+func SetGpuUsageMetric(metric string) {
+	GpuUsageMetric = metric
+}

pkg/metrics/consts/conts.go

@@ -90,4 +90,8 @@ var (
 		config.CgroupfsSystemCPU,
 		config.CgroupfsUserCPU,
 	}
+	GPUMetricNames = []string{
+		config.GPUSMUtilization,
+		config.GPUMemUtilization,
+	}
 )

pkg/metrics/container/metrics.go

@@ -79,6 +79,10 @@ func (c *collector) initMetrics() {
 		c.descriptions[name] = desc
 		c.collectors[name] = metricfactory.NewPromCounter(desc)
 	}
+	for name, desc := range metricfactory.GPUUsageMetricsPromDesc(context) {
+		c.descriptions[name] = desc
+		c.collectors[name] = metricfactory.NewPromCounter(desc)
+	}
 
 	desc := metricfactory.MetricsPromDesc(context, "joules", "_total", "", consts.ContainerEnergyLabels)
 	c.descriptions["total"] = desc

pkg/metrics/metricfactory/metric_factory.go

@@ -114,6 +114,16 @@ func NodeCPUFrequencyMetricsPromDesc(context string) (descriptions map[string]*p
 	return descriptions
 }
 
+func GPUUsageMetricsPromDesc(context string) (descriptions map[string]*prometheus.Desc) {
+	descriptions = make(map[string]*prometheus.Desc)
+	if config.EnabledGPU {
+		for _, name := range consts.GPUMetricNames {
+			descriptions[name] = resMetricsPromDesc(context, name, "nvidia-nvml")
+		}
+	}
+	return descriptions
+}
+
 func resMetricsPromDesc(context, name, source string) (desc *prometheus.Desc) {
 	var labels []string
 	switch context {

pkg/metrics/process/metrics.go

@@ -78,6 +78,10 @@ func (c *collector) initMetrics() {
 		c.descriptions[name] = desc
 		c.collectors[name] = metricfactory.NewPromCounter(desc)
 	}
+	for name, desc := range metricfactory.GPUUsageMetricsPromDesc(context) {
+		c.descriptions[name] = desc
+		c.collectors[name] = metricfactory.NewPromCounter(desc)
+	}
 }
 
 func (c *collector) Describe(ch chan<- *prometheus.Desc) {

pkg/metrics/utils/utils.go

@@ -66,6 +66,12 @@ func CollectResUtilizationMetrics(ch chan<- prometheus.Metric, instance interfac
 			CollectResUtil(ch, instance, collectorName, collectors[collectorName])
 		}
 	}
+
+	if config.EnabledGPU {
+		for _, collectorName := range consts.GPUMetricNames {
+			CollectResUtil(ch, instance, collectorName, collectors[collectorName])
+		}
+	}
 }
 
 func collect(ch chan<- prometheus.Metric, collector metricfactory.PromMetric, value float64, labelValues []string) {

pkg/sensors/accelerator/gpu/source/gpu_nvml.go

@@ -31,6 +31,8 @@ import (
 var (
 	// List of GPU identifiers for the device
 	devices []interface{}
+	// bool to check if the process utilization collection is supported
+	processUtilizationSupported bool = true
 )
 
 type GPUNvml struct {
@@ -116,29 +118,56 @@ func (n *GPUNvml) GetProcessResourceUtilizationPerDevice(device interface{}, sin
 	processAcceleratorMetrics := map[uint32]ProcessUtilizationSample{}
 	lastUtilizationTimestamp := uint64(time.Now().Add(-1*since).UnixNano() / 1000)
 
-	processUtilizationSample, ret := device.(nvml.Device).GetProcessUtilization(lastUtilizationTimestamp)
-	if ret != nvml.SUCCESS {
-		if ret == nvml.ERROR_NOT_FOUND {
-			// ignore the error if there is no process running in the GPU
-			return nil, nil
+	if processUtilizationSupported {
+		processUtilizationSample, ret := device.(nvml.Device).GetProcessUtilization(lastUtilizationTimestamp)
+		if ret != nvml.SUCCESS {
+			if ret == nvml.ERROR_NOT_FOUND {
+				// ignore the error if there is no process running in the GPU
+				return nil, nil
+			}
+			processUtilizationSupported = false
+		} else {
+			for _, pinfo := range processUtilizationSample {
+				// pid 0 means no data.
+				if pinfo.Pid != 0 {
+					processAcceleratorMetrics[pinfo.Pid] = ProcessUtilizationSample{
+						Pid:       pinfo.Pid,
+						TimeStamp: pinfo.TimeStamp,
+						SmUtil:    pinfo.SmUtil,
+						MemUtil:   pinfo.MemUtil,
+						EncUtil:   pinfo.EncUtil,
+						DecUtil:   pinfo.DecUtil,
+					}
+				}
+			}
 		}
-		return nil, fmt.Errorf("failed to get processes' utilization on device %v: %v", device, nvml.ErrorString(ret))
 	}
-
-	for _, pinfo := range processUtilizationSample {
-		// pid 0 means no data.
-		if pinfo.Pid != 0 {
-			processAcceleratorMetrics[pinfo.Pid] = ProcessUtilizationSample{
-				Pid:       pinfo.Pid,
-				TimeStamp: pinfo.TimeStamp,
-				SmUtil:    pinfo.SmUtil,
-				MemUtil:   pinfo.MemUtil,
-				EncUtil:   pinfo.EncUtil,
-				DecUtil:   pinfo.DecUtil,
+	if !processUtilizationSupported { // if processUtilizationSupported is false, try deviceGetMPSComputeRunningProcesses_v3 to use memory usage to ratio power usage
+		config.GpuUsageMetric = config.GPUMemUtilization
+		processInfo, ret := device.(nvml.Device).GetComputeRunningProcesses()
+		if ret != nvml.SUCCESS {
+			if ret == nvml.ERROR_NOT_FOUND {
+				// ignore the error if there is no process running in the GPU
+				return nil, nil
+			}
+			return nil, fmt.Errorf("failed to get processes' utilization on device %v: %v", device, nvml.ErrorString(ret))
+		}
+		memoryInfo, ret := device.(nvml.Device).GetMemoryInfo()
+		if ret != nvml.SUCCESS {
+			return nil, fmt.Errorf("failed to get memory info on device %v: %v", device, nvml.ErrorString(ret))
+		}
+		// convert processInfo to processUtilizationSample
+		for _, pinfo := range processInfo {
+			// pid 0 means no data.
+			if pinfo.Pid != 0 {
+				processAcceleratorMetrics[pinfo.Pid] = ProcessUtilizationSample{
+					Pid:     pinfo.Pid,
+					MemUtil: uint32(pinfo.UsedGpuMemory * 100 / memoryInfo.Total),
+				}
+				klog.V(5).Infof("pid: %d, memUtil: %d\n", pinfo.Pid, processAcceleratorMetrics[pinfo.Pid].MemUtil)
 			}
 		}
 	}
-
 	return processAcceleratorMetrics, nil
 }