feat: negative metrics spike handling

NetApp · Jan 27, 2025 · bcf659a · bcf659a
1 parent c7eed53
commit bcf659a
Show file tree

Hide file tree

Showing 25 changed files with 1,265 additions and 123 deletions.
diff --git a/cmd/collectors/keyperf/keyperf.go b/cmd/collectors/keyperf/keyperf.go
@@ -445,7 +445,7 @@ func (kp *KeyPerf) cookCounters(curMat *matrix.Matrix, prevMat *matrix.Matrix) (
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = curMat.Delta(key, prevMat, kp.Logger); err != nil {
+		if skips, err = curMat.Delta(key, prevMat, cachedData, kp.Logger); err != nil {
 			kp.Logger.Error("Calculate delta", slogx.Err(err), slog.String("key", key))
 			continue
 		}

diff --git a/cmd/collectors/keyperf/keyperf_test.go b/cmd/collectors/keyperf/keyperf_test.go
@@ -19,6 +19,45 @@ const (
 	pollerName = "test"
 )
 
+func TestSkipsSequence(t *testing.T) {
+	conf.TestLoadHarvestConfig("testdata/config.yml")
+	kp := newKeyPerf("Volume", "volume.yaml")
+
+	// First Poll
+	t.Log("Running First Poll")
+	kp.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData1.json", 0, 0)
+	if t.Failed() {
+		t.Fatal("First Poll failed")
+	}
+
+	// Complete Poll
+	t.Log("Running Complete Poll")
+	kp.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData1.json", 1, 12)
+	if t.Failed() {
+		t.Fatal("Complete Poll failed")
+	}
+
+	// Skips Poll
+	t.Log("Running Skips Poll")
+	kp.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData2.json", 1, 10)
+	if t.Failed() {
+		t.Fatal("Skips Poll failed")
+	}
+
+	// Skips Poll 2
+	t.Log("Running Skips Poll 2")
+	kp.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData3.json", 1, 10)
+	if t.Failed() {
+		t.Fatal("Skips Poll 2 failed")
+	}
+
+	t.Log("Running Poll 3")
+	kp.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData4.json", 1, 12)
+	if t.Failed() {
+		t.Fatal("Poll 3 failed")
+	}
+}
+
 func TestPartialAggregationSequence(t *testing.T) {
 	conf.TestLoadHarvestConfig("testdata/config.yml")
 	kp := newKeyPerf("Volume", "volume.yaml")

diff --git a/cmd/collectors/keyperf/testdata/skips/pollData1.json b/cmd/collectors/keyperf/testdata/skips/pollData1.json
@@ -0,0 +1,61 @@
+{
+  "records": [
+    {
+      "uuid": "03613247-54ac-4127-a512-afb331f1c207",
+      "name": "osc_vol01",
+      "style": "flexvol",
+      "svm": {
+        "name": "osc"
+      },
+      "statistics": {
+        "timestamp": "2025-01-27T07:00:16Z",
+        "status": "ok",
+        "latency_raw": {
+          "other": 1092346493,
+          "total": 15717305666890,
+          "read": 15711857370185,
+          "write": 4355950212
+        },
+        "iops_raw": {
+          "read": 15749383688,
+          "write": 10437797,
+          "other": 7805846,
+          "total": 15767627331
+        },
+        "throughput_raw": {
+          "read": 1032148923371343,
+          "write": 54906830868,
+          "other": 0,
+          "total": 1032203830202211
+        },
+        "cloud": {
+          "timestamp": "2025-01-27T07:00:16Z",
+          "status": "ok",
+          "iops_raw": {
+            "read": 192314,
+            "write": 18302,
+            "other": 0,
+            "total": 210616
+          },
+          "latency_raw": {
+            "read": 13811084776,
+            "write": 78515374639,
+            "other": 0,
+            "total": 92326459415
+          }
+        }
+      },
+      "_links": {
+        "self": {
+          "href": "/api/storage/volumes/03613247-54ac-4127-a512-afb331f1c207"
+        }
+      }
+    }
+  ],
+  "num_records": 1,
+  "_links": {
+    "self": {
+      "href": "/api/storage/volumes?fields=statistics,svm.name,style&name=osc_vol01"
+    }
+  }
+}
diff --git a/cmd/collectors/keyperf/testdata/skips/pollData2.json b/cmd/collectors/keyperf/testdata/skips/pollData2.json
@@ -0,0 +1,61 @@
+{
+  "records": [
+    {
+      "uuid": "03613247-54ac-4127-a512-afb331f1c207",
+      "name": "osc_vol01",
+      "style": "flexvol",
+      "svm": {
+        "name": "osc"
+      },
+      "statistics": {
+        "timestamp": "2025-01-27T07:00:16Z",
+        "status": "ok",
+        "latency_raw": {
+          "other": 1092346493,
+          "total": 15717305666890,
+          "read": 15711857370185,
+          "write": 4355950212
+        },
+        "iops_raw": {
+          "read": 157,
+          "write": 10437797,
+          "other": 7805846,
+          "total": 15767627331
+        },
+        "throughput_raw": {
+          "read": 1032148923371343,
+          "write": 54906830868,
+          "other": 0,
+          "total": 1032203830202211
+        },
+        "cloud": {
+          "timestamp": "2025-01-27T07:00:16Z",
+          "status": "ok",
+          "iops_raw": {
+            "read": 192314,
+            "write": 18302,
+            "other": 0,
+            "total": 210616
+          },
+          "latency_raw": {
+            "read": 13811084776,
+            "write": 78515374639,
+            "other": 0,
+            "total": 92326459415
+          }
+        }
+      },
+      "_links": {
+        "self": {
+          "href": "/api/storage/volumes/03613247-54ac-4127-a512-afb331f1c207"
+        }
+      }
+    }
+  ],
+  "num_records": 1,
+  "_links": {
+    "self": {
+      "href": "/api/storage/volumes?fields=statistics,svm.name,style&name=osc_vol01"
+    }
+  }
+}
diff --git a/cmd/collectors/keyperf/testdata/skips/pollData3.json b/cmd/collectors/keyperf/testdata/skips/pollData3.json
@@ -0,0 +1,61 @@
+{
+  "records": [
+    {
+      "uuid": "03613247-54ac-4127-a512-afb331f1c207",
+      "name": "osc_vol01",
+      "style": "flexvol",
+      "svm": {
+        "name": "osc"
+      },
+      "statistics": {
+        "timestamp": "2025-01-27T07:00:16Z",
+        "status": "ok",
+        "latency_raw": {
+          "other": 1092346493,
+          "total": 15717305666890,
+          "read": 15711857370185,
+          "write": 4355950212
+        },
+        "iops_raw": {
+          "read": 15749383688,
+          "write": 10437797,
+          "other": 7805846,
+          "total": 15767627331
+        },
+        "throughput_raw": {
+          "read": 1032148923371343,
+          "write": 54906830868,
+          "other": 0,
+          "total": 1032203830202211
+        },
+        "cloud": {
+          "timestamp": "2025-01-27T07:00:16Z",
+          "status": "ok",
+          "iops_raw": {
+            "read": 192314,
+            "write": 18302,
+            "other": 0,
+            "total": 210616
+          },
+          "latency_raw": {
+            "read": 13811084776,
+            "write": 78515374639,
+            "other": 0,
+            "total": 92326459415
+          }
+        }
+      },
+      "_links": {
+        "self": {
+          "href": "/api/storage/volumes/03613247-54ac-4127-a512-afb331f1c207"
+        }
+      }
+    }
+  ],
+  "num_records": 1,
+  "_links": {
+    "self": {
+      "href": "/api/storage/volumes?fields=statistics,svm.name,style&name=osc_vol01"
+    }
+  }
+}
diff --git a/cmd/collectors/keyperf/testdata/skips/pollData4.json b/cmd/collectors/keyperf/testdata/skips/pollData4.json
@@ -0,0 +1,61 @@
+{
+  "records": [
+    {
+      "uuid": "03613247-54ac-4127-a512-afb331f1c207",
+      "name": "osc_vol01",
+      "style": "flexvol",
+      "svm": {
+        "name": "osc"
+      },
+      "statistics": {
+        "timestamp": "2025-01-27T07:00:16Z",
+        "status": "ok",
+        "latency_raw": {
+          "other": 1092346493,
+          "total": 15717305666890,
+          "read": 15711857370185,
+          "write": 4355950212
+        },
+        "iops_raw": {
+          "read": 15759383688,
+          "write": 10437797,
+          "other": 7805846,
+          "total": 15767627331
+        },
+        "throughput_raw": {
+          "read": 1032148923371343,
+          "write": 54906830868,
+          "other": 0,
+          "total": 1032203830202211
+        },
+        "cloud": {
+          "timestamp": "2025-01-27T07:00:16Z",
+          "status": "ok",
+          "iops_raw": {
+            "read": 192314,
+            "write": 18302,
+            "other": 0,
+            "total": 210616
+          },
+          "latency_raw": {
+            "read": 13811084776,
+            "write": 78515374639,
+            "other": 0,
+            "total": 92326459415
+          }
+        }
+      },
+      "_links": {
+        "self": {
+          "href": "/api/storage/volumes/03613247-54ac-4127-a512-afb331f1c207"
+        }
+      }
+    }
+  ],
+  "num_records": 1,
+  "_links": {
+    "self": {
+      "href": "/api/storage/volumes?fields=statistics,svm.name,style&name=osc_vol01"
+    }
+  }
+}
diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go
@@ -1221,7 +1221,7 @@ func (r *RestPerf) cookCounters(curMat *matrix.Matrix, prevMat *matrix.Matrix) (
 
 	// Calculate timestamp delta first since many counters require it for postprocessing.
 	// Timestamp has "raw" property, so it isn't post-processed automatically
-	if _, err = curMat.Delta("timestamp", prevMat, r.Logger); err != nil {
+	if _, err = curMat.Delta("timestamp", prevMat, cachedData, r.Logger); err != nil {
 		r.Logger.Error("(timestamp) calculate delta:", slogx.Err(err))
 	}
 
@@ -1251,7 +1251,7 @@ func (r *RestPerf) cookCounters(curMat *matrix.Matrix, prevMat *matrix.Matrix) (
 		}
 
 		// all other properties - first calculate delta
-		if skips, err = curMat.Delta(key, prevMat, r.Logger); err != nil {
+		if skips, err = curMat.Delta(key, prevMat, cachedData, r.Logger); err != nil {
 			r.Logger.Error("Calculate delta:", slogx.Err(err), slog.String("key", key))
 			continue
 		}

diff --git a/cmd/collectors/restperf/restperf_test.go b/cmd/collectors/restperf/restperf_test.go
@@ -352,7 +352,13 @@ func TestPartialAggregationSequence(t *testing.T) {
 
 func newRestPerf(object string, path string) *RestPerf {
 	var err error
-	opts := options.New(options.WithConfPath("testdata/conf"))
+	homePath := "../../../"
+	conf.TestLoadHarvestConfig("testdata/config.yml")
+	opts := options.New(options.WithConfPath(homePath + "/conf"))
+	// An additional histogram metric is being tested, so load the data from the test dataset.
+	if path == "volume.yaml" {
+		opts = options.New(options.WithConfPath("testdata/conf"))
+	}
 	opts.Poller = pollerName
 	opts.HomePath = "testdata"
 	opts.IsTest = true
@@ -415,7 +421,7 @@ func TestQosVolume(t *testing.T) {
 			pollDataPath1: "testdata/qos-volume-poll-1.json",
 			pollDataPath2: "testdata/qos-volume-poll-2.json",
 			numInstances:  9,
-			numMetrics:    234,
+			numMetrics:    243,
 			sum:           18,
 		},
 	}
@@ -487,6 +493,50 @@ func TestQosVolume(t *testing.T) {
 	}
 }
 
+func TestSkipsSequence(t *testing.T) {
+	r := newRestPerf("Qtree", "qtree.yaml")
+
+	counters := jsonToPerfRecords("testdata/skips/pollCounter.json")
+	_, err := r.pollCounter(counters[0].Records.Array(), 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// First Poll
+	t.Log("Running First Poll")
+	r.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData1.json", 0, 0)
+	if t.Failed() {
+		t.Fatal("First Poll failed")
+	}
+
+	// Complete Poll
+	t.Log("Running Complete Poll")
+	r.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData1.json", 1, 4)
+	if t.Failed() {
+		t.Fatal("Complete Poll failed")
+	}
+
+	// Skips Poll
+	t.Log("Running Skips Poll")
+	r.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData2.json", 1, 3)
+	if t.Failed() {
+		t.Fatal("Skips Poll failed")
+	}
+
+	// Skips Poll 2
+	t.Log("Running Skips Poll 2")
+	r.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData3.json", 1, 3)
+	if t.Failed() {
+		t.Fatal("Skips Poll 2 failed")
+	}
+
+	t.Log("Running Poll 3")
+	r.testPollInstanceAndDataWithMetrics(t, "testdata/skips/pollData4.json", 1, 4)
+	if t.Failed() {
+		t.Fatal("Poll 3 failed")
+	}
+}
+
 func processAndCookCounters(r *RestPerf, pollData []rest.PerfRecord, prevMat *matrix.Matrix) (map[string]*matrix.Matrix, uint64, error) {
 	curMat := prevMat.Clone(matrix.With{Data: false, Metrics: true, Instances: true, ExportInstances: true})
 	curMat.Reset()