Fix pod metrics that don't get deleted when the pod is deleted (#1796)

pkg/controllers/metrics/pod/controller.go

@@ -147,7 +147,17 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reco
 	if err := c.kubeClient.Get(ctx, req.NamespacedName, pod); err != nil {
 		if errors.IsNotFound(err) {
 			c.pendingPods.Delete(req.NamespacedName.String())
+			// Delete the unstarted metric since the pod is deleted
+			podUnstartedTimeSeconds.Delete(map[string]string{
+				podName:      req.Name,
+				podNamespace: req.Namespace,
+			})
 			c.unscheduledPods.Delete(req.NamespacedName.String())
+			// Delete the unbound metric since the pod is deleted
+			podCurrentUnboundTimeSeconds.Delete(map[string]string{
+				podName:      req.Name,
+				podNamespace: req.Namespace,
+			})
 			c.metricStore.Delete(req.NamespacedName.String())
 		}
 		return reconcile.Result{}, client.IgnoreNotFound(err)

pkg/controllers/metrics/pod/suite_test.go

@@ -177,6 +177,35 @@ var _ = Describe("Pod Metrics", func() {
 		_, found = FindMetricWithLabelValues("karpenter_pods_startup_duration_seconds", nil)
 		Expect(found).To(BeTrue())
 	})
+	It("should delete pod unstarted time and pod unbound duration metric on pod delete", func() {
+		p := test.Pod()
+		p.Status.Phase = corev1.PodPending
+		ExpectApplied(ctx, env.Client, p)
+		ExpectReconcileSucceeded(ctx, podController, client.ObjectKeyFromObject(p))
+		_, found := FindMetricWithLabelValues("karpenter_pods_current_unbound_time_seconds", map[string]string{
+			"name":      p.GetName(),
+			"namespace": p.GetNamespace(),
+		})
+		Expect(found).To(BeTrue())
+		_, found = FindMetricWithLabelValues("karpenter_pods_unstarted_time_seconds", map[string]string{
+			"name":      p.GetName(),
+			"namespace": p.GetNamespace(),
+		})
+		Expect(found).To(BeTrue())
+
+		ExpectDeleted(ctx, env.Client, p)
+		ExpectReconcileSucceeded(ctx, podController, client.ObjectKeyFromObject(p))
+		_, found = FindMetricWithLabelValues("karpenter_pods_current_unbound_time_seconds", map[string]string{
+			"name":      p.GetName(),
+			"namespace": p.GetNamespace(),
+		})
+		Expect(found).To(BeFalse())
+		_, found = FindMetricWithLabelValues("karpenter_pods_unstarted_time_seconds", map[string]string{
+			"name":      p.GetName(),
+			"namespace": p.GetNamespace(),
+		})
+		Expect(found).To(BeFalse())
+	})
 	It("should delete the pod state metric on pod delete", func() {
 		p := test.Pod()
 		ExpectApplied(ctx, env.Client, p)