Skip to content

Commit

Permalink
Separate prometheus label for scaledObject and scaledJob
Browse files Browse the repository at this point in the history
  • Loading branch information
@yoongon
yoongon committed Oct 31, 2023
1 parent 84c81dc commit 07c254e
Showing 1 changed file with 84 additions and 29 deletions.
113 changes: 84 additions & 29 deletions pkg/prommetrics/prommetrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ const (
)

var (
metricLabels = []string{"namespace", "metric", "scaledObject", "scaler", "scalerIndex", "type"}
buildInfo = prometheus.NewGaugeVec(
metricLabelsForScaledObject = []string{"namespace", "metric", "scaledObject", "scaler", "scalerIndex"}
metricLabelsForScaledJob = []string{"namespace", "metric", "scaledJob", "scaler", "scalerIndex"}
buildInfo = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Name: "build_info",
Expand All @@ -57,41 +58,77 @@ var (
},
[]string{},
)
scalerMetricsValue = prometheus.NewGaugeVec(
scalerMetricsValueForScaledObject = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "metrics_value",
Help: "Metric Value used for HPA",
},
metricLabels,
metricLabelsForScaledObject,
)
scalerMetricsLatency = prometheus.NewGaugeVec(
scalerMetricsValueForScaledJob = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "metrics_value",
Help: "Metric Value used for HPA",
},
metricLabelsForScaledJob,
)
scalerMetricsLatencyForScaledObject = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "metrics_latency",
Help: "Scaler Metrics Latency",
},
metricLabels,
metricLabelsForScaledObject,
)
scalerActive = prometheus.NewGaugeVec(
scalerMetricsLatencyForScaledJob = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "metrics_latency",
Help: "Scaler Metrics Latency",
},
metricLabelsForScaledJob,
)
scalerActiveForScaledObject = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "active",
Help: "Activity of a Scaler Metric",
},
metricLabels,
metricLabelsForScaledObject,
)
scalerErrors = prometheus.NewCounterVec(
scalerActiveForScaledJob = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "active",
Help: "Activity of a Scaler Metric",
},
metricLabelsForScaledJob,
)
scalerErrorsForScaledObject = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "errors",
Help: "Number of scaler errors",
},
metricLabels,
metricLabelsForScaledObject,
)
scalerErrorsForScaledJob = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: DefaultPromMetricsNamespace,
Subsystem: "scaler",
Name: "errors",
Help: "Number of scaler errors",
},
metricLabelsForScaledJob,
)
scaledObjectErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Expand Down Expand Up @@ -143,14 +180,20 @@ var (

func init() {
metrics.Registry.MustRegister(scalerErrorsTotal)
metrics.Registry.MustRegister(scalerMetricsValue)
metrics.Registry.MustRegister(scalerMetricsLatency)
metrics.Registry.MustRegister(internalLoopLatency)
metrics.Registry.MustRegister(scalerActive)
metrics.Registry.MustRegister(scalerErrors)
metrics.Registry.MustRegister(scaledObjectErrors)
metrics.Registry.MustRegister(scaledJobErrors)

metrics.Registry.MustRegister(scalerMetricsValueForScaledObject)
metrics.Registry.MustRegister(scalerMetricsLatencyForScaledObject)
metrics.Registry.MustRegister(scalerActiveForScaledObject)
metrics.Registry.MustRegister(scalerErrorsForScaledObject)

metrics.Registry.MustRegister(scalerMetricsValueForScaledJob)
metrics.Registry.MustRegister(scalerMetricsLatencyForScaledJob)
metrics.Registry.MustRegister(scalerActiveForScaledJob)
metrics.Registry.MustRegister(scalerErrorsForScaledJob)

metrics.Registry.MustRegister(triggerTotalsGaugeVec)
metrics.Registry.MustRegister(crdTotalsGaugeVec)
metrics.Registry.MustRegister(buildInfo)
Expand All @@ -160,17 +203,31 @@ func init() {

// RecordScalerMetric create a measurement of the external metric used by the HPA
func RecordScalerMetric(namespace string, scaledResource string, scaler string, scalerIndex int, metric string, isScaledObject bool, value float64) {
scalerMetricsValue.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric, getResourceType(isScaledObject))).Set(value)
if isScaledObject {
scalerMetricsValueForScaledObject.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(value)
} else {
scalerMetricsValueForScaledJob.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(value)
}

}

// RecordScalerLatency create a measurement of the latency to external metric
func RecordScalerLatency(namespace string, scaledResource string, scaler string, scalerIndex int, metric string, isScaledObject bool, value float64) {
scalerMetricsLatency.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric, getResourceType(isScaledObject))).Set(value)
if isScaledObject {
scalerMetricsLatencyForScaledObject.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(value)
} else {
scalerMetricsLatencyForScaledJob.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(value)
}

}

// RecordScaledObjectLatency create a measurement of the latency executing scalable object loop
func RecordScalableObjectLatency(namespace string, name string, isScaledObject bool, value float64) {
internalLoopLatency.WithLabelValues(namespace, getResourceType(isScaledObject), name).Set(value)
resourceType := "scaledjob"
if isScaledObject {
resourceType = "scaledobject"
}
internalLoopLatency.WithLabelValues(namespace, resourceType, name).Set(value)
}

// RecordScalerActive create a measurement of the activity of the scaler
Expand All @@ -180,13 +237,18 @@ func RecordScalerActive(namespace string, scaledResource string, scaler string,
activeVal = 1
}

scalerActive.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric, getResourceType(isScaledObject))).Set(float64(activeVal))
if isScaledObject {
scalerActiveForScaledObject.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(float64(activeVal))
} else {
scalerActiveForScaledJob.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Set(float64(activeVal))
}

}

// RecordScalerError counts the number of errors occurred in trying to get an external metric used by the HPA
func RecordScalerError(namespace string, scaledResource string, scaler string, scalerIndex int, metric string, isScaledObject bool, err error) {
if err != nil {
scalerErrors.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric, getResourceType(isScaledObject))).Inc()
scalerErrorsForScaledObject.With(getLabels(namespace, scaledResource, scaler, scalerIndex, metric)).Inc()
if isScaledObject {
RecordScaledObjectError(namespace, scaledResource, err)
} else {
Expand All @@ -196,7 +258,7 @@ func RecordScalerError(namespace string, scaledResource string, scaler string, s
return
}
// initialize metric with 0 if not already set
_, errscaler := scalerErrors.GetMetricWith(getLabels(namespace, scaledResource, scaler, scalerIndex, metric, getResourceType(isScaledObject)))
_, errscaler := scalerErrorsForScaledObject.GetMetricWith(getLabels(namespace, scaledResource, scaler, scalerIndex, metric))
if errscaler != nil {
log.Error(errscaler, "Unable to write to metrics to Prometheus Server: %v")
}
Expand Down Expand Up @@ -237,8 +299,8 @@ func RecordBuildInfo() {
buildInfo.WithLabelValues(version.Version, version.GitCommit, runtime.Version(), runtime.GOOS, runtime.GOARCH).Set(1)
}

func getLabels(namespace string, scaledObject string, scaler string, scalerIndex int, metric string, resourceType string) prometheus.Labels {
return prometheus.Labels{"namespace": namespace, "scaledObject": scaledObject, "scaler": scaler, "scalerIndex": strconv.Itoa(scalerIndex), "metric": metric, "type": resourceType}
func getLabels(namespace string, scaledObject string, scaler string, scalerIndex int, metric string) prometheus.Labels {
return prometheus.Labels{"namespace": namespace, "scaledObject": scaledObject, "scaler": scaler, "scalerIndex": strconv.Itoa(scalerIndex), "metric": metric}
}

func IncrementTriggerTotal(triggerType string) {
Expand Down Expand Up @@ -268,10 +330,3 @@ func DecrementCRDTotal(crdType, namespace string) {

crdTotalsGaugeVec.WithLabelValues(crdType, namespace).Dec()
}

func getResourceType(isScaledObject bool) string {
if isScaledObject {
return "scaledobject"
}
return "scaledjob"
}

0 comments on commit 07c254e

Please sign in to comment.