Skip to content

Commit

Permalink
feat: --expose-per-cert-error-metrics for detailled errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@arcln
arcln committed Jun 28, 2021
1 parent e4744fa commit 0b4f573
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 54 deletions.
9 changes: 8 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ The following metrics are available:
For advanced configuration, see the program's `--help` :

```
Usage: x509-certificate-exporter [-hv] [--debug] [-d value] [--exclude-label value] [--exclude-namespace value] [--expose-relative-metrics] [-f value] [--include-label value] [--include-namespace value] [-k value] [-p value] [-s value] [--trim-path-components value] [--watch-kube-secrets] [parameters ...]
Usage: x509-certificate-exporter [-hv] [--debug] [-d value] [--exclude-label value] [--exclude-namespace value] [--expose-per-cert-error-metrics] [--expose-relative-metrics] [-f value] [--include-label value] [--include-namespace value] [-k value] [-l value] [--max-cache-duration value] [-p value] [-s value] [--trim-path-components value] [--watch-kube-secrets] [parameters ...]
--debug enable debug mode
-d, --watch-dir=value
watch one or more directory which contains x509 certificate
Expand All @@ -66,6 +66,9 @@ Usage: x509-certificate-exporter [-hv] [--debug] [-d value] [--exclude-label val
--exclude-namespace=value
removes the given kube namespace from the watch list
(applied after --include-namespace)
--expose-per-cert-error-metrics
expose additionnal error metric for each certificate
indicating wether it has failure(s)
--expose-relative-metrics
expose additionnal metrics with relative durations instead
of absolute timestamps
Expand All @@ -83,6 +86,10 @@ Usage: x509-certificate-exporter [-hv] [--debug] [-d value] [--exclude-label val
watch one or more Kubernetes client configuration (kind
Config) which contains embedded x509 certificates or PEM
file paths
-l, --expose-labels=value
--max-cache-duration=value
maximum cache duration for kube secrets. cache is per
namespace and randomized to avoid massive requests.
-p, --port=value prometheus exporter listening port [9793]
-s, --secret-type=value
one or more kubernetes secret type & key to watch (e.g.
Expand Down
2 changes: 2 additions & 0 deletions cmd/x509-certificate-exporter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ func main() {
debug := getopt.BoolLong("debug", 0, "enable debug mode")
trimPathComponents := getopt.IntLong("trim-path-components", 0, 0, "remove <n> leading component(s) from path(s) in label(s)")
exposeRelativeMetrics := getopt.BoolLong("expose-relative-metrics", 0, "expose additionnal metrics with relative durations instead of absolute timestamps")
exposeErrorMetrics := getopt.BoolLong("expose-per-cert-error-metrics", 0, "expose additionnal error metric for each certificate indicating wether it has failure(s)")
exposeLabels := getopt.StringLong("expose-labels", 'l', "one or more comma-separated labels to enable (defaults to all if not specified)")

maxCacheDuration := durationFlag(0)
Expand Down Expand Up @@ -76,6 +77,7 @@ func main() {
TrimPathComponents: *trimPathComponents,
MaxCacheDuration: time.Duration(maxCacheDuration),
ExposeRelativeMetrics: *exposeRelativeMetrics,
ExposeErrorMetrics: *exposeErrorMetrics,
KubeSecretTypes: kubeSecretTypes,
KubeIncludeNamespaces: kubeIncludeNamespaces,
KubeExcludeNamespaces: kubeExcludeNamespaces,
Expand Down
127 changes: 84 additions & 43 deletions internal/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ var (
certValidSinceHelp = "Indicates the elapsed time since the certificate's not before timestamp"
certValidSinceDesc = prometheus.NewDesc(certValidSinceMetric, certValidSinceHelp, nil, nil)

certErrorMetric = "x509_cert_error"
certErrorHelp = "Indicates wether the corresponding secret has read failure(s)"
certErrorDesc = prometheus.NewDesc(certErrorMetric, certErrorHelp, nil, nil)

certErrorsMetric = "x509_read_errors"
certErrorsHelp = "Indicates the number of read failure(s)"
certErrorsDesc = prometheus.NewDesc(certErrorsMetric, certErrorsHelp, nil, nil)
Expand All @@ -52,24 +56,50 @@ func (collector *collector) Describe(ch chan<- *prometheus.Desc) {
ch <- certExpiresInDesc
ch <- certValidSinceDesc
}

if collector.exporter.ExposeErrorMetrics {
ch <- certErrorDesc
}
}

func (collector *collector) Collect(ch chan<- prometheus.Metric) {
certRefs, certErrors := collector.exporter.parseAllCertificates()

for index, err := range certErrors {
if err.err != nil {
log.Debugf("read error %d: %+v", index+1, err.err)
}
}

for _, certRef := range certRefs {
for _, cert := range certRef.certificates {
metrics := collector.getMetricsForCertificate(cert, certRef)
for _, metric := range metrics {
ch <- metric
}
}

if collector.exporter.ExposeErrorMetrics && len(certRef.certificates) > 0 {
labelKeys, labelValues := collector.unzipLabels(collector.getBaseLabels(certRef))

ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(certErrorMetric, certErrorHelp, labelKeys, nil),
prometheus.GaugeValue,
0,
labelValues...,
)
}
}

for index, err := range certErrors {
if err.err != nil {
log.Debugf("read error %d: %+v", index+1, err.err)
}

if collector.exporter.ExposeErrorMetrics && err.ref != nil {
labelKeys, labelValues := collector.unzipLabels(collector.getBaseLabels(err.ref))

ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(certErrorMetric, certErrorHelp, labelKeys, nil),
prometheus.GaugeValue,
1,
labelValues...,
)
}
}

ch <- prometheus.MustNewConstMetric(
Expand All @@ -80,27 +110,9 @@ func (collector *collector) Collect(ch chan<- prometheus.Metric) {
}

func (collector *collector) getMetricsForCertificate(certData *parsedCertificate, ref *certificateRef) []prometheus.Metric {
labels := map[string]string{
"serial_number": certData.cert.SerialNumber.String(),
}

if ref.format != certificateFormatKubeSecret {
trimComponentsCount := collector.exporter.TrimPathComponents
pathComponents := strings.Split(ref.path, "/")
prefix := ""
if pathComponents[0] == "" {
trimComponentsCount++
prefix = "/"
}

labels["filename"] = filepath.Base(ref.path)
labels["filepath"] = path.Join(prefix, path.Join(pathComponents[trimComponentsCount:]...))
} else {
labels["secret_name"] = filepath.Base(ref.path)
labels["secret_namespace"] = strings.Split(ref.path, "/")[1]
labels["secret_key"] = ref.kubeSecretKey
}
labels := collector.getBaseLabels(ref)

labels["serial_number"] = certData.cert.SerialNumber.String()
fillLabelsFromName(&certData.cert.Issuer, "issuer", labels)
fillLabelsFromName(&certData.cert.Subject, "subject", labels)

Expand All @@ -118,23 +130,7 @@ func (collector *collector) getMetricsForCertificate(certData *parsedCertificate
expired = 1.
}

labelKeys := []string{}
labelValues := []string{}
for key, value := range labels {
if collector.exporter.ExposeLabels == nil {
labelKeys = append(labelKeys, key)
labelValues = append(labelValues, value)
continue
}

for _, label := range collector.exporter.ExposeLabels {
if label == key {
labelKeys = append(labelKeys, key)
labelValues = append(labelValues, value)
}
}
}

labelKeys, labelValues := collector.unzipLabels(labels)
metrics := []prometheus.Metric{
prometheus.MustNewConstMetric(
prometheus.NewDesc(certExpiredMetric, certExpiredHelp, labelKeys, nil),
Expand Down Expand Up @@ -175,6 +171,51 @@ func (collector *collector) getMetricsForCertificate(certData *parsedCertificate
return metrics
}

func (collector *collector) getBaseLabels(ref *certificateRef) map[string]string {
labels := map[string]string{}

if ref.format != certificateFormatKubeSecret {
trimComponentsCount := collector.exporter.TrimPathComponents
pathComponents := strings.Split(ref.path, "/")
prefix := ""
if pathComponents[0] == "" {
trimComponentsCount++
prefix = "/"
}

labels["filename"] = filepath.Base(ref.path)
labels["filepath"] = path.Join(prefix, path.Join(pathComponents[trimComponentsCount:]...))
} else {
labels["secret_name"] = filepath.Base(ref.path)
labels["secret_namespace"] = strings.Split(ref.path, "/")[1]
labels["secret_key"] = ref.kubeSecretKey
}

return labels
}

func (collector *collector) unzipLabels(labels map[string]string) ([]string, []string) {
labelKeys := []string{}
labelValues := []string{}

for key, value := range labels {
if collector.exporter.ExposeLabels == nil {
labelKeys = append(labelKeys, key)
labelValues = append(labelValues, value)
continue
}

for _, label := range collector.exporter.ExposeLabels {
if label == key {
labelKeys = append(labelKeys, key)
labelValues = append(labelValues, value)
}
}
}

return labelKeys, labelValues
}

func fillLabelsFromName(name *pkix.Name, prefix string, output map[string]string) {
if len(name.Country) > 0 {
output[fmt.Sprintf("%s_C", prefix)] = name.Country[0]
Expand Down
26 changes: 16 additions & 10 deletions internal/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type Exporter struct {
TrimPathComponents int
MaxCacheDuration time.Duration
ExposeRelativeMetrics bool
ExposeErrorMetrics bool
ExposeLabels []string
KubeSecretTypes []string
KubeIncludeNamespaces []string
Expand Down Expand Up @@ -111,13 +112,10 @@ func (exporter *Exporter) DiscoverCertificates() {
func (exporter *Exporter) parseAllCertificates() ([]*certificateRef, []*certificateError) {
output := []*certificateRef{}
outputErrors := []*certificateError{}
raiseError := func(err error) {
outputErrors = append(outputErrors, &certificateError{
err: err,
})

if exporter.isDiscovery {
log.Warn(err)
raiseError := func(err *certificateError) {
outputErrors = append(outputErrors, err)
if exporter.isDiscovery && err.err != nil {
log.Warn(err.err)
}
}

Expand All @@ -139,7 +137,10 @@ func (exporter *Exporter) parseAllCertificates() ([]*certificateRef, []*certific
for _, dir := range exporter.Directories {
files, err := os.ReadDir(dir)
if err != nil {
raiseError(fmt.Errorf("failed to open directory \"%s\", %s", dir, err.Error()))
raiseError(&certificateError{
err: fmt.Errorf("failed to open directory \"%s\", %s", dir, err.Error()),
})

continue
}

Expand All @@ -159,7 +160,9 @@ func (exporter *Exporter) parseAllCertificates() ([]*certificateRef, []*certific
certs, errs := exporter.parseAllKubeSecrets()
output = append(output, certs...)
for _, err := range errs {
raiseError(err)
raiseError(&certificateError{
err: err,
})
}
}

Expand All @@ -174,7 +177,10 @@ func (exporter *Exporter) parseAllCertificates() ([]*certificateRef, []*certific
err = fmt.Errorf("no certificate(s) found in \"%s\"", cert.path)
}

raiseError(err)
raiseError(&certificateError{
err: err,
ref: cert,
})
} else if exporter.isDiscovery {
log.Infof("%d valid certificate(s) found in \"%s\"", len(cert.certificates), cert.path)
}
Expand Down
34 changes: 34 additions & 0 deletions internal/exporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,40 @@ func TestCorruptedCertInYAML(t *testing.T) {
})
}

func TestErrorMetrics(t *testing.T) {
_, filename, _, _ := runtime.Caller(0)

test := func(enableErrorMetrics bool) {
testRequest(t, &Exporter{
Directories: []string{path.Join(filepath.Dir(filename), "../test")},
ExposeErrorMetrics: enableErrorMetrics,
}, func(metrics []model.MetricFamily) {
errorMetric := getMetricsForName(metrics, "x509_cert_error")

if enableErrorMetrics {
errors := 0
for _, metric := range errorMetric {
if metric.GetGauge().GetValue() > 0 {
errors++
}
}

assert.Len(t, errorMetric, 21, "missing x509_read_error metrics")
assert.Equal(t, 17, errors, "missing x509_read_error metrics")
} else {
assert.Len(t, errorMetric, 0, "unexpected x509_read_error metrics")
}

errorsMetric := getMetricsForName(metrics, "x509_read_errors")
assert.Len(t, errorsMetric, 1, "missing x509_read_errors metric")
assert.Equal(t, errorsMetric[0].GetGauge().GetValue(), 17., "invalid x509_read_errors value")
})
}

test(false)
test(true)
}

func TestBindAddrAlreadyInUse(t *testing.T) {
listener, _ := net.Listen("tcp", ":9793")
e := &Exporter{Port: 9793}
Expand Down

0 comments on commit 0b4f573

Please sign in to comment.