Skip to content

Commit

Permalink
feat: add hasSourceTenantsForMetrics validator, refactor, optimize do…
Browse files Browse the repository at this point in the history
…cs html rendering, add some additional tests

Signed-off-by: Martin Chodur <[email protected]>
  • Loading branch information
FUSAKLA committed Dec 5, 2023
1 parent bc7c0eb commit b3e2e23
Show file tree
Hide file tree
Showing 23 changed files with 495 additions and 178 deletions.
7 changes: 5 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ PROMRUVAL_BIN := ./promruval
E2E_TESTS_VALIDATIONS_FILE := examples/validation.yaml
E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE := examples/additional-validation.yaml
E2E_TESTS_RULES_FILES := examples/rules/*.yaml
E2E_TESTS_DOCS_FILE_MD := examples/human_readable.md
E2E_TESTS_DOCS_FILE_HTML := examples/human_readable.html

all: deps lint build test e2e-test
all: clean deps lint build test e2e-test

$(TMP_DIR):
mkdir -p $(TMP_DIR)
Expand All @@ -36,7 +38,8 @@ build:

e2e-test: build
$(PROMRUVAL_BIN) validate --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) $(E2E_TESTS_RULES_FILES)
$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE)
$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) > $(E2E_TESTS_DOCS_FILE_MD)
$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) --output=html > $(E2E_TESTS_DOCS_FILE_HTML)

docker: build
docker build -t fusakla/promruval .
Expand Down
43 changes: 43 additions & 0 deletions examples/human_readable.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@

<h1>Validation rules</h1>

<h2><a href="#check-severity-label">check-severity-label</a></h2>
<ul>
<li>Alert has labels: <code>severity</code></li>
<li>Alert label <code>severity</code> has one of the allowed values: <code>info</code>,<code>warning</code>,<code>critical</code></li>
<li>Alert if rule has label <code>severity</code> with value <code>info</code> , it cannot have label <code>page</code></li>
<li>Alert expression can be successfully evaluated on the live Prometheus instance</li>
<li>Alert expression uses only labels that are actually present in Prometheus</li>
<li>Alert expression selectors actually matches any series in Prometheus</li>
<li>Alert expression does not use data older than <code>6h0m0s</code></li>
</ul>

<h2><a href="#check-team-label">check-team-label</a></h2>
<ul>
<li>Alert has labels: <code>xxx</code></li>
<li>Alert label <code>team</code> has one of the allowed values: <code>[email protected]</code></li>
</ul>

<h2><a href="#check-playbook-annotation">check-playbook-annotation</a></h2>
<ul>
<li>Alert has any of these annotations: <code>playbook</code>,<code>link</code></li>
<li>Alert Annotation <code>link</code> is a valid URL and does not return HTTP status 404</li>
</ul>

<h2><a href="#check-alert-title">check-alert-title</a></h2>
<ul>
<li>Alert has all of these annotations: <code>title</code></li>
</ul>

<h2><a href="#check-prometheus-limitations">check-prometheus-limitations</a></h2>
<ul>
<li>All rules expression does not use data older than <code>6h0m0s</code></li>
<li>All rules does not use any of the <code>cluster</code>,<code>locality</code>,<code>prometheus-type</code>,<code>replica</code> labels is in its expression</li>
<li>All rules verifies if the rule group, the rule belongs to, has the required source_tenants configured, according to the mapping of metric names to tenants: <code>k8s</code>:<code>^container_.*|kube_.*$</code></li>
</ul>

<h2><a href="#another-checks">another-checks</a></h2>
<ul>
<li>All rules labels does not have empty values</li>
</ul>

31 changes: 31 additions & 0 deletions examples/human_readable.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@

Validation rules:

check-severity-label
- Alert has labels: `severity`
- Alert label `severity` has one of the allowed values: `info`,`warning`,`critical`
- Alert if rule has label `severity` with value `info` , it cannot have label `page`
- Alert expression can be successfully evaluated on the live Prometheus instance
- Alert expression uses only labels that are actually present in Prometheus
- Alert expression selectors actually matches any series in Prometheus
- Alert expression does not use data older than `6h0m0s`

check-team-label
- Alert has labels: `xxx`
- Alert label `team` has one of the allowed values: `[email protected]`

check-playbook-annotation
- Alert has any of these annotations: `playbook`,`link`
- Alert Annotation `link` is a valid URL and does not return HTTP status 404

check-alert-title
- Alert has all of these annotations: `title`

check-prometheus-limitations
- All rules expression does not use data older than `6h0m0s`
- All rules does not use any of the `cluster`,`locality`,`prometheus-type`,`replica` labels is in its expression
- All rules verifies if the rule group, the rule belongs to, has the required source_tenants configured, according to the mapping of metric names to tenants: `k8s`:`^container_.*|kube_.*$`

another-checks
- All rules labels does not have empty values

3 changes: 2 additions & 1 deletion examples/rules/rules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,14 @@ groups:
disabled_validation_rules: check-team-label,check-prometheus-limitations

- name: testIgnoreValidationsInExpr
source_tenants: ["k8s"]
rules:
- alert: test
expr: |
# Comment before.
# Comment on the same line. ignore_validations: labelHasAllowedValue
# Comment after.
foo{
kube_pod_labels{
# ignore_validations: expressionSelectorsMatchesAnything, hasLabels
}
for: 1m
Expand Down
16 changes: 10 additions & 6 deletions examples/validation.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ validationRules:
validations:
- type: hasLabels
params:
labels: [ "severity" ]
labels: ["severity"]
- type: labelHasAllowedValue
params:
label: "severity"
allowedValues: [ "info", "warning", "critical" ]
allowedValues: ["info", "warning", "critical"]
- type: exclusiveLabels
params:
firstLabel: severity
Expand All @@ -35,7 +35,7 @@ validationRules:
validations:
- type: hasLabels
params:
labels: [ "xxx" ]
labels: ["xxx"]
- type: labelHasAllowedValue
params:
label: "team"
Expand All @@ -47,7 +47,7 @@ validationRules:
validations:
- type: hasAnyOfAnnotations
params:
annotations: [ "playbook", "link" ]
annotations: ["playbook", "link"]
- type: annotationIsValidURL
params:
annotation: "link"
Expand All @@ -58,7 +58,7 @@ validationRules:
validations:
- type: hasAnnotations
params:
annotations: [ "title" ]
annotations: ["title"]

- name: check-prometheus-limitations
scope: All rules
Expand All @@ -68,4 +68,8 @@ validationRules:
limit: "6h"
- type: expressionDoesNotUseLabels
params:
labels: [ "cluster", "locality", "prometheus-type", "replica" ]
labels: ["cluster", "locality", "prometheus-type", "replica"]
- type: hasSourceTenantsForMetrics
params:
sourceTenants:
k8s: "container_.*|kube_.*"
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ require (
github.com/prometheus/common v0.45.0
github.com/prometheus/prometheus v0.48.0
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.8.4
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/yaml.v3 v3.0.1
gotest.tools v2.2.0+incompatible
Expand Down Expand Up @@ -55,14 +57,12 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common/sigv4 v0.1.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/goleak v1.3.0 // indirect
golang.org/x/crypto v0.15.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/oauth2 v0.14.0 // indirect
golang.org/x/sync v0.5.0 // indirect
Expand Down
15 changes: 8 additions & 7 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/fusakla/promruval/v2/pkg/prometheus"
"github.com/fusakla/promruval/v2/pkg/report"
"github.com/fusakla/promruval/v2/pkg/validate"
"github.com/fusakla/promruval/v2/pkg/validationrule"
"github.com/fusakla/promruval/v2/pkg/validator"
log "github.com/sirupsen/logrus"
"gopkg.in/alecthomas/kingpin.v2"
Expand Down Expand Up @@ -55,22 +56,22 @@ func loadConfigFile(configFilePath string) (*config.Config, error) {
return validationConfig, nil
}

func validationRulesFromConfig(config *config.Config) ([]*validate.ValidationRule, error) {
var validationRules []*validate.ValidationRule
func validationRulesFromConfig(config *config.Config) ([]*validationrule.ValidationRule, error) {
var validationRules []*validationrule.ValidationRule
rulesIteration:
for _, rule := range config.ValidationRules {
for _, validationRule := range config.ValidationRules {
for _, disabledRule := range *disabledRules {
if disabledRule == rule.Name {
if disabledRule == validationRule.Name {
continue rulesIteration
}
}
for _, enabledRule := range *enabledRules {
if enabledRule != rule.Name {
if enabledRule != validationRule.Name {
continue rulesIteration
}
}
newRule := validate.NewValidationRule(rule.Name, rule.Scope)
for _, validatorConfig := range rule.Validations {
newRule := validationrule.New(validationRule.Name, validationRule.Scope)
for _, validatorConfig := range validationRule.Validations {
newValidator, err := validator.NewFromConfig(validatorConfig)
if err != nil {
return nil, fmt.Errorf("loading validator config: %w", err)
Expand Down
3 changes: 2 additions & 1 deletion pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ package config

import (
"fmt"
"gopkg.in/yaml.v3"
"time"

"gopkg.in/yaml.v3"

"github.com/creasty/defaults"
)

Expand Down
2 changes: 1 addition & 1 deletion pkg/report/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

type ValidationRule interface {
Name() string
Scope() string
Scope() config.ValidationScope
ValidationTexts() []string
}

Expand Down
13 changes: 10 additions & 3 deletions pkg/report/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"html/template"
"regexp"
)

var htmlTemplate = `
Expand All @@ -13,7 +14,7 @@ var htmlTemplate = `
<h2><a href="#{{.Name}}">{{.Name}}</a></h2>
<ul>
{{- range .Validations }}
<li>{{$currentRule.Scope}} {{.}}</li>
<li>{{$currentRule.Scope}} {{. | backticksToCodeTag }}</li>
{{- end }}
</ul>
{{- end }}
Expand All @@ -40,6 +41,12 @@ Validation rules:
{{- end }}
`

var customFuncs = template.FuncMap{
"backticksToCodeTag": func(s string) template.HTML {
return template.HTML(regexp.MustCompile("`([^`]+)`").ReplaceAllString(s, "<code>$1</code>"))
},
}

type templateRule struct {
Name string
Scope string
Expand All @@ -55,7 +62,7 @@ func ValidationDocs(validationRules []ValidationRule, format string) (string, er
for _, rule := range validationRules {
data.Rules = append(data.Rules, templateRule{
Name: rule.Name(),
Scope: rule.Scope(),
Scope: string(rule.Scope()),
Validations: rule.ValidationTexts(),
})
}
Expand All @@ -72,7 +79,7 @@ func ValidationDocs(validationRules []ValidationRule, format string) (string, er
return "", fmt.Errorf("unsupported format type %s", format)
}

tmpl, err := template.New("docs").Parse(templateToUse)
tmpl, err := template.New("docs").Funcs(customFuncs).Parse(templateToUse)
if err != nil {
return "", err
}
Expand Down
82 changes: 82 additions & 0 deletions pkg/unmarshaler/unmarshaler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package unmarshaler

import (
"strings"

"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/rulefmt"
"gopkg.in/yaml.v3"
)

type fakeTestFile struct {
RuleFiles []yaml.Node `yaml:"rule_files,omitempty"`
EvaluationInterval yaml.Node `yaml:"evaluation_interval,omitempty"`
GroupEvalOrder []yaml.Node `yaml:"group_eval_order,omitempty"`
Tests []yaml.Node `yaml:"tests,omitempty"`
}

type RulesFile struct {
Groups []RuleGroup `yaml:"groups"`
fakeTestFile // Just so we can unmarshal also PromQL test files but ignore them because it has no Groups
}

type RuleGroup struct {
Name string `yaml:"name"`
Interval model.Duration `yaml:"interval,omitempty"`
PartialResponseStrategy string `yaml:"partial_response_strategy,omitempty"`
SourceTenants []string `yaml:"source_tenants,omitempty"`
Rules []ruleWithComment `yaml:"rules"`
}

type ruleWithComment struct {
node yaml.Node
rule rulefmt.RuleNode
}

func (r *ruleWithComment) OriginalRule() rulefmt.Rule {
return rulefmt.Rule{
Record: r.rule.Record.Value,
Alert: r.rule.Alert.Value,
Expr: r.rule.Expr.Value,
For: r.rule.For,
Labels: r.rule.Labels,
Annotations: r.rule.Annotations,
}
}

func (r *ruleWithComment) UnmarshalYAML(value *yaml.Node) error {
err := value.Decode(&r.node)
if err != nil {
return err
}
err = value.Decode(&r.rule)
if err != nil {
return err
}
return nil
}

func (r *ruleWithComment) DisabledValidators(commentPrefix string) []string {
commentPrefix += ":"
var disabledValidators []string
allComments := strings.Split(r.node.HeadComment, "\n")
for _, line := range strings.Split(r.rule.Expr.Value, "\n") {
before, comment, found := strings.Cut(line, "#")
if !found || strings.TrimSpace(before) != "" {
continue
}
allComments = append(allComments, comment)
}
for _, comment := range allComments {
_, csv, found := strings.Cut(comment, commentPrefix)
if !found {
continue
}
validators := strings.Split(csv, ",")
for _, v := range validators {
vv := strings.TrimSpace(v)
disabledValidators = append(disabledValidators, vv)
}
}
return disabledValidators
}
Loading

0 comments on commit b3e2e23

Please sign in to comment.