feat: use gomplate.Template. Remove templating package (#1160)

* feat: use gomplate.Template. Remove templating package [skip ci] * chore: bump gomplate * chore: bump commons kommons & flanksource/yaml.v3 * chore: replace sprint() with string() in cel expression * chore: bump gomplate to support duration funcs * chore: bump gomplate * fix: var use in exec fixture * chore: go mod tidy * chore: replace len() with size() for cel expression * chore: bump gomplate * chore: check for elastic search cluster health * chore: use different image for probing and adding health status logs * chore: logs for elastic search in post setup * chore: separate search fixtures
flanksource · Jul 27, 2023 · b2e3152 · b2e3152
1 parent 66bdb7f
commit b2e3152
Show file tree

Hide file tree

Showing 31 changed files with 1,153 additions and 3,390 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -18,6 +18,7 @@ jobs:
           - minimal --skip-all
           - k8s
           - datasources
+          - search
           - git
           # - restic
     runs-on: ubuntu-latest

diff --git a/api/v1/common.go b/api/v1/common.go
@@ -11,6 +11,7 @@ import (
 	"github.com/c2h5oh/datasize"
 	"github.com/flanksource/commons/duration"
 	"github.com/flanksource/duty/types"
+	"github.com/flanksource/gomplate/v3"
 )
 
 type Duration string
@@ -220,6 +221,16 @@ func (t Template) IsEmpty() bool {
 	return t.Template == "" && t.JSONPath == "" && t.Expression == "" && t.Javascript == ""
 }
 
+// Convert to gomplate.Template
+func (t Template) Gomplate() gomplate.Template {
+	return gomplate.Template{
+		Template:   t.Template,
+		JSONPath:   t.JSONPath,
+		Expression: t.Expression,
+		Javascript: t.Javascript,
+	}
+}
+
 // +k8s:deepcopy-gen=false
 type DisplayTemplate interface {
 	GetDisplayTemplate() Template

diff --git a/checks/common.go b/checks/common.go
@@ -11,7 +11,7 @@ import (
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
 	"github.com/flanksource/canary-checker/pkg/utils"
-	"github.com/flanksource/canary-checker/templating"
+	"github.com/flanksource/gomplate/v3"
 	"github.com/robfig/cron/v3"
 )
 
@@ -52,7 +52,7 @@ func def(a, b string) string {
 }
 
 func template(ctx *context.Context, template v1.Template) (string, error) {
-	return templating.Template(ctx.Environment, template)
+	return gomplate.RunTemplate(ctx.Environment, template.Gomplate())
 }
 
 func transform(ctx *context.Context, in *pkg.CheckResult) ([]*pkg.CheckResult, error) {

diff --git a/checks/http.go b/checks/http.go
@@ -169,9 +169,10 @@ func (c *HTTPChecker) Check(ctx *context.Context, extConfig external.Check) pkg.
 		"code":    status,
 		"headers": resp.GetHeaders(),
 		"elapsed": time.Since(start),
-		"sslAge":  age,
 		"content": body,
+		"sslAge":  utils.Deref(age),
 	}
+
 	if resp.IsJSON() {
 		json, err := resp.AsJSON()
 		if err != nil {

diff --git a/cmd/root.go b/cmd/root.go
@@ -6,8 +6,8 @@ import (
 	"github.com/flanksource/canary-checker/pkg/cache"
 	"github.com/flanksource/canary-checker/pkg/db"
 	"github.com/flanksource/canary-checker/pkg/runner"
-	"github.com/flanksource/canary-checker/templating"
 	"github.com/flanksource/commons/logger"
+	gomplate "github.com/flanksource/gomplate/v3"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -17,7 +17,7 @@ var Root = &cobra.Command{
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		logger.UseZap(cmd.Flags())
 		for _, script := range sharedLibrary {
-			if err := templating.LoadSharedLibrary(script); err != nil {
+			if err := gomplate.LoadSharedLibrary(script); err != nil {
 				logger.Errorf("Failed to load shared library %s: %v", script, err)
 			}
 		}

diff --git a/fixtures/datasources/_post_setup.sh b/fixtures/datasources/_post_setup.sh
@@ -1,4 +1,2 @@
 #!/bin/bash
 
-echo "Running kubectl wait for elasticsearch"
-kubectl -n default wait --for=condition=ready pod -l app=elasticsearch --timeout=5m
diff --git a/fixtures/datasources/_setup.yaml b/fixtures/datasources/_setup.yaml
@@ -580,173 +580,6 @@ metadata:
   namespace: default
 ---
 apiVersion: v1
-kind: Secret
-metadata:
-  name: search
-  namespace: default
-stringData:
-  OPENSEARCH_USERNAME: admin
-  OPENSEARCH_PASSWORD: secret
-  ELASTIC_SEARCH_USERNAME: admin
-  ELASTIC_SEARCH_PASSWORD: secret
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: elasticsearch
-  name: elasticsearch
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: elasticsearch
-  template:
-    metadata:
-      labels:
-        app: elasticsearch
-    spec:
-      containers:
-        - name: elasticsearch
-          image: docker.elastic.co/elasticsearch/elasticsearch:8.1.3
-          env:
-            - name: ES_JAVA_OPTS
-              value: "-Xms256m -Xmx256m"
-            - name: network.bind_host
-              value: 0.0.0.0
-            - name: network.host
-              value: 0.0.0.0
-            - name: discovery.type
-              value: single-node
-            - name: xpack.security.enabled
-              value: "false"
-          imagePullPolicy: IfNotPresent
-          ports:
-            - containerPort: 9200
-              name: http
-              protocol: TCP
-            - containerPort: 9300
-              name: transport
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 9200
-            initialDelaySeconds: 10
-        - name: populate-db
-          image: debian:bookworm
-          command: ["/bin/sh", "-c"]
-          readinessProbe:
-            exec:
-              command:
-                - ls
-                - /tmp/done
-          args:
-            # We install wait-for-it and wait for elasticsearch to be ready and then
-            # populate it with dummy data
-            - >
-              apt update && apt install -y curl;
-              curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -o wait-for-it.sh;
-              chmod +x wait-for-it.sh; ./wait-for-it.sh localhost:9200 --timeout=0;
-              curl -X PUT localhost:9200/index;
-              curl -X POST localhost:9200/index/_doc -d '{"system": {"role": "api"}}' -H 'Content-Type: application/json';
-              touch /tmp/done;
-              sleep infinity
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: elasticsearch
-  namespace: default
-spec:
-  ports:
-    - name: elasticsearch
-      port: 9200
-      targetPort: 9200
-      protocol: TCP
-  selector:
-    app: elasticsearch
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: opensearch
-  name: opensearch
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: opensearch
-  template:
-    metadata:
-      labels:
-        app: opensearch
-    spec:
-      containers:
-        - name: opensearch
-          image: opensearchproject/opensearch:2.7.0
-          env:
-            - name: ES_JAVA_OPTS
-              value: "-Xms256m -Xmx256m"
-            - name: network.bind_host
-              value: 0.0.0.0
-            - name: network.host
-              value: 0.0.0.0
-            - name: discovery.type
-              value: single-node
-            - name: DISABLE_SECURITY_PLUGIN
-              value: "true"
-          imagePullPolicy: IfNotPresent
-          ports:
-            - containerPort: 9200
-              name: http
-              protocol: TCP
-            - containerPort: 9300
-              name: transport
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 9200
-            initialDelaySeconds: 10
-        - name: populate-db
-          image: debian:bookworm
-          command: ["/bin/sh", "-c"]
-          readinessProbe:
-            exec:
-              command:
-                - ls
-                - /tmp/done
-          args:
-            # We install wait-for-it and wait for elasticsearch to be ready and then
-            # populate it with dummy data
-            - >
-              apt update && apt install -y curl;
-              curl https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -o wait-for-it.sh;
-              chmod +x wait-for-it.sh; ./wait-for-it.sh localhost:9200 --timeout=0;
-              curl -X PUT localhost:9200/index;
-              curl -X POST localhost:9200/index/_doc -d '{"system": {"role": "api", "version": "v1.0"}}' -H 'Content-Type: application/json';
-              touch /tmp/done; 
-              sleep infinity
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: opensearch
-  namespace: default
-spec:
-  ports:
-    - name: opensearch
-      port: 9200
-      targetPort: 9200
-      protocol: TCP
-  selector:
-    app: opensearch
----
-apiVersion: v1
 kind: Namespace
 metadata:
   name: ldap

diff --git a/fixtures/datasources/kustomization.yaml b/fixtures/datasources/kustomization.yaml
@@ -14,8 +14,4 @@ resources:
   - prometheus.yaml
   - redis_fail.yaml
   - redis_pass.yaml
-  - elasticsearch_fail.yaml
-  - elasticsearch_pass.yaml
-  - opensearch_fail.yaml
-  - opensearch_pass.yaml
   - alertmanager_fail.yaml
diff --git a/fixtures/git/git_check_pass.yaml b/fixtures/git/git_check_pass.yaml
@@ -8,7 +8,7 @@ spec:
     - query: "SELECT * FROM github_repo_checks('flanksource/duty') where branch='main'"
       name: github-check
       test:
-        expr: len(results) > 0
+        expr: size(results) > 0
       githubToken:
         valueFrom:
           secretKeyRef:

diff --git a/fixtures/minimal/exec_fail.yaml b/fixtures/minimal/exec_fail.yaml
@@ -7,7 +7,10 @@ metadata:
 spec:
   interval: 30
   exec:
-    - description: "exec dummy check"
-      name: exec-fail-check
+    - name: exec-fail-check
+      description: "exec dummy check"
       script: |
-        eche "hi there"
+        echo "hi there"
+      test:
+        expr: 'results.stdout == "hello"'
+
diff --git a/fixtures/minimal/exec_pass.yaml b/fixtures/minimal/exec_pass.yaml
@@ -5,9 +5,9 @@ metadata:
 spec:
   interval: 30
   exec:
-   - description: "exec dummy check"
-     script: |
-       echo "hello"
-     name: exec-pass-check
-     test:
-       expr: 'results.Stdout == "hello"'
+    - name: exec-pass-check
+      description: "exec dummy check"
+      script: |
+        echo "hello"
+      test:
+        expr: 'results.stdout == "hello"'
diff --git a/fixtures/minimal/http_fail.yaml b/fixtures/minimal/http_fail.yaml
@@ -13,6 +13,6 @@ spec:
     - endpoint: https://httpbin.demo.aws.flanksource.com/status/200
       name: http fail test expr check
       display:
-        expr: sprint(code) + " should be 500"
+        expr: string(code) + " should be 500"
       test:
         expr: code == 500
diff --git a/fixtures/minimal/http_fail_connection.yaml b/fixtures/minimal/http_fail_connection.yaml
@@ -7,12 +7,12 @@ metadata:
 spec:
   interval: 30
   http:
-    - connection: 'connection://http/500'
+    - connection: 'connection://HTTP/500'
       name: http fail response code check
       responseCodes: [200]
-    - connection: 'connection://http/200'
+    - connection: 'connection://HTTP/200'
       name: http fail test expr check
       display:
-        expr: sprint(code) + " should be 500"
+        expr: string(code) + " should be 500"
       test:
         expr: code == 500
diff --git a/fixtures/minimal/http_pass_single.yaml b/fixtures/minimal/http_pass_single.yaml
@@ -19,6 +19,6 @@ spec:
     - name: http-expr-tests
       url: https://httpbin.demo.aws.flanksource.com/status/200
       test:
-        expr: "code in [200,201,301] and sslAge > Duration('7d')"
+        expr: "code in [200,201,301] && sslAge > Duration('7d')"
       display:
         template: "code={{.code}}, age={{.sslAge}}"
diff --git a/fixtures/search/_post_setup.sh b/fixtures/search/_post_setup.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "Running kubectl wait for elasticsearch"
+kubectl -n default wait --for=condition=ready pod -l app=elasticsearch --timeout=5m
+
+echo "Fetching elastic search health";
+curl -s "http://elasticsearch.default.svc.cluster.local:9200/_cluster/health" -H 'Content-Type: application/json';
+curl -s "http://elasticsearch.default.svc.cluster.local:9200/_cluster/allocation/explain" -H 'Content-Type: application/json';
+
+echo "Fetching populate-db logs from elasticsearch pod";
+kubectl logs -n default -l app=elasticsearch -c populate-db