Merge pull request #333 from flanksource/git-check

feat: add support askgit

.github/workflows/test.yml

@@ -27,6 +27,7 @@ jobs:
           - minimal --skip-all
           - k8s
           - datasources
+          - git
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -52,6 +53,7 @@ jobs:
       - name: Test
         env:
           KUBERNETES_VERSION: v1.20.7
+          GH_TOKEN: ${{ secrets.CHECKRUNS_TOKEN }}
         run: ./test/e2e.sh fixtures/${{matrix.suite}}
       - name: Publish Unit Test Results
         uses: EnricoMi/publish-unit-test-result-action@v1

Dockerfile

@@ -38,6 +38,12 @@ RUN curl -L https://mirrors.estointernet.in/apache//jmeter/binaries/apache-jmete
 
 ENV PATH /apache-jmeter-5.4.1/bin/:$PATH
 
+# Install askgit binaries
+RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz && \
+    tar xf askgit.tar.gz && \
+    mv askgit /usr/local/bin/askgit && \
+    rm askgit.tar.gz
+
 # install CA certificates
 RUN apt-get update && \
   apt-get install -y ca-certificates && \

api/v1/canary_types.go

@@ -61,6 +61,7 @@ type CanarySpec struct {
 	Prometheus     []PrometheusCheck     `yaml:"prometheus,omitempty" json:"prometheus,omitempty"`
 	MongoDB        []MongoDBCheck        `yaml:"mongodb,omitempty" json:"mongodb,omitempty"`
 	CloudWatch     []CloudWatchCheck     `yaml:"cloudwatch,omitempty" json:"cloudwatch,omitempty"`
+	GitHub         []GitHubCheck         `yaml:"github,omitempty" json:"github,omitempty"`
 	// interval (in seconds) to run checks on Deprecated in favor of Schedule
 	Interval uint64 `yaml:"interval,omitempty" json:"interval,omitempty"`
 	// Schedule to run checks on. Supports all cron expression, example: '30 3-6,20-23 * * *'. For more info about cron expression syntax see https://en.wikipedia.org/wiki/Cron
@@ -151,6 +152,9 @@ func (spec CanarySpec) GetAllChecks() []external.Check {
 	for _, check := range spec.CloudWatch {
 		checks = append(checks, check)
 	}
+	for _, check := range spec.GitHub {
+		checks = append(checks, check)
+	}
 	return checks
 }
 

api/v1/checks.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/flanksource/canary-checker/api/external"
 	"github.com/flanksource/kommons"
@@ -638,6 +639,27 @@ func (c MongoDBCheck) GetType() string {
 	return "mongodb"
 }
 
+// Git executes a SQL style query against a github repo using https://github.com/askgitdev/askgit
+type Git struct {
+	GitHubCheck `yaml:",inline" json:",inline"`
+}
+
+type GitHubCheck struct {
+	Description `yaml:",inline" json:",inline"`
+	Templatable `yaml:",inline" json:",inline"`
+	// Query to be executed. Please see https://github.com/askgitdev/askgit for more details regarding syntax
+	Query       string          `yaml:"query" json:"query"`
+	GithubToken *kommons.EnvVar `yaml:"githubToken,omitempty" json:"githubToken,omitempty"`
+}
+
+func (c GitHubCheck) GetType() string {
+	return "github"
+}
+
+func (c GitHubCheck) GetEndpoint() string {
+	return strings.ReplaceAll(c.Query, " ", "-")
+}
+
 /*
 [include:minimal/http_pass.yaml]
 */
@@ -914,4 +936,5 @@ var AllChecks = []external.Check{
 	GCSBucketCheck{},
 	MongoDBCheck{},
 	CloudWatchCheck{},
+	GitHubCheck{},
 }

checks/checker.go

@@ -46,6 +46,7 @@ var All = []Checker{
 	&MongoDBChecker{},
 	&GCSBucketChecker{},
 	&CloudWatchChecker{},
+	&GitHubChecker{},
 	NewPodChecker(),
 	NewNamespaceChecker(),
 	NewTCPChecker(),

checks/github.go

@@ -0,0 +1,66 @@
+package checks
+
+import (
+	"encoding/json"
+	"fmt"
+	osExec "os/exec"
+	"strings"
+
+	"github.com/flanksource/canary-checker/api/context"
+	"github.com/flanksource/canary-checker/api/external"
+	v1 "github.com/flanksource/canary-checker/api/v1"
+	"github.com/flanksource/canary-checker/pkg"
+)
+
+func init() {
+	//register metrics here
+}
+
+type GitHubChecker struct {
+}
+
+func (c *GitHubChecker) Type() string {
+	return "github"
+}
+
+func (c *GitHubChecker) Run(ctx *context.Context) []*pkg.CheckResult {
+	var results []*pkg.CheckResult
+	for _, conf := range ctx.Canary.Spec.GitHub {
+		result := c.Check(ctx, conf)
+		if result != nil {
+			results = append(results, result)
+		}
+	}
+	return results
+}
+
+func (c *GitHubChecker) Check(ctx *context.Context, extConfig external.Check) *pkg.CheckResult {
+	check := extConfig.(v1.GitHubCheck)
+	checkResult := pkg.Success(check, ctx.Canary)
+	_, githubToken, err := ctx.Kommons.GetEnvValue(*check.GithubToken, ctx.Canary.GetNamespace())
+	if err != nil {
+		return checkResult.Failf("error fetching github token: %v", err)
+	}
+	askGitCmd := fmt.Sprintf("GITHUB_TOKEN=%v askgit \"%v\" --format json", githubToken, check.Query)
+	cmd := osExec.Command("bash", "-c", askGitCmd)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return checkResult.Failf("error executing askgit command: %v", err)
+	}
+	rows := string(output)
+	var rowResults = make([]map[string]string, 0)
+	for _, row := range strings.Split(rows, "\n") {
+		if row == "" {
+			continue
+		}
+		var rowResult map[string]string
+		err := json.Unmarshal([]byte(row), &rowResult)
+		if err != nil {
+			return checkResult.Failf("error parsing askgit result: %v", err)
+		}
+
+		rowResults = append(rowResults, rowResult)
+	}
+	checkResult.AddDetails(rowResults)
+	return checkResult
+}

config/deploy/crd.yaml

@@ -766,6 +766,75 @@ spec:
                       - bucket
                     type: object
                   type: array
+                github:
+                  items:
+                    properties:
+                      description:
+                        description: Description for the check
+                        type: string
+                      display:
+                        properties:
+                          expr:
+                            type: string
+                          jsonPath:
+                            type: string
+                          template:
+                            type: string
+                        type: object
+                      githubToken:
+                        properties:
+                          name:
+                            type: string
+                          value:
+                            type: string
+                          valueFrom:
+                            properties:
+                              configMapKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                  - key
+                                type: object
+                              secretKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                  - key
+                                type: object
+                            type: object
+                        type: object
+                      icon:
+                        description: Icon for overwriting default icon on the dashboard
+                        type: string
+                      name:
+                        description: Name of the check
+                        type: string
+                      query:
+                        description: Query to be executed. Please see https://github.com/askgitdev/askgit for more details regarding syntax
+                        type: string
+                      test:
+                        properties:
+                          expr:
+                            type: string
+                          jsonPath:
+                            type: string
+                          template:
+                            type: string
+                        type: object
+                    required:
+                      - query
+                    type: object
+                  type: array
                 helm:
                   items:
                     properties:

config/deploy/manifests.yaml

@@ -773,6 +773,75 @@ spec:
                       - bucket
                     type: object
                   type: array
+                github:
+                  items:
+                    properties:
+                      description:
+                        description: Description for the check
+                        type: string
+                      display:
+                        properties:
+                          expr:
+                            type: string
+                          jsonPath:
+                            type: string
+                          template:
+                            type: string
+                        type: object
+                      githubToken:
+                        properties:
+                          name:
+                            type: string
+                          value:
+                            type: string
+                          valueFrom:
+                            properties:
+                              configMapKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                  - key
+                                type: object
+                              secretKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                  - key
+                                type: object
+                            type: object
+                        type: object
+                      icon:
+                        description: Icon for overwriting default icon on the dashboard
+                        type: string
+                      name:
+                        description: Name of the check
+                        type: string
+                      query:
+                        description: Query to be executed. Please see https://github.com/askgitdev/askgit for more details regarding syntax
+                        type: string
+                      test:
+                        properties:
+                          expr:
+                            type: string
+                          jsonPath:
+                            type: string
+                          template:
+                            type: string
+                        type: object
+                    required:
+                      - query
+                    type: object
+                  type: array
                 helm:
                   items:
                     properties:

fixtures/git/_setup.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# Install askgit
+curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz
+tar xf askgit.tar.gz
+sudo mv askgit /usr/bin/askgit
+sudo chmod +x /usr/bin/askgit
+rm askgit.tar.gz
+
+#verification
+which askgit
+askgit --help
+
+# creating a GITHUB_TOKEN Secret
+kubectl create secret generic github-token --from-literal=GITHUB_TOKEN="${GH_TOKEN}" --namespace default
+
+kubectl get secret github-token -o yaml --namespace default