Merge pull request #319 from flanksource/fix-errors

CI fixes

.github/workflows/lint.yml

@@ -43,7 +43,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
       - name: setup node

.github/workflows/test.yml

@@ -27,9 +27,6 @@ jobs:
           - minimal --skip-all
           - k8s
           - datasources
-          - quarantine
-          - restic
-          - s3
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -48,6 +45,10 @@ jobs:
           restore-keys: |
             cache-
       - run: make bin
+      - name: Install restic
+        run: |
+          sudo apt-get install restic
+          sudo restic self-update
       - name: Test
         env:
           KUBERNETES_VERSION: v1.20.7

Makefile

@@ -170,7 +170,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/[email protected] ;\
+	go install sigs.k8s.io/controller-tools/cmd/[email protected] ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen
@@ -187,7 +187,7 @@ ifeq (, $(shell which kustomize))
 	KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$KUSTOMIZE_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/kustomize/kustomize/[email protected] ;\
+	go install sigs.k8s.io/kustomize/kustomize/[email protected] ;\
 	rm -rf $$KUSTOMIZE_GEN_TMP_DIR ;\
 	}
 KUSTOMIZE=$(GOBIN)/kustomize
@@ -201,7 +201,7 @@ endif
 	_TMP_DIR=$$(mktemp -d) ;\
 	cd $$_TMP_DIR ;\
 	go mod init tmp   ;\
-	go get github.com/jstemmer/go-junit-report   ;\
+	go install github.com/jstemmer/go-junit-report   ;\
 	rm -rf $$_TMP_DIR
 	cp $(GOBIN)/go-junit-report .bin/go-junit-report ;\
 
@@ -212,7 +212,7 @@ endif
 		ln -s apache-jmeter-5.4.1/bin/jmeter .bin/jmeter
 
 .bin/restic:
-	wget -nv  https://github.com/restic/restic/releases/download/v0.12.0/restic_0.12.0_$(OS)_$(ARCH).bz2 -O .bin/restic.bz2 && \
+	wget -nv  https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_$(OS)_$(ARCH).bz2 -O .bin/restic.bz2 && \
     bunzip2  .bin/restic.bz2 && \
     chmod +x .bin/restic
 
@@ -224,7 +224,7 @@ endif
 	wget -q https://github.com/flanksource/karina/releases/download/v0.50.0/karina_$(OS)-$(ARCH) -O .bin/karina && \
 	chmod +x .bin/karina
 
-.bin/yq:
+.bin/yq: .bin
 	curl -sSLo .bin/yq https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_$(OS)_$(ARCH) && chmod +x .bin/yq
 YQ = $(realpath ./.bin/yq)
 

api/v1/checks.go

@@ -144,8 +144,6 @@ type S3BucketCheck struct {
 	FolderTest    `yaml:",inline" json:",inline"`
 	Filter        FolderFilter `yaml:"filter,omitempty" json:"filter,omitempty"`
 	Bucket        string       `yaml:"bucket" json:"bucket"`
-	// glob path to restrict matches to a subset
-	ObjectPath string `yaml:"objectPath" json:"objectPath,omitempty"`
 	// Use path style path: http://s3.amazonaws.com/BUCKET/KEY instead of http://BUCKET.s3.amazonaws.com/KEY
 	UsePathStyle bool `yaml:"usePathStyle" json:"usePathStyle,omitempty"`
 }
@@ -464,7 +462,7 @@ type HelmCheck struct {
 	Chartmuseum string          `yaml:"chartmuseum" json:"chartmuseum,omitempty"`
 	Project     string          `yaml:"project,omitempty" json:"project,omitempty"`
 	Auth        *Authentication `yaml:"auth,omitempty" json:"auth,omitempty"`
-	CaFile      *string         `yaml:"cafile,omitempty" json:"cafile,omitempty"`
+	CaFile      string          `yaml:"cafile,omitempty" json:"cafile,omitempty"`
 }
 
 func (c HelmCheck) GetEndpoint() string {

checks/helm.go

@@ -44,8 +44,8 @@ func (c *HelmChecker) Run(ctx *context.Context) []*pkg.CheckResult {
 func (c *HelmChecker) Check(ctx *context.Context, extConfig external.Check) *pkg.CheckResult {
 	config := extConfig.(v1.HelmCheck)
 	start := time.Now()
+	result := pkg.Success(config)
 	var uploadOK, downloadOK = true, true
-	chartmuseum := fmt.Sprintf("%s/chartrepo/%s/", config.Chartmuseum, config.Project)
 	logger.Tracef("Uploading test chart")
 	namespace := ctx.Canary.Namespace
 	var err error
@@ -54,95 +54,50 @@ func (c *HelmChecker) Check(ctx *context.Context, extConfig external.Check) *pkg
 		return Failf(config, "failed to fetch auth details: %v", err)
 	}
 	client, _ := pusher.NewClient(
-		pusher.URL(chartmuseum),
+		pusher.URL(config.Chartmuseum),
 		pusher.Username(auth.Username.Value),
 		pusher.Password(auth.Password.Value),
 		pusher.ContextPath(""),
 		pusher.Timeout(60),
-		pusher.CAFile(*config.CaFile))
+		pusher.CAFile(config.CaFile))
 	chartPath, err := createTestChart()
 	if err != nil {
-		return &pkg.CheckResult{
-			Pass:     false,
-			Check:    config,
-			Invalid:  true,
-			Duration: 0,
-			Message:  fmt.Sprintf("Failed to create test chart: %v", err),
-		}
+		return result.ErrorMessage(err).StartTime(start)
 	}
 	response, err := client.UploadChartPackage(*chartPath, false)
 	if err != nil {
-		return &pkg.CheckResult{
-			Check:    config,
-			Pass:     false,
-			Invalid:  true,
-			Duration: 0,
-			Message:  fmt.Sprintf("Failed to check: %v", err),
-		}
+		return result.ErrorMessage(err).StartTime(start)
 	}
-
 	defer func() {
 		response.Close = true
 	}()
-
 	if response.StatusCode != 201 {
-		return &pkg.CheckResult{
-			Check:    config,
-			Pass:     false,
-			Invalid:  false,
-			Duration: 0,
-			Message:  "Failed to push test chart",
-		}
-	}
-
-	if err != nil {
-		return &pkg.CheckResult{
-			Check:    config,
-			Pass:     false,
-			Invalid:  true,
-			Duration: 0,
-			Message:  fmt.Sprintf("Failed to get user: %v", err),
-		}
+		return result.ErrorMessage(fmt.Errorf("failed to upload test chart. Error code: %d", response.StatusCode)).StartTime(start)
 	}
 
 	defer os.RemoveAll("./test-chart-0.1.0.tgz") // nolint: errcheck
 
 	iCli := action.NewPull()
-	if config.CaFile != nil {
-		iCli.CaFile = *config.CaFile
+	if config.CaFile != "" {
+		iCli.CaFile = config.CaFile
 	}
 	kubeconfigPath := pkg.GetKubeconfig()
 	iCli.Settings = &cli.EnvSettings{
 		KubeConfig: kubeconfigPath,
 	}
-
 	logger.Tracef("Pulling test chart")
-	url, err := url.Parse(chartmuseum)
+	url, err := url.Parse(config.Chartmuseum)
 	if err != nil {
-		return &pkg.CheckResult{
-			Check:    config,
-			Pass:     false,
-			Invalid:  true,
-			Duration: 0,
-			Message:  fmt.Sprintf("Failed to parse chartmuseum url: %v", err),
-		}
+		return result.ErrorMessage(err).StartTime(start)
 	}
 	url.Path = path.Join(url.Path, "charts/test-chart-0.1.0.tgz")
 	_, err = iCli.Run(url.String())
 	if err != nil {
-		return &pkg.CheckResult{
-			Check:    config,
-			Pass:     false,
-			Invalid:  false,
-			Duration: 0,
-		}
+		return result.ErrorMessage(err).StartTime(start)
 	}
 
-	defer cleanUp("test-chart", chartmuseum, config, auth.Username.Value, auth.Password.Value) // nolint: errcheck
+	defer cleanUp("test-chart", config.Chartmuseum, config, auth.Username.Value, auth.Password.Value) // nolint: errcheck
 
-	if err != nil {
-		logger.Warnf("Failed to perform cleanup: %v", err)
-	}
 	elapsed := time.Since(start)
 	return &pkg.CheckResult{
 		Check:    config,
@@ -153,7 +108,7 @@ func (c *HelmChecker) Check(ctx *context.Context, extConfig external.Check) *pkg
 }
 
 func cleanUp(chartname string, chartmuseum string, config v1.HelmCheck, username, password string) error {
-	caCert, err := ioutil.ReadFile(*config.CaFile)
+	caCert, err := ioutil.ReadFile(config.CaFile)
 	if err != nil {
 		return fmt.Errorf("failed to read certificate file: %v", err)
 	}

checks/junit.go

@@ -65,7 +65,7 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod {
 	pod.Spec = check.Spec
 	for _, container := range pod.Spec.Containers {
 		if len(container.Command) > 0 {
-			// attemp to wrap the command so that it always completes, allowing for access to junit results
+			// attempt to wrap the command so that it always completes, allowing for access to junit results
 			container.Args = []string{fmt.Sprintf(`
 			set -e
 			EXIT_CODE=0
@@ -115,7 +115,7 @@ func newPod(ctx *context.Context, check v1.JunitCheck) *corev1.Pod {
 }
 
 func deletePod(ctx *context.Context, pod *corev1.Pod) {
-	if ctx.Canary.Annotations["skipDelete"] == "true" {
+	if ctx.Canary.Annotations["skipDelete"] == "true" { // nolint: goconst
 		return
 	}
 	if err := ctx.Kommons.DeleteByKind(podKind, pod.Namespace, pod.Name); err != nil {

checks/s3_bucket.go

@@ -101,7 +101,7 @@ type S3 struct {
 	Bucket string
 }
 
-func (conn *S3) CheckFolder(ctx *context.Context, path string, filter v1.FolderFilter) (*FolderCheck, error) {
+func (conn *S3) CheckFolder(ctx *context.Context, filter v1.FolderFilter) (*FolderCheck, error) {
 	result := FolderCheck{}
 
 	var marker *string = nil
@@ -180,7 +180,7 @@ func (c *S3BucketChecker) Check(ctx *context.Context, extConfig external.Check)
 		}),
 		Bucket: bucket.Bucket,
 	}
-	folders, err := client.CheckFolder(ctx, bucket.ObjectPath, bucket.Filter)
+	folders, err := client.CheckFolder(ctx, bucket.Filter)
 	if err != nil {
 		return result.ErrorMessage(fmt.Errorf("failed to retrieve s3://%s: %v", bucket.Bucket, err))
 	}

config/deploy/crd.yaml

@@ -5666,9 +5666,6 @@ spec:
                       name:
                         description: Name of the check
                         type: string
-                      objectPath:
-                        description: glob path to restrict matches to a subset
-                        type: string
                       region:
                         type: string
                       secretKey:

config/deploy/manifests.yaml

@@ -5673,9 +5673,6 @@ spec:
                       name:
                         description: Name of the check
                         type: string
-                      objectPath:
-                        description: glob path to restrict matches to a subset
-                        type: string
                       region:
                         type: string
                       secretKey:

fixtures/restic/_setup.sh → fixtures/datasources/_setup.sh

@@ -1,7 +1,10 @@
+#!/bin/bash
+
+set -e
 
 restic version
 # Initialize Restic Repo
 # Do not fail if it already exists
 RESTIC_PASSWORD="S0m3p@sswd" AWS_ACCESS_KEY_ID="minio" AWS_SECRET_ACCESS_KEY="minio123" restic --cacert .certs/ingress-ca.crt -r s3:https://minio.${DOMAIN}/restic-canary-checker init || true
 #take some backup in restic
-RESTIC_PASSWORD="S0m3p@sswd" AWS_ACCESS_KEY_ID="minio" AWS_SECRET_ACCESS_KEY="minio123" restic --cacert .certs/ingress-ca.crt -r s3:https://minio.${DOMAIN}/restic-canary-checker backup $(pwd)
+RESTIC_PASSWORD="S0m3p@sswd" AWS_ACCESS_KEY_ID="minio" AWS_SECRET_ACCESS_KEY="minio123" restic --cacert .certs/ingress-ca.crt -r s3:https://minio.${DOMAIN}/restic-canary-checker backup "$(pwd)"

fixtures/datasources/_setup.yaml

@@ -548,3 +548,65 @@ kind: ConfigMap
 metadata:
   name: jmeter-fail-test
   namespace: default
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: chartmuseum
+  name: chartmuseum
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: chartmuseum
+  template:
+    metadata:
+      labels:
+        app: chartmuseum
+    spec:
+      containers:
+        - image: ghcr.io/helm/chartmuseum:v0.13.1
+          name: chartmuseum
+          imagePullPolicy: "IfNotPresent"
+          ports:
+            - containerPort: 8080
+          env:
+            - name: STORAGE
+              value: local
+            - name: STORAGE_LOCAL_ROOTDIR
+              value: /tmp/charts
+            - name: BASIC_AUTH_USER
+              value: admin
+            - name: BASIC_AUTH_PASS
+              value: passwd
+            - name: ALLOW_OVERWRITE
+              value: "true"
+            - name: AUTH_ANONYMOUS_GET
+              value: "true"
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: chartmuseum
+  namespace: default
+spec:
+  ports:
+    - name: chartmuseum
+      port: 8080
+      targetPort: 8080
+      protocol: TCP
+  selector:
+    app: chartmuseum
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: aws-credentials
+  namespace: default
+stringData:
+  AWS_ACCESS_KEY_ID: minio
+  AWS_SECRET_ACCESS_KEY: minio123
+---

fixtures/datasources/helm_pass.yaml

@@ -0,0 +1,14 @@
+apiVersion: canaries.flanksource.com/v1
+kind: Canary
+metadata:
+  name: helm-pass
+spec:
+  interval: 30
+  helm:
+    - chartmuseum: http://chartmuseum.default:8080
+      project: library
+      auth:
+        username:
+          value: admin
+        password:
+          value: passwd

fixtures/restic/restic_fail.yaml → fixtures/datasources/restic_fail.yaml

@@ -1,5 +1,5 @@
 restic:
-  - repository: s3:https://minio.127.0.0.1.nip.io/restic-canary-checker
+  - repository: s3:http://minio.minio:9000/restic-canary-checker
     password:
       value: S0m3p@sswd
     maxAge: 10s