Merge branch 'master' into use-duty-types

.github/workflows/test.yml

@@ -17,6 +17,7 @@ jobs:
           - k8s
           - datasources
           - git
+          # - restic
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -35,11 +36,6 @@ jobs:
           restore-keys: |
             cache-
       - run: make bin
-      - name: Install restic
-        run: |
-          sudo apt update
-          sudo apt install -y restic
-          sudo restic self-update
       - name: Test
         env:
           KUBERNETES_VERSION: v1.20.7

Dockerfile

@@ -32,33 +32,40 @@ RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key
       fonts-freefont-ttf \
       --no-install-recommends
 
+  RUN apt-get update && apt-get upgrade -y && \
+      rm -Rf /var/lib/apt/lists/* && \
+      apt-get clean
 
-RUN curl -L https://github.com/restic/restic/releases/download/v0.12.0/restic_0.12.0_linux_amd64.bz2 -o restic.bz2 && \
+ENV RESTIC_VERSION=0.15.2
+RUN curl -L https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2 -o restic.bz2 && \
   bunzip2  /app/restic.bz2 && \
   chmod +x /app/restic && \
   mv /app/restic /usr/local/bin/ && \
   rm -rf /app/restic.bz2
 
-#Install jmeter
-RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-5.4.3.zip -o apache-jmeter-5.4.3.zip && \
-  unzip apache-jmeter-5.4.3.zip -d /opt && \
-  rm apache-jmeter-5.4.3.zip
+ENV JMETER_VERSION=5.5
+RUN curl -L https://dlcdn.apache.org//jmeter/binaries/apache-jmeter-${JMETER_VERSION}.zip -o apache-jmeter-${JMETER_VERSION}.zip && \
+  unzip apache-jmeter-${JMETER_VERSION}.zip -d /opt && \
+  rm apache-jmeter-${JMETER_VERSION}.zip
 
-ENV PATH /opt/apache-jmeter-5.4.3/bin/:$PATH
+ENV PATH /opt/apache-jmeter-${JMETER_VERSION}/bin/:$PATH
 
 
 RUN curl -L https://github.com/flanksource/askgit/releases/download/v0.4.8-flanksource/askgit-linux-amd64.tar.gz -o askgit.tar.gz && \
     tar xf askgit.tar.gz && \
     mv askgit /usr/local/bin/askgit && \
-    rm askgit.tar.gz
+    rm askgit.tar.gz && \
+    wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.18_amd64.deb && \
+    dpkg -i libssl1.1_1.1.1f-1ubuntu2.18_amd64.deb && \
+    rm libssl1.1_1.1.1f-1ubuntu2.18_amd64.deb
 
 ENV K6_VERSION=v0.44.0
 RUN curl -L https://github.com/grafana/k6/releases/download/${K6_VERSION}/k6-${K6_VERSION}-linux-amd64.tar.gz -o k6.tar.gz && \
     tar xvf k6.tar.gz && \
     mv k6-${K6_VERSION}-linux-amd64/k6 /usr/local/bin/k6 && \
     rm k6.tar.gz
 
-RUN curl -Lsf https://sh.benthos.dev | bash -s -- 3.56.0
+RUN curl -Lsf https://sh.benthos.dev | bash -s -- 4.15.0
 
 RUN curl -L https://github.com/multiprocessio/dsq/releases/download/v0.23.0/dsq-linux-x64-v0.23.0.zip -o dsq.zip && \
     unzip dsq.zip && \

Makefile

@@ -201,16 +201,16 @@ endif
 		ln -s apache-jmeter-5.4.3/bin/jmeter .bin/jmeter
 
 .bin/restic:
-	wget -nv  https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_$(OS)_$(ARCH).bz2 -O .bin/restic.bz2 && \
-    bunzip2  .bin/restic.bz2 && \
-    chmod +x .bin/restic
+	curl -sSLo /usr/local/bin/restic.bz2  https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_$(OS)_$(ARCH).bz2   && \
+    bunzip2  /usr/local/bin/restic.bz2  && \
+    chmod +x /usr/local/bin/restic
 
 .bin/wait4x:
 	wget -nv https://github.com/atkrad/wait4x/releases/download/v0.3.0/wait4x-$(OS)-$(ARCH) -O .bin/wait4x && \
   chmod +x .bin/wait4x
 
 .bin/karina:
-	wget -q https://github.com/flanksource/karina/releases/download/v0.50.0/karina_$(OS)-$(ARCH) -O .bin/karina && \
+	curl -sSLo .bin/karina https://github.com/flanksource/karina/releases/download/v0.50.0/karina_$(OS)-$(ARCH) && \
 	chmod +x .bin/karina
 
 .bin/yq: .bin
@@ -224,8 +224,11 @@ ifeq ($(OS), darwin)
 	brew install --cask macfuse
 	brew install datawire/blackbird/telepresence-legacy
 else
-	sudo curl -fL https://app.getambassador.io/download/tel2/linux/amd64/latest/telepresence -o /usr/local/bin/telepresence
-	sudo chmod a+x /usr/local/bin/telepresence
+	sudo apt-get install -y conntrack
+	wget https://s3.amazonaws.com/datawire-static-files/telepresence/telepresence-0.109.tar.gz
+	tar xzf telepresence-0.109.tar.gz
+	sudo mv telepresence-0.109/bin/telepresence /usr/local/bin/
+	sudo mv telepresence-0.109/libexec/sshuttle-telepresence /usr/local/bin/
 endif
 endif
 

api/context/context.go

@@ -2,12 +2,17 @@ package context
 
 import (
 	gocontext "context"
+	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/commons/logger"
+	"github.com/flanksource/duty"
+	"github.com/flanksource/duty/models"
 	"github.com/flanksource/kommons"
+	"gorm.io/gorm"
 )
 
 type KubernetesContext struct {
@@ -25,6 +30,7 @@ type Context struct {
 	Canary      v1.Canary
 	Environment map[string]interface{}
 	logger.Logger
+	db *gorm.DB
 }
 
 func (ctx *Context) String() string {
@@ -41,6 +47,37 @@ func (ctx *Context) WithDeadline(deadline time.Time) (*Context, gocontext.Cancel
 	return ctx, fn
 }
 
+func (ctx *Context) HydrateConnectionByURL(connectionName string) (*models.Connection, error) {
+	if !strings.HasPrefix(connectionName, "connection://") {
+		return nil, nil
+	}
+	if connectionName == "" {
+		return nil, nil
+	}
+
+	if ctx.db == nil {
+		return nil, errors.New("db has not been initialized")
+	}
+
+	k8sClient, err := ctx.Kommons.GetClientset()
+	if err != nil {
+		return nil, err
+	}
+
+	connection, err := duty.HydratedConnectionByURL(ctx, ctx.db, k8sClient, ctx.Namespace, connectionName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Connection name was explicitly provided but was not found.
+	// That's an error.
+	if connection == nil {
+		return nil, fmt.Errorf("connection %s not found", connectionName)
+	}
+
+	return connection, nil
+}
+
 func NewKubernetesContext(client *kommons.Client, namespace string) *KubernetesContext {
 	if namespace == "" {
 		namespace = "default"
@@ -64,11 +101,13 @@ func (ctx *KubernetesContext) Clone() *KubernetesContext {
 	}
 }
 
-func New(client *kommons.Client, canary v1.Canary) *Context {
+func New(client *kommons.Client, db *gorm.DB, canary v1.Canary) *Context {
 	if canary.Namespace == "" {
 		canary.Namespace = "default"
 	}
+
 	return &Context{
+		db:          db,
 		Context:     gocontext.Background(),
 		Kommons:     client,
 		Namespace:   canary.GetNamespace(),
@@ -86,6 +125,18 @@ func (ctx *Context) IsTrace() bool {
 	return ctx.Canary.IsTrace()
 }
 
+func (ctx *Context) Debugf(format string, args ...interface{}) {
+	if ctx.IsDebug() {
+		ctx.Logger.Infof(format, args...)
+	}
+}
+
+func (ctx *Context) Tracef(format string, args ...interface{}) {
+	if ctx.IsTrace() {
+		ctx.Logger.Infof(format, args...)
+	}
+}
+
 func (ctx *Context) New(environment map[string]interface{}) *Context {
 	return &Context{
 		Context:     ctx.Context,

api/v1/canary_types.go

@@ -72,6 +72,7 @@ type CanarySpec struct {
 	ConfigDB       []ConfigDBCheck       `yaml:"configDB,omitempty" json:"configDB,omitempty"`
 	Elasticsearch  []ElasticsearchCheck  `yaml:"elasticsearch,omitempty" json:"elasticsearch,omitempty"`
 	AlertManager   []AlertManagerCheck   `yaml:"alertmanager,omitempty" json:"alertmanager,omitempty"`
+	Dynatrace      []DynatraceCheck      `yaml:"dynatrace,omitempty" json:"dynatrace,omitempty"`
 	AzureDevops    []AzureDevopsCheck    `yaml:"azureDevops,omitempty" json:"azureDevops,omitempty"`
 	// interval (in seconds) to run checks on Deprecated in favor of Schedule
 	Interval uint64 `yaml:"interval,omitempty" json:"interval,omitempty"`
@@ -191,6 +192,9 @@ func (spec CanarySpec) GetAllChecks() []external.Check {
 	for _, check := range spec.AzureDevops {
 		checks = append(checks, check)
 	}
+	for _, check := range spec.Dynatrace {
+		checks = append(checks, check)
+	}
 	return checks
 }
 
@@ -205,7 +209,7 @@ func (spec CanarySpec) GetSchedule() string {
 }
 
 func (c Canary) IsTrace() bool {
-	return c.Annotations != nil && c.Annotations["debug"] == "true" //nolint
+	return c.Annotations != nil && c.Annotations["trace"] == "true" //nolint
 }
 
 func (c Canary) IsDebug() bool {