feat: add support for env vars & mounting in exec check (#1353)

* feat: add support for env vars in exec check * feat: impl checkout on exec action * chore: rename fixtures. add _pass suffix * chore: use connection's AsGoGetterURL * bump: duty * fix: build * chore: expect aws exec test to fail
flanksource · Oct 19, 2023 · 603e32c · 603e32c
1 parent d93e99a
commit 603e32c
Show file tree

Hide file tree

Showing 19 changed files with 646 additions and 123 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,3 +20,4 @@ postgres-db/
 ui/scripts/
 Chart.lock
 chart/charts/
+.downloads
diff --git a/api/v1/checks.go b/api/v1/checks.go
@@ -968,13 +968,28 @@ type ExecConnections struct {
 	Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
 }
 
+type GitCheckout struct {
+	URL         string       `yaml:"url,omitempty" json:"url,omitempty"`
+	Connection  string       `yaml:"connection,omitempty" json:"connection,omitempty"`
+	Username    types.EnvVar `yaml:"username,omitempty" json:"username,omitempty"`
+	Password    types.EnvVar `yaml:"password,omitempty" json:"password,omitempty"`
+	Certificate types.EnvVar `yaml:"certificate,omitempty" json:"certificate,omitempty"`
+	// Destination is the full path to where the contents of the URL should be downloaded to.
+	// If left empty, the sha256 hash of the URL will be used as the dir name.
+	Destination string `yaml:"destination,omitempty" json:"destination,omitempty"`
+}
+
 type ExecCheck struct {
 	Description `yaml:",inline" json:",inline"`
 	Templatable `yaml:",inline" json:",inline"`
 	// Script can be a inline script or a path to a script that needs to be executed
 	// On windows executed via powershell and in darwin and linux executed using bash
 	Script      string          `yaml:"script" json:"script"`
 	Connections ExecConnections `yaml:"connections,omitempty" json:"connections,omitempty"`
+	// EnvVars are the environment variables that are accesible to exec processes
+	EnvVars []types.EnvVar `yaml:"env,omitempty" json:"env,omitempty"`
+	// Checkout details the git repository that should be mounted to the process
+	Checkout *GitCheckout `yaml:"checkout,omitempty" json:"checkout,omitempty"`
 }
 
 func (c ExecCheck) GetType() string {

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/checks/exec.go b/checks/exec.go
@@ -3,19 +3,19 @@ package checks
 import (
 	"bytes"
 	"fmt"
-	"math/rand"
 	"os"
 	osExec "os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
-	textTemplate "text/template"
 
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/canary-checker/api/external"
 	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg"
-	"github.com/flanksource/commons/logger"
+	"github.com/flanksource/commons/files"
+	"github.com/flanksource/commons/hash"
+	"github.com/flanksource/duty/models"
 )
 
 type ExecChecker struct {
@@ -36,102 +36,179 @@ func (c *ExecChecker) Run(ctx *context.Context) pkg.Results {
 	for _, conf := range ctx.Canary.Spec.Exec {
 		results = append(results, c.Check(ctx, conf)...)
 	}
+
 	return results
 }
 
+type execEnv struct {
+	envs       []string
+	mountPoint string
+}
+
+func (c *ExecChecker) prepareEnvironment(ctx *context.Context, check v1.ExecCheck) (*execEnv, error) {
+	var result execEnv
+
+	for _, env := range check.EnvVars {
+		val, err := ctx.GetEnvValueFromCache(env)
+		if err != nil {
+			return nil, fmt.Errorf("error fetching env value (name=%s): %w", env.Name, err)
+		}
+
+		result.envs = append(result.envs, fmt.Sprintf("%s=%s", env.Name, val))
+	}
+
+	if check.Checkout != nil {
+		sourceURL := check.Checkout.URL
+
+		if connection, err := ctx.HydrateConnectionByURL(check.Checkout.Connection); err != nil {
+			return nil, fmt.Errorf("error hydrating connection: %w", err)
+		} else if connection != nil {
+			goGetterURL, err := connection.AsGoGetterURL()
+			if err != nil {
+				return nil, fmt.Errorf("error getting go getter URL: %w", err)
+			}
+			sourceURL = goGetterURL
+		}
+
+		if sourceURL == "" {
+			return nil, fmt.Errorf("error checking out. missing URL")
+		}
+
+		result.mountPoint = check.Checkout.Destination
+		if result.mountPoint == "" {
+			pwd, _ := os.Getwd()
+			result.mountPoint = filepath.Join(pwd, ".downloads", hash.Sha256Hex(sourceURL))
+		}
+
+		if err := files.Getter(sourceURL, result.mountPoint); err != nil {
+			return nil, fmt.Errorf("error checking out %s: %w", sourceURL, err)
+		}
+	}
+
+	return &result, nil
+}
+
 func (c *ExecChecker) Check(ctx *context.Context, extConfig external.Check) pkg.Results {
 	check := extConfig.(v1.ExecCheck)
+
+	env, err := c.prepareEnvironment(ctx, check)
+	if err != nil {
+		return []*pkg.CheckResult{pkg.Fail(check, ctx.Canary).Failf("something went wrong while preparing exec env: %v", err)}
+	}
+
 	switch runtime.GOOS {
 	case "windows":
-		return execPowershell(check, ctx)
+		return execPowershell(ctx, check, env)
 	default:
-		return execBash(check, ctx)
+		return execBash(ctx, check, env)
 	}
 }
 
-func execPowershell(check v1.ExecCheck, ctx *context.Context) pkg.Results {
+func execPowershell(ctx *context.Context, check v1.ExecCheck, envParams *execEnv) pkg.Results {
 	result := pkg.Success(check, ctx.Canary)
 	ps, err := osExec.LookPath("powershell.exe")
 	if err != nil {
 		result.Failf("powershell not found")
 	}
+
 	args := []string{check.Script}
-	cmd := osExec.Command(ps, args...)
+	cmd := osExec.CommandContext(ctx, ps, args...)
+	if len(envParams.envs) != 0 {
+		cmd.Env = append(os.Environ(), envParams.envs...)
+	}
+	if envParams.mountPoint != "" {
+		cmd.Dir = envParams.mountPoint
+	}
+
+	return runCmd(cmd, result)
+}
+
+func execBash(ctx *context.Context, check v1.ExecCheck, envParams *execEnv) pkg.Results {
+	result := pkg.Success(check, ctx.Canary)
+	fields := strings.Fields(check.Script)
+	if len(fields) == 0 {
+		return []*pkg.CheckResult{result.Failf("no script provided")}
+	}
+
+	cmd := osExec.CommandContext(ctx, "bash", "-c", check.Script)
+	if len(envParams.envs) != 0 {
+		cmd.Env = append(os.Environ(), envParams.envs...)
+	}
+	if envParams.mountPoint != "" {
+		cmd.Dir = envParams.mountPoint
+	}
+
+	if err := setupConnection(ctx, check, cmd); err != nil {
+		return []*pkg.CheckResult{result.Failf("failed to setup connection: %v", err)}
+	}
+
 	return runCmd(cmd, result)
 }
 
 func setupConnection(ctx *context.Context, check v1.ExecCheck, cmd *osExec.Cmd) error {
+	var envPreps []models.EnvPrep
+
 	if check.Connections.AWS != nil {
 		if err := check.Connections.AWS.Populate(ctx, ctx.Kubernetes, ctx.Namespace); err != nil {
 			return fmt.Errorf("failed to hydrate aws connection: %w", err)
 		}
 
-		configPath, err := saveConfig(awsConfigTemplate, check.Connections.AWS)
-		defer os.RemoveAll(filepath.Dir(configPath))
-		if err != nil {
-			return fmt.Errorf("failed to store AWS credentials: %w", err)
-		}
-
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, "AWS_EC2_METADATA_DISABLED=true") // https://github.com/aws/aws-cli/issues/5262#issuecomment-705832151
-		cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%s", configPath))
-		if check.Connections.AWS.Region != "" {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("AWS_DEFAULT_REGION=%s", check.Connections.AWS.Region))
+		c := models.Connection{
+			Type:     models.ConnectionTypeAWS,
+			Username: check.Connections.AWS.AccessKey.ValueStatic,
+			Password: check.Connections.AWS.SecretKey.ValueStatic,
+			Properties: map[string]string{
+				"region": check.Connections.AWS.Region,
+			},
 		}
+		envPreps = append(envPreps, c.AsEnv(ctx))
 	}
 
 	if check.Connections.Azure != nil {
 		if err := check.Connections.Azure.HydrateConnection(ctx); err != nil {
 			return fmt.Errorf("failed to hydrate connection %w", err)
 		}
 
-		// login with service principal
-		runCmd := osExec.Command("az", "login", "--service-principal", "--username", check.Connections.Azure.ClientID.ValueStatic, "--password", check.Connections.Azure.ClientSecret.ValueStatic, "--tenant", check.Connections.Azure.TenantID)
-		if err := runCmd.Run(); err != nil {
-			return fmt.Errorf("failed to login: %w", err)
+		c := models.Connection{
+			Type:     models.ConnectionTypeAzure,
+			Username: check.Connections.Azure.ClientID.ValueStatic,
+			Password: check.Connections.Azure.ClientSecret.ValueStatic,
+			Properties: map[string]string{
+				"tenant": check.Connections.Azure.TenantID,
+			},
 		}
+		envPreps = append(envPreps, c.AsEnv(ctx))
 	}
 
 	if check.Connections.GCP != nil {
 		if err := check.Connections.GCP.HydrateConnection(ctx); err != nil {
 			return fmt.Errorf("failed to hydrate connection %w", err)
 		}
 
-		configPath, err := saveConfig(gcloudConfigTemplate, check.Connections.GCP)
-		defer os.RemoveAll(filepath.Dir(configPath))
-		if err != nil {
-			return fmt.Errorf("failed to store gcloud credentials: %w", err)
+		c := models.Connection{
+			Type:        models.ConnectionTypeGCP,
+			Certificate: check.Connections.GCP.Credentials.ValueStatic,
+			URL:         check.Connections.GCP.Endpoint,
 		}
+		envPreps = append(envPreps, c.AsEnv(ctx))
+	}
 
-		// to configure gcloud CLI to use the service account specified in GOOGLE_APPLICATION_CREDENTIALS,
-		// we need to explicitly activate it
-		runCmd := osExec.Command("gcloud", "auth", "activate-service-account", "--key-file", configPath)
-		if err := runCmd.Run(); err != nil {
-			return fmt.Errorf("failed to activate GCP service account: %w", err)
+	for _, envPrep := range envPreps {
+		preRuns, err := envPrep.Inject(ctx, cmd)
+		if err != nil {
+			return err
 		}
 
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("GOOGLE_APPLICATION_CREDENTIALS=%s", configPath))
+		for _, run := range preRuns {
+			if err := run.Run(); err != nil {
+				return err
+			}
+		}
 	}
 
 	return nil
 }
 
-func execBash(check v1.ExecCheck, ctx *context.Context) pkg.Results {
-	result := pkg.Success(check, ctx.Canary)
-	fields := strings.Fields(check.Script)
-	if len(fields) == 0 {
-		return []*pkg.CheckResult{result.Failf("no script provided")}
-	}
-
-	cmd := osExec.Command("bash", "-c", check.Script)
-	if err := setupConnection(ctx, check, cmd); err != nil {
-		return []*pkg.CheckResult{result.Failf("failed to setup connection: %v", err)}
-	}
-
-	return runCmd(cmd, result)
-}
-
 func runCmd(cmd *osExec.Cmd, result *pkg.CheckResult) (results pkg.Results) {
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
@@ -151,40 +228,3 @@ func runCmd(cmd *osExec.Cmd, result *pkg.CheckResult) (results pkg.Results) {
 	results = append(results, result)
 	return results
 }
-
-func saveConfig(configTemplate *textTemplate.Template, view any) (string, error) {
-	dirPath := filepath.Join(".creds", fmt.Sprintf("cred-%d", rand.Intn(10000000)))
-	if err := os.MkdirAll(dirPath, 0700); err != nil {
-		return "", err
-	}
-
-	configPath := fmt.Sprintf("%s/credentials", dirPath)
-	logger.Tracef("Creating credentials file: %s", configPath)
-
-	file, err := os.Create(configPath)
-	if err != nil {
-		return configPath, err
-	}
-	defer file.Close()
-
-	if err := configTemplate.Execute(file, view); err != nil {
-		return configPath, err
-	}
-
-	return configPath, nil
-}
-
-var (
-	awsConfigTemplate    *textTemplate.Template
-	gcloudConfigTemplate *textTemplate.Template
-)
-
-func init() {
-	awsConfigTemplate = textTemplate.Must(textTemplate.New("").Parse(`[default]
-aws_access_key_id = {{.AccessKey.ValueStatic}}
-aws_secret_access_key = {{.SecretKey.ValueStatic}}
-{{if .SessionToken.ValueStatic}}aws_session_token={{.SessionToken.ValueStatic}}{{end}}
-`))
-
-	gcloudConfigTemplate = textTemplate.Must(textTemplate.New("").Parse(`{{.Credentials}}`))
-}
diff --git a/checks/http.go b/checks/http.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/flanksource/canary-checker/api/context"
 	"github.com/flanksource/commons/http"
+	"github.com/flanksource/commons/http/middlewares"
 	"github.com/flanksource/duty/models"
 	gomplate "github.com/flanksource/gomplate/v3"
 
@@ -80,7 +81,12 @@ func (c *HTTPChecker) generateHTTPRequest(ctx *context.Context, check v1.HTTPChe
 	}
 
 	if check.Oauth2 != nil {
-		client.OAuth(connection.Username, connection.Password, check.Oauth2.TokenURL, check.Oauth2.Scopes...)
+		client.OAuth(middlewares.OauthConfig{
+			ClientID:     connection.Username,
+			ClientSecret: connection.Password,
+			TokenURL:     check.Oauth2.TokenURL,
+			Scopes:       check.Oauth2.Scopes,
+		})
 	}
 
 	client.NTLM(check.NTLM)