[WIP] Feature/poc GitHub actions

cmd/job-executor-service/main.go

@@ -46,6 +46,8 @@ type envConfig struct {
 	DefaultResourceRequestsMemory string `envconfig:"DEFAULT_RESOURCE_REQUESTS_MEMORY"`
 	// Respond with .finished event if no configuration found
 	AlwaysSendFinishedEvent string `envconfig:"ALWAYS_SEND_FINISHED_EVENT"`
+	// Container Registry
+	ContainerRegistry string `envconfig:"CONTAINER_REGISTRY"`
 }
 
 // ServiceName specifies the current services name (e.g., used as source when sending CloudEvents)
@@ -104,6 +106,7 @@ func processKeptnCloudEvent(ctx context.Context, event cloudevents.Event) error
 			InitContainerImage:          env.InitContainerImage,
 			DefaultResourceRequirements: DefaultResourceRequirements,
 			AlwaysSendFinishedEvent:     false,
+			ContainerRegistry:           env.ContainerRegistry,
 		},
 	}
 

deploy/registry.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    app: registry
+  name: registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: registry
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: registry
+    spec:
+      containers:
+      - image: registry:2
+        name: registry
+        ports:
+        - containerPort: 5000
+        resources: {}
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    app: registry
+  name: registry
+spec:
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000
+  selector:
+    app: registry
+status:
+  loadBalancer: {}

deploy/service.yaml

@@ -46,6 +46,8 @@ spec:
               value: "50m"
             - name: DEFAULT_RESOURCE_REQUESTS_MEMORY
               value: "128Mi"
+            - name: CONTAINER_REGISTRY
+              value: "TODO"
         - name: distributor
           image: keptn/distributor:0.8.4
           livenessProbe:

go.mod

@@ -18,6 +18,7 @@ require (
 	golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4 // indirect
 	golang.org/x/tools v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	gotest.tools v2.2.0+incompatible
 	honnef.co/go/tools v0.1.3 // indirect
 	k8s.io/api v0.22.0

helm/templates/configmap.yaml

@@ -9,3 +9,4 @@ data:
   default_resource_limits_memory: "512Mi"
   default_resource_requests_cpu: "50m"
   default_resource_requests_memory: "128Mi"
+  container_registry: "TODO"

helm/templates/deployment.yaml

@@ -82,6 +82,11 @@ spec:
               configMapKeyRef:
                 name: job-service-config
                 key: default_resource_requests_memory
+          - name: CONTAINER_REGISTRY
+            valueFrom:
+             configMapKeyRef:
+               name: job-service-config
+               key: container_registry
           livenessProbe:
             httpGet:
               path: /health

pkg/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"gopkg.in/yaml.v2"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
 	"regexp"
 
 	"github.com/PaesslerAG/jsonpath"
@@ -18,10 +19,11 @@ type Config struct {
 
 // Action contains a action within the config which needs to be triggered
 type Action struct {
-	Name   string  `yaml:"name"`
-	Events []Event `yaml:"events"`
-	Tasks  []Task  `yaml:"tasks"`
-	Silent bool    `yaml:"silent"`
+	Name   string       `yaml:"name"`
+	Events []Event      `yaml:"events"`
+	Tasks  []Task       `yaml:"tasks"`
+	Silent bool         `yaml:"silent"`
+	Steps  []model.Step `yaml:"steps"`
 }
 
 // Event defines a keptn event which determines if an Action should be triggered
@@ -85,7 +87,20 @@ func NewConfig(yamlContent []byte) (*Config, error) {
 		return nil, fmt.Errorf("apiVersion %v is not supported, use %v", *config.APIVersion, supportedAPIVersion)
 	}
 
-	return &config, nil
+	err = config.checkConfigs()
+
+	return &config, err
+}
+
+func (c *Config) checkConfigs() error {
+
+	// check if the actions don't contain steps and tasks -> that should not be valid
+	for _, action := range c.Actions {
+		if len(action.Steps) > 0 && len(action.Tasks) > 0 {
+			return fmt.Errorf("action %v contains steps and tasks", action.Name)
+		}
+	}
+	return nil
 }
 
 // IsEventMatch indicated whether a given event matches the config

pkg/eventhandler/eventhandler_test.go

@@ -196,3 +196,10 @@ func TestStartK8sJobSilent(t *testing.T) {
 	err = fakeEventSender.AssertSentEventTypes([]string{})
 	assert.NilError(t, err)
 }
+
+func TestGetGithubProjectName(t *testing.T) {
+
+	eh := EventHandler{}
+	assert.Equal(t, "aquasecurity/trivy-action", eh.getGithubProjectName("aquasecurity/trivy-action@master"))
+	assert.Equal(t, "aquasecurity/trivy-action", eh.getGithubProjectName("aquasecurity/trivy-action"))
+}

pkg/eventhandler/eventhandlers.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"keptn-sandbox/job-executor-service/pkg/config"
+	"keptn-sandbox/job-executor-service/pkg/github"
 	"keptn-sandbox/job-executor-service/pkg/k8sutils"
 	"log"
 	"math"
 	"strconv"
+	"strings"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2" // make sure to use v2 cloudevents here
 	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
@@ -73,13 +75,88 @@ func (eh *EventHandler) HandleEvent() error {
 	}
 
 	log.Printf("Match found for event %s of type %s. Starting k8s job to run action '%s'", eh.Event.Context.GetID(), eh.Event.Type(), action.Name)
+	log.Printf("action: %+v", *action)
 
 	k8s := k8sutils.NewK8s(eh.JobSettings.JobNamespace)
+	err = k8s.ConnectToCluster()
+	if err != nil {
+		log.Printf("Error while connecting to cluster: %s\n", err)
+		return err
+	}
+
+	err = eh.handleGithubAction(k8s, action)
+	if err != nil {
+		log.Printf("An error occurred while handling GitHub action: %s", err)
+		return err
+	}
+
+	log.Printf("executing action: %+v", *action)
 	eh.startK8sJob(k8s, action, eventAsInterface)
 
 	return nil
 }
 
+func (eh *EventHandler) handleGithubAction(k8s k8sutils.K8s, action *config.Action) error {
+	for _, step := range action.Steps {
+
+		log.Printf("Handling step: %+v", step)
+
+		githubProjectName := eh.getGithubProjectName(step.Uses)
+
+		// TODO using step.Name here is a bad idea because it can contain whitespaces etc.
+		imageLocation, err := k8s.CreateImageBuilder(step.Name, githubProjectName, eh.JobSettings.ContainerRegistry)
+		log.Printf("imageLocation: %v", imageLocation)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err = k8s.DeleteK8sJob(step.Name)
+			if err != nil {
+				log.Printf("Error while deleting job: %s\n", err.Error())
+			}
+		}()
+
+		jobErr := k8s.AwaitK8sJobDone(step.Name, defaultMaxPollCount, pollIntervalInSeconds)
+		if jobErr != nil {
+			log.Println(err)
+		}
+
+		err, githubAction := github.GetActionYaml(githubProjectName)
+		if err != nil {
+			return err
+		}
+
+		args, err := github.PrepareArgs(step.With, githubAction.Inputs, githubAction.Runs.Args)
+		if err != nil {
+			return err
+		}
+
+		task := config.Task{
+			Name:  githubAction.Name,
+			Files: nil,
+			Image: imageLocation,
+			Cmd:   nil,
+			Args:  args,
+			Env:   nil,
+		}
+
+		log.Printf("task: %+v", task)
+
+		action.Tasks = append(action.Tasks, task)
+	}
+	return nil
+}
+
+func (eh *EventHandler) getGithubProjectName(uses string) string {
+	githubProjectName := uses
+	index := strings.LastIndex(githubProjectName, "@")
+	if index > 0 {
+		githubProjectName = githubProjectName[:index]
+	}
+	return githubProjectName
+}
+
 func (eh *EventHandler) createEventPayloadAsInterface() (map[string]interface{}, error) {
 
 	var eventDataAsInterface interface{}
@@ -114,15 +191,6 @@ func (eh *EventHandler) startK8sJob(k8s k8sutils.K8s, action *config.Action, jso
 		}
 	}
 
-	err := k8s.ConnectToCluster()
-	if err != nil {
-		log.Printf("Error while connecting to cluster: %s\n", err.Error())
-		if !action.Silent {
-			sendTaskFailedEvent(eh.Keptn, "", eh.ServiceName, err, "")
-		}
-		return
-	}
-
 	allJobLogs := []jobLogs{}
 
 	for index, task := range action.Tasks {

pkg/file/file.go

@@ -27,6 +27,11 @@ func MountFiles(actionName string, taskName string, fs afero.Fs, configService k
 		return fmt.Errorf("no action found with name '%s'", actionName)
 	}
 
+	if len(action.Steps) >= 0 {
+		log.Printf("mounting files for github actions is not supported")
+		return nil
+	}
+
 	found, task := action.FindTaskByName(taskName)
 	if !found {
 		return fmt.Errorf("no task found with name '%s'", taskName)

pkg/github/github.go

@@ -0,0 +1,36 @@
+package github
+
+import (
+	"fmt"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
+	"strings"
+)
+
+func PrepareArgs(with map[string]string, inputs map[string]model.Input, args []string) ([]string, error) {
+	var filledArgs []string
+
+	for inputKey, inputValue := range inputs {
+		argKey := fmt.Sprintf("inputs.%s", inputKey)
+
+		for _, arg := range args {
+			if strings.Contains(arg, argKey) {
+
+				argValue := inputValue.Default
+				if withValue, ok := with[inputKey]; ok {
+					argValue = withValue
+				} else {
+					if inputValue.Required {
+						return nil, fmt.Errorf("required input '%s' not provided", inputKey)
+					}
+				}
+
+				splittedArg := strings.Split(arg, "$")
+				arg := strings.TrimSpace(splittedArg[0])
+				filledArgs = append(filledArgs, arg)
+				filledArgs = append(filledArgs, argValue)
+			}
+		}
+	}
+
+	return filledArgs, nil
+}

pkg/github/github_test.go

@@ -0,0 +1,43 @@
+package github
+
+import (
+	"gotest.tools/assert"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
+	"log"
+	"testing"
+)
+
+func TestPrepareArgs(t *testing.T) {
+	with := map[string]string{"scan-type": "banana", "format": "cucumber"}
+	inputs := map[string]model.Input{
+		"scan-type": {
+			Required: false,
+			Default:  "",
+		},
+		"format": {
+			Required: false,
+			Default:  "",
+		},
+		"template": {
+			Required: false,
+			Default:  "table",
+		},
+	}
+	args := []string{"-a ${{ inputs.scan-type }}", "-b ${{ inputs.format }}", "-c ${{ inputs.template }}"}
+	k8sArgs, err := PrepareArgs(with, inputs, args)
+	assert.NilError(t, err)
+	log.Printf("%v", k8sArgs)
+}
+
+func TestPrepareArgs_RequiredInput(t *testing.T) {
+	with := map[string]string{}
+	inputs := map[string]model.Input{
+		"scan-type": {
+			Required: true,
+			Default:  "",
+		},
+	}
+	args := []string{"-a ${{ inputs.scan-type }}"}
+	_, err := PrepareArgs(with, inputs, args)
+	assert.Error(t, err, "required input 'scan-type' not provided")
+}