[WIP] Feature/poc GitHub actions

cmd/job-executor-service/main.go

@@ -46,6 +46,8 @@ type envConfig struct {
 	DefaultResourceRequestsMemory string `envconfig:"DEFAULT_RESOURCE_REQUESTS_MEMORY"`
 	// Respond with .finished event if no configuration found
 	AlwaysSendFinishedEvent string `envconfig:"ALWAYS_SEND_FINISHED_EVENT"`
+	// Container Registry
+	ContainerRegistry string `envconfig:"CONTAINER_REGISTRY"`
 }
 
 // ServiceName specifies the current services name (e.g., used as source when sending CloudEvents)
@@ -104,6 +106,7 @@ func processKeptnCloudEvent(ctx context.Context, event cloudevents.Event) error
 			InitContainerImage:          env.InitContainerImage,
 			DefaultResourceRequirements: DefaultResourceRequirements,
 			AlwaysSendFinishedEvent:     false,
+			ContainerRegistry:           env.ContainerRegistry,
 		},
 	}
 

deploy/registry.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    app: registry
+  name: registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: registry
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: registry
+    spec:
+      containers:
+      - image: registry:2
+        name: registry
+        ports:
+        - containerPort: 5000
+        resources: {}
+status: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    app: registry
+  name: registry
+spec:
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000
+  selector:
+    app: registry
+status:
+  loadBalancer: {}

go.mod

@@ -18,6 +18,7 @@ require (
 	golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4 // indirect
 	golang.org/x/tools v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	gotest.tools v2.2.0+incompatible
 	honnef.co/go/tools v0.1.3 // indirect
 	k8s.io/api v0.22.0

pkg/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"gopkg.in/yaml.v2"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
 	"regexp"
 
 	"github.com/PaesslerAG/jsonpath"
@@ -18,10 +19,11 @@ type Config struct {
 
 // Action contains a action within the config which needs to be triggered
 type Action struct {
-	Name   string  `yaml:"name"`
-	Events []Event `yaml:"events"`
-	Tasks  []Task  `yaml:"tasks"`
-	Silent bool    `yaml:"silent"`
+	Name   string       `yaml:"name"`
+	Events []Event      `yaml:"events"`
+	Tasks  []Task       `yaml:"tasks"`
+	Silent bool         `yaml:"silent"`
+	Steps  []model.Step `yaml:"steps"`
 }
 
 // Event defines a keptn event which determines if an Action should be triggered
@@ -85,7 +87,20 @@ func NewConfig(yamlContent []byte) (*Config, error) {
 		return nil, fmt.Errorf("apiVersion %v is not supported, use %v", *config.APIVersion, supportedAPIVersion)
 	}
 
-	return &config, nil
+	err = config.checkConfigs()
+
+	return &config, err
+}
+
+func (c *Config) checkConfigs() error {
+
+	// check if the actions don't contain steps and tasks -> that should not be valid
+	for _, action := range c.Actions {
+		if len(action.Steps) > 0 && len(action.Tasks) > 0 {
+			return fmt.Errorf("action %v contains steps and tasks", action.Name)
+		}
+	}
+	return nil
 }
 
 // IsEventMatch indicated whether a given event matches the config

pkg/eventhandler/eventhandler_test.go

@@ -196,3 +196,10 @@ func TestStartK8sJobSilent(t *testing.T) {
 	err = fakeEventSender.AssertSentEventTypes([]string{})
 	assert.NilError(t, err)
 }
+
+func TestGetGithubProjectName(t *testing.T) {
+
+	eh := EventHandler{}
+	assert.Equal(t, "aquasecurity/trivy-action", eh.getGithubProjectName("aquasecurity/trivy-action@master"))
+	assert.Equal(t, "aquasecurity/trivy-action", eh.getGithubProjectName("aquasecurity/trivy-action"))
+}

pkg/eventhandler/eventhandlers.go

@@ -4,10 +4,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"keptn-sandbox/job-executor-service/pkg/config"
+	"keptn-sandbox/job-executor-service/pkg/github"
 	"keptn-sandbox/job-executor-service/pkg/k8sutils"
 	"log"
 	"math"
 	"strconv"
+	"strings"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2" // make sure to use v2 cloudevents here
 	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
@@ -73,13 +75,88 @@ func (eh *EventHandler) HandleEvent() error {
 	}
 
 	log.Printf("Match found for event %s of type %s. Starting k8s job to run action '%s'", eh.Event.Context.GetID(), eh.Event.Type(), action.Name)
+	log.Printf("action: %+v", *action)
 
 	k8s := k8sutils.NewK8s(eh.JobSettings.JobNamespace)
+	err = k8s.ConnectToCluster()
+	if err != nil {
+		log.Printf("Error while connecting to cluster: %s\n", err)
+		return err
+	}
+
+	err = eh.handleGithubAction(k8s, action)
+	if err != nil {
+		log.Printf("An error occurred while handling GitHub action: %s", err)
+		return err
+	}
+
+	log.Printf("executing action: %+v", *action)
 	eh.startK8sJob(k8s, action, eventAsInterface)
 
 	return nil
 }
 
+func (eh *EventHandler) handleGithubAction(k8s k8sutils.K8s, action *config.Action) error {
+	for _, step := range action.Steps {
+
+		log.Printf("Handling step: %+v", step)
+
+		githubProjectName := eh.getGithubProjectName(step.Uses)
+
+		// TODO using step.Name here is a bad idea because it can contain whitespaces etc.
+		imageLocation, err := k8s.CreateImageBuilder(step.Name, githubProjectName, eh.JobSettings.ContainerRegistry)
+		log.Printf("imageLocation: %v", imageLocation)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err = k8s.DeleteK8sJob(step.Name)
+			if err != nil {
+				log.Printf("Error while deleting job: %s\n", err.Error())
+			}
+		}()
+
+		jobErr := k8s.AwaitK8sJobDone(step.Name, defaultMaxPollCount, pollIntervalInSeconds)
+		if jobErr != nil {
+			log.Println(err)
+		}
+
+		err, githubAction := github.GetActionYaml(githubProjectName)
+		if err != nil {
+			return err
+		}
+
+		args, err := github.PrepareArgs(step.With, githubAction.Inputs, githubAction.Runs.Args)
+		if err != nil {
+			return err
+		}
+
+		task := config.Task{
+			Name:  githubAction.Name,
+			Files: nil,
+			Image: imageLocation,
+			Cmd:   nil,
+			Args:  args,
+			Env:   nil,
+		}
+
+		log.Printf("task: %+v", task)
+
+		action.Tasks = append(action.Tasks, task)
+	}
+	return nil
+}
+
+func (eh *EventHandler) getGithubProjectName(uses string) string {
+	githubProjectName := uses
+	index := strings.LastIndex(githubProjectName, "@")
+	if index > 0 {
+		githubProjectName = githubProjectName[:index]
+	}
+	return githubProjectName
+}
+
 func (eh *EventHandler) createEventPayloadAsInterface() (map[string]interface{}, error) {
 
 	var eventDataAsInterface interface{}
@@ -114,15 +191,6 @@ func (eh *EventHandler) startK8sJob(k8s k8sutils.K8s, action *config.Action, jso
 		}
 	}
 
-	err := k8s.ConnectToCluster()
-	if err != nil {
-		log.Printf("Error while connecting to cluster: %s\n", err.Error())
-		if !action.Silent {
-			sendTaskFailedEvent(eh.Keptn, "", eh.ServiceName, err, "")
-		}
-		return
-	}
-
 	allJobLogs := []jobLogs{}
 
 	for index, task := range action.Tasks {

pkg/github/github.go

@@ -0,0 +1,39 @@
+package github
+
+import (
+	"fmt"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
+	"log"
+	"strings"
+)
+
+func PrepareArgs(with map[string]string, inputs map[string]model.Input, args []string) ([]string, error) {
+	var filledArgs []string
+
+	for inputKey, inputValue := range inputs {
+		argKey := fmt.Sprintf("inputs.%s", inputKey)
+		log.Printf("argKey: %v", argKey)
+
+		for _, arg := range args {
+			if strings.Contains(arg, argKey) {
+				log.Printf("matched argKey: %v", argKey)
+
+				argValue := inputValue.Default
+				if withValue, ok := with[inputKey]; ok {
+					argValue = withValue
+				} else {
+					if inputValue.Required {
+						return nil, fmt.Errorf("required input '%s' not provided", inputKey)
+					}
+				}
+
+				splittedArg := strings.Split(arg, "$")
+				arg := strings.TrimSpace(splittedArg[0])
+				filledArgs = append(filledArgs, arg)
+				filledArgs = append(filledArgs, argValue)
+			}
+		}
+	}
+
+	return filledArgs, nil
+}

pkg/github/github_test.go

@@ -0,0 +1,43 @@
+package github
+
+import (
+	"gotest.tools/assert"
+	"keptn-sandbox/job-executor-service/pkg/github/model"
+	"log"
+	"testing"
+)
+
+func TestPrepareArgs(t *testing.T) {
+	with := map[string]string{"scan-type": "banana", "format": "cucumber"}
+	inputs := map[string]model.Input{
+		"scan-type": {
+			Required: false,
+			Default:  "",
+		},
+		"format": {
+			Required: false,
+			Default:  "",
+		},
+		"template": {
+			Required: false,
+			Default:  "table",
+		},
+	}
+	args := []string{"-a ${{ inputs.scan-type }}", "-b ${{ inputs.format }}", "-c ${{ inputs.template }}"}
+	k8sArgs, err := PrepareArgs(with, inputs, args)
+	assert.NilError(t, err)
+	log.Printf("%v", k8sArgs)
+}
+
+func TestPrepareArgs_RequiredInput(t *testing.T) {
+	with := map[string]string{}
+	inputs := map[string]model.Input{
+		"scan-type": {
+			Required: true,
+			Default:  "",
+		},
+	}
+	args := []string{"-a ${{ inputs.scan-type }}"}
+	_, err := PrepareArgs(with, inputs, args)
+	assert.Error(t, err, "required input 'scan-type' not provided")
+}

pkg/github/model/action.go

@@ -0,0 +1,63 @@
+package model
+
+import (
+	"gopkg.in/yaml.v3"
+	"io"
+)
+
+// ActionRunsUsing is the type of runner for the action
+type ActionRunsUsing string
+
+const (
+	// ActionRunsUsingNode12 for running with node12
+	ActionRunsUsingNode12 = "node12"
+	// ActionRunsUsingDocker for running with docker
+	ActionRunsUsingDocker = "docker"
+	// ActionRunsUsingComposite for running composite
+	ActionRunsUsingComposite = "composite"
+)
+
+// ActionRuns are a field in Action
+type ActionRuns struct {
+	Using      ActionRunsUsing   `yaml:"using"`
+	Env        map[string]string `yaml:"env"`
+	Main       string            `yaml:"main"`
+	Image      string            `yaml:"image"`
+	Entrypoint []string          `yaml:"entrypoint"`
+	Args       []string          `yaml:"args"`
+	Steps      []Step            `yaml:"steps"`
+}
+
+// Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action.
+type Action struct {
+	Name        string            `yaml:"name"`
+	Author      string            `yaml:"author"`
+	Description string            `yaml:"description"`
+	Inputs      map[string]Input  `yaml:"inputs"`
+	Outputs     map[string]Output `yaml:"outputs"`
+	Runs        ActionRuns        `yaml:"runs"`
+	Branding    struct {
+		Color string `yaml:"color"`
+		Icon  string `yaml:"icon"`
+	} `yaml:"branding"`
+}
+
+// Input parameters allow you to specify data that the action expects to use during runtime. GitHub stores input parameters as environment variables. Input ids with uppercase letters are converted to lowercase during runtime. We recommended using lowercase input ids.
+type Input struct {
+	Description string `yaml:"description"`
+	Required    bool   `yaml:"required"`
+	Default     string `yaml:"default"`
+}
+
+// Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input.
+type Output struct {
+	Description string `yaml:"description"`
+	Value       string `yaml:"value"`
+}
+
+// ReadAction reads an action from a reader
+func ReadAction(in io.Reader) (*Action, error) {
+	a := new(Action)
+	err := yaml.NewDecoder(in).Decode(a)
+	return a, err
+}