Skip to content

Commit

Permalink
feat: improved resource referencing and added useful volume resource …
Browse files Browse the repository at this point in the history
…support

Signed-off-by: Ben Meier <[email protected]>
  • Loading branch information
astromechza committed Mar 1, 2024
1 parent e5a668f commit 3d4b57c
Show file tree
Hide file tree
Showing 10 changed files with 318 additions and 160 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
github.com/compose-spec/compose-go v1.6.0
github.com/imdario/mergo v0.3.13
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1
github.com/score-spec/score-go v1.1.0
github.com/spf13/cobra v1.6.0
github.com/stretchr/testify v1.8.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down
7 changes: 2 additions & 5 deletions internal/command/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,11 +232,8 @@ func run(cmd *cobra.Command, args []string) error {
//
log.Print("Writing .env file template...\n")

envVars := make([]string, 0, len(vars))
for key, val := range vars {
if val == nil {
val = ""
}
envVars := make([]string, 0)
for key, val := range vars.Accessed() {
var envVar = fmt.Sprintf("%s=%v\n", key, val)
envVars = append(envVars, envVar)
}
Expand Down
3 changes: 3 additions & 0 deletions internal/command/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,9 @@ func TestRunExample04(t *testing.T) {
source: data
target: /usr/share/nginx/html
read_only: true
volumes:
data:
driver: local
`

assert.Equal(t, expectedOutput, stdout)
Expand Down
60 changes: 43 additions & 17 deletions internal/compose/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,26 +18,45 @@ import (
)

// ConvertSpec converts SCORE specification into docker-compose configuration.
func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, error) {
ctx, err := buildContext(spec.Metadata, spec.Resources)
if err != nil {
return nil, nil, fmt.Errorf("preparing context: %w", err)
}
func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error) {
// Track any uses of the environment resource or resources that are overridden with an env provider using the tracker.
envVarTracker := NewEnvVarTracker()

workloadName, ok := spec.Metadata["name"].(string)
if !ok || len(workloadName) == 0 {
return nil, nil, errors.New("workload metadata is missing a name")
var project = compose.Project{
Services: make(compose.Services, 0, len(spec.Containers)),
}

if len(spec.Containers) == 0 {
return nil, nil, errors.New("workload does not have any containers to convert into a compose service")
// this map holds the results of the provisioning process
resources := make(map[string]ResourceWithOutputs)

// The first thing we must do is validate or create the resources this workload depends on.
// NOTE: this will soon be replaced by a much more sophisticated resource provisioning system!
for resourceName, resourceSpec := range spec.Resources {
if resourceSpec.Type == "environment" {
if DerefOr(resourceSpec.Class, "default") != "default" {
return nil, nil, fmt.Errorf("resources.%s: '%s.%s' is not supported in score-compose", resourceName, resourceSpec.Type, *resourceSpec.Class)
}
resources[resourceName] = envVarTracker
} else if resourceSpec.Type == "volume" && DerefOr(resourceSpec.Class, "default") == "default" {
if project.Volumes == nil {
project.Volumes = make(compose.Volumes)
}
// replace this with a real provisioner later
project.Volumes[resourceName] = compose.VolumeConfig{Driver: "local"}
resources[resourceName] = &StaticResource{Outputs: map[string]interface{}{}}
} else {
// TODO: only enable this if the type.class is in an allow-list or the allow-list is '*' - otherwise return an error
resources[resourceName] = envVarTracker.GenerateResource(resourceName)
}
}

var project = compose.Project{
Services: make(compose.Services, 0, len(spec.Containers)),
ctx, err := buildContext(spec.Metadata, resources)
if err != nil {
return nil, nil, fmt.Errorf("preparing context: %w", err)
}

externalVars := ExternalVariables(ctx.ListEnvVars())
// This is already validated by spec validation
workloadName, _ := spec.Metadata["name"].(string)

var ports []compose.ServicePortConfig
if spec.Service != nil && len(spec.Service.Ports) > 0 {
Expand Down Expand Up @@ -70,8 +89,11 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err

var env = make(compose.MappingWithEquals, len(cSpec.Variables))
for key, val := range cSpec.Variables {
var envVarVal = ctx.Substitute(val)
env[key] = &envVarVal
resolved, err := ctx.Substitute(val)
if err != nil {
return nil, nil, fmt.Errorf("containers.%s.variables.%s: %w", containerName, key, err)
}
env[key] = &resolved
}

// NOTE: Sorting is necessary for DeepEqual call within our Unit Tests to work reliably
Expand All @@ -87,9 +109,13 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err
if vol.Path != nil && *vol.Path != "" {
return nil, nil, fmt.Errorf("can't mount named volume with sub path '%s': %w", *vol.Path, errors.New("not supported"))
}
resolvedVolumeSource, err := ctx.Substitute(vol.Source)
if err != nil {
return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: %w", containerName, idx, err)
}
volumes[idx] = compose.ServiceVolumeConfig{
Type: "volume",
Source: ctx.Substitute(vol.Source),
Source: resolvedVolumeSource,
Target: vol.Target,
ReadOnly: DerefOr(vol.ReadOnly, false),
}
Expand Down Expand Up @@ -119,5 +145,5 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err

project.Services = append(project.Services, svc)
}
return &project, externalVars, nil
return &project, envVarTracker, nil
}
26 changes: 15 additions & 11 deletions internal/compose/convert_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func TestScoreConvert(t *testing.T) {
Name string
Source *score.Workload
Project *compose.Project
Vars ExternalVariables
Vars map[string]string
Error error
}{
// Success path
Expand Down Expand Up @@ -93,7 +93,7 @@ func TestScoreConvert(t *testing.T) {
},
},
},
Vars: ExternalVariables{},
Vars: map[string]string{},
},
{
Name: "Should convert all resources references",
Expand Down Expand Up @@ -126,7 +126,7 @@ func TestScoreConvert(t *testing.T) {
"app-db": {
Type: "postgress",
},
"dns": {
"some-dns": {
Type: "dns",
},
"data": {
Expand All @@ -142,7 +142,7 @@ func TestScoreConvert(t *testing.T) {
Environment: compose.MappingWithEquals{
"DEBUG": stringPtr("${DEBUG}"),
"LOGS_LEVEL": stringPtr("${LOGS_LEVEL}"),
"DOMAIN_NAME": stringPtr(""),
"DOMAIN_NAME": stringPtr("${SOME_DNS_DOMAIN_NAME}"),
"CONNECTION_STRING": stringPtr("postgresql://${APP_DB_HOST}:${APP_DB_PORT}/${APP_DB_NAME}"),
},
Volumes: []compose.ServiceVolumeConfig{
Expand All @@ -155,12 +155,16 @@ func TestScoreConvert(t *testing.T) {
},
},
},
Volumes: map[string]compose.VolumeConfig{
"data": {Driver: "local"},
},
},
Vars: ExternalVariables{
"DEBUG": "",
"APP_DB_HOST": "",
"APP_DB_PORT": "",
"APP_DB_NAME": "",
Vars: map[string]string{
"DEBUG": "",
"APP_DB_HOST": "",
"APP_DB_PORT": "",
"APP_DB_NAME": "",
"SOME_DNS_DOMAIN_NAME": "",
},
},
{
Expand Down Expand Up @@ -213,7 +217,7 @@ func TestScoreConvert(t *testing.T) {
},
},
},
Vars: ExternalVariables{},
Vars: map[string]string{},
},

// Errors handling
Expand Down Expand Up @@ -260,7 +264,7 @@ func TestScoreConvert(t *testing.T) {
//
assert.NoError(t, err)
assert.Equal(t, tt.Project, proj)
assert.Equal(t, tt.Vars, vars)
assert.Equal(t, tt.Vars, vars.Accessed())
}
})
}
Expand Down
61 changes: 61 additions & 0 deletions internal/compose/envvar_tracker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package compose

import (
"maps"
"os"
"strings"
)

type EnvVarTracker struct {
lookup func(key string) (string, bool)
accessed map[string]string
}

func NewEnvVarTracker() *EnvVarTracker {
return &EnvVarTracker{
lookup: os.LookupEnv,
accessed: make(map[string]string),
}
}

func (e *EnvVarTracker) Accessed() map[string]string {
return maps.Clone(e.accessed)
}

// the env var tracker is a resource itself (an environment resource)
var _ ResourceWithOutputs = (*EnvVarTracker)(nil)

func (e *EnvVarTracker) LookupOutput(keys ...string) (interface{}, error) {
if len(keys) == 0 {
panic("requires at least 1 key")
}
envVarKey := strings.ToUpper(strings.Join(keys, "_"))
envVarKey = strings.ReplaceAll(envVarKey, "-", "_")
if v, ok := e.lookup(envVarKey); ok {
e.accessed[envVarKey] = v
} else {
e.accessed[envVarKey] = ""
}
return "${" + envVarKey + "}", nil
}

type envVarResourceTracker struct {
prefix string
inner *EnvVarTracker
}

func (e *envVarResourceTracker) LookupOutput(keys ...string) (interface{}, error) {
next := make([]string, 1+len(keys))
next[0] = e.prefix
for i, k := range keys {
next[1+i] = k
}
return e.inner.LookupOutput(next...)
}

func (e *EnvVarTracker) GenerateResource(resName string) ResourceWithOutputs {
return &envVarResourceTracker{
inner: e,
prefix: resName,
}
}
30 changes: 30 additions & 0 deletions internal/compose/resources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package compose

import (
"fmt"
"strings"
)

type ResourceWithOutputs interface {
LookupOutput(keys ...string) (interface{}, error)
}

type StaticResource struct {
Outputs map[string]interface{}
}

func (sr *StaticResource) LookupOutput(keys ...string) (interface{}, error) {
resolvedValue := interface{}(sr.Outputs)
remainingKeys := keys
for partIndex, part := range remainingKeys {
mapV, ok := resolvedValue.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("cannot lookup a key in %T", resolvedValue)
}
resolvedValue, ok = mapV[part]
if !ok {
return nil, fmt.Errorf("output '%s' does not exist", strings.Join(keys[:partIndex], "."))
}
}
return resolvedValue, nil
}
Loading

0 comments on commit 3d4b57c

Please sign in to comment.