feat: improved resource referencing and throw errors for invalid expressions

examples/04-extras/README.md

@@ -20,7 +20,7 @@ containers:
   hello:
     image: nginx
     volumes:
-      - source: ${resources.data}
+      - source: data
         target: /usr/share/nginx/html
         readOnly: true
 
@@ -50,7 +50,7 @@ services:
         target: /usr/share/nginx/html
 ```
 
-This compose service configuration references a volume called `data`, that should be available in the target environment by the time service starts.
+This compose service configuration references a volume called `data`, that should be available in the target environment by the time the service starts.
 
 If running the service with the Docker, the volume can be created manually ([read more](https://docs.docker.com/storage/volumes/#create-and-manage-volumes)).
 

examples/04-extras/score.yaml

@@ -13,7 +13,7 @@ containers:
   hello:
     image: nginx
     volumes:
-      - source: ${resources.data}
+      - source: data
         target: /usr/share/nginx/html
         readOnly: true
 

go.mod

@@ -7,9 +7,9 @@ toolchain go1.21.0
 require (
 	github.com/compose-spec/compose-go v1.6.0
 	github.com/imdario/mergo v0.3.13
-	github.com/mitchellh/mapstructure v1.5.0
 	github.com/score-spec/score-go v1.1.0
 	github.com/spf13/cobra v1.6.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
 	github.com/tidwall/sjson v1.2.5
 	gopkg.in/yaml.v3 v3.0.1
@@ -20,10 +20,10 @@ require (
 	github.com/distribution/distribution/v3 v3.0.0-20220725133111-4bf3547399eb // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect

internal/command/run.go

@@ -232,11 +232,8 @@ func run(cmd *cobra.Command, args []string) error {
 		//
 		log.Print("Writing .env file template...\n")
 
-		envVars := make([]string, 0, len(vars))
-		for key, val := range vars {
-			if val == nil {
-				val = ""
-			}
+		envVars := make([]string, 0)
+		for key, val := range vars.Accessed() {
 			var envVar = fmt.Sprintf("%s=%v\n", key, val)
 			envVars = append(envVars, envVar)
 		}

internal/command/run_test.go

@@ -138,6 +138,10 @@ resources:
         data: here
   resource-two2:
     type: Resource-Two
+  volume-name:
+    type: volume
+  volume-two:
+    type: volume
 `), 0600))
 	stdout, stderr, err := executeAndResetCommand(context.Background(), rootCmd, []string{"run", "--file", filepath.Join(td, "score.yaml"), "--output", filepath.Join(td, "compose.yaml")})
 	assert.NoError(t, err)

internal/compose/convert.go

@@ -18,12 +18,7 @@ import (
 )
 
 // ConvertSpec converts SCORE specification into docker-compose configuration.
-func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, error) {
-	ctx, err := buildContext(spec.Metadata, spec.Resources)
-	if err != nil {
-		return nil, nil, fmt.Errorf("preparing context: %w", err)
-	}
-
+func ConvertSpec(spec *score.Workload) (*compose.Project, *EnvVarTracker, error) {
 	workloadName, ok := spec.Metadata["name"].(string)
 	if !ok || len(workloadName) == 0 {
 		return nil, nil, errors.New("workload metadata is missing a name")
@@ -37,7 +32,29 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err
 		Services: make(compose.Services, 0, len(spec.Containers)),
 	}
 
-	externalVars := ExternalVariables(ctx.ListEnvVars())
+	// Track any uses of the environment resource or resources that are overridden with an env provider using the tracker.
+	envVarTracker := new(EnvVarTracker)
+	resources := make(map[string]ResourceWithOutputs)
+	// The first thing we must do is validate or create the resources this workload depends on.
+	// NOTE: this will soon be replaced by a much more sophisticated resource provisioning system!
+	for resourceName, resourceSpec := range spec.Resources {
+		if resourceSpec.Type == "environment" {
+			if DerefOr(resourceSpec.Class, "default") != "default" {
+				return nil, nil, fmt.Errorf("resources.%s: '%s.%s' is not supported in score-compose", resourceName, resourceSpec.Type, *resourceSpec.Class)
+			}
+			resources[resourceName] = envVarTracker
+		} else if resourceSpec.Type == "volume" && DerefOr(resourceSpec.Class, "default") == "default" {
+			resources[resourceName] = resourceWithStaticOutputs{}
+		} else {
+			// TODO: only enable this if the type.class is in an allow-list or the allow-list is '*' - otherwise return an error
+			resources[resourceName] = envVarTracker.GenerateResource(resourceName)
+		}
+	}
+
+	ctx, err := buildContext(spec.Metadata, resources)
+	if err != nil {
+		return nil, nil, fmt.Errorf("preparing context: %w", err)
+	}
 
 	var ports []compose.ServicePortConfig
 	if spec.Service != nil && len(spec.Service.Ports) > 0 {
@@ -70,8 +87,11 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err
 
 		var env = make(compose.MappingWithEquals, len(cSpec.Variables))
 		for key, val := range cSpec.Variables {
-			var envVarVal = ctx.Substitute(val)
-			env[key] = &envVarVal
+			resolved, err := ctx.Substitute(val)
+			if err != nil {
+				return nil, nil, fmt.Errorf("containers.%s.variables.%s: %w", containerName, key, err)
+			}
+			env[key] = &resolved
 		}
 
 		// NOTE: Sorting is necessary for DeepEqual call within our Unit Tests to work reliably
@@ -87,9 +107,22 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err
 				if vol.Path != nil && *vol.Path != "" {
 					return nil, nil, fmt.Errorf("can't mount named volume with sub path '%s': %w", *vol.Path, errors.New("not supported"))
 				}
+
+				// TODO: deprecate this - volume should be linked directly
+				resolvedVolumeSource, err := ctx.Substitute(vol.Source)
+				if err != nil {
+					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: %w", containerName, idx, err)
+				}
+
+				if res, ok := spec.Resources[resolvedVolumeSource]; !ok {
+					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' does not exist", containerName, idx, resolvedVolumeSource)
+				} else if res.Type != "volume" {
+					return nil, nil, fmt.Errorf("containers.%s.volumes[%d].source: resource '%s' is not a volume", containerName, idx, resolvedVolumeSource)
+				}
+
 				volumes[idx] = compose.ServiceVolumeConfig{
 					Type:     "volume",
-					Source:   ctx.Substitute(vol.Source),
+					Source:   resolvedVolumeSource,
 					Target:   vol.Target,
 					ReadOnly: DerefOr(vol.ReadOnly, false),
 				}
@@ -119,5 +152,5 @@ func ConvertSpec(spec *score.Workload) (*compose.Project, ExternalVariables, err
 
 		project.Services = append(project.Services, svc)
 	}
-	return &project, externalVars, nil
+	return &project, envVarTracker, nil
 }

internal/compose/convert_test.go

@@ -25,7 +25,7 @@ func TestScoreConvert(t *testing.T) {
 		Name    string
 		Source  *score.Workload
 		Project *compose.Project
-		Vars    ExternalVariables
+		Vars    map[string]string
 		Error   error
 	}{
 		// Success path
@@ -93,7 +93,6 @@ func TestScoreConvert(t *testing.T) {
 					},
 				},
 			},
-			Vars: ExternalVariables{},
 		},
 		{
 			Name: "Should convert all resources references",
@@ -112,7 +111,7 @@ func TestScoreConvert(t *testing.T) {
 						},
 						Volumes: []score.ContainerVolumesElem{
 							{
-								Source:   "${resources.data}",
+								Source:   "data",
 								Target:   "/mnt/data",
 								ReadOnly: Ref(true),
 							},
@@ -126,7 +125,7 @@ func TestScoreConvert(t *testing.T) {
 					"app-db": {
 						Type: "postgress",
 					},
-					"dns": {
+					"some-dns": {
 						Type: "dns",
 					},
 					"data": {
@@ -142,7 +141,7 @@ func TestScoreConvert(t *testing.T) {
 						Environment: compose.MappingWithEquals{
 							"DEBUG":             stringPtr("${DEBUG}"),
 							"LOGS_LEVEL":        stringPtr("${LOGS_LEVEL}"),
-							"DOMAIN_NAME":       stringPtr(""),
+							"DOMAIN_NAME":       stringPtr("${SOME_DNS_DOMAIN_NAME}"),
 							"CONNECTION_STRING": stringPtr("postgresql://${APP_DB_HOST}:${APP_DB_PORT}/${APP_DB_NAME}"),
 						},
 						Volumes: []compose.ServiceVolumeConfig{
@@ -156,11 +155,12 @@ func TestScoreConvert(t *testing.T) {
 					},
 				},
 			},
-			Vars: ExternalVariables{
-				"DEBUG":       "",
-				"APP_DB_HOST": "",
-				"APP_DB_PORT": "",
-				"APP_DB_NAME": "",
+			Vars: map[string]string{
+				"DEBUG":                "",
+				"APP_DB_HOST":          "",
+				"APP_DB_PORT":          "",
+				"APP_DB_NAME":          "",
+				"SOME_DNS_DOMAIN_NAME": "",
 			},
 		},
 		{
@@ -213,7 +213,6 @@ func TestScoreConvert(t *testing.T) {
 					},
 				},
 			},
-			Vars: ExternalVariables{},
 		},
 
 		// Errors handling
@@ -229,7 +228,7 @@ func TestScoreConvert(t *testing.T) {
 						Image: "busybox",
 						Volumes: []score.ContainerVolumesElem{
 							{
-								Source:   "${resources.data}",
+								Source:   "data",
 								Target:   "/mnt/data",
 								Path:     Ref("sub/path"),
 								ReadOnly: Ref(true),
@@ -245,6 +244,35 @@ func TestScoreConvert(t *testing.T) {
 			},
 			Error: errors.New("not supported"),
 		},
+
+		{
+			Name: "Should report an error for volume that doesn't exist in resources",
+			Source: &score.Workload{
+				Metadata: score.WorkloadMetadata{"name": "test"},
+				Containers: score.WorkloadContainers{
+					"test": score.Container{
+						Image:   "busybox",
+						Volumes: []score.ContainerVolumesElem{{Source: "data", Target: "/mnt/data"}},
+					},
+				},
+			},
+			Error: errors.New("containers.test.volumes[0].source: resource 'data' does not exist"),
+		},
+
+		{
+			Name: "Should report an error for volume resource that isn't a volume",
+			Source: &score.Workload{
+				Metadata: score.WorkloadMetadata{"name": "test"},
+				Containers: score.WorkloadContainers{
+					"test": score.Container{
+						Image:   "busybox",
+						Volumes: []score.ContainerVolumesElem{{Source: "data", Target: "/mnt/data"}},
+					},
+				},
+				Resources: map[string]score.Resource{"data": {Type: "thing"}},
+			},
+			Error: errors.New("containers.test.volumes[0].source: resource 'data' is not a volume"),
+		},
 	}
 
 	for _, tt := range tests {
@@ -260,7 +288,7 @@ func TestScoreConvert(t *testing.T) {
 				//
 				assert.NoError(t, err)
 				assert.Equal(t, tt.Project, proj)
-				assert.Equal(t, tt.Vars, vars)
+				assert.Equal(t, tt.Vars, vars.Accessed())
 			}
 		})
 	}

internal/compose/envvar_tracker.go

@@ -0,0 +1,73 @@
+package compose
+
+import (
+	"maps"
+	"os"
+	"strings"
+)
+
+// EnvVarTracker is used to provide the `environment` resource type. This tracks what keys are accessed and replaces
+// them with outputs that are environment variable references that docker compose will support.
+// This keeps track of which keys were accessed so that we can produce a reference file or list of keys for the user
+// to understand what inputs docker compose will require at launch time.
+type EnvVarTracker struct {
+	// lookup is an environment variable lookup function, if nil this will be defaulted to os.LookupEnv
+	lookup func(key string) (string, bool)
+	// accessed is the map of accessed environment variables and the value they had at access time
+	accessed map[string]string
+}
+
+func (e *EnvVarTracker) Accessed() map[string]string {
+	return maps.Clone(e.accessed)
+}
+
+// the env var tracker is a resource itself (an environment resource)
+var _ ResourceWithOutputs = (*EnvVarTracker)(nil)
+
+func (e *EnvVarTracker) LookupOutput(keys ...string) (interface{}, error) {
+	if len(keys) == 0 {
+		panic("requires at least 1 key")
+	}
+	envVarKey := strings.ToUpper(strings.Join(keys, "_"))
+
+	// in theory we can replace more unexpected characters
+	envVarKey = strings.ReplaceAll(envVarKey, "-", "_")
+	envVarKey = strings.ReplaceAll(envVarKey, ".", "_")
+
+	if e.lookup == nil {
+		e.lookup = os.LookupEnv
+	}
+	if e.accessed == nil {
+		e.accessed = make(map[string]string, 1)
+	}
+
+	if v, ok := e.lookup(envVarKey); ok {
+		e.accessed[envVarKey] = v
+	} else {
+		e.accessed[envVarKey] = ""
+	}
+	return "${" + envVarKey + "}", nil
+}
+
+func (e *EnvVarTracker) GenerateResource(resName string) ResourceWithOutputs {
+	return &envVarResourceTracker{
+		inner:  e,
+		prefix: resName,
+	}
+}
+
+// envVarResourceTracker is a child object of EnvVarTracker and is used as a fallback behavior for resource types
+// that are not supported natively: we treat them like environment variables instead with a prefix of the resource name.
+type envVarResourceTracker struct {
+	prefix string
+	inner  *EnvVarTracker
+}
+
+func (e *envVarResourceTracker) LookupOutput(keys ...string) (interface{}, error) {
+	next := make([]string, 1+len(keys))
+	next[0] = e.prefix
+	for i, k := range keys {
+		next[1+i] = k
+	}
+	return e.inner.LookupOutput(next...)
+}

internal/compose/resources.go

@@ -0,0 +1,31 @@
+package compose
+
+import "fmt"
+
+// ResourceWithOutputs is an interface that resource implementations in the future may provide.
+// The keys here are the parts of a .-separated path traversal down a tree to return some data from the outputs of
+// the provisioned resource. If an error occurs looking up the output, an error should be thrown.
+// nil is a valid result since some resources may return null in their outputs.
+type ResourceWithOutputs interface {
+	LookupOutput(keys ...string) (interface{}, error)
+}
+
+type resourceWithStaticOutputs map[string]interface{}
+
+func (r resourceWithStaticOutputs) LookupOutput(keys ...string) (interface{}, error) {
+	var resolvedValue interface{}
+	resolvedValue = (map[string]interface{})(r)
+	for _, k := range keys {
+		mapV, ok := resolvedValue.(map[string]interface{})
+		if !ok {
+			return "", fmt.Errorf("cannot lookup key '%s', context is not a map", k)
+		}
+		resolvedValue, ok = mapV[k]
+		if !ok {
+			return "", fmt.Errorf("key '%s' not found", k)
+		}
+	}
+	return resolvedValue, nil
+}
+
+var _ ResourceWithOutputs = (resourceWithStaticOutputs)(nil)