Skip to content

Commit

Permalink
fix: lagoon.persistent.name override checks
Browse files Browse the repository at this point in the history
  • Loading branch information
@shreddedbacon
shreddedbacon committed Oct 12, 2024
1 parent 50c4ebf commit 1c4046f
Show file tree
Hide file tree
Showing 21 changed files with 825 additions and 82 deletions.
33 changes: 33 additions & 0 deletions cmd/template_lagoonservices_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,39 @@ func TestTemplateLagoonServices(t *testing.T) {
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-spot-affinity",
},
{
name: "test-basic-persistent-name",
description: "tests a basic deployment with a persistent name",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/basic/lagoon.persistent-name.yml",
ImageReferences: map[string]string{
"basic": "harbor.example/example-project/main/basic@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-persistent-name",
},
{
name: "test-basic-persistent-names",
description: "tests a basic deployment with two services with persistent name",
args: testdata.GetSeedData(
testdata.TestData{
ProjectName: "example-project",
EnvironmentName: "main",
Branch: "main",
LagoonYAML: "internal/testdata/basic/lagoon.persistent-name-2.yml",
ImageReferences: map[string]string{
"basic": "harbor.example/example-project/main/basic@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
"basic2": "harbor.example/example-project/main/basic2@sha256:b2001babafaa8128fe89aa8fd11832cade59931d14c3de5b3ca32e2a010fbaa8",
},
}, true),
templatePath: "testoutput",
want: "internal/testdata/basic/service-templates/test-basic-persistent-names",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions internal/generator/buildvalues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ type ServiceValues struct {
IsDBaaS bool `json:"isDBaaS"`
IsSingle bool `json:"isSingle"`
AdditionalVolumes []ServiceVolume `json:"additonalVolumes,omitempty"`
CreateDefaultVolume bool `json:"createDefaultVolume"`
}

type ImageBuild struct {
Expand Down
6 changes: 6 additions & 0 deletions internal/generator/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,12 @@ func NewGenerator(
return nil, err
}

// strip out duplicate default volumes
err = flagDefaultVolumeCreation(&buildValues)
if err != nil {
return nil, err
}

if imageCacheBuildArgsJSON != "" {
err = json.Unmarshal([]byte(imageCacheBuildArgsJSON), &buildValues.ImageCacheBuildArguments)
if err != nil {
Expand Down
74 changes: 37 additions & 37 deletions internal/generator/services.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,43 +249,6 @@ func composeToServiceValues(
lagoonOverrideName = composeService
}

// check if the service has any persistent labels, this is the path that the volume will be mounted to
servicePersistentPath := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent")
if servicePersistentPath == "" {
// if there is no persistent path, check if the service type has a default path
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentPath = val.Volumes.PersistentVolumePath
// check if the service type provides or consumes a default persistent volume
if (val.ProvidesPersistentVolume || val.ConsumesPersistentVolume) && servicePersistentPath == "" {
return nil, fmt.Errorf("label lagoon.persistent not defined for service %s, no valid mount path was found", composeService)
}
}
}
servicePersistentName := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.name")
if servicePersistentName == "" && servicePersistentPath != "" {
// if there is a persistent path defined, then set the persistent name to be the compose service if no persistent name is provided
// persistent name is used by joined services like nginx/php or cli or worker pods to mount another service volume
servicePersistentName = lagoonOverrideName
}
servicePersistentSize := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.size")
if servicePersistentSize == "" {
// if there is no persistent size, check if the service type has a default size allocated
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentSize = val.Volumes.PersistentVolumeSize
// check if the service type provides persistent volume, and that a size was detected
if val.ProvidesPersistentVolume && servicePersistentSize == "" {
return nil, fmt.Errorf("label lagoon.persistent.size not defined for service %s, no valid size was found", composeService)
}
}
}
if servicePersistentSize != "" {
// check the provided size is a valid resource size for kubernetes
_, err := ValidateResourceSize(servicePersistentSize)
if err != nil {
return nil, fmt.Errorf("provided persistent volume size for %s is not valid: %v", servicePersistentName, err)
}
}

baseimage := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.base.image")
if baseimage != "" {
// First, let's ensure that the structure of the base image is valid
Expand Down Expand Up @@ -381,6 +344,43 @@ func composeToServiceValues(
}
}

// check if the service has any persistent labels, this is the path that the volume will be mounted to
servicePersistentPath := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent")
if servicePersistentPath == "" {
// if there is no persistent path, check if the service type has a default path
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentPath = val.Volumes.PersistentVolumePath
// check if the service type provides or consumes a default persistent volume
if (val.ProvidesPersistentVolume || val.ConsumesPersistentVolume) && servicePersistentPath == "" {
return nil, fmt.Errorf("label lagoon.persistent not defined for service %s, no valid mount path was found", composeService)
}
}
}
servicePersistentName := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.name")
if servicePersistentName == "" && servicePersistentPath != "" {
// if there is a persistent path defined, then set the persistent name to be the compose service if no persistent name is provided
// persistent name is used by joined services like nginx/php or cli or worker pods to mount another service volume
servicePersistentName = lagoonOverrideName
}
servicePersistentSize := lagoon.CheckDockerComposeLagoonLabel(composeServiceValues.Labels, "lagoon.persistent.size")
if servicePersistentSize == "" {
// if there is no persistent size, check if the service type has a default size allocated
if val, ok := servicetypes.ServiceTypes[lagoonType]; ok {
servicePersistentSize = val.Volumes.PersistentVolumeSize
// check if the service type provides persistent volume, and that a size was detected
if val.ProvidesPersistentVolume && servicePersistentSize == "" {
return nil, fmt.Errorf("label lagoon.persistent.size not defined for service %s, no valid size was found", composeService)
}
}
}
if servicePersistentSize != "" {
// check the provided size is a valid resource size for kubernetes
_, err := ValidateResourceSize(servicePersistentSize)
if err != nil {
return nil, fmt.Errorf("provided persistent volume size for %s is not valid: %v", servicePersistentName, err)
}
}

// calculate if this service needs any additional volumes attached from the calculated build volumes
// additional volumes can only be attached to certain
serviceVolumes, err := calculateServiceVolumes(buildValues, lagoonType, servicePersistentName, composeServiceValues.Labels)
Expand Down
64 changes: 53 additions & 11 deletions internal/generator/services_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"testing"
"time"

"github.com/andreyvit/diff"
composetypes "github.com/compose-spec/compose-go/types"
"github.com/uselagoon/build-deploy-tool/internal/dbaasclient"
"github.com/uselagoon/build-deploy-tool/internal/helpers"
Expand All @@ -19,10 +20,11 @@ func Test_composeToServiceValues(t *testing.T) {
composeServiceValues composetypes.ServiceConfig
}
tests := []struct {
name string
args args
want *ServiceValues
wantErr bool
name string
description string
args args
want *ServiceValues
wantErr bool
}{
{
name: "test1",
Expand Down Expand Up @@ -201,7 +203,9 @@ func Test_composeToServiceValues(t *testing.T) {
wantErr: true,
},
{
name: "test4 - variable servicetypes type override",
name: "test-variable-servicetypes-type-override",
description: `this test should fail to change the nginx service from nginx-php to nginx-php-persistent
this is because there is no lagoon.persistent labels on the dockercompose service to suit this change`,
args: args{
buildValues: &BuildValues{
Namespace: "example-project-main",
Expand Down Expand Up @@ -231,10 +235,46 @@ func Test_composeToServiceValues(t *testing.T) {
},
},
},
want: nil,
wantErr: true,
},
{
name: "test-variable-servicetypes-type-override-b",
description: `this test should pasee and change the nginx service from nginx to basic
this is because there is nothing really special between these two services besides the port`,
args: args{
buildValues: &BuildValues{
Namespace: "example-project-main",
Project: "example-project",
ImageRegistry: "harbor.example",
Environment: "main",
Branch: "main",
BuildType: "branch",
ServiceTypeOverrides: &lagoon.EnvironmentVariable{
Name: "LAGOON_SERVICE_TYPES",
Value: "nginx:basic,mariadb:mariadb-dbaas",
},
LagoonYAML: lagoon.YAML{
Environments: lagoon.Environments{
"main": lagoon.Environment{},
},
},
},
composeService: "nginx",
composeServiceValues: composetypes.ServiceConfig{
Labels: composetypes.Labels{
"lagoon.type": "nginx",
},
Build: &composetypes.BuildConfig{
Context: ".",
Dockerfile: "../testdata/basic/docker/basic.dockerfile",
},
},
},
want: &ServiceValues{
Name: "nginx",
OverrideName: "nginx",
Type: "nginx-php-persistent",
Type: "basic",
AutogeneratedRoutesEnabled: true,
AutogeneratedRoutesTLSAcme: true,
InPodCronjobs: []lagoon.Cronjob{},
Expand All @@ -245,7 +285,6 @@ func Test_composeToServiceValues(t *testing.T) {
DockerFile: "../testdata/basic/docker/basic.dockerfile",
BuildImage: "harbor.example/example-project/main/nginx:latest",
},
BackupsEnabled: true,
},
},
{
Expand Down Expand Up @@ -558,6 +597,9 @@ func Test_composeToServiceValues(t *testing.T) {
Type: "mariadb-single",
AutogeneratedRoutesEnabled: false,
AutogeneratedRoutesTLSAcme: false,
PersistentVolumePath: "/var/lib/mysql",
PersistentVolumeName: "mariadb",
PersistentVolumeSize: "5Gi",
DBaaSEnvironment: "development2",
InPodCronjobs: []lagoon.Cronjob{},
NativeCronjobs: []lagoon.Cronjob{},
Expand Down Expand Up @@ -1252,10 +1294,10 @@ func Test_composeToServiceValues(t *testing.T) {
t.Errorf("composeToServiceValues() error = %v, wantErr %v", err, tt.wantErr)
return
}
lValues, _ := json.Marshal(got)
wValues, _ := json.Marshal(tt.want)
if !reflect.DeepEqual(string(lValues), string(wValues)) {
t.Errorf("composeToServiceValues() = %v, want %v", string(lValues), string(wValues))
f1, _ := json.MarshalIndent(got, "", " ")
r1, _ := json.MarshalIndent(tt.want, "", " ")
if !reflect.DeepEqual(f1, r1) {
t.Errorf("composeToServiceValues() = \n%v", diff.LineDiff(string(r1), string(f1)))
}
})
}
Expand Down
22 changes: 22 additions & 0 deletions internal/generator/volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,25 @@ func calculateServiceVolumes(buildValues *BuildValues, lagoonType, servicePersis

return serviceVolumes, nil
}

// flagDefaultVolumeCreation checks services for duplicate default volumes and flags only the ones that need to be created
// this will ensure that a service type that provides a volume will create that volume
// even if the `lagoon.persistent.name` is different to the default name being the service name
// due to the way labels in docker-compose can be added to services for lagoon, this allows multiple types to mount one shared volume by name
// from one main service, across multiple services by using the `lagoon.persistent.name` value
func flagDefaultVolumeCreation(
buildValues *BuildValues,
) error {
vols := make(map[string]bool)
for idx, service := range buildValues.Services {
if sType, ok := servicetypes.ServiceTypes[service.Type]; ok && service.PersistentVolumeName != "" {
if _, ok := vols[service.PersistentVolumeName]; !ok && sType.ProvidesPersistentVolume {
// this volume is the one to be created by the first service that provides a persistent volume
service.CreateDefaultVolume = true
vols[service.PersistentVolumeName] = true
buildValues.Services[idx] = service
}
}
}
return nil
}
Loading

0 comments on commit 1c4046f

Please sign in to comment.