feat: add secret discovery (#60)

* feat: add secret discovery

this also adds some performance by avoiding multiple I/O on files

closes #26

* fix: wrong paths for discovered secrets, add docs

README.md

@@ -57,7 +57,6 @@ docker stack deploy --compose-file docker-compose.yaml swarm-cd
 This will start SwarmCD, it will periodically check the stack repo
 for new changes, pulling them and updating the stack.
 
-
 ## Manage Encrypted Secrets Using SOPS
 
 You can use [sops](https://github.com/getsops/sops) to encrypt secrets in git repos and
@@ -113,6 +112,19 @@ secrets:
 This way, SwarmCD will decrypt the files each time before it updates
 the stack.
 
+### Automatic SOPS secrets detection
+
+Instead of specifying the paths of every single secrets you need to decrypt,
+you can use the `sops_secrets_discovery: true` option:
+
+- in the `config.yaml` file to enable it globally
+- in the `stacks.yaml` file for the individual stacks.
+
+Please note that:
+
+- if the global setting is set to `true`, it ignores individual stacks overrides.
+- if the stack-level setting is set to `true`, it ignores the `sops_files` setting altogether.
+
 ## Connect SwarmCD to a remote docker socket
 
 You can use the `DOCKER_HOST` environment variable to point SwarmCD to a remote docker socket,

docs/README.md

@@ -1,6 +1,7 @@
 # Documentation
 
-Here you can find configuration file references for 
+Here you can find configuration file references for:
+
 - [repos.yaml](repos.yaml)
 - [stacks.yaml](stacks.yaml)
-- [config.yaml](config.yaml)
+- [config.yaml](config.yaml)

docs/config.yaml

@@ -7,6 +7,9 @@ update_interval: 120
 # The path where SwarmCD will checkout repos
 repos_path: repos/
 
+# Automatically detect secrets to decrypt with SOPS
+sops_secrets_discovery: true
+
 # Automatically rotate configs and secrets
 # when the change. Adds a hash to config
 # and secret names
@@ -19,4 +22,3 @@ repos:
 # You can define stacks here instead of 
 # defining a separate stacks.yaml file
 stacks:
-

docs/stacks.yaml

@@ -22,4 +22,6 @@ stack-name:
   # before updating stack
   sops_files:
     - path/to/sops/encrypted/file
-
+  # Enable the automatic secret discovery
+  # alternative to sops_files
+  sops_secrets_discovery: false

swarmcd/init.go

@@ -19,7 +19,6 @@ type StackStatus struct {
 	RepoURL  string
 }
 
-
 var config *util.Config = &util.Configs
 
 var logger *slog.Logger = util.Logger
@@ -44,7 +43,7 @@ func Init() (err error) {
 	return
 }
 
-func initRepos() (error) {
+func initRepos() error {
 	for repoName, repoConfig := range config.RepoConfigs {
 		repoPath := path.Join(config.ReposPath, repoName)
 		auth, err := createHTTPBasicAuth(repoName)
@@ -98,7 +97,8 @@ func initStacks() error {
 		if !ok {
 			return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo)
 		}
-		swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile)
+		discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery
+		swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets)
 		stacks = append(stacks, swarmStack)
 		stackStatus[stack] = &StackStatus{}
 		stackStatus[stack].RepoURL = stackRepo.url

swarmcd/stack.go

@@ -1,6 +1,7 @@
 package swarmcd
 
 import (
+	"bytes"
 	"crypto/md5"
 	"fmt"
 	"log/slog"
@@ -14,22 +15,24 @@ import (
 )
 
 type swarmStack struct {
-	name        string
-	repo        *stackRepo
-	branch      string
-	composePath string
-	sopsFiles   []string
-	valuesFile  string
+	name            string
+	repo            *stackRepo
+	branch          string
+	composePath     string
+	sopsFiles       []string
+	valuesFile      string
+	discoverSecrets bool
 }
 
-func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string) *swarmStack {
+func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool) *swarmStack {
 	return &swarmStack{
-		name:        name,
-		repo:        repo,
-		branch:      branch,
-		composePath: composePath,
-		sopsFiles:   sopsFiles,
-		valuesFile:  valuesFile,
+		name:            name,
+		repo:            repo,
+		branch:          branch,
+		composePath:     composePath,
+		sopsFiles:       sopsFiles,
+		valuesFile:      valuesFile,
+		discoverSecrets: discoverSecrets,
 	}
 }
 
@@ -46,92 +49,138 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) {
 	}
 	log.Debug("changes pulled", "revision", revision)
 
-	log.Debug("decrypting sops files...")
-	err = swarmStack.decryptSopsFiles()
+	log.Debug("reading stack file...")
+	stackBytes, err := swarmStack.readStack()
 	if err != nil {
-		return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err)
+		return
 	}
 
 	if swarmStack.valuesFile != "" {
 		log.Debug("rendering template...")
-		err = swarmStack.renderComposeTemplate()
-		if err != nil {
-			return
-		}
+		stackBytes, err = swarmStack.renderComposeTemplate(stackBytes)
+	}
+	if err != nil {
+		return
 	}
 
-	log.Debug("rotating configs and secrets...")
-	err = swarmStack.rotateConfigsAndSecrets()
+	log.Debug("parsing stack content...")
+	stackContents, err := swarmStack.parseStackString([]byte(stackBytes))
 	if err != nil {
 		return
 	}
 
-	log.Debug("deploying stack...")
-	err = swarmStack.deployStack()
+	log.Debug("decrypting secrets...")
+	err = swarmStack.decryptSopsFiles(stackContents)
+	if err != nil {
+		return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err)
+	}
+
+	log.Debug("rotating configs and secrets...")
+	err = swarmStack.rotateConfigsAndSecrets(stackContents)
 	if err != nil {
 		return
 	}
-	return
-}
 
-func (swarmStack *swarmStack) decryptSopsFiles() (err error) {
-	for _, sopsFile := range swarmStack.sopsFiles {
-		err = util.DecryptFile(path.Join(swarmStack.repo.path, sopsFile))
-		if err != nil {
-			return
-		}
+	log.Debug("writing stack to file...")
+	err = swarmStack.writeStack(stackContents)
+	if err != nil {
+		return
 	}
+
+	log.Debug("deploying stack...")
+	err = swarmStack.deployStack()
 	return
 }
 
-func (swarmStack *swarmStack) deployStack() error {
-	cmd := stack.NewStackCommand(dockerCli)
-	cmd.SetArgs([]string{
-		"deploy", "--detach", "--with-registry-auth", "-c",
-		path.Join(swarmStack.repo.path, swarmStack.composePath),
-		swarmStack.name,
-	})
-	// To stop printing errors and usage message to stdout
-	cmd.SilenceErrors = true
-	cmd.SilenceUsage = true
-	err := cmd.Execute()
+func (swarmStack *swarmStack) readStack() ([]byte, error) {
+	composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath)
+	composeFileBytes, err := os.ReadFile(composeFile)
 	if err != nil {
-		return fmt.Errorf("could not deploy stack %s: %s", swarmStack.name, err)
+		return nil, fmt.Errorf("could not read compose file %s: %w", composeFile, err)
 	}
-	return nil
+	return composeFileBytes, nil
 }
 
-func (swarmStack *swarmStack) rotateConfigsAndSecrets() error {
-	composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath)
-	composeFileBytes, err := os.ReadFile(composeFile)
+func (swarmStack *swarmStack) renderComposeTemplate(templateContents []byte) ([]byte, error) {
+	valuesFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.valuesFile)
+	valuesBytes, err := os.ReadFile(valuesFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not read %s stack values file: %w", swarmStack.name, err)
+	}
+	var valuesMap map[string]any
+	yaml.Unmarshal(valuesBytes, &valuesMap)
+	templ, err := template.New(swarmStack.name).Parse(string(templateContents[:]))
+	if err != nil {
+		return nil, fmt.Errorf("could not parse %s stack compose file as a Go template: %w", swarmStack.name, err)
+	}
+	var stackContents bytes.Buffer
+	err = templ.Execute(&stackContents, map[string]map[string]any{"Values": valuesMap})
 	if err != nil {
-		return fmt.Errorf("could not read compose file %s: %w", composeFile, err)
+		return nil, fmt.Errorf("error rending %s stack compose template: %w", swarmStack.name, err)
 	}
+	return stackContents.Bytes(), nil
+}
+
+func (swarmStack *swarmStack) parseStackString(stackContent []byte) (map[string]any, error) {
 	var composeMap map[string]any
-	err = yaml.Unmarshal(composeFileBytes, &composeMap)
+	err := yaml.Unmarshal(stackContent, &composeMap)
 	if err != nil {
-		return fmt.Errorf("could not parse yaml file %s: %w", composeFile, err)
+		return nil, fmt.Errorf("could not parse stack yaml: %w", err)
 	}
+	return composeMap, nil
+}
 
+func (swarmStack *swarmStack) decryptSopsFiles(composeMap map[string]any) (err error) {
+	var sopsFiles []string
+	if !swarmStack.discoverSecrets {
+		sopsFiles = swarmStack.sopsFiles
+	} else {
+		sopsFiles, err = discoverSecrets(composeMap, swarmStack.composePath)
+		if err != nil {
+			return
+		}
+	}
+	for _, sopsFile := range sopsFiles {
+		err = util.DecryptFile(path.Join(swarmStack.repo.path, sopsFile))
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func discoverSecrets(composeMap map[string]any, composePath string) ([]string, error) {
+	var sopsFiles []string
+	if secrets, ok := composeMap["secrets"].(map[string]any); ok {
+		for secretName, secret := range secrets {
+			secretMap, ok := secret.(map[string]any)
+			if !ok {
+				return nil, fmt.Errorf("invalid compose file: %s secret must be a map", secretName)
+			}
+			secretFile, ok := secretMap["file"].(string)
+			if !ok {
+				return nil, fmt.Errorf("invalid compose file: %s file field must be a string", secretName)
+			}
+			objectDir := path.Join(path.Dir(composePath), secretFile)
+			sopsFiles = append(sopsFiles, objectDir)
+		}
+	}
+	return sopsFiles, nil
+}
+
+func (swarmStack *swarmStack) rotateConfigsAndSecrets(composeMap map[string]any) error {
 	if configs, ok := composeMap["configs"].(map[string]any); ok {
-		err = swarmStack.rotateObjects(configs)
+		err := swarmStack.rotateObjects(configs)
 		if err != nil {
 			return fmt.Errorf("could not rotate one or more config files of stack %s: %w", swarmStack.name, err)
 		}
 	}
 	if secrets, ok := composeMap["secrets"].(map[string]any); ok {
-		err = swarmStack.rotateObjects(secrets)
+		err := swarmStack.rotateObjects(secrets)
 		if err != nil {
 			return fmt.Errorf("could not rotate one or more secret files of stack %s: %w", swarmStack.name, err)
 		}
 	}
-
-	composeFileBytes, err = yaml.Marshal(composeMap)
-	if err != nil {
-		return fmt.Errorf("could not store comopse file as yaml after calculating hashes for stack %s", swarmStack.name)
-	}
-	fileInfo, _ := os.Stat(composeFile)
-	os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode())
 	return nil
 }
 
@@ -157,26 +206,31 @@ func (swarmStack *swarmStack) rotateObjects(objects map[string]any) error {
 	return nil
 }
 
-func (swarmStack *swarmStack) renderComposeTemplate() error {
-	composeFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.composePath)
-	valuesFile := path.Join(config.ReposPath, swarmStack.repo.path, swarmStack.valuesFile)
-	valuesBytes, err := os.ReadFile(valuesFile)
+func (swarmStack *swarmStack) writeStack(composeMap map[string]any) error {
+	composeFileBytes, err := yaml.Marshal(composeMap)
 	if err != nil {
-		return fmt.Errorf("could not read %s stack values file: %w", swarmStack.name, err)
+		return fmt.Errorf("could not store compose file as yaml after calculating hashes for stack %s", swarmStack.name)
 	}
-	var valuesMap map[string]any
-	yaml.Unmarshal(valuesBytes, &valuesMap)
-	templ, err := template.New(path.Base(composeFile)).ParseFiles(composeFile)
-	if err != nil {
-		return fmt.Errorf("could not parse %s stack compose file as a Go template: %w", swarmStack.name, err)
-	}
-	composeFileWriter, err := os.Create(composeFile)
-	if err != nil {
-		return fmt.Errorf("could not open %s stack compose file: %w", swarmStack.name, err)
-	}
-	err = templ.Execute(composeFileWriter, map[string]map[string]any{"Values": valuesMap})
+	composeFile := path.Join(swarmStack.repo.path, swarmStack.composePath)
+	fileInfo, _ := os.Stat(composeFile)
+	os.WriteFile(composeFile, composeFileBytes, fileInfo.Mode())
+	return nil
+}
+
+func (swarmStack *swarmStack) deployStack() error {
+	cmd := stack.NewStackCommand(dockerCli)
+	cmd.SetArgs([]string{
+		"deploy", "--detach", "--with-registry-auth", "-c",
+		path.Join(swarmStack.repo.path, swarmStack.composePath),
+		swarmStack.name,
+	})
+	// To stop printing errors and
+	// usage message to stdout
+	cmd.SilenceErrors = true
+	cmd.SilenceUsage = true
+	err := cmd.Execute()
 	if err != nil {
-		return fmt.Errorf("error rending %s stack compose template: %w", swarmStack.name, err)
+		return fmt.Errorf("could not deploy stack %s: %s", swarmStack.name, err)
 	}
 	return nil
 }