Export: Allow deleting manifests of specific envs when merging exports (

#871)

If, for example, `env1` and `env2` were deployed. `env2` is renamed to `env3`, the "merge" export would only export env3, causing manifest conflicts because `env2` manifests were removed

The feature is explained more in-depth in the docs changes

cmd/tk/export.go

@@ -40,6 +40,7 @@ func exportCmd() *cli.Command {
 		panic(err)
 	}
 	mergeStrategy := cmd.Flags().String("merge-strategy", "", "What to do when exporting to an existing directory. The default setting is to disallow exporting to an existing directory. Values: 'fail-on-conflicts', 'replace-envs'")
+	mergeDeletedEnvs := cmd.Flags().StringArray("merge-deleted-envs", nil, "Tanka main files that have been deleted. This is used when using a merge strategy to also delete the files of these deleted environments.")
 
 	vars := workflowFlags(cmd.Flags())
 	getJsonnetOpts := jsonnetFlags(cmd.Flags())
@@ -65,8 +66,9 @@ func exportCmd() *cli.Command {
 				Filters:     filters,
 				Name:        vars.name,
 			},
-			Selector:    getLabelSelector(),
-			Parallelism: *parallel,
+			Selector:         getLabelSelector(),
+			Parallelism:      *parallel,
+			MergeDeletedEnvs: *mergeDeletedEnvs,
 		}
 
 		if opts.MergeStrategy, err = determineMergeStrategy(*merge, *mergeStrategy); err != nil {

docs/docs/exporting.md

@@ -113,6 +113,31 @@ When these flags are passed, Tanka will:
 2. Generate the manifests for the targeted environments into the output directory.
 3. Add in the new manifests entries into the `manifest.json` file and re-export it.
 
+#### Finding out which environments to export
+
+Tanka provides the `tk tool importers` command to figure out which `main.jsonnet` need to be re-exported based on what files were modified in a workspace.
+
+If, for example, the `lib/my-lib/main.libsonnet` file was modified, you could run the command like this to find which files to export:
+
+```console
+# Find out which envs to re-export
+$ tk tool importers lib/my-lib/main.libsonnet
+my-repo-path/jsonnet/environments/my-env/main.jsonnet
+
+# Re-export the envs
+$ tk export myoutputdir my-repo-path/jsonnet/environments/my-env/main.jsonnet --merge-strategy=replace-envs
+```
+
+Note that deleted environments need special consideration when doing this.
+The `tk tool importers` utility only works with existing files so deleting an environment will result in stale `manifest.json` entries and moving an environment will result in manifest conflicts.
+In order to correctly handle deleted environments, they need to be passed to the export command:
+
+```console
+$ tk export myoutputdir my-repo-path/jsonnet/environments/my-new-env-path/main.jsonnet --merge-strategy=replace-envs \
+  --merge-deleted-envs my-repo-path/jsonnet/environments/my-old-env-path/main.jsonnet \
+  --merge-deleted-envs my-repo-path/jsonnet/environments/other-deleted-env-path/main.jsonnet
+```
+
 ### Using a memory ballast
 
 _Read [this blog post](https://blog.twitch.tv/en/2019/04/10/go-memory-ballast-how-i-learnt-to-stop-worrying-and-love-the-heap/) for more information about memory ballasts._

pkg/tanka/export.go

@@ -56,6 +56,9 @@ type ExportEnvOpts struct {
 	// - fail-on-conflicts: fail when an exported file already exists
 	// - replace-envs: delete files previously exported by the targeted envs and re-export them
 	MergeStrategy ExportMergeStrategy
+	// Environments (main.jsonnet files) that have been deleted since the last export.
+	// This is used when using a merge strategy to delete the files of these deleted environments.
+	MergeDeletedEnvs []string
 }
 
 func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnvOpts) error {
@@ -73,11 +76,16 @@ func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnv
 
 	// delete files previously exported by the targeted envs.
 	if opts.MergeStrategy == ExportMergeStrategyReplaceEnvs {
-		if err := deletePreviouslyExportedManifests(to, envs); err != nil {
+		if err := deletePreviouslyExportedManifestsFromTankaEnvs(to, envs); err != nil {
 			return fmt.Errorf("deleting previously exported manifests: %w", err)
 		}
 	}
 
+	// delete files that were exported by environments that have been deleted since the last export.
+	if err := deletePreviouslyExportedManifests(to, opts.MergeDeletedEnvs); err != nil {
+		return fmt.Errorf("deleting previously exported manifests from deleted environments: %w", err)
+	}
+
 	// get all environments for paths
 	loadedEnvs, err := parallelLoadEnvironments(envs, parallelOpts{
 		Opts:        opts.Opts,
@@ -174,7 +182,24 @@ func dirEmpty(dir string) (bool, error) {
 	return false, err
 }
 
-func deletePreviouslyExportedManifests(path string, envs []*v1alpha1.Environment) error {
+func deletePreviouslyExportedManifestsFromTankaEnvs(path string, envs []*v1alpha1.Environment) error {
+	envNames := []string{}
+	for _, env := range envs {
+		envNames = append(envNames, env.Metadata.Namespace)
+	}
+	return deletePreviouslyExportedManifests(path, envNames)
+}
+
+func deletePreviouslyExportedManifests(path string, tankaEnvNames []string) error {
+	if len(tankaEnvNames) == 0 {
+		return nil
+	}
+
+	envNamesMap := make(map[string]struct{})
+	for _, envName := range tankaEnvNames {
+		envNamesMap[envName] = struct{}{}
+	}
+
 	fileToEnvMap := make(map[string]string)
 
 	manifestFilePath := filepath.Join(path, manifestFile)
@@ -190,14 +215,9 @@ func deletePreviouslyExportedManifests(path string, envs []*v1alpha1.Environment
 		return err
 	}
 
-	envNames := make(map[string]struct{})
-	for _, env := range envs {
-		envNames[env.Metadata.Namespace] = struct{}{}
-	}
-
 	var deletedManifestKeys []string
 	for exportedManifest, manifestEnv := range fileToEnvMap {
-		if _, ok := envNames[manifestEnv]; ok {
+		if _, ok := envNamesMap[manifestEnv]; ok {
 			deletedManifestKeys = append(deletedManifestKeys, exportedManifest)
 			if err := os.Remove(filepath.Join(path, exportedManifest)); err != nil {
 				return err

pkg/tanka/export_test.go

@@ -121,6 +121,25 @@ func TestExportEnvironments(t *testing.T) {
     "static/updated-deployment.yaml": "test-export-envs/static-env/main.jsonnet",
     "static/updated-service.yaml": "test-export-envs/static-env/main.jsonnet"
 }`)
+
+	// Re-export and delete the files of one env
+	opts.Opts.ExtCode = jsonnet.InjectedCode{
+		"deploymentName": "'updated-again-deployment'",
+		"serviceName":    "'updated-again-service'",
+	}
+	opts.MergeDeletedEnvs = []string{"test-export-envs/inline-envs/main.jsonnet"}
+	require.NoError(t, ExportEnvironments(staticEnv, tempDir, opts))
+	checkFiles(t, tempDir, []string{
+		filepath.Join(tempDir, "static", "updated-again-deployment.yaml"),
+		filepath.Join(tempDir, "static", "updated-again-service.yaml"),
+		filepath.Join(tempDir, "manifest.json"),
+	})
+	manifestContent, err = os.ReadFile(filepath.Join(tempDir, "manifest.json"))
+	require.NoError(t, err)
+	assert.Equal(t, string(manifestContent), `{
+    "static/updated-again-deployment.yaml": "test-export-envs/static-env/main.jsonnet",
+    "static/updated-again-service.yaml": "test-export-envs/static-env/main.jsonnet"
+}`)
 }
 
 func BenchmarkExportEnvironmentsWithReplaceEnvs(b *testing.B) {