Skip to content

Commit

Permalink
Add --key flag for generate commands to specify resource key (#1165)
Browse files Browse the repository at this point in the history
## Changes
Add --key for generate commands to specify resource key.

Also, resource config files are now not prefixed anymore.

## Tests
Integration tests passed

---------

Co-authored-by: Pieter Noordhuis <[email protected]>
  • Loading branch information
andrewnester and pietern authored Jan 31, 2024
1 parent 5fda017 commit b28432a
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 8 deletions.
3 changes: 3 additions & 0 deletions cmd/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
)

func newGenerateCommand() *cobra.Command {
var key string

cmd := &cobra.Command{
Use: "generate",
Short: "Generate bundle configuration",
Expand All @@ -15,5 +17,6 @@ func newGenerateCommand() *cobra.Command {

cmd.AddCommand(generate.NewGenerateJobCommand())
cmd.AddCommand(generate.NewGeneratePipelineCommand())
cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`)
return cmd
}
8 changes: 6 additions & 2 deletions cmd/bundle/generate/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,18 @@ func TestGeneratePipelineCommand(t *testing.T) {

srcDir := filepath.Join(root, "src")
cmd.Flag("source-dir").Value.Set(srcDir)

var key string
cmd.Flags().StringVar(&key, "key", "test_pipeline", "")

err := cmd.RunE(cmd, []string{})
require.NoError(t, err)

data, err := os.ReadFile(filepath.Join(configDir, "pipeline_test_pipeline.yml"))
data, err := os.ReadFile(filepath.Join(configDir, "test_pipeline.yml"))
require.NoError(t, err)
require.Equal(t, fmt.Sprintf(`resources:
pipelines:
pipeline_test_pipeline:
test_pipeline:
name: test-pipeline
libraries:
- notebook:
Expand Down
6 changes: 5 additions & 1 deletion cmd/bundle/generate/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@ func NewGenerateJobCommand() *cobra.Command {
return err
}

jobKey := fmt.Sprintf("job_%s", textutil.NormalizeString(job.Settings.Name))
jobKey := cmd.Flag("key").Value.String()
if jobKey == "" {
jobKey = textutil.NormalizeString(job.Settings.Name)
}

result := map[string]dyn.Value{
"resources": dyn.V(map[string]dyn.Value{
"jobs": dyn.V(map[string]dyn.Value{
Expand Down
10 changes: 7 additions & 3 deletions cmd/bundle/generate/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,15 @@ func NewGeneratePipelineCommand() *cobra.Command {
return err
}

jobKey := fmt.Sprintf("pipeline_%s", textutil.NormalizeString(pipeline.Name))
pipelineKey := cmd.Flag("key").Value.String()
if pipelineKey == "" {
pipelineKey = textutil.NormalizeString(pipeline.Name)
}

result := map[string]dyn.Value{
"resources": dyn.V(map[string]dyn.Value{
"pipelines": dyn.V(map[string]dyn.Value{
jobKey: v,
pipelineKey: v,
}),
}),
}
Expand All @@ -77,7 +81,7 @@ func NewGeneratePipelineCommand() *cobra.Command {
return err
}

filename := filepath.Join(configDir, fmt.Sprintf("%s.yml", jobKey))
filename := filepath.Join(configDir, fmt.Sprintf("%s.yml", pipelineKey))
err = yamlsaver.SaveAsYAML(result, filename, force)
if err != nil {
return err
Expand Down
2 changes: 1 addition & 1 deletion internal/bundle/generate_job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func TestAccGenerateFromExistingJobAndDeploy(t *testing.T) {
_, err = os.Stat(filepath.Join(bundleRoot, "src", "test.py"))
require.NoError(t, err)

matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "job_generated_job_*.yml"))
matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "generated_job_*.yml"))
require.NoError(t, err)
require.Len(t, matches, 1)

Expand Down
2 changes: 1 addition & 1 deletion internal/bundle/generate_pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func TestAccGenerateFromExistingPipelineAndDeploy(t *testing.T) {
_, err = os.Stat(filepath.Join(bundleRoot, "src", "test.py"))
require.NoError(t, err)

matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "pipeline_generated_pipeline_*.yml"))
matches, err := filepath.Glob(filepath.Join(bundleRoot, "resources", "generated_pipeline_*.yml"))
require.NoError(t, err)
require.Len(t, matches, 1)

Expand Down

0 comments on commit b28432a

Please sign in to comment.