From bb347a1dc56d0fb9cb878e1f73bec830c308d807 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 8 Oct 2024 14:15:42 +0000 Subject: [PATCH 1/9] start: allow users to call job start command to start up a previously stopped job --- command/commands.go | 10 ++ command/job_start.go | 252 ++++++++++++++++++++++++++++++++++++++ command/job_start_test.go | 200 ++++++++++++++++++++++++++++++ 3 files changed, 462 insertions(+) create mode 100644 command/job_start.go create mode 100644 command/job_start_test.go diff --git a/command/commands.go b/command/commands.go index 7a46c08d484..9fb41f4ad98 100644 --- a/command/commands.go +++ b/command/commands.go @@ -521,6 +521,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "job start": func() (cli.Command, error) { + return &JobStartCommand{ + Meta: meta, + }, nil + }, "job validate": func() (cli.Command, error) { return &JobValidateCommand{ Meta: meta, @@ -1071,6 +1076,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "start": func() (cli.Command, error) { + return &JobStartCommand{ + Meta: meta, + }, nil + }, "system": func() (cli.Command, error) { return &SystemCommand{ Meta: meta, diff --git a/command/job_start.go b/command/job_start.go new file mode 100644 index 00000000000..43479515343 --- /dev/null +++ b/command/job_start.go @@ -0,0 +1,252 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/api/contexts" + "github.com/posener/complete" + "os" + "strings" + "sync" +) + +type JobStartCommand struct { + Meta +} + +func (c *JobStartCommand) Help() string { + helpText := ` +Usage: nomad job start [options] +Alias: nomad start + + Start an existing stopped job. This command is used to start a previously stopped job's + most recent running version up. Upon successful start, an interactive + monitor session will start to display log lines as the job starts up its + allocations based on its most recent running version. It is safe to exit the monitor + early using ctrl+c. + + When ACLs are enabled, this command requires a token with the 'submit-job' + and 'read-job' capabilities for the job's namespace. The 'list-jobs' + capability is required to run the command with job prefixes instead of exact + job IDs. + + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Start Options: + + -detach + Return immediately instead of entering monitor mode. After the + job start command is submitted, a new evaluation ID is printed to the + screen, which can be used to examine the evaluation using the eval-status + command. + + -consul-token + The Consul token used to verify that the caller has access to the Service + Identity policies associated in the targeted version of the job. + + -vault-token + The Vault token used to verify that the caller has access to the Vault + policies in the targeted version of the job. + + -verbose + Display full information. +` + return strings.TrimSpace(helpText) +} + +func (c *JobStartCommand) Synopsis() string { + return "Start a stopped job" +} + +func (c *JobStartCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-detach": complete.PredictNothing, + "-verbose": complete.PredictNothing, + }) +} + +func (c *JobStartCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictFunc(func(a complete.Args) []string { + client, err := c.Meta.Client() + if err != nil { + return nil + } + + resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Jobs, nil) + if err != nil { + return []string{} + } + return resp.Matches[contexts.Jobs] + }) +} +func (c *JobStartCommand) Name() string { return "job start" } + +func (c *JobStartCommand) Run(args []string) int { + var detach, verbose bool + var consulToken, vaultToken string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&detach, "detach", false, "") + flags.BoolVar(&verbose, "verbose", false, "") + flags.StringVar(&consulToken, "consul-token", "", "") + flags.StringVar(&vaultToken, "vault-token", "", "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got at least one job + args = flags.Args() + if len(args) < 1 { + c.Ui.Error("This command takes at least one argument: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + var jobIDs []string + for _, jobID := range flags.Args() { + jobIDs = append(jobIDs, strings.TrimSpace(jobID)) + } + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + statusCh := make(chan int, len(jobIDs)) + var wg sync.WaitGroup + + for _, jobID := range jobIDs { + jobID := jobID + + wg.Add(1) + go func() { + defer wg.Done() + + // Truncate the id unless full length is requested + length := shortId + if verbose { + length = fullId + } + + // Check if the job exists and has been stopped + jobId, namespace, err := c.JobIDByPrefix(client, jobID, nil) + if err != nil { + c.Ui.Error(err.Error()) + statusCh <- 1 + return + } + job, err := c.JobByPrefix(client, jobId, nil) + if err != nil { + c.Ui.Error(err.Error()) + statusCh <- 1 + return + } + if *job.Status != "dead" { + c.Ui.Error(fmt.Sprintf("Job %v has not been stopped and has following status: %v", *job.Name, *job.Status)) + statusCh <- 1 + return + + } + + // Get all versions associated to current job + q := &api.QueryOptions{Namespace: namespace} + + versions, _, _, err := client.Jobs().Versions(jobID, true, q) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s", err)) + statusCh <- 1 + } + + // Find the most recent running version for this job + var chosenVersion *api.Job + var chosenIndex uint64 + versionAvailable := false + for i := len(versions) - 1; i >= 0; i-- { + if *versions[i].Status == "running" { + chosenVersion = versions[i] + chosenIndex = uint64(i) + versionAvailable = true + } + + } + if !versionAvailable { + c.Ui.Error(fmt.Sprintf("No previous running versions of job %v, %s", chosenVersion, err)) + statusCh <- 1 + return + } + + // Parse the Consul token + if consulToken == "" { + // Check the environment variable + consulToken = os.Getenv("CONSUL_HTTP_TOKEN") + } + + // Parse the Vault token + if vaultToken == "" { + // Check the environment variable + vaultToken = os.Getenv("VAULT_TOKEN") + } + + // Revert to most recent running version! + m := &api.WriteOptions{Namespace: namespace} + resp, _, err := client.Jobs().Revert(jobID, chosenIndex, nil, m, consulToken, vaultToken) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s, %v", err, chosenIndex)) + statusCh <- 1 + return + } + if *job.Name == "bridge2" { + c.Ui.Output(fmt.Sprintf("HERE")) + + } + + // Nothing to do + evalCreated := resp.EvalID != "" + + if !evalCreated { + statusCh <- 0 + return + } + + if detach { + c.Ui.Output("Evaluation ID: " + resp.EvalID) + statusCh <- 0 + return + } + + mon := newMonitor(c.Ui, client, length) + statusCh <- mon.monitor(resp.EvalID) + + }() + } + // users will still see + // errors if any while we + // wait for the goroutines + // to finish processing + wg.Wait() + + // close the channel to ensure + // the range statement below + // doesn't go on indefinitely + close(statusCh) + + // return a non-zero exit code + // if even a single job start fails + for status := range statusCh { + if status != 0 { + return status + } + } + return 0 +} diff --git a/command/job_start_test.go b/command/job_start_test.go new file mode 100644 index 00000000000..aac3c018728 --- /dev/null +++ b/command/job_start_test.go @@ -0,0 +1,200 @@ +package command + +import ( + "encoding/json" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/posener/complete" + "os" + "path/filepath" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/command/agent" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/mitchellh/cli" + "github.com/shoenig/test/must" +) + +var _ cli.Command = (*JobStartCommand)(nil) + +func TestJobStartCommand_Fails(t *testing.T) { + ci.Parallel(t) + + srv, _, addr := testServer(t, true, func(c *agent.Config) { + c.DevMode = true + }) + defer srv.Shutdown() + + ui := cli.NewMockUi() + cmd := &JobStartCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + code := cmd.Run([]string{"-bad", "-flag"}) + must.One(t, code) + + out := ui.ErrorWriter.String() + must.StrContains(t, out, "flag provided but not defined: -bad") + + ui.ErrorWriter.Reset() + + // Fails on nonexistent job ID + code = cmd.Run([]string{"-address=" + addr, "non-existent"}) + must.One(t, code) + + out = ui.ErrorWriter.String() + must.StrContains(t, out, "No job(s) with prefix or ID") + + ui.ErrorWriter.Reset() + + // Fails on connection failure + code = cmd.Run([]string{"-address=nope", "n"}) + must.One(t, code) + + out = ui.ErrorWriter.String() + must.StrContains(t, out, "Error querying job prefix") + + // Fails on attempting to start a job that's not been stopped + jobID := uuid.Generate() + jobFilePath := filepath.Join(os.TempDir(), jobID+".nomad") + + t.Cleanup(func() { + _ = os.Remove(jobFilePath) + }) + job := testJob(jobID) + job.TaskGroups[0].Tasks[0].Resources.MemoryMB = pointer.Of(16) + job.TaskGroups[0].Tasks[0].Resources.DiskMB = pointer.Of(32) + job.TaskGroups[0].Tasks[0].Resources.CPU = pointer.Of(10) + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "30s", + } + + jobJSON, err := json.MarshalIndent(job, "", " ") + must.NoError(t, err) + + jobFile := jobFilePath + err = os.WriteFile(jobFile, []byte(jobJSON), 0o644) + must.NoError(t, err) + + runCmd := &JobRunCommand{Meta: Meta{Ui: ui}} + code = runCmd.Run([]string{"-address", addr, "-json", jobFile}) + must.Zero(t, code, + must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), + must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), + ) + + code = cmd.Run([]string{"-address=" + addr, jobID}) + must.One(t, code) + out = ui.ErrorWriter.String() + must.StrContains(t, out, "has not been stopped and has following status:") + +} + +func TestStartCommand_ManyJobs(t *testing.T) { + ci.Parallel(t) + + srv, _, addr := testServer(t, true, func(c *agent.Config) { + c.DevMode = true + }) + defer srv.Shutdown() + + // the number of jobs we want to run + numJobs := 10 + + // create and run a handful of jobs + jobIDs := make([]string, 0, numJobs) + for i := 0; i < numJobs; i++ { + jobID := uuid.Generate() + jobIDs = append(jobIDs, jobID) + } + + jobFilePath := func(jobID string) string { + return filepath.Join(os.TempDir(), jobID+".nomad") + } + + // cleanup job files we will create + t.Cleanup(func() { + for _, jobID := range jobIDs { + _ = os.Remove(jobFilePath(jobID)) + } + }) + + // record cli output + ui := cli.NewMockUi() + + for _, jobID := range jobIDs { + job := testJob(jobID) + job.TaskGroups[0].Tasks[0].Resources.MemoryMB = pointer.Of(16) + job.TaskGroups[0].Tasks[0].Resources.DiskMB = pointer.Of(32) + job.TaskGroups[0].Tasks[0].Resources.CPU = pointer.Of(10) + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "30s", + } + + jobJSON, err := json.MarshalIndent(job, "", " ") + must.NoError(t, err) + + jobFile := jobFilePath(jobID) + err = os.WriteFile(jobFile, []byte(jobJSON), 0o644) + must.NoError(t, err) + + cmd := &JobRunCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-address", addr, "-json", jobFile}) + must.Zero(t, code, + must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), + must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), + ) + } + + // helper for stopping a list of jobs + stop := func(args ...string) (stdout string, stderr string, code int) { + cmd := &JobStopCommand{Meta: Meta{Ui: ui}} + code = cmd.Run(args) + return ui.OutputWriter.String(), ui.ErrorWriter.String(), code + } + // helper for starting a list of jobs + start := func(args ...string) (stdout string, stderr string, code int) { + cmd := &JobStartCommand{Meta: Meta{Ui: ui}} + code = cmd.Run(args) + return ui.OutputWriter.String(), ui.ErrorWriter.String(), code + } + + // stop all jobs in one command + args := []string{"-address", addr, "-detach"} + args = append(args, jobIDs...) + stdout, stderr, code := stop(args...) + must.Zero(t, code, + must.Sprintf("job stop stdout: %s", stdout), + must.Sprintf("job stop stderr: %s", stderr), + ) + + // start all jobs again in one command + stdout, stderr, code = start(args...) + must.Zero(t, code, + must.Sprintf("job start stdout: %s", stdout), + must.Sprintf("job start stderr: %s", stderr), + ) +} +func TestStartCommand_AutocompleteArgs(t *testing.T) { + ci.Parallel(t) + + srv, _, url := testServer(t, true, nil) + defer srv.Shutdown() + + ui := cli.NewMockUi() + cmd := &JobStartCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Create a fake job + state := srv.Agent.Server().State() + j := mock.Job() + must.NoError(t, state.UpsertJob(structs.MsgTypeTestSetup, 1000, nil, j)) + + prefix := j.ID[:len(j.ID)-5] + args := complete.Args{Last: prefix} + predictor := cmd.AutocompleteArgs() + + res := predictor.Predict(args) + must.Len(t, 1, res) + must.Eq(t, j.ID, res[0]) +} From a5e5c66c54c00b0d650e95392db97bd38b988671 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Tue, 8 Oct 2024 15:36:43 +0000 Subject: [PATCH 2/9] quick clean up --- command/job_start.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/command/job_start.go b/command/job_start.go index 43479515343..1772eff776e 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -206,10 +206,6 @@ func (c *JobStartCommand) Run(args []string) int { statusCh <- 1 return } - if *job.Name == "bridge2" { - c.Ui.Output(fmt.Sprintf("HERE")) - - } // Nothing to do evalCreated := resp.EvalID != "" From a36a5d8c6dadb99c05c628d3de72a8737f5c533e Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 24 Oct 2024 15:01:09 +0000 Subject: [PATCH 3/9] clean up and remove stable condition --- command/job_start.go | 16 ++++++++-------- command/job_start_test.go | 24 +++++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/command/job_start.go b/command/job_start.go index 1772eff776e..ce03fd4ecf3 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -139,7 +139,7 @@ func (c *JobStartCommand) Run(args []string) int { length = fullId } - // Check if the job exists and has been stopped + // Check if the job exists and has been stopped (status is dead) jobId, namespace, err := c.JobIDByPrefix(client, jobID, nil) if err != nil { c.Ui.Error(err.Error()) @@ -153,8 +153,8 @@ func (c *JobStartCommand) Run(args []string) int { return } if *job.Status != "dead" { - c.Ui.Error(fmt.Sprintf("Job %v has not been stopped and has following status: %v", *job.Name, *job.Status)) - statusCh <- 1 + c.Ui.Info(fmt.Sprintf("Job %v has not been stopped and has the following status: %v", *job.Name, *job.Status)) + statusCh <- 0 return } @@ -169,19 +169,18 @@ func (c *JobStartCommand) Run(args []string) int { } // Find the most recent running version for this job - var chosenVersion *api.Job var chosenIndex uint64 versionAvailable := false for i := len(versions) - 1; i >= 0; i-- { - if *versions[i].Status == "running" { - chosenVersion = versions[i] + if *versions[i].Status == "running" && *versions[i].Stop == true { chosenIndex = uint64(i) versionAvailable = true + break } } if !versionAvailable { - c.Ui.Error(fmt.Sprintf("No previous running versions of job %v, %s", chosenVersion, err)) + c.Ui.Error(fmt.Sprintf("No previous running versions of job %v, %s", *job.Name, err)) statusCh <- 1 return } @@ -200,9 +199,10 @@ func (c *JobStartCommand) Run(args []string) int { // Revert to most recent running version! m := &api.WriteOptions{Namespace: namespace} + resp, _, err := client.Jobs().Revert(jobID, chosenIndex, nil, m, consulToken, vaultToken) if err != nil { - c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s, %v", err, chosenIndex)) + c.Ui.Error(fmt.Sprintf("Error retrieving job version %v for job %s: %s,", chosenIndex, jobID, err)) statusCh <- 1 return } diff --git a/command/job_start_test.go b/command/job_start_test.go index aac3c018728..b32a499196b 100644 --- a/command/job_start_test.go +++ b/command/job_start_test.go @@ -2,19 +2,18 @@ package command import ( "encoding/json" - "github.com/hashicorp/nomad/nomad/mock" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/posener/complete" - "os" - "path/filepath" - "testing" - "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/command/agent" "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/cli" + "github.com/posener/complete" "github.com/shoenig/test/must" + "os" + "path/filepath" + "testing" ) var _ cli.Command = (*JobStartCommand)(nil) @@ -55,7 +54,7 @@ func TestJobStartCommand_Fails(t *testing.T) { out = ui.ErrorWriter.String() must.StrContains(t, out, "Error querying job prefix") - // Fails on attempting to start a job that's not been stopped + // Info on attempting to start a job that's not been stopped jobID := uuid.Generate() jobFilePath := filepath.Join(os.TempDir(), jobID+".nomad") @@ -85,8 +84,8 @@ func TestJobStartCommand_Fails(t *testing.T) { ) code = cmd.Run([]string{"-address=" + addr, jobID}) - must.One(t, code) - out = ui.ErrorWriter.String() + must.Zero(t, code) + out = ui.OutputWriter.String() must.StrContains(t, out, "has not been stopped and has following status:") } @@ -98,7 +97,6 @@ func TestStartCommand_ManyJobs(t *testing.T) { c.DevMode = true }) defer srv.Shutdown() - // the number of jobs we want to run numJobs := 10 @@ -140,11 +138,13 @@ func TestStartCommand_ManyJobs(t *testing.T) { must.NoError(t, err) cmd := &JobRunCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-address", addr, "-json", jobFile}) must.Zero(t, code, must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), ) + } // helper for stopping a list of jobs @@ -175,7 +175,9 @@ func TestStartCommand_ManyJobs(t *testing.T) { must.Sprintf("job start stdout: %s", stdout), must.Sprintf("job start stderr: %s", stderr), ) + } + func TestStartCommand_AutocompleteArgs(t *testing.T) { ci.Parallel(t) From 0d0740f46dcf7d37977ab4a63357736659e091e7 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 24 Oct 2024 15:04:17 +0000 Subject: [PATCH 4/9] copyright headers --- command/job_start_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/command/job_start_test.go b/command/job_start_test.go index b32a499196b..e5061525cac 100644 --- a/command/job_start_test.go +++ b/command/job_start_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package command import ( From f2da6d41c28e899872a46204d154cb7878817032 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 24 Oct 2024 16:44:26 +0000 Subject: [PATCH 5/9] tidy conditions --- command/job_start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/job_start.go b/command/job_start.go index ce03fd4ecf3..6f7b812b0b5 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -172,7 +172,7 @@ func (c *JobStartCommand) Run(args []string) int { var chosenIndex uint64 versionAvailable := false for i := len(versions) - 1; i >= 0; i-- { - if *versions[i].Status == "running" && *versions[i].Stop == true { + if *versions[i].Status == "running" && *versions[i].Stop { chosenIndex = uint64(i) versionAvailable = true break From 35e15db142d7b7db1c1f539cc04100f98f0c5234 Mon Sep 17 00:00:00 2001 From: Martina Santangelo Date: Thu, 31 Oct 2024 13:01:26 -0400 Subject: [PATCH 6/9] test versions, add changelog --- .changelog/24150.txt | 3 ++ command/commands.go | 2 +- command/job_start.go | 44 +++++++++-------------- command/job_start_test.go | 73 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 90 insertions(+), 32 deletions(-) create mode 100644 .changelog/24150.txt diff --git a/.changelog/24150.txt b/.changelog/24150.txt new file mode 100644 index 00000000000..cab219ca49a --- /dev/null +++ b/.changelog/24150.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: Added job start command to allow starting a stopped job from the cli +``` diff --git a/command/commands.go b/command/commands.go index ec979a165a9..d516b158f27 100644 --- a/command/commands.go +++ b/command/commands.go @@ -523,7 +523,7 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { }, "job start": func() (cli.Command, error) { return &JobStartCommand{ - Meta: meta, + Meta: meta, }, nil }, "job tag": func() (cli.Command, error) { diff --git a/command/job_start.go b/command/job_start.go index 6f7b812b0b5..cec42d2ae09 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -5,16 +5,18 @@ package command import ( "fmt" - "github.com/hashicorp/nomad/api" - "github.com/hashicorp/nomad/api/contexts" - "github.com/posener/complete" "os" "strings" "sync" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/api/contexts" + "github.com/posener/complete" ) type JobStartCommand struct { Meta + versionSelected uint64 } func (c *JobStartCommand) Help() string { @@ -23,8 +25,8 @@ Usage: nomad job start [options] Alias: nomad start Start an existing stopped job. This command is used to start a previously stopped job's - most recent running version up. Upon successful start, an interactive - monitor session will start to display log lines as the job starts up its + most recent running version. Upon successful start, an interactive + monitor session will start to display log lines as the job starts its allocations based on its most recent running version. It is safe to exit the monitor early using ctrl+c. @@ -33,7 +35,6 @@ Alias: nomad start capability is required to run the command with job prefixes instead of exact job IDs. - General Options: ` + generalOptionsUsage(usageOptsDefault) + ` @@ -126,8 +127,7 @@ func (c *JobStartCommand) Run(args []string) int { statusCh := make(chan int, len(jobIDs)) var wg sync.WaitGroup - for _, jobID := range jobIDs { - jobID := jobID + for _, jobIDPrefix := range jobIDs { wg.Add(1) go func() { @@ -139,14 +139,7 @@ func (c *JobStartCommand) Run(args []string) int { length = fullId } - // Check if the job exists and has been stopped (status is dead) - jobId, namespace, err := c.JobIDByPrefix(client, jobID, nil) - if err != nil { - c.Ui.Error(err.Error()) - statusCh <- 1 - return - } - job, err := c.JobByPrefix(client, jobId, nil) + job, err := c.JobByPrefix(client, jobIDPrefix, nil) if err != nil { c.Ui.Error(err.Error()) statusCh <- 1 @@ -160,9 +153,9 @@ func (c *JobStartCommand) Run(args []string) int { } // Get all versions associated to current job - q := &api.QueryOptions{Namespace: namespace} + q := &api.QueryOptions{Namespace: *job.Namespace} - versions, _, _, err := client.Jobs().Versions(jobID, true, q) + versions, _, _, err := client.Jobs().Versions(*job.ID, true, q) if err != nil { c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s", err)) statusCh <- 1 @@ -179,35 +172,32 @@ func (c *JobStartCommand) Run(args []string) int { } } + c.versionSelected = chosenIndex if !versionAvailable { - c.Ui.Error(fmt.Sprintf("No previous running versions of job %v, %s", *job.Name, err)) + c.Ui.Error(fmt.Sprintf("No previous running versions of job %v", *job.Name)) statusCh <- 1 return } - // Parse the Consul token if consulToken == "" { - // Check the environment variable consulToken = os.Getenv("CONSUL_HTTP_TOKEN") } - // Parse the Vault token if vaultToken == "" { - // Check the environment variable vaultToken = os.Getenv("VAULT_TOKEN") } // Revert to most recent running version! - m := &api.WriteOptions{Namespace: namespace} + m := &api.WriteOptions{Namespace: *job.Namespace} - resp, _, err := client.Jobs().Revert(jobID, chosenIndex, nil, m, consulToken, vaultToken) + resp, _, err := client.Jobs().Revert(*job.ID, chosenIndex, nil, m, consulToken, vaultToken) if err != nil { - c.Ui.Error(fmt.Sprintf("Error retrieving job version %v for job %s: %s,", chosenIndex, jobID, err)) + c.Ui.Error(fmt.Sprintf("Error retrieving job version %v for job %s: %s,", chosenIndex, *job.ID, err)) statusCh <- 1 return } - // Nothing to do + // Nothing to do: periodic or dispatch job evalCreated := resp.EvalID != "" if !evalCreated { diff --git a/command/job_start_test.go b/command/job_start_test.go index e5061525cac..1c005847d1a 100644 --- a/command/job_start_test.go +++ b/command/job_start_test.go @@ -5,6 +5,10 @@ package command import ( "encoding/json" + "os" + "path/filepath" + "testing" + "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/command/agent" "github.com/hashicorp/nomad/helper/pointer" @@ -14,9 +18,6 @@ import ( "github.com/mitchellh/cli" "github.com/posener/complete" "github.com/shoenig/test/must" - "os" - "path/filepath" - "testing" ) var _ cli.Command = (*JobStartCommand)(nil) @@ -89,7 +90,7 @@ func TestJobStartCommand_Fails(t *testing.T) { code = cmd.Run([]string{"-address=" + addr, jobID}) must.Zero(t, code) out = ui.OutputWriter.String() - must.StrContains(t, out, "has not been stopped and has following status:") + must.StrContains(t, out, "has not been stopped and has the following status:") } @@ -181,6 +182,70 @@ func TestStartCommand_ManyJobs(t *testing.T) { } +func TestStartCommand_StartCorrectVersion(t *testing.T) { + ci.Parallel(t) + + srv, _, addr := testServer(t, true, func(c *agent.Config) { + c.DevMode = true + }) + defer srv.Shutdown() + + jobID := uuid.Generate() + + jobFilePath := filepath.Join(os.TempDir(), jobID+".nomad") + + t.Cleanup(func() { + _ = os.Remove(jobFilePath) + }) + + ui := cli.NewMockUi() + + job := testNomadServiceJob(jobID) + job.TaskGroups[0].Tasks[0].Resources.MemoryMB = pointer.Of(16) + job.TaskGroups[0].Tasks[0].Resources.DiskMB = pointer.Of(32) + job.TaskGroups[0].Tasks[0].Resources.CPU = pointer.Of(10) + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "30s", + } + + jobJSON, err := json.MarshalIndent(job, "", " ") + must.NoError(t, err) + + jobFile := jobFilePath + err = os.WriteFile(jobFile, []byte(jobJSON), 0o644) + must.NoError(t, err) + + cmd := &JobRunCommand{Meta: Meta{Ui: ui}} + + code := cmd.Run([]string{"-address", addr, "-json", jobFile}) + must.Zero(t, code, + must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), + must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), + ) + + args := []string{"-address", addr, "-detach"} + args = append(args, jobID) + expectedVersions := []uint64{0, 2, 4} + stopCmd := &JobStopCommand{Meta: Meta{Ui: ui}} + startCmd := &JobStartCommand{Meta: Meta{Ui: ui}} + + // for multiple cycles of starting/stopping a job, check that the correct, most recent running version is picked + for i := range 3 { + code = stopCmd.Run(args) + must.Zero(t, code, + must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), + must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), + ) + code = startCmd.Run(args) + must.Zero(t, code, + must.Sprintf("job start stdout: %s", ui.OutputWriter.String()), + must.Sprintf("job start stderr: %s", ui.ErrorWriter.String()), + ) + must.Eq(t, expectedVersions[i], startCmd.versionSelected) + } + +} + func TestStartCommand_AutocompleteArgs(t *testing.T) { ci.Parallel(t) From 8f0e67b19f2a519dedc03c8b567bd487bf5ca92d Mon Sep 17 00:00:00 2001 From: Martina Santangelo Date: Thu, 7 Nov 2024 13:05:49 -0500 Subject: [PATCH 7/9] selecting version fixes --- command/job_start.go | 18 ++++++------ command/job_start_test.go | 59 +++++++++++---------------------------- 2 files changed, 27 insertions(+), 50 deletions(-) diff --git a/command/job_start.go b/command/job_start.go index cec42d2ae09..05a1dbbf8ad 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -145,6 +145,7 @@ func (c *JobStartCommand) Run(args []string) int { statusCh <- 1 return } + if *job.Status != "dead" { c.Ui.Info(fmt.Sprintf("Job %v has not been stopped and has the following status: %v", *job.Name, *job.Status)) statusCh <- 0 @@ -161,18 +162,18 @@ func (c *JobStartCommand) Run(args []string) int { statusCh <- 1 } - // Find the most recent running version for this job - var chosenIndex uint64 + // Find the most recent version for this job that has not been stopped + var chosenVersion uint64 versionAvailable := false - for i := len(versions) - 1; i >= 0; i-- { - if *versions[i].Status == "running" && *versions[i].Stop { - chosenIndex = uint64(i) + for i := range versions { + if !*versions[i].Stop { + chosenVersion = *versions[i].Version versionAvailable = true break } } - c.versionSelected = chosenIndex + c.versionSelected = chosenVersion if !versionAvailable { c.Ui.Error(fmt.Sprintf("No previous running versions of job %v", *job.Name)) statusCh <- 1 @@ -190,12 +191,13 @@ func (c *JobStartCommand) Run(args []string) int { // Revert to most recent running version! m := &api.WriteOptions{Namespace: *job.Namespace} - resp, _, err := client.Jobs().Revert(*job.ID, chosenIndex, nil, m, consulToken, vaultToken) + resp, _, err := client.Jobs().Revert(*job.ID, chosenVersion, nil, m, consulToken, vaultToken) if err != nil { - c.Ui.Error(fmt.Sprintf("Error retrieving job version %v for job %s: %s,", chosenIndex, *job.ID, err)) + c.Ui.Error(fmt.Sprintf("Error retrieving job version %v for job %s: %s,", chosenVersion, *job.ID, err)) statusCh <- 1 return } + versions, _, _, err = client.Jobs().Versions(*job.ID, true, q) // Nothing to do: periodic or dispatch job evalCreated := resp.EvalID != "" diff --git a/command/job_start_test.go b/command/job_start_test.go index 1c005847d1a..9857cbd12c7 100644 --- a/command/job_start_test.go +++ b/command/job_start_test.go @@ -127,12 +127,6 @@ func TestStartCommand_ManyJobs(t *testing.T) { for _, jobID := range jobIDs { job := testJob(jobID) - job.TaskGroups[0].Tasks[0].Resources.MemoryMB = pointer.Of(16) - job.TaskGroups[0].Tasks[0].Resources.DiskMB = pointer.Of(32) - job.TaskGroups[0].Tasks[0].Resources.CPU = pointer.Of(10) - job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ - "run_for": "30s", - } jobJSON, err := json.MarshalIndent(job, "", " ") must.NoError(t, err) @@ -182,60 +176,41 @@ func TestStartCommand_ManyJobs(t *testing.T) { } -func TestStartCommand_StartCorrectVersion(t *testing.T) { +func TestStartCommand_MultipleCycles(t *testing.T) { ci.Parallel(t) - srv, _, addr := testServer(t, true, func(c *agent.Config) { + srv, client, addr := testServer(t, true, func(c *agent.Config) { c.DevMode = true }) - defer srv.Shutdown() - - jobID := uuid.Generate() - - jobFilePath := filepath.Join(os.TempDir(), jobID+".nomad") - t.Cleanup(func() { - _ = os.Remove(jobFilePath) - }) + defer srv.Shutdown() ui := cli.NewMockUi() - job := testNomadServiceJob(jobID) - job.TaskGroups[0].Tasks[0].Resources.MemoryMB = pointer.Of(16) - job.TaskGroups[0].Tasks[0].Resources.DiskMB = pointer.Of(32) - job.TaskGroups[0].Tasks[0].Resources.CPU = pointer.Of(10) - job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ - "run_for": "30s", + job1 := testJob("job-start-test") + resp, _, err := client.Jobs().Register(job1, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + if code := waitForSuccess(ui, client, fullId, t, resp.EvalID); code != 0 { + t.Fatalf("status code non zero saw %d", code) } - - jobJSON, err := json.MarshalIndent(job, "", " ") - must.NoError(t, err) - - jobFile := jobFilePath - err = os.WriteFile(jobFile, []byte(jobJSON), 0o644) - must.NoError(t, err) - - cmd := &JobRunCommand{Meta: Meta{Ui: ui}} - - code := cmd.Run([]string{"-address", addr, "-json", jobFile}) - must.Zero(t, code, - must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), - must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), - ) args := []string{"-address", addr, "-detach"} - args = append(args, jobID) - expectedVersions := []uint64{0, 2, 4} + args = append(args, "job-start-test") + expectedVersions := []uint64{0, 2, 4, 6, 8, 10} stopCmd := &JobStopCommand{Meta: Meta{Ui: ui}} startCmd := &JobStartCommand{Meta: Meta{Ui: ui}} - // for multiple cycles of starting/stopping a job, check that the correct, most recent running version is picked - for i := range 3 { - code = stopCmd.Run(args) + // check multiple cycles of starting/stopping a job result in the correct version selected + for i := range 6 { + + code := stopCmd.Run(args) must.Zero(t, code, must.Sprintf("job stop stdout: %s", ui.OutputWriter.String()), must.Sprintf("job stop stderr: %s", ui.ErrorWriter.String()), ) + code = startCmd.Run(args) must.Zero(t, code, must.Sprintf("job start stdout: %s", ui.OutputWriter.String()), From 150ad92dfe82307cb477a086956b83247f8fa0b1 Mon Sep 17 00:00:00 2001 From: Martina Santangelo Date: Thu, 7 Nov 2024 13:09:04 -0500 Subject: [PATCH 8/9] doc fix --- command/job_start.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/command/job_start.go b/command/job_start.go index 05a1dbbf8ad..aabec927eef 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -25,9 +25,9 @@ Usage: nomad job start [options] Alias: nomad start Start an existing stopped job. This command is used to start a previously stopped job's - most recent running version. Upon successful start, an interactive + most recent version. Upon successful start, an interactive monitor session will start to display log lines as the job starts its - allocations based on its most recent running version. It is safe to exit the monitor + allocations based on its most recent version. It is safe to exit the monitor early using ctrl+c. When ACLs are enabled, this command requires a token with the 'submit-job' @@ -174,8 +174,9 @@ func (c *JobStartCommand) Run(args []string) int { } c.versionSelected = chosenVersion + if !versionAvailable { - c.Ui.Error(fmt.Sprintf("No previous running versions of job %v", *job.Name)) + c.Ui.Error(fmt.Sprintf("No previous available versions of job %v", *job.Name)) statusCh <- 1 return } From 5126d670f055ff32d0bde42025c9415e158c8b3d Mon Sep 17 00:00:00 2001 From: Martina Santangelo Date: Thu, 7 Nov 2024 13:19:28 -0500 Subject: [PATCH 9/9] clean up --- command/job_start.go | 1 - 1 file changed, 1 deletion(-) diff --git a/command/job_start.go b/command/job_start.go index aabec927eef..cef5ce0b454 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -198,7 +198,6 @@ func (c *JobStartCommand) Run(args []string) int { statusCh <- 1 return } - versions, _, _, err = client.Jobs().Versions(*job.ID, true, q) // Nothing to do: periodic or dispatch job evalCreated := resp.EvalID != ""