Drivers: add work_dir config to exec/raw_exec/java drivers (#24249)

---------

Co-authored-by: wurosh <[email protected]>
Co-authored-by: Michael Schurter <[email protected]>
Co-authored-by: Tim Gross <[email protected]>

.changelog/24249.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+drivers: Add work_dir config to exec/rawexec/java drivers for setting the working directory of processes in a task
+```

drivers/exec/driver.go

@@ -97,6 +97,7 @@ var (
 		"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
 		"cap_add":  hclspec.NewAttr("cap_add", "list(string)", false),
 		"cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false),
+		"work_dir": hclspec.NewAttr("work_dir", "string", false),
 	})
 
 	// driverCapabilities represents the RPC response for what features are
@@ -211,6 +212,9 @@ type TaskConfig struct {
 
 	// CapDrop is a set of linux capabilities to disable.
 	CapDrop []string `codec:"cap_drop"`
+
+	// WorkDir is the working directory inside the chroot
+	WorkDir string `codec:"work_dir"`
 }
 
 func (tc *TaskConfig) validate() error {
@@ -237,6 +241,10 @@ func (tc *TaskConfig) validate() error {
 		return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
 	}
 
+	if tc.WorkDir != "" && !filepath.IsAbs(tc.WorkDir) {
+		return fmt.Errorf("work_dir must be absolute but got relative path %q", tc.WorkDir)
+	}
+
 	return nil
 }
 
@@ -517,6 +525,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		NoPivotRoot:      d.config.NoPivotRoot,
 		Resources:        cfg.Resources,
 		TaskDir:          cfg.TaskDir().Dir,
+		WorkDir:          driverConfig.WorkDir,
 		StdoutPath:       cfg.StdoutPath,
 		StderrPath:       cfg.StderrPath,
 		Mounts:           cfg.Mounts,

drivers/exec/driver_test.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/lib/cgroupslib"
 	"github.com/hashicorp/nomad/client/lib/numalib"
 	ctestutils "github.com/hashicorp/nomad/client/testutil"
@@ -127,6 +128,50 @@ func TestExecDriver_Fingerprint(t *testing.T) {
 	}
 }
 
+func TestExecDriver_WorkDir(t *testing.T) {
+	ci.Parallel(t)
+
+	ctestutils.ExecCompatible(t)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	d := newExecDriverTest(t, ctx)
+	harness := dtestutil.NewDriverHarness(t, d)
+	allocID := uuid.Generate()
+	task := &drivers.TaskConfig{
+		AllocID:   allocID,
+		ID:        uuid.Generate(),
+		Name:      "test",
+		Resources: testResources(allocID, "test"),
+	}
+
+	workDir := filepath.Join("/", allocdir.TaskLocal)
+	tc := &TaskConfig{
+		Command: "/bin/cat",
+		Args:    []string{"foo.txt"},
+		WorkDir: workDir,
+	}
+	must.NoError(t, task.EncodeConcreteDriverConfig(&tc))
+
+	cleanup := harness.MkAllocDir(task, false)
+	defer cleanup()
+
+	must.NoError(t, os.WriteFile(filepath.Join(task.TaskDir().Dir, allocdir.TaskLocal, "foo.txt"), []byte("foo"), 660))
+
+	handle, _, err := harness.StartTask(task)
+	must.NoError(t, err)
+
+	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
+	must.NoError(t, err)
+
+	// Task will fail if cat cannot find the file, which would only happen
+	// if the task's WorkDir was setup incorrectly
+	result := <-ch
+	must.Zero(t, result.ExitCode)
+	must.NoError(t, harness.DestroyTask(task.ID, true))
+}
+
 func TestExecDriver_StartWait(t *testing.T) {
 	ci.Parallel(t)
 	ctestutils.ExecCompatible(t)
@@ -735,11 +780,13 @@ func TestConfig_ParseAllHCL(t *testing.T) {
 config {
   command = "/bin/bash"
   args = ["-c", "echo hello"]
+  work_dir = "/root"
 }`
 
 	expected := &TaskConfig{
 		Command: "/bin/bash",
 		Args:    []string{"-c", "echo hello"},
+		WorkDir: "/root",
 	}
 
 	var tc *TaskConfig
@@ -1013,4 +1060,18 @@ func TestDriver_TaskConfig_validate(t *testing.T) {
 			}).validate())
 		}
 	})
+
+	t.Run("work_dir", func(t *testing.T) {
+		for _, tc := range []struct {
+			workDir string
+			exp     error
+		}{
+			{workDir: "/foo", exp: nil},
+			{workDir: "foo", exp: errors.New(`work_dir must be absolute but got relative path "foo"`)},
+		} {
+			must.Eq(t, tc.exp, (&TaskConfig{
+				WorkDir: tc.workDir,
+			}).validate())
+		}
+	})
 }

drivers/java/driver.go

@@ -98,6 +98,7 @@ var (
 		"ipc_mode":    hclspec.NewAttr("ipc_mode", "string", false),
 		"cap_add":     hclspec.NewAttr("cap_add", "list(string)", false),
 		"cap_drop":    hclspec.NewAttr("cap_drop", "list(string)", false),
+		"work_dir":    hclspec.NewAttr("work_dir", "string", false),
 	})
 
 	// driverCapabilities is returned by the Capabilities RPC and indicates what
@@ -189,6 +190,9 @@ type TaskConfig struct {
 
 	// CapDrop is a set of linux capabilities to disable.
 	CapDrop []string `codec:"cap_drop"`
+
+	// WorkDir is the working directory for the task
+	WorkDir string `coded:"work_dir"`
 }
 
 func (tc *TaskConfig) validate() error {
@@ -215,6 +219,9 @@ func (tc *TaskConfig) validate() error {
 		return fmt.Errorf("cap_drop configured with capabilities not supported by system: %s", badDrops)
 	}
 
+	if tc.WorkDir != "" && !filepath.IsAbs(tc.WorkDir) {
+		return fmt.Errorf("work_dir must be an absolute path: %s", tc.WorkDir)
+	}
 	return nil
 }
 
@@ -496,6 +503,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		ResourceLimits:   true,
 		Resources:        cfg.Resources,
 		TaskDir:          cfg.TaskDir().Dir,
+		WorkDir:          driverConfig.WorkDir,
 		StdoutPath:       cfg.StdoutPath,
 		StderrPath:       cfg.StderrPath,
 		Mounts:           cfg.Mounts,

drivers/java/driver_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/hashicorp/nomad/plugins/drivers"
 	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
 	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 )
 
@@ -209,6 +210,49 @@ func TestJavaDriver_Class_Start_Wait(t *testing.T) {
 	require.NoError(t, harness.DestroyTask(task.ID, true))
 }
 
+func TestJavaDriver_WorkDir(t *testing.T) {
+	ci.Parallel(t)
+	javaCompatible(t)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	d := newJavaDriverTest(t, ctx)
+
+	harness := dtestutil.NewDriverHarness(t, d)
+
+	tc := &TaskConfig{
+		JarPath: "../demoapp.jar",
+		Args:    []string{"1"},
+		JvmOpts: []string{"-Xmx64m", "-Xms32m"},
+		WorkDir: "/local",
+	}
+
+	task := basicTask(t, "demo-app", tc)
+
+	cleanup := harness.MkAllocDir(task, true)
+	defer cleanup()
+
+	copyFile("./test-resources/demoapp.jar", filepath.Join(task.TaskDir().Dir, "demoapp.jar"), t)
+
+	handle, _, err := harness.StartTask(task)
+	must.NoError(t, err)
+
+	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
+	must.NoError(t, err)
+	result := <-ch
+	must.Nil(t, result.Err)
+
+	must.Zero(t, result.ExitCode)
+
+	// Get the stdout of the process and assert that it's not empty
+	stdout, err := os.ReadFile(filepath.Join(task.TaskDir().LogDir, "demo-app.stdout.0"))
+	must.NoError(t, err)
+	must.Eq(t, string(stdout), "Hello, the current working directory is: /local\n")
+
+	must.NoError(t, harness.DestroyTask(task.ID, true))
+}
+
 func TestJavaCmdArgs(t *testing.T) {
 	ci.Parallel(t)
 
@@ -521,4 +565,18 @@ func TestDriver_TaskConfig_validate(t *testing.T) {
 			}).validate())
 		}
 	})
+
+	t.Run("work_dir", func(t *testing.T) {
+		for _, tc := range []struct {
+			workDir string
+			exp     error
+		}{
+			{workDir: "/goodpath", exp: nil},
+			{workDir: "badpath", exp: errors.New("work_dir must be an absolute path: badpath")},
+		} {
+			require.Equal(t, tc.exp, (&TaskConfig{
+				WorkDir: tc.workDir,
+			}).validate())
+		}
+	})
 }

drivers/java/test-resources/Hello.java

@@ -5,7 +5,7 @@
 
 public class Hello {
     public static void main(String[] args) {
-        System.out.println("Hello");
+        System.out.println("Hello, the current working directory is: " + System.getProperty("user.dir"));
         int seconds = 5;
         if (args.length != 0) {
             seconds = Integer.parseInt(args[0]);

drivers/rawexec/driver.go

@@ -94,6 +94,7 @@ var (
 		"cgroup_v2_override": hclspec.NewAttr("cgroup_v2_override", "string", false),
 		"cgroup_v1_override": hclspec.NewAttr("cgroup_v1_override", "list(map(string))", false),
 		"oom_score_adj":      hclspec.NewAttr("oom_score_adj", "number", false),
+		"work_dir":           hclspec.NewAttr("work_dir", "string", false),
 	})
 
 	// capabilities is returned by the Capabilities RPC and indicates what
@@ -172,6 +173,25 @@ type TaskConfig struct {
 
 	// OOMScoreAdj sets the oom_score_adj on Linux systems
 	OOMScoreAdj int `codec:"oom_score_adj"`
+
+	// WorkDir sets the working directory of the task
+	WorkDir string `codec:"work_dir"`
+}
+
+func (t *TaskConfig) validate() error {
+	// ensure only one of cgroups_v1_override and cgroups_v2_override have been
+	// configured; must check here because task config validation cannot happen
+	// on the server.
+	if len(t.OverrideCgroupV1) > 0 && t.OverrideCgroupV2 != "" {
+		return errors.New("only one of cgroups_v1_override and cgroups_v2_override may be set")
+	}
+	if t.OOMScoreAdj < 0 {
+		return errors.New("oom_score_adj must not be negative")
+	}
+	if t.WorkDir != "" && !filepath.IsAbs(t.WorkDir) {
+		return errors.New("work_dir must be an absolute path")
+	}
+	return nil
 }
 
 // TaskState is the state which is encoded in the handle returned in
@@ -352,8 +372,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
 	}
 
-	if driverConfig.OOMScoreAdj < 0 {
-		return nil, nil, fmt.Errorf("oom_score_adj must not be negative")
+	driverConfig.OverrideCgroupV2 = cgroupslib.CustomPathCG2(driverConfig.OverrideCgroupV2)
+
+	if err := driverConfig.validate(); err != nil {
+		return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
 	}
 
 	if err := d.Validate(*cfg); err != nil {
@@ -383,22 +405,16 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		Env:              cfg.EnvList(),
 		User:             cfg.User,
 		TaskDir:          cfg.TaskDir().Dir,
+		WorkDir:          driverConfig.WorkDir,
 		StdoutPath:       cfg.StdoutPath,
 		StderrPath:       cfg.StderrPath,
 		NetworkIsolation: cfg.NetworkIsolation,
 		Resources:        cfg.Resources.Copy(),
-		OverrideCgroupV2: cgroupslib.CustomPathCG2(driverConfig.OverrideCgroupV2),
+		OverrideCgroupV2: driverConfig.OverrideCgroupV2,
 		OverrideCgroupV1: driverConfig.OverrideCgroupV1,
 		OOMScoreAdj:      int32(driverConfig.OOMScoreAdj),
 	}
 
-	// ensure only one of cgroups_v1_override and cgroups_v2_override have been
-	// configured; must check here because task config validation cannot happen
-	// on the server.
-	if len(execCmd.OverrideCgroupV1) > 0 && execCmd.OverrideCgroupV2 != "" {
-		return nil, nil, errors.New("only one of cgroups_v1_override and cgroups_v2_override may be set")
-	}
-
 	ps, err := exec.Launch(execCmd)
 	if err != nil {
 		pluginClient.Kill()