Allow workspace create to create links to external directories

lib/ramble/docs/workspace.rst

@@ -61,6 +61,37 @@ in a specified directory. To create an anonymous workspace, use:
 
     $ ramble workspace create -d <path_to_workspace>
 
+.. _workspace-links:
+
+^^^^^^^^^^^^^^^
+Workspace Links
+^^^^^^^^^^^^^^^
+
+In order to save disk space, sometimes it can be useful to share internal
+workspace directories across workspaces when they reuse aspects of each others
+workflows. Ramble provides a way to create a new workspace where the inputs and
+software directories are symbolic links to external directories (whether in a
+workspace or not), to help minimize duplication of files across workspaces.
+
+To use this, when creating a workspace you can use the ``--inputs-dir`` and
+``--software-dir`` argument to provide paths for the source of these symbolic
+links.
+
+As an example:
+
+.. code-block:: console
+
+  $ ramble workspace create -d foo
+  $ ramble workspace create -d bar --software-dir foo/software --inputs-dir foo/inputs
+
+In the above example, two workspaces are created (``foo`` and ``bar``). The
+workspace named ``bar`` has symbolic links for its ``software`` and ``inputs``
+directories that link to the same named directories in the ``foo`` workspace.
+
+Additionally, these directories do not need to be part of any workspace, and
+could instead be external directories used to have a common storage location
+for software environments and input files.
+
 .. _workspace-structure:
 
 -------------------

lib/ramble/ramble/cmd/workspace.py

@@ -215,15 +215,24 @@ def workspace_create_setup_parser(subparser):
     subparser.add_argument(
         '-d', '--dir', action='store_true',
         help='create a workspace in a specific directory')
+    subparser.add_argument(
+        '--software-dir', metavar='dir',
+        help='external directory to link as software directory in workspace')
+    subparser.add_argument(
+        '--inputs-dir', metavar='dir',
+        help='external directory to link as inputs directory in workspace')
 
 
 def workspace_create(args):
     _workspace_create(args.create_workspace, args.dir,
-                      args.config, args.template_execute)
+                      args.config, args.template_execute,
+                      software_dir=args.software_dir,
+                      inputs_dir=args.inputs_dir)
 
 
 def _workspace_create(name_or_path, dir=False,
-                      config=None, template_execute=None):
+                      config=None, template_execute=None,
+                      software_dir=None, inputs_dir=None):
     """Create a new workspace
 
     Arguments:
@@ -235,6 +244,10 @@ def _workspace_create(name_or_path, dir=False,
                       generate the workspace
         template_execute (str): Path to a template execute script to
                                 create the workspace with
+        software_dir (str): Path to software dir that should be linked
+                            instead of creating a new directory.
+        inputs_dir (str): Path to inputs dir that should be linked
+                          instead of creating a new directory.
     """
 
     # Sanity check file paths, to avoid half-creating an incomplete workspace
@@ -268,7 +281,7 @@ def _workspace_create(name_or_path, dir=False,
         logger.msg("You can activate this workspace with:")
         logger.msg(f"  ramble workspace activate {name_or_path}")
 
-    workspace.write()
+    workspace.write(inputs_dir=inputs_dir, software_dir=software_dir)
 
     if config:
         with open(config, 'r') as f:

lib/ramble/ramble/test/cmd/workspace.py

@@ -114,6 +114,21 @@ def check_results(ws):
     assert os.path.exists(os.path.join(ws.root, fn + '.yaml'))
 
 
+def test_workspace_create_links(mutable_mock_workspace_path, tmpdir):
+    import pathlib
+    expected_links = ['software', 'inputs']
+    with tmpdir.as_cwd():
+        workspace('create', '-d', 'foo')
+        workspace('create', '-d', 'bar', '--inputs-dir', 'foo/inputs',
+                  '--software-dir', 'foo/software')
+
+        for expected_link in expected_links:
+            assert os.path.exists(os.path.join('bar', expected_link))
+            assert os.path.islink(os.path.join('bar', expected_link))
+            resolved_path = pathlib.PosixPath(os.path.join('bar', expected_link)).resolve()
+            assert str(resolved_path) == os.path.abspath(os.path.join('foo', expected_link))
+
+
 def test_workspace_activate_fails(mutable_mock_workspace_path):
     workspace('create', 'foo')
     out = workspace('activate', 'foo')

lib/ramble/ramble/workspace/workspace.py

@@ -668,7 +668,7 @@ def _read_auxiliary_software_file(self, name, f):
         """Read an auxiliary software file for generated software directories"""
         self._auxiliary_software_files[name] = f
 
-    def write(self):
+    def write(self, software_dir=None, inputs_dir=None):
         """Write an in-memory workspace to its location on disk."""
 
         # Ensure required directory structure exists
@@ -677,8 +677,17 @@ def write(self):
         fs.mkdirp(self.auxiliary_software_dir)
         fs.mkdirp(self.log_dir)
         fs.mkdirp(self.experiment_dir)
-        fs.mkdirp(self.input_dir)
-        fs.mkdirp(self.software_dir)
+
+        if inputs_dir:
+            os.symlink(os.path.abspath(inputs_dir), self.input_dir, target_is_directory=True)
+        elif not os.path.exists(self.input_dir):
+            fs.mkdirp(self.input_dir)
+
+        if software_dir:
+            os.symlink(os.path.abspath(software_dir), self.software_dir, target_is_directory=True)
+        elif not os.path.exists(self.software_dir):
+            fs.mkdirp(self.software_dir)
+
         fs.mkdirp(self.shared_dir)
         fs.mkdirp(self.shared_license_dir)
 

share/ramble/ramble-completion.bash

@@ -664,7 +664,7 @@ _ramble_workspace_deactivate() {
 _ramble_workspace_create() {
     if $list_options
     then
-        RAMBLE_COMPREPLY="-h --help -c --config -t --template_execute -d --dir"
+        RAMBLE_COMPREPLY="-h --help -c --config -t --template_execute -d --dir --software-dir --inputs-dir"
     else
         RAMBLE_COMREPLY=""
     fi