Nested Workspaces template

.github/workflows/task_runner_basic_e2e.yml

@@ -26,7 +26,7 @@ on:
         options:
           - all
           - torch_cnn_mnist
-          - keras_cnn_mnist
+          - keras/cnn_mnist
       python_version:
         description: "Python version"
         required: false
@@ -88,17 +88,17 @@ jobs:
           # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run.
           # Thus these models are excluded from the matrix for now.
           # Default combination if no input is provided (i.e. 'all' is selected).
-          # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12]
+          # * TLS - models [torch_cnn_mnist, keras/cnn_mnist] and python versions [3.10, 3.11, 3.12]
           # * Non-TLS - models [torch_cnn_mnist] and python version [3.10]
-          # * No client auth - models [keras_cnn_mnist] and python version [3.10]
+          # * No client auth - models [keras/cnn_mnist] and python version [3.10]
           # * Memory logs - models [torch_cnn_mnist] and python version [3.10]
           # ---------------------------------------------------------------
           echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT"
 
           if [ "${{ env.MODEL_NAME }}" == "all" ]; then
-            echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT"
+            echo "models_for_tls=[\"torch_cnn_mnist\", \"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT"
             echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT"
-            echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT"
+            echo "models_for_no_client_auth=[\"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT"
             echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT"
           else
             echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"

.github/workflows/task_runner_dockerized_ws_e2e.yml

@@ -32,7 +32,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       matrix:
-        model_name: ["keras_cnn_mnist"]
+        model_name: ["keras/cnn_mnist"]
         python_version: ["3.10", "3.11", "3.12"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
@@ -73,7 +73,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       matrix:
-        model_name: ["keras_cnn_mnist"]
+        model_name: ["keras/cnn_mnist"]
         python_version: ["3.10"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
@@ -114,7 +114,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       matrix:
-        model_name: ["keras_cnn_mnist"]
+        model_name: ["keras/cnn_mnist"]
         python_version: ["3.10"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
@@ -155,7 +155,7 @@ jobs:
     timeout-minutes: 15
     strategy:
       matrix:
-        model_name: ["keras_cnn_mnist"]
+        model_name: ["keras/cnn_mnist"]
         python_version: ["3.10"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 

.github/workflows/tr_docker_gramine_direct.yml

@@ -27,7 +27,7 @@ jobs:
 
     - name: Create workspace image
       run: |
-        fx workspace create --prefix example_workspace --template keras_cnn_mnist
+        fx workspace create --prefix example_workspace --template keras/cnn_mnist
         cd example_workspace
         fx plan initialize -a localhost
 

.github/workflows/tr_docker_native.yml

@@ -27,7 +27,7 @@ jobs:
 
     - name: Create workspace image
       run: |
-        fx workspace create --prefix example_workspace --template keras_cnn_mnist
+        fx workspace create --prefix example_workspace --template keras/cnn_mnist
         cd example_workspace
         fx plan initialize -a localhost
         fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }}

.github/workflows/ubuntu.yml

@@ -53,4 +53,4 @@ jobs:
           pip install .
       - name: Test TaskRunner API
         run: |
-          python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+          python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model

.github/workflows/windows.yml

@@ -52,4 +52,4 @@ jobs:
           pip install .
       - name: Test TaskRunner API
         run: |
-          python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+          python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model

Jenkinsfile

@@ -6,13 +6,13 @@ def snykData = [
     // CN-14619 snyk test CLI does not support -f in requirements.txt file
     // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch_cnn_histology/requirements.txt',
     'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch_cnn_histology/src/requirements.txt',
-    'openfl-workspace_keras_nlp': 'openfl-workspace/keras_nlp/requirements.txt',
+    'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt',
     'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch_cnn_mnist/requirements.txt',
     'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch_unet_kvasir/requirements.txt',
     'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt',
     'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt',
     'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt',
-    'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras_cnn_mnist/requirements.txt',
+    'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/cnn_mnist/requirements.txt',
     'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt',
     'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt',
     'openfl-tutorials_interactive_api_pytorch_histology_envoy': 'openfl-tutorials/interactive_api/PyTorch_Histology/envoy/requirements.txt',

docs/about/features_index/taskrunner.rst

@@ -150,18 +150,18 @@ STEP 1: Create a Workspace
 		$ fx
 
 
-2. 	This example uses the :code:`keras_cnn_mnist` template.
+2. 	This example uses the :code:`keras/cnn_mnist` template.
 
-	Set the environment variables to use the :code:`keras_cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory.
+	Set the environment variables to use the :code:`keras/cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory.
 
     .. code-block:: shell
 
-        $ export WORKSPACE_TEMPLATE=keras_cnn_mnist
+        $ export WORKSPACE_TEMPLATE=keras/cnn_mnist
         $ export WORKSPACE_PATH=${HOME}/my_federation
 
 3.	Decide a workspace template, which are end-to-end federated learning training demonstrations. The following is a sample of available templates:
 
- - :code:`keras_cnn_mnist`: a workspace with a simple `Keras <http://keras.io/>`__ CNN model that will download the `MNIST <http://yann.lecun.com/exdb/mnist/>`_ dataset and train in a federation.
+ - :code:`keras/cnn_mnist`: a workspace with a simple `Keras <http://keras.io/>`__ CNN model that will download the `MNIST <http://yann.lecun.com/exdb/mnist/>`_ dataset and train in a federation.
  - :code:`tf_2dunet`: a workspace with a simple `TensorFlow <http://tensorflow.org>`__ CNN model that will use the `BraTS <https://www.med.upenn.edu/sbia/brats2017/data.html>`_ dataset and train in a federation.
  - :code:`tf_cnn_histology`: a workspace with a simple `TensorFlow <http://tensorflow.org>`__ CNN model that will download the `Colorectal Histology <https://zenodo.org/record/53169#.XGZemKwzbmG>`_ dataset and train in a federation.
  - :code:`torch_cnn_histology`: a workspace with a simple `PyTorch <http://pytorch.org/>`__ CNN model that will download the `Colorectal Histology <https://zenodo.org/record/53169#.XGZemKwzbmG>`_ dataset and train in a federation.

docs/releases.md

@@ -133,7 +133,7 @@ The OpenFL v1.2 release contains the following updates:
 - New [Interactive Python API](https://github.com/securefederatedai/openfl/tree/main/openfl-tutorials/deprecated/interactive_api) (experimental)
 - Example FedProx algorithm implementation for PyTorch and Tensorflow
 - `AggregationFunctionInterface` for custom aggregation functions 
-- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_nlp)
+- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras/nlp)
 - Fixed lossy compression pipelines and added an [example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_cnn_with_compression) for usage
 - Bug fixes and documentation improvements
 

docs/tutorials/taskrunner.ipynb

@@ -36,7 +36,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!fx workspace create --prefix ./mnist_example --template keras_cnn_mnist\n",
+    "!fx workspace create --prefix ./mnist_example --template keras/cnn_mnist\n",
     "%cd ./mnist_example"
    ]
   },

openfl-workspace/keras/nlp/.workspace

@@ -0,0 +1,2 @@
+current_plan_name: default
+

openfl-workspace/torch_cnn_histology/.workspace

@@ -0,0 +1,2 @@
+current_plan_name: default
+

openfl-workspace/torch_cnn_histology_fedcurv/.workspace

@@ -0,0 +1,2 @@
+current_plan_name: default
+

openfl-workspace/torch_cnn_mnist/.workspace

@@ -0,0 +1,2 @@
+current_plan_name: default
+

openfl-workspace/xgb_higgs/.workspace

@@ -0,0 +1,2 @@
+current_plan_name: default
+

openfl/interface/workspace.py

@@ -97,11 +97,15 @@ def get_templates():
         list: A list of default templates.
     """
 
-    return [
-        d.name
-        for d in WORKSPACE.glob("*")
-        if d.is_dir() and d.name not in ["__pycache__", "workspace", "experimental"]
-    ]
+    templates = []
+    excluded_dirs = ["workspace", "experimental"]
+    for root, _, files in os.walk(WORKSPACE):
+        if any(file.endswith(".workspace") for file in files):
+            dir_path = os.path.relpath(root, WORKSPACE)
+            dir_path = dir_path.replace(os.sep, "/")
+            if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs):
+                templates.append(dir_path)
+    return templates
 
 
 @workspace.command(name="create")

openfl/native/native.py

@@ -216,7 +216,7 @@ def init(
         workspace_template (str): The template that should be used as the
             basis for the experiment.  Defaults to 'default'.
             Other options include are any of the template names
-            [keras_cnn_mnist, tf_2dunet, tf_cnn_histology,
+            [keras/cnn_mnist, tf_2dunet, tf_cnn_histology,
             mtorch_cnn_histology, torch_cnn_mnist].
         log_level (str): Log level for logging. METRIC level is available.
             Defaults to 'INFO'.

tests/end_to_end/README.md

@@ -55,10 +55,10 @@ For example, to run Task runner (bare metal approach) with - torch_cnn_mnist mod
 python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls
 ```
 
-And, to run Task runner (via dockerized workspace) with keras_cnn_mnist, 2 collaborators, 3 rounds:
+And, to run Task runner (via dockerized workspace) with keras/cnn_mnist, 2 collaborators, 3 rounds:
 
 ```sh
-python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras_cnn_mnist
+python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/cnn_mnist
 ```
 
 ### Fixture and marker mapping:

tests/end_to_end/utils/constants.py

@@ -11,7 +11,7 @@ class ModelName(Enum):
     # IMP - The model name must be same (and in uppercase) as the model value.
     # This is used to identify the model in the tests.
     TORCH_CNN_MNIST = "torch_cnn_mnist"
-    KERAS_CNN_MNIST = "keras_cnn_mnist"
+    KERAS_CNN_MNIST = "keras/cnn_mnist"
     TORCH_CNN_HISTOLOGY = "torch_cnn_histology"
     XGB_HIGGS = "xgb_higgs"
 

tests/github/test_double_ws_export.py

@@ -22,7 +22,7 @@ def main():
         for entry in iterator:
             if entry.name not in ['__init__.py', 'workspace', 'default']:
                 workspace_choice.append(entry.name)
-    parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice)
+    parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice)
     parser.add_argument('--fed_workspace', default='fed_work12345alpha81671')
     parser.add_argument('--col1', default='one123dragons')
     parser.add_argument('--col2', default='beta34unicorns')

tests/github/test_gandlf.py

@@ -21,7 +21,7 @@ def exec(command, directory):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--template', default='keras_cnn_mnist')
+    parser.add_argument('--template', default='keras/cnn_mnist')
     parser.add_argument('--fed_workspace', default='fed_work12345alpha81671')
     parser.add_argument('--col1', default='one')
     parser.add_argument('--col2', default='two')

tests/github/test_hello_federation.py

@@ -17,11 +17,14 @@ def main():
     # Test the pipeline
     parser = argparse.ArgumentParser()
     workspace_choice = []
-    with os.scandir('openfl-workspace') as iterator:
-        for entry in iterator:
-            if entry.name not in ['__init__.py', 'workspace', 'default']:
-                workspace_choice.append(entry.name)
-    parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice)
+    excluded_dirs = ['workspace', 'default', "experimental"]
+    for root, _, files in os.walk('openfl-workspace'):
+        if any(file.endswith(".workspace") for file in files):
+            dir_path = os.path.relpath(root, 'openfl-workspace')
+            dir_path = dir_path.replace(os.sep, '/')
+            if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs):
+                workspace_choice.append(dir_path)
+    parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice)
     parser.add_argument('--fed_workspace', default='fed_work12345alpha81671')
     parser.add_argument('--col1', default='one123dragons')
     parser.add_argument('--col2', default='beta34unicorns')