From 5df77512bab2387de2419b03a6b9feb98be118e3 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:28:56 -0800 Subject: [PATCH 01/15] code changes Signed-off-by: yes --- .github/workflows/task_runner_basic_e2e.yml | 10 +++++----- .../workflows/task_runner_dockerized_ws_e2e.yml | 8 ++++---- .github/workflows/tr_docker_gramine_direct.yml | 2 +- .github/workflows/tr_docker_native.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- Jenkinsfile | 4 ++-- docs/about/features_index/taskrunner.rst | 8 ++++---- docs/releases.md | 2 +- docs/tutorials/taskrunner.ipynb | 2 +- openfl/interface/workspace.py | 14 +++++++++----- openfl/native/native.py | 2 +- tests/end_to_end/README.md | 4 ++-- tests/end_to_end/utils/constants.py | 2 +- tests/github/test_double_ws_export.py | 2 +- tests/github/test_gandlf.py | 2 +- tests/github/test_hello_federation.py | 13 ++++++++----- 17 files changed, 44 insertions(+), 37 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 5b64de09df..caeaf23088 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -26,7 +26,7 @@ on: options: - all - torch_cnn_mnist - - keras_cnn_mnist + - keras/cnn_mnist python_version: description: "Python version" required: false @@ -88,17 +88,17 @@ jobs: # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. # Default combination if no input is provided (i.e. 'all' is selected). - # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] + # * TLS - models [torch_cnn_mnist, keras/cnn_mnist] and python versions [3.10, 3.11, 3.12] # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] - # * No client auth - models [keras_cnn_mnist] and python version [3.10] + # * No client auth - models [keras/cnn_mnist] and python version [3.10] # * Memory logs - models [torch_cnn_mnist] and python version [3.10] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" if [ "${{ env.MODEL_NAME }}" == "all" ]; then - echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"torch_cnn_mnist\", \"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" else echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index ce677c5789..896b87400d 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10", "3.11", "3.12"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -73,7 +73,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -114,7 +114,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -155,7 +155,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 169189a42d..9ecbe90819 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras_cnn_mnist + fx workspace create --prefix example_workspace --template keras/cnn_mnist cd example_workspace fx plan initialize -a localhost diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index 9fcb9b8759..b80e5ca161 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras_cnn_mnist + fx workspace create --prefix example_workspace --template keras/cnn_mnist cd example_workspace fx plan initialize -a localhost fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 617c004751..ebaa206867 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -53,4 +53,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model + python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5f3ffa1220..5bf4fb2bf4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,4 +52,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 73f919c844..fd7db70d21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,13 +6,13 @@ def snykData = [ // CN-14619 snyk test CLI does not support -f in requirements.txt file // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch_cnn_histology/requirements.txt', 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch_cnn_histology/src/requirements.txt', - 'openfl-workspace_keras_nlp': 'openfl-workspace/keras_nlp/requirements.txt', + 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch_cnn_mnist/requirements.txt', 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch_unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', - 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras_cnn_mnist/requirements.txt', + 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/cnn_mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_histology_envoy': 'openfl-tutorials/interactive_api/PyTorch_Histology/envoy/requirements.txt', diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index f8730f463b..255a63d904 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -150,18 +150,18 @@ STEP 1: Create a Workspace $ fx -2. This example uses the :code:`keras_cnn_mnist` template. +2. This example uses the :code:`keras/cnn_mnist` template. - Set the environment variables to use the :code:`keras_cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. + Set the environment variables to use the :code:`keras/cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. .. code-block:: shell - $ export WORKSPACE_TEMPLATE=keras_cnn_mnist + $ export WORKSPACE_TEMPLATE=keras/cnn_mnist $ export WORKSPACE_PATH=${HOME}/my_federation 3. Decide a workspace template, which are end-to-end federated learning training demonstrations. The following is a sample of available templates: - - :code:`keras_cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`keras/cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. - :code:`tf_2dunet`: a workspace with a simple `TensorFlow `__ CNN model that will use the `BraTS `_ dataset and train in a federation. - :code:`tf_cnn_histology`: a workspace with a simple `TensorFlow `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - :code:`torch_cnn_histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. diff --git a/docs/releases.md b/docs/releases.md index 30ed029ea6..539336ceb1 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -133,7 +133,7 @@ The OpenFL v1.2 release contains the following updates: - New [Interactive Python API](https://github.com/securefederatedai/openfl/tree/main/openfl-tutorials/deprecated/interactive_api) (experimental) - Example FedProx algorithm implementation for PyTorch and Tensorflow - `AggregationFunctionInterface` for custom aggregation functions -- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_nlp) +- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras/nlp) - Fixed lossy compression pipelines and added an [example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_cnn_with_compression) for usage - Bug fixes and documentation improvements diff --git a/docs/tutorials/taskrunner.ipynb b/docs/tutorials/taskrunner.ipynb index d19fcdc6d0..a95236e17b 100644 --- a/docs/tutorials/taskrunner.ipynb +++ b/docs/tutorials/taskrunner.ipynb @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fx workspace create --prefix ./mnist_example --template keras_cnn_mnist\n", + "!fx workspace create --prefix ./mnist_example --template keras/cnn_mnist\n", "%cd ./mnist_example" ] }, diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 522ff99b5f..8881ec9d8f 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -97,11 +97,15 @@ def get_templates(): list: A list of default templates. """ - return [ - d.name - for d in WORKSPACE.glob("*") - if d.is_dir() and d.name not in ["__pycache__", "workspace", "experimental"] - ] + templates = [] + excluded_dirs = ["workspace", "experimental"] + for root, _, files in os.walk(WORKSPACE): + if any(file.endswith(".workspace") for file in files): + dir_path = os.path.relpath(root, WORKSPACE) + dir_path = dir_path.replace(os.sep, "/") + if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): + templates.append(dir_path) + return templates @workspace.command(name="create") diff --git a/openfl/native/native.py b/openfl/native/native.py index 117058abbb..c1b3813f92 100644 --- a/openfl/native/native.py +++ b/openfl/native/native.py @@ -216,7 +216,7 @@ def init( workspace_template (str): The template that should be used as the basis for the experiment. Defaults to 'default'. Other options include are any of the template names - [keras_cnn_mnist, tf_2dunet, tf_cnn_histology, + [keras/cnn_mnist, tf_2dunet, tf_cnn_histology, mtorch_cnn_histology, torch_cnn_mnist]. log_level (str): Log level for logging. METRIC level is available. Defaults to 'INFO'. diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 191cfd0db4..0495629c1e 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -55,10 +55,10 @@ For example, to run Task runner (bare metal approach) with - torch_cnn_mnist mod python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls ``` -And, to run Task runner (via dockerized workspace) with keras_cnn_mnist, 2 collaborators, 3 rounds: +And, to run Task runner (via dockerized workspace) with keras/cnn_mnist, 2 collaborators, 3 rounds: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras_cnn_mnist +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/cnn_mnist ``` ### Fixture and marker mapping: diff --git a/tests/end_to_end/utils/constants.py b/tests/end_to_end/utils/constants.py index a9f66e47ef..971174c381 100644 --- a/tests/end_to_end/utils/constants.py +++ b/tests/end_to_end/utils/constants.py @@ -11,7 +11,7 @@ class ModelName(Enum): # IMP - The model name must be same (and in uppercase) as the model value. # This is used to identify the model in the tests. TORCH_CNN_MNIST = "torch_cnn_mnist" - KERAS_CNN_MNIST = "keras_cnn_mnist" + KERAS_CNN_MNIST = "keras/cnn_mnist" TORCH_CNN_HISTOLOGY = "torch_cnn_histology" XGB_HIGGS = "xgb_higgs" diff --git a/tests/github/test_double_ws_export.py b/tests/github/test_double_ws_export.py index 95c9440b31..7e9b42bec3 100644 --- a/tests/github/test_double_ws_export.py +++ b/tests/github/test_double_ws_export.py @@ -22,7 +22,7 @@ def main(): for entry in iterator: if entry.name not in ['__init__.py', 'workspace', 'default']: workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') diff --git a/tests/github/test_gandlf.py b/tests/github/test_gandlf.py index a57f9f53a0..08e80e2118 100644 --- a/tests/github/test_gandlf.py +++ b/tests/github/test_gandlf.py @@ -21,7 +21,7 @@ def exec(command, directory): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--template', default='keras_cnn_mnist') + parser.add_argument('--template', default='keras/cnn_mnist') parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one') parser.add_argument('--col2', default='two') diff --git a/tests/github/test_hello_federation.py b/tests/github/test_hello_federation.py index e6b84b8de2..a3ef7296a8 100644 --- a/tests/github/test_hello_federation.py +++ b/tests/github/test_hello_federation.py @@ -17,11 +17,14 @@ def main(): # Test the pipeline parser = argparse.ArgumentParser() workspace_choice = [] - with os.scandir('openfl-workspace') as iterator: - for entry in iterator: - if entry.name not in ['__init__.py', 'workspace', 'default']: - workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice) + excluded_dirs = ['workspace', 'default', "experimental"] + for root, _, files in os.walk('openfl-workspace'): + if any(file.endswith(".workspace") for file in files): + dir_path = os.path.relpath(root, 'openfl-workspace') + dir_path = dir_path.replace(os.sep, '/') + if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): + workspace_choice.append(dir_path) + parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') From bf71c5f05639095f6ddbad4c196b5ed013406d9b Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:32:13 -0800 Subject: [PATCH 02/15] code changes Signed-off-by: yes --- openfl-workspace/{keras_2dunet => keras/2dunet}/.workspace | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/README.md | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/cols.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/data.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/defaults | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/plan.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/requirements.txt | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/__init__.py | 0 .../{keras_2dunet => keras/2dunet}/src/brats_utils.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/dataloader.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/nii_reader.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/taskrunner.py | 0 openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/.workspace | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/cols.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/data.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/defaults | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/plan.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/requirements.txt | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/__init__.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/dataloader.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/mnist_utils.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/taskrunner.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/.workspace | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/cols.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/data.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/plan.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/requirements.txt | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/__init__.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader_utils.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/taskrunner.py | 0 31 files changed, 0 insertions(+), 0 deletions(-) rename openfl-workspace/{keras_2dunet => keras/2dunet}/.workspace (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/README.md (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/cols.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/data.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/defaults (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/plan.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/requirements.txt (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/__init__.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/brats_utils.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/dataloader.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/nii_reader.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/taskrunner.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/.workspace (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/cols.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/data.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/defaults (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/plan.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/requirements.txt (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/__init__.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/dataloader.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/mnist_utils.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/taskrunner.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/.workspace (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/cols.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/data.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/plan.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/requirements.txt (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/__init__.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader_utils.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/taskrunner.py (100%) diff --git a/openfl-workspace/keras_2dunet/.workspace b/openfl-workspace/keras/2dunet/.workspace similarity index 100% rename from openfl-workspace/keras_2dunet/.workspace rename to openfl-workspace/keras/2dunet/.workspace diff --git a/openfl-workspace/keras_2dunet/README.md b/openfl-workspace/keras/2dunet/README.md similarity index 100% rename from openfl-workspace/keras_2dunet/README.md rename to openfl-workspace/keras/2dunet/README.md diff --git a/openfl-workspace/keras_2dunet/plan/cols.yaml b/openfl-workspace/keras/2dunet/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/cols.yaml rename to openfl-workspace/keras/2dunet/plan/cols.yaml diff --git a/openfl-workspace/keras_2dunet/plan/data.yaml b/openfl-workspace/keras/2dunet/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/data.yaml rename to openfl-workspace/keras/2dunet/plan/data.yaml diff --git a/openfl-workspace/keras_2dunet/plan/defaults b/openfl-workspace/keras/2dunet/plan/defaults similarity index 100% rename from openfl-workspace/keras_2dunet/plan/defaults rename to openfl-workspace/keras/2dunet/plan/defaults diff --git a/openfl-workspace/keras_2dunet/plan/plan.yaml b/openfl-workspace/keras/2dunet/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/plan.yaml rename to openfl-workspace/keras/2dunet/plan/plan.yaml diff --git a/openfl-workspace/keras_2dunet/requirements.txt b/openfl-workspace/keras/2dunet/requirements.txt similarity index 100% rename from openfl-workspace/keras_2dunet/requirements.txt rename to openfl-workspace/keras/2dunet/requirements.txt diff --git a/openfl-workspace/keras_2dunet/src/__init__.py b/openfl-workspace/keras/2dunet/src/__init__.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/__init__.py rename to openfl-workspace/keras/2dunet/src/__init__.py diff --git a/openfl-workspace/keras_2dunet/src/brats_utils.py b/openfl-workspace/keras/2dunet/src/brats_utils.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/brats_utils.py rename to openfl-workspace/keras/2dunet/src/brats_utils.py diff --git a/openfl-workspace/keras_2dunet/src/dataloader.py b/openfl-workspace/keras/2dunet/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/dataloader.py rename to openfl-workspace/keras/2dunet/src/dataloader.py diff --git a/openfl-workspace/keras_2dunet/src/nii_reader.py b/openfl-workspace/keras/2dunet/src/nii_reader.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/nii_reader.py rename to openfl-workspace/keras/2dunet/src/nii_reader.py diff --git a/openfl-workspace/keras_2dunet/src/taskrunner.py b/openfl-workspace/keras/2dunet/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/taskrunner.py rename to openfl-workspace/keras/2dunet/src/taskrunner.py diff --git a/openfl-workspace/keras_cnn_mnist/.workspace b/openfl-workspace/keras/cnn_mnist/.workspace similarity index 100% rename from openfl-workspace/keras_cnn_mnist/.workspace rename to openfl-workspace/keras/cnn_mnist/.workspace diff --git a/openfl-workspace/keras_cnn_mnist/plan/cols.yaml b/openfl-workspace/keras/cnn_mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/cols.yaml rename to openfl-workspace/keras/cnn_mnist/plan/cols.yaml diff --git a/openfl-workspace/keras_cnn_mnist/plan/data.yaml b/openfl-workspace/keras/cnn_mnist/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/data.yaml rename to openfl-workspace/keras/cnn_mnist/plan/data.yaml diff --git a/openfl-workspace/keras_cnn_mnist/plan/defaults b/openfl-workspace/keras/cnn_mnist/plan/defaults similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/defaults rename to openfl-workspace/keras/cnn_mnist/plan/defaults diff --git a/openfl-workspace/keras_cnn_mnist/plan/plan.yaml b/openfl-workspace/keras/cnn_mnist/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/plan.yaml rename to openfl-workspace/keras/cnn_mnist/plan/plan.yaml diff --git a/openfl-workspace/keras_cnn_mnist/requirements.txt b/openfl-workspace/keras/cnn_mnist/requirements.txt similarity index 100% rename from openfl-workspace/keras_cnn_mnist/requirements.txt rename to openfl-workspace/keras/cnn_mnist/requirements.txt diff --git a/openfl-workspace/keras_cnn_mnist/src/__init__.py b/openfl-workspace/keras/cnn_mnist/src/__init__.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/__init__.py rename to openfl-workspace/keras/cnn_mnist/src/__init__.py diff --git a/openfl-workspace/keras_cnn_mnist/src/dataloader.py b/openfl-workspace/keras/cnn_mnist/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/dataloader.py rename to openfl-workspace/keras/cnn_mnist/src/dataloader.py diff --git a/openfl-workspace/keras_cnn_mnist/src/mnist_utils.py b/openfl-workspace/keras/cnn_mnist/src/mnist_utils.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/mnist_utils.py rename to openfl-workspace/keras/cnn_mnist/src/mnist_utils.py diff --git a/openfl-workspace/keras_cnn_mnist/src/taskrunner.py b/openfl-workspace/keras/cnn_mnist/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/taskrunner.py rename to openfl-workspace/keras/cnn_mnist/src/taskrunner.py diff --git a/openfl-workspace/keras_nlp/.workspace b/openfl-workspace/keras/nlp/.workspace similarity index 100% rename from openfl-workspace/keras_nlp/.workspace rename to openfl-workspace/keras/nlp/.workspace diff --git a/openfl-workspace/keras_nlp/plan/cols.yaml b/openfl-workspace/keras/nlp/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/cols.yaml rename to openfl-workspace/keras/nlp/plan/cols.yaml diff --git a/openfl-workspace/keras_nlp/plan/data.yaml b/openfl-workspace/keras/nlp/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/data.yaml rename to openfl-workspace/keras/nlp/plan/data.yaml diff --git a/openfl-workspace/keras_nlp/plan/plan.yaml b/openfl-workspace/keras/nlp/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/plan.yaml rename to openfl-workspace/keras/nlp/plan/plan.yaml diff --git a/openfl-workspace/keras_nlp/requirements.txt b/openfl-workspace/keras/nlp/requirements.txt similarity index 100% rename from openfl-workspace/keras_nlp/requirements.txt rename to openfl-workspace/keras/nlp/requirements.txt diff --git a/openfl-workspace/keras_nlp/src/__init__.py b/openfl-workspace/keras/nlp/src/__init__.py similarity index 100% rename from openfl-workspace/keras_nlp/src/__init__.py rename to openfl-workspace/keras/nlp/src/__init__.py diff --git a/openfl-workspace/keras_nlp/src/dataloader.py b/openfl-workspace/keras/nlp/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_nlp/src/dataloader.py rename to openfl-workspace/keras/nlp/src/dataloader.py diff --git a/openfl-workspace/keras_nlp/src/dataloader_utils.py b/openfl-workspace/keras/nlp/src/dataloader_utils.py similarity index 100% rename from openfl-workspace/keras_nlp/src/dataloader_utils.py rename to openfl-workspace/keras/nlp/src/dataloader_utils.py diff --git a/openfl-workspace/keras_nlp/src/taskrunner.py b/openfl-workspace/keras/nlp/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_nlp/src/taskrunner.py rename to openfl-workspace/keras/nlp/src/taskrunner.py From 3a557169d1926fce1399026a976e0b9b9827a534 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:38:37 -0800 Subject: [PATCH 03/15] code changes Signed-off-by: yes --- openfl-workspace/gandlf_seg_test/.workspace | 2 ++ openfl-workspace/torch_cnn_histology/.workspace | 2 ++ openfl-workspace/torch_cnn_histology_fedcurv/.workspace | 2 ++ openfl-workspace/torch_cnn_mnist/.workspace | 2 ++ openfl-workspace/xgb_higgs/.workspace | 2 ++ 5 files changed, 10 insertions(+) create mode 100644 openfl-workspace/gandlf_seg_test/.workspace create mode 100644 openfl-workspace/torch_cnn_histology/.workspace create mode 100644 openfl-workspace/torch_cnn_histology_fedcurv/.workspace create mode 100644 openfl-workspace/torch_cnn_mnist/.workspace create mode 100644 openfl-workspace/xgb_higgs/.workspace diff --git a/openfl-workspace/gandlf_seg_test/.workspace b/openfl-workspace/gandlf_seg_test/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/gandlf_seg_test/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_histology/.workspace b/openfl-workspace/torch_cnn_histology/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_histology/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/.workspace b/openfl-workspace/torch_cnn_histology_fedcurv/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_histology_fedcurv/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_mnist/.workspace b/openfl-workspace/torch_cnn_mnist/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_mnist/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/xgb_higgs/.workspace b/openfl-workspace/xgb_higgs/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/xgb_higgs/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + From 138ca06f1da9017e23bcabbd5cae855e9448dc53 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:14:12 -0800 Subject: [PATCH 04/15] code changes Signed-off-by: yes --- .github/workflows/straggler-handling.yml | 2 +- .github/workflows/task_runner_basic_e2e.yml | 22 +- .../task_runner_dockerized_ws_e2e.yml | 8 +- .github/workflows/task_runner_fedeval_e2e.yml | 8 +- .github/workflows/taskrunner.yml | 2 +- .../workflows/taskrunner_eden_pipeline.yml | 2 +- .../workflows/tr_docker_gramine_direct.yml | 2 +- .github/workflows/tr_docker_native.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- Jenkinsfile | 12 +- docs/about/features_index/fed_eval.rst | 8 +- docs/about/features_index/taskrunner.rst | 14 +- .../advanced_topics/log_metric_callback.rst | 2 +- .../straggler_handling_algorithms.rst | 2 +- docs/releases.md | 4 +- docs/tutorials/taskrunner.ipynb | 2 +- .../{keras/cnn_mnist => JAX}/.workspace | 0 .../{keras/cnn_mnist => JAX}/plan/cols.yaml | 0 .../{keras/cnn_mnist => JAX}/plan/data.yaml | 0 .../{keras/cnn_mnist => JAX}/plan/defaults | 0 .../{keras/cnn_mnist => JAX}/plan/plan.yaml | 0 .../{keras/cnn_mnist => JAX}/requirements.txt | 0 .../{keras/cnn_mnist => JAX}/src/__init__.py | 0 .../cnn_mnist => JAX}/src/dataloader.py | 0 .../cnn_mnist => JAX}/src/mnist_utils.py | 0 .../cnn_mnist => JAX}/src/taskrunner.py | 0 .../mnist}/.workspace | 0 .../mnist}/plan/cols.yaml | 0 openfl-workspace/keras/mnist/plan/data.yaml | 7 + .../mnist}/plan/defaults | 0 openfl-workspace/keras/mnist/plan/plan.yaml | 46 ++ openfl-workspace/keras/mnist/requirements.txt | 3 + .../mnist}/src/__init__.py | 0 .../keras/mnist/src/dataloader.py | 47 ++ .../keras/mnist/src/mnist_utils.py | 118 +++++ .../keras/mnist/src/taskrunner.py | 78 +++ .../histology}/.workspace | 0 .../histology}/plan/cols.yaml | 0 .../histology}/plan/data.yaml | 0 .../histology}/plan/plan.yaml | 0 .../histology}/requirements.txt | 0 .../histology}/src/__init__.py | 0 .../histology}/src/dataloader.py | 0 .../histology}/src/taskrunner.py | 0 .../histology_fedcurv}/.workspace | 0 .../histology_fedcurv}/README.md | 4 +- .../histology_fedcurv}/plan/cols.yaml | 0 .../histology_fedcurv}/plan/data.yaml | 0 .../histology_fedcurv}/plan/plan.yaml | 0 .../histology_fedcurv}/requirements.txt | 0 .../torch/histology_fedcurv/src/__init__.py | 3 + .../histology_fedcurv}/src/dataloader.py | 0 .../histology_fedcurv}/src/taskrunner.py | 0 .../llm_horovod}/.workspace | 0 .../llm_horovod}/LLM_Horovod.MD | 6 +- .../llm_horovod}/plan/cols.yaml | 0 .../llm_horovod}/plan/data.yaml | 0 .../llm_horovod}/plan/defaults | 0 .../llm_horovod}/plan/plan.yaml | 0 .../llm_horovod}/requirements.txt | 0 .../llm_horovod}/setup_env.sh | 2 +- .../llm_horovod}/src/InHorovodLLMTrainer.py | 0 .../llm_horovod}/src/InHorovodrun.py | 0 .../llm_horovod}/src/__init__.py | 0 .../llm_horovod}/src/emotion_utils.py | 0 .../llm_horovod}/src/model_utils.py | 0 .../llm_horovod}/src/pt_model.py | 0 .../llm_horovod}/src/ptemotion_inmemory.py | 0 .../mnist}/.workspace | 0 .../mnist}/README.md | 4 +- .../mnist}/plan/cols.yaml | 0 .../mnist}/plan/data.yaml | 0 .../mnist}/plan/defaults | 0 .../mnist}/plan/plan.yaml | 0 .../mnist}/requirements.txt | 0 .../mnist}/src/__init__.py | 0 .../mnist}/src/cnn_model.py | 0 .../mnist}/src/dataloader.py | 0 .../mnist}/src/taskrunner.py | 0 .../mnist_eden_compression}/.workspace | 0 .../mnist_eden_compression}/plan/cols.yaml | 0 .../mnist_eden_compression}/plan/data.yaml | 0 .../mnist_eden_compression}/plan/defaults | 0 .../mnist_eden_compression}/plan/plan.yaml | 0 .../mnist_eden_compression}/requirements.txt | 0 .../mnist_eden_compression}/src/__init__.py | 0 .../src/mnist_utils.py | 0 .../mnist_eden_compression}/src/pt_cnn.py | 0 .../src/ptmnist_inmemory.py | 0 .../mnist_fed_eval}/.workspace | 0 .../mnist_fed_eval}/plan/cols.yaml | 0 .../mnist_fed_eval}/plan/data.yaml | 0 .../mnist_fed_eval}/plan/defaults | 0 .../mnist_fed_eval}/plan/plan.yaml | 0 .../mnist_fed_eval}/requirements.txt | 0 .../mnist_fed_eval}/src/__init__.py | 0 .../mnist_fed_eval}/src/mnist_utils.py | 0 .../mnist_fed_eval}/src/pt_cnn.py | 0 .../mnist_fed_eval}/src/ptmnist_inmemory.py | 0 .../mnist_straggler_check}/.workspace | 0 .../mnist_straggler_check/plan/cols.yaml | 5 + .../mnist_straggler_check}/plan/data.yaml | 0 .../mnist_straggler_check}/plan/defaults | 0 .../mnist_straggler_check}/plan/plan.yaml | 0 .../mnist_straggler_check}/requirements.txt | 0 .../mnist_straggler_check}/src/__init__.py | 0 .../mnist_straggler_check}/src/mnist_utils.py | 0 .../mnist_straggler_check}/src/pt_cnn.py | 0 .../src/ptmnist_inmemory.py | 0 .../template}/.workspace | 0 .../template}/plan/cols.yaml | 0 .../template}/plan/data.yaml | 0 .../template}/plan/defaults | 0 .../template}/plan/plan.yaml | 0 .../template}/requirements.txt | 0 .../template}/src/__init__.py | 0 .../template}/src/dataloader.py | 0 .../template}/src/taskrunner.py | 0 openfl-workspace/torch/unet_kvasir/.workspace | 2 + .../unet_kvasir}/plan/cols.yaml | 0 .../unet_kvasir}/plan/data.yaml | 0 .../torch/unet_kvasir/plan/defaults | 2 + .../unet_kvasir}/plan/plan.yaml | 0 .../unet_kvasir}/requirements.txt | 0 .../unet_kvasir}/src/__init__.py | 0 .../unet_kvasir}/src/dataloader.py | 0 .../unet_kvasir}/src/pt_unet_parts.py | 0 .../unet_kvasir}/src/taskrunner.py | 0 openfl/federated/data/loader_jax.py | 138 +++++ openfl/federated/task/runner_jax.py | 489 ++++++++++++++++++ openfl/native/native.py | 4 +- tests/end_to_end/README.md | 8 +- tests/end_to_end/utils/conftest_helper.py | 2 +- tests/end_to_end/utils/constants.py | 8 +- tests/github/test_double_ws_export.py | 2 +- tests/github/test_gandlf.py | 2 +- tests/github/test_hello_federation.py | 2 +- 138 files changed, 1008 insertions(+), 70 deletions(-) rename openfl-workspace/{keras/cnn_mnist => JAX}/.workspace (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/cols.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/data.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/defaults (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/plan.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/requirements.txt (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/__init__.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/dataloader.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/mnist_utils.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_histology => keras/mnist}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => keras/mnist}/plan/cols.yaml (100%) create mode 100644 openfl-workspace/keras/mnist/plan/data.yaml rename openfl-workspace/{torch_cnn_mnist_eden_compression => keras/mnist}/plan/defaults (100%) create mode 100644 openfl-workspace/keras/mnist/plan/plan.yaml create mode 100644 openfl-workspace/keras/mnist/requirements.txt rename openfl-workspace/{torch_cnn_histology => keras/mnist}/src/__init__.py (100%) create mode 100644 openfl-workspace/keras/mnist/src/dataloader.py create mode 100644 openfl-workspace/keras/mnist/src/mnist_utils.py create mode 100644 openfl-workspace/keras/mnist/src/taskrunner.py rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology}/.workspace (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/histology_fedcurv}/.workspace (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/README.md (90%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/requirements.txt (100%) create mode 100644 openfl-workspace/torch/histology_fedcurv/src/__init__.py rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/.workspace (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/LLM_Horovod.MD (92%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/llm_horovod}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/llm_horovod}/plan/defaults (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/plan/plan.yaml (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/requirements.txt (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/setup_env.sh (72%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/InHorovodLLMTrainer.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/InHorovodrun.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/src/__init__.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/emotion_utils.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/model_utils.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/pt_model.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/ptemotion_inmemory.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/README.md (97%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/cnn_model.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_eden_compression}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_eden_compression}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/.workspace (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_fed_eval}/plan/data.yaml (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_fed_eval}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/mnist_straggler_check}/.workspace (100%) create mode 100644 openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml rename openfl-workspace/{torch_llm_horovod => torch/mnist_straggler_check}/plan/data.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/mnist_straggler_check}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/requirements.txt (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_straggler_check}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_template => torch/template}/.workspace (100%) rename openfl-workspace/{torch_template => torch/template}/plan/cols.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/plan/data.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/plan/defaults (100%) rename openfl-workspace/{torch_template => torch/template}/plan/plan.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/requirements.txt (100%) rename openfl-workspace/{torch_template => torch/template}/src/__init__.py (100%) rename openfl-workspace/{torch_template => torch/template}/src/dataloader.py (100%) rename openfl-workspace/{torch_template => torch/template}/src/taskrunner.py (100%) create mode 100644 openfl-workspace/torch/unet_kvasir/.workspace rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/cols.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/data.yaml (100%) create mode 100644 openfl-workspace/torch/unet_kvasir/plan/defaults rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/plan.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/requirements.txt (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/__init__.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/dataloader.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/pt_unet_parts.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/taskrunner.py (100%) create mode 100644 openfl/federated/data/loader_jax.py create mode 100644 openfl/federated/task/runner_jax.py diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml index b5bb6996ce..5dba77278e 100644 --- a/.github/workflows/straggler-handling.yml +++ b/.github/workflows/straggler-handling.yml @@ -35,4 +35,4 @@ jobs: pip install . - name: Test Straggler Handling Interface run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist_straggler_check --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 + python -m tests.github.test_hello_federation --template torch/mnist_straggler_check --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index caeaf23088..5430c5695c 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -25,8 +25,8 @@ on: type: choice options: - all - - torch_cnn_mnist - - keras/cnn_mnist + - torch/mnist + - keras/mnist python_version: description: "Python version" required: false @@ -85,21 +85,21 @@ jobs: id: input_selection run: | # --------------------------------------------------------------- - # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. + # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. # Default combination if no input is provided (i.e. 'all' is selected). - # * TLS - models [torch_cnn_mnist, keras/cnn_mnist] and python versions [3.10, 3.11, 3.12] - # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] - # * No client auth - models [keras/cnn_mnist] and python version [3.10] - # * Memory logs - models [torch_cnn_mnist] and python version [3.10] + # * TLS - models [torch/mnist, keras/mnist] and python versions [3.10, 3.11, 3.12] + # * Non-TLS - models [torch/mnist] and python version [3.10] + # * No client auth - models [keras/mnist] and python version [3.10] + # * Memory logs - models [torch/mnist] and python version [3.10] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" if [ "${{ env.MODEL_NAME }}" == "all" ]; then - echo "models_for_tls=[\"torch_cnn_mnist\", \"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"torch/mnist\", \"keras/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" else echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index 896b87400d..23febe3e64 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10", "3.11", "3.12"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -73,7 +73,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -114,7 +114,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -155,7 +155,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/task_runner_fedeval_e2e.yml b/.github/workflows/task_runner_fedeval_e2e.yml index fce6408686..88bd114836 100644 --- a/.github/workflows/task_runner_fedeval_e2e.yml +++ b/.github/workflows/task_runner_fedeval_e2e.yml @@ -34,9 +34,9 @@ jobs: timeout-minutes: 30 strategy: matrix: - # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. + # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - model_name: ["torch_cnn_mnist", "keras_cnn_mnist"] + model_name: ["torch/mnist", "keras_cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -77,9 +77,9 @@ jobs: timeout-minutes: 30 strategy: matrix: - # Testing this scenario only for torch_cnn_mnist model and python 3.10 + # Testing this scenario only for torch/mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ["torch_cnn_mnist"] + model_name: ["torch/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml index 11692a4b5c..1df43888eb 100644 --- a/.github/workflows/taskrunner.yml +++ b/.github/workflows/taskrunner.yml @@ -32,4 +32,4 @@ jobs: pip install . - name: Task Runner API run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist --fed_workspace aggregator --col1 collaborator1 --col2 collaborator2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template torch/mnist --fed_workspace aggregator --col1 collaborator1 --col2 collaborator2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml index 02eea0e2dc..bfacb7a61d 100644 --- a/.github/workflows/taskrunner_eden_pipeline.yml +++ b/.github/workflows/taskrunner_eden_pipeline.yml @@ -31,4 +31,4 @@ jobs: pip install . - name: Test TaskRunner API with Eden Compression run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist_eden_compression --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 + python -m tests.github.test_hello_federation --template torch/mnist_eden_compression --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 9ecbe90819..ca9155a647 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras/cnn_mnist + fx workspace create --prefix example_workspace --template keras/mnist cd example_workspace fx plan initialize -a localhost diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index b80e5ca161..7824897e03 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras/cnn_mnist + fx workspace create --prefix example_workspace --template keras/mnist cd example_workspace fx plan initialize -a localhost fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index ebaa206867..f82193a433 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -53,4 +53,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model + python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5bf4fb2bf4..e33ca1e09c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,4 +52,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index fd7db70d21..bd97ab7f2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,17 +2,17 @@ def snykData = [ 'openfl-docker': 'openfl-docker/Dockerfile.base', 'openfl': 'setup.py', 'openfl-workspace_tf_2dunet': 'openfl-workspace/tf_2dunet/requirements.txt', - 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt', + 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch/mnist_straggler_check/requirements.txt', // CN-14619 snyk test CLI does not support -f in requirements.txt file - // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch_cnn_histology/requirements.txt', - 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch_cnn_histology/src/requirements.txt', + // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch/histology/requirements.txt', + 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch/histology/src/requirements.txt', 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', - 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch_cnn_mnist/requirements.txt', - 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch_unet_kvasir/requirements.txt', + 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', + 'openfl-workspace_torch/unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', - 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/cnn_mnist/requirements.txt', + 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_histology_envoy': 'openfl-tutorials/interactive_api/PyTorch_Histology/envoy/requirements.txt', diff --git a/docs/about/features_index/fed_eval.rst b/docs/about/features_index/fed_eval.rst index 3f8e6b8637..01dca906ba 100644 --- a/docs/about/features_index/fed_eval.rst +++ b/docs/about/features_index/fed_eval.rst @@ -26,7 +26,7 @@ Example Using the Task Runner API (Aggregator-based Workflow) The following steps can be leveraged to achieve practical e2e usage of FedEval -*N.B*: We will be using torch_cnn_mnist plan itself for both training and with some minor changes for evaluation as well +*N.B*: We will be using torch/mnist plan itself for both training and with some minor changes for evaluation as well *Prerequisites*: Please ensure that OpenFL version==1.7 is installed or you can also choose to install latest from source. @@ -48,13 +48,13 @@ With OpenFL version==1.7 aggregator start command is enhanced to have an optiona --help Show this message and exit. 1. **Setup** -We will use the `torch_cnn_mnist` workspace for training +We will use the `torch/mnist` workspace for training Let's first configure a workspace with all necesary certificates .. code-block:: shell - fx workspace create --prefix ./cnn_train_eval --template torch_cnn_mnist + fx workspace create --prefix ./cnn_train_eval --template torch/mnist cd cnn_train_eval fx workspace certify fx aggregator generate-cert-request @@ -416,7 +416,7 @@ The updated plan post initialization with edits to make it ready for evaluation metrics: - loss -We have done following changes to the initialized torch_cnn_mnist plan in the new workspace: +We have done following changes to the initialized torch/mnist plan in the new workspace: - Set the rounds_to_train to 1 as evaluation needs just one round of federation run across the collaborators - Removed all other training related tasks from assigner settings except "aggregated_model_validation" Now let's replace the ``init.pbuf`` with the previously saved ``trained_model.pbuf`` diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index 255a63d904..0956a2cd8e 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -88,7 +88,7 @@ Each YAML top-level section contains the following subsections: The following is an example of a **plan.yaml**: -.. literalinclude:: ../../../openfl-workspace/torch_cnn_mnist/plan/plan.yaml +.. literalinclude:: ../../../openfl-workspace/torch/mnist/plan/plan.yaml :language: yaml @@ -150,22 +150,22 @@ STEP 1: Create a Workspace $ fx -2. This example uses the :code:`keras/cnn_mnist` template. +2. This example uses the :code:`keras/mnist` template. - Set the environment variables to use the :code:`keras/cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. + Set the environment variables to use the :code:`keras/mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. .. code-block:: shell - $ export WORKSPACE_TEMPLATE=keras/cnn_mnist + $ export WORKSPACE_TEMPLATE=keras/mnist $ export WORKSPACE_PATH=${HOME}/my_federation 3. Decide a workspace template, which are end-to-end federated learning training demonstrations. The following is a sample of available templates: - - :code:`keras/cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`keras/mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. - :code:`tf_2dunet`: a workspace with a simple `TensorFlow `__ CNN model that will use the `BraTS `_ dataset and train in a federation. - :code:`tf_cnn_histology`: a workspace with a simple `TensorFlow `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - - :code:`torch_cnn_histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - - :code:`torch_cnn_mnist`: a workspace with a simple `PyTorch `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`torch/histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. + - :code:`torch/mnist`: a workspace with a simple `PyTorch `__ CNN model that will download the `MNIST `_ dataset and train in a federation. See the complete list of available templates. diff --git a/docs/developer_guide/advanced_topics/log_metric_callback.rst b/docs/developer_guide/advanced_topics/log_metric_callback.rst index a449feba61..35256da30f 100644 --- a/docs/developer_guide/advanced_topics/log_metric_callback.rst +++ b/docs/developer_guide/advanced_topics/log_metric_callback.rst @@ -83,7 +83,7 @@ For logging through Tensorboard, enable the parameter :code:`write_logs : true` settings : write_logs : true -Follow the steps below to write your custom callback function instead. As an example, a full implementation can be found at `Federated_Pytorch_MNIST_Tutorial.ipynb `_ and in the **torch_cnn_mnist** workspace. +Follow the steps below to write your custom callback function instead. As an example, a full implementation can be found at `Federated_Pytorch_MNIST_Tutorial.ipynb `_ and in the **torch/mnist** workspace. 1. Define the callback function, like how you defined in Python API, in the **src** directory in your workspace. diff --git a/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst b/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst index cdfa2b6a2c..3ec03cc68c 100644 --- a/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst +++ b/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst @@ -29,7 +29,7 @@ The following are the straggler handling algorithms supported in OpenFL: Demonstration of adding the straggler handling interface ========================================================= -The example template, **torch_cnn_mnist_straggler_check**, uses the ``PercentageBasedStragglerHandling``. To gain a better understanding of how experiments perform, you can modify the **percent_collaborators_needed** or **minimum_reporting** parameter in the template **plan.yaml** or even choose **CutoffTimeBasedStragglerHandling** function instead: +The example template, **torch/mnist_straggler_check**, uses the ``PercentageBasedStragglerHandling``. To gain a better understanding of how experiments perform, you can modify the **percent_collaborators_needed** or **minimum_reporting** parameter in the template **plan.yaml** or even choose **CutoffTimeBasedStragglerHandling** function instead: .. code-block:: yaml diff --git a/docs/releases.md b/docs/releases.md index 539336ceb1..bcd0f19a39 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -15,7 +15,7 @@ - **FL Workspace Dockerization**: Revised Task Runner API workspace dockerization process, with TEE-ready containers (using Gramine and Intel® Software Guard Extensions). Follow the [updated instructions](https://github.com/securefederatedai/openfl/blob/develop/openfl-docker/README.md) to enhance the privacy and security of your FL experiments. -- **Federated Evaluation via TaskRunner API**: OpenFL 1.7 further simplifies the creation of Federated Evaluation experiments via the TaskRunner API (see the example [FedEval workspace](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_cnn_mnist_fed_eval)). +- **Federated Evaluation via TaskRunner API**: OpenFL 1.7 further simplifies the creation of Federated Evaluation experiments via the TaskRunner API (see the example [FedEval workspace](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch/mnist_fed_eval)). - **Keras 3 API**: Upgrading the base TaskRunner classes and example workspaces to Keras 3 for building state-of-the-art FL experiments with TensorFlow (more backends to be included in the upcoming OpenFL releases). @@ -28,7 +28,7 @@ ### New Features and APIs: - **Federated LLM fine-tuning**: - - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters + - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch/llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters - **Neuralchat-7b fine-tuning**: Learn how to fine-tune [neuralchat-7b](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/LLM/neuralchat) using the Intel® Extension for Transformers and the workflow interface. - **Workflow API enhancements**: Introducing an experimental [Workspace Export](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/workflow/1001_Workspace_Creation_from_JupyterNotebook.ipynb) feature that can be used to transform a Workflow API-based FL experiment into the TaskRunner API format for running in a distributed deployment. There is also groundwork laid for a future FederatedRuntime implementation for Workflow API, in addition to the currently supported LocalRuntime. diff --git a/docs/tutorials/taskrunner.ipynb b/docs/tutorials/taskrunner.ipynb index a95236e17b..a1874eaf5a 100644 --- a/docs/tutorials/taskrunner.ipynb +++ b/docs/tutorials/taskrunner.ipynb @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fx workspace create --prefix ./mnist_example --template keras/cnn_mnist\n", + "!fx workspace create --prefix ./mnist_example --template keras/mnist\n", "%cd ./mnist_example" ] }, diff --git a/openfl-workspace/keras/cnn_mnist/.workspace b/openfl-workspace/JAX/.workspace similarity index 100% rename from openfl-workspace/keras/cnn_mnist/.workspace rename to openfl-workspace/JAX/.workspace diff --git a/openfl-workspace/keras/cnn_mnist/plan/cols.yaml b/openfl-workspace/JAX/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/cols.yaml rename to openfl-workspace/JAX/plan/cols.yaml diff --git a/openfl-workspace/keras/cnn_mnist/plan/data.yaml b/openfl-workspace/JAX/plan/data.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/data.yaml rename to openfl-workspace/JAX/plan/data.yaml diff --git a/openfl-workspace/keras/cnn_mnist/plan/defaults b/openfl-workspace/JAX/plan/defaults similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/defaults rename to openfl-workspace/JAX/plan/defaults diff --git a/openfl-workspace/keras/cnn_mnist/plan/plan.yaml b/openfl-workspace/JAX/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/plan.yaml rename to openfl-workspace/JAX/plan/plan.yaml diff --git a/openfl-workspace/keras/cnn_mnist/requirements.txt b/openfl-workspace/JAX/requirements.txt similarity index 100% rename from openfl-workspace/keras/cnn_mnist/requirements.txt rename to openfl-workspace/JAX/requirements.txt diff --git a/openfl-workspace/keras/cnn_mnist/src/__init__.py b/openfl-workspace/JAX/src/__init__.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/__init__.py rename to openfl-workspace/JAX/src/__init__.py diff --git a/openfl-workspace/keras/cnn_mnist/src/dataloader.py b/openfl-workspace/JAX/src/dataloader.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/dataloader.py rename to openfl-workspace/JAX/src/dataloader.py diff --git a/openfl-workspace/keras/cnn_mnist/src/mnist_utils.py b/openfl-workspace/JAX/src/mnist_utils.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/mnist_utils.py rename to openfl-workspace/JAX/src/mnist_utils.py diff --git a/openfl-workspace/keras/cnn_mnist/src/taskrunner.py b/openfl-workspace/JAX/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/taskrunner.py rename to openfl-workspace/JAX/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_histology/.workspace b/openfl-workspace/keras/mnist/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_histology/.workspace rename to openfl-workspace/keras/mnist/.workspace diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/cols.yaml b/openfl-workspace/keras/mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/cols.yaml rename to openfl-workspace/keras/mnist/plan/cols.yaml diff --git a/openfl-workspace/keras/mnist/plan/data.yaml b/openfl-workspace/keras/mnist/plan/data.yaml new file mode 100644 index 0000000000..257c7825fe --- /dev/null +++ b/openfl-workspace/keras/mnist/plan/data.yaml @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +# collaborator_name,data_directory_path +one,1 + + diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/defaults b/openfl-workspace/keras/mnist/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/defaults rename to openfl-workspace/keras/mnist/plan/defaults diff --git a/openfl-workspace/keras/mnist/plan/plan.yaml b/openfl-workspace/keras/mnist/plan/plan.yaml new file mode 100644 index 0000000000..54867f4578 --- /dev/null +++ b/openfl-workspace/keras/mnist/plan/plan.yaml @@ -0,0 +1,46 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +aggregator : + defaults : plan/defaults/aggregator.yaml + template : openfl.component.Aggregator + settings : + init_state_path : save/init.pbuf + best_state_path : save/best.pbuf + last_state_path : save/last.pbuf + rounds_to_train : 10 + +collaborator : + defaults : plan/defaults/collaborator.yaml + template : openfl.component.Collaborator + settings : + delta_updates : false + opt_treatment : RESET + +data_loader : + defaults : plan/defaults/data_loader.yaml + template : src.dataloader.KerasMNISTInMemory + settings : + collaborator_count : 2 + data_group_name : mnist + batch_size : 256 + +task_runner : + defaults : plan/defaults/task_runner.yaml + template : src.taskrunner.KerasCNN + +network : + defaults : plan/defaults/network.yaml + +assigner : + defaults : plan/defaults/assigner.yaml + +tasks : + defaults : plan/defaults/tasks_keras.yaml + +compression_pipeline : + defaults : plan/defaults/compression_pipeline.yaml + # To use different Compression Pipeline, uncomment the following lines + # template : openfl.pipelines.KCPipeline + # settings : + # n_clusters : 6 diff --git a/openfl-workspace/keras/mnist/requirements.txt b/openfl-workspace/keras/mnist/requirements.txt new file mode 100644 index 0000000000..858a7dc3c8 --- /dev/null +++ b/openfl-workspace/keras/mnist/requirements.txt @@ -0,0 +1,3 @@ +keras==3.8.0 +tensorflow==2.18.0 + diff --git a/openfl-workspace/torch_cnn_histology/src/__init__.py b/openfl-workspace/keras/mnist/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/__init__.py rename to openfl-workspace/keras/mnist/src/__init__.py diff --git a/openfl-workspace/keras/mnist/src/dataloader.py b/openfl-workspace/keras/mnist/src/dataloader.py new file mode 100644 index 0000000000..040e8091c9 --- /dev/null +++ b/openfl-workspace/keras/mnist/src/dataloader.py @@ -0,0 +1,47 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from openfl.federated import KerasDataLoader +from .mnist_utils import load_mnist_shard + + +class KerasMNISTInMemory(KerasDataLoader): + """Data Loader for MNIST Dataset.""" + + def __init__(self, data_path, batch_size, **kwargs): + """ + Initialize. + + Args: + data_path: File path for the dataset + batch_size (int): The batch size for the data loader + **kwargs: Additional arguments, passed to super init and load_mnist_shard + """ + super().__init__(batch_size, **kwargs) + + # TODO: We should be downloading the dataset shard into a directory + # TODO: There needs to be a method to ask how many collaborators and + # what index/rank is this collaborator. + # Then we have a way to automatically shard based on rank and size of + # collaborator list. + try: + int(data_path) + except: + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) + + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + + self.X_train = X_train + self.y_train = y_train + self.X_valid = X_valid + self.y_valid = y_valid + + self.num_classes = num_classes diff --git a/openfl-workspace/keras/mnist/src/mnist_utils.py b/openfl-workspace/keras/mnist/src/mnist_utils.py new file mode 100644 index 0000000000..d19e13d9dd --- /dev/null +++ b/openfl-workspace/keras/mnist/src/mnist_utils.py @@ -0,0 +1,118 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from logging import getLogger + +import numpy as np +from tensorflow.python.keras.utils.data_utils import get_file + +logger = getLogger(__name__) + + +def one_hot(labels, classes): + """ + One Hot encode a vector. + + Args: + labels (list): List of labels to onehot encode + classes (int): Total number of categorical classes + + Returns: + np.array: Matrix of one-hot encoded labels + """ + return np.eye(classes)[labels] + + +def _load_raw_datashards(shard_num, collaborator_count): + """ + Load the raw data by shard. + + Returns tuples of the dataset shard divided into training and validation. + + Args: + shard_num (int): The shard number to use + collaborator_count (int): The number of collaborators in the federation + + Returns: + 2 tuples: (image, label) of the training, validation dataset + """ + origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' + path = get_file('mnist.npz', + origin=origin_folder + 'mnist.npz', + file_hash='731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1') + + with np.load(path) as f: + # get all of mnist + X_train_tot = f['x_train'] + y_train_tot = f['y_train'] + + X_valid_tot = f['x_test'] + y_valid_tot = f['y_test'] + + # create the shards + shard_num = int(shard_num) + X_train = X_train_tot[shard_num::collaborator_count] + y_train = y_train_tot[shard_num::collaborator_count] + + X_valid = X_valid_tot[shard_num::collaborator_count] + y_valid = y_valid_tot[shard_num::collaborator_count] + + return (X_train, y_train), (X_valid, y_valid) + + +def load_mnist_shard(shard_num, collaborator_count, categorical=True, + channels_last=True, **kwargs): + """ + Load the MNIST dataset. + + Args: + shard_num (int): The shard to use from the dataset + collaborator_count (int): The number of collaborators in the federation + categorical (bool): True = convert the labels to one-hot encoded + vectors (Default = True) + channels_last (bool): True = The input images have the channels + last (Default = True) + **kwargs: Additional parameters to pass to the function + + Returns: + list: The input shape + int: The number of classes + numpy.ndarray: The training data + numpy.ndarray: The training labels + numpy.ndarray: The validation data + numpy.ndarray: The validation labels + """ + img_rows, img_cols = 28, 28 + num_classes = 10 + + (X_train, y_train), (X_valid, y_valid) = _load_raw_datashards( + shard_num, collaborator_count + ) + + if channels_last: + X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) + X_valid = X_valid.reshape(X_valid.shape[0], img_rows, img_cols, 1) + input_shape = (img_rows, img_cols, 1) + else: + X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) + X_valid = X_valid.reshape(X_valid.shape[0], 1, img_rows, img_cols) + input_shape = (1, img_rows, img_cols) + + X_train = X_train.astype('float32') + X_valid = X_valid.astype('float32') + X_train /= 255 + X_valid /= 255 + + logger.info(f'MNIST > X_train Shape : {X_train.shape}') + logger.info(f'MNIST > y_train Shape : {y_train.shape}') + logger.info(f'MNIST > Train Samples : {X_train.shape[0]}') + logger.info(f'MNIST > Valid Samples : {X_valid.shape[0]}') + + if categorical: + # convert class vectors to binary class matrices + y_train = one_hot(y_train, num_classes) + y_valid = one_hot(y_valid, num_classes) + + return input_shape, num_classes, X_train, y_train, X_valid, y_valid diff --git a/openfl-workspace/keras/mnist/src/taskrunner.py b/openfl-workspace/keras/mnist/src/taskrunner.py new file mode 100644 index 0000000000..165861033c --- /dev/null +++ b/openfl-workspace/keras/mnist/src/taskrunner.py @@ -0,0 +1,78 @@ +# Copyright (C) 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from keras.models import Sequential +from keras.layers import Conv2D +from keras.layers import Dense +from keras.layers import Flatten + +from openfl.federated import KerasTaskRunner + + +class KerasCNN(KerasTaskRunner): + """A basic convolutional neural network model.""" + + def __init__(self, **kwargs): + """ + Initialize. + + Args: + **kwargs: Additional parameters to pass to the function + """ + super().__init__(**kwargs) + + self.model = self.build_model(self.feature_shape, self.data_loader.num_classes, **kwargs) + + self.initialize_tensorkeys_for_functions() + + self.model.summary(print_fn=self.logger.info) + + self.logger.info(f'Train Set Size : {self.get_train_data_size()}') + self.logger.info(f'Valid Set Size : {self.get_valid_data_size()}') + + def build_model(self, + input_shape, + num_classes, + conv_kernel_size=(4, 4), + conv_strides=(2, 2), + conv1_channels_out=16, + conv2_channels_out=32, + final_dense_inputsize=100, + **kwargs): + """ + Define the model architecture. + + Args: + input_shape (numpy.ndarray): The shape of the data + num_classes (int): The number of classes of the dataset + + Returns: + keras.models.Sequential: The model defined in Keras + + """ + model = Sequential() + + model.add(Conv2D(conv1_channels_out, + kernel_size=conv_kernel_size, + strides=conv_strides, + activation='relu', + input_shape=input_shape)) + + model.add(Conv2D(conv2_channels_out, + kernel_size=conv_kernel_size, + strides=conv_strides, + activation='relu')) + + model.add(Flatten()) + + model.add(Dense(final_dense_inputsize, activation='relu')) + + model.add(Dense(num_classes, activation='softmax')) + + model.compile(loss="categorical_crossentropy", + optimizer="adam", + metrics=["accuracy"]) + + return model diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/.workspace b/openfl-workspace/torch/histology/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/.workspace rename to openfl-workspace/torch/histology/.workspace diff --git a/openfl-workspace/torch_cnn_histology/plan/cols.yaml b/openfl-workspace/torch/histology/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/cols.yaml rename to openfl-workspace/torch/histology/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_histology/plan/data.yaml b/openfl-workspace/torch/histology/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/data.yaml rename to openfl-workspace/torch/histology/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_histology/plan/plan.yaml b/openfl-workspace/torch/histology/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/plan.yaml rename to openfl-workspace/torch/histology/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_histology/requirements.txt b/openfl-workspace/torch/histology/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_histology/requirements.txt rename to openfl-workspace/torch/histology/requirements.txt diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/__init__.py b/openfl-workspace/torch/histology/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/__init__.py rename to openfl-workspace/torch/histology/src/__init__.py diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch/histology/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/dataloader.py rename to openfl-workspace/torch/histology/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_histology/src/taskrunner.py b/openfl-workspace/torch/histology/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/taskrunner.py rename to openfl-workspace/torch/histology/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist/.workspace b/openfl-workspace/torch/histology_fedcurv/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist/.workspace rename to openfl-workspace/torch/histology_fedcurv/.workspace diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/README.md b/openfl-workspace/torch/histology_fedcurv/README.md similarity index 90% rename from openfl-workspace/torch_cnn_histology_fedcurv/README.md rename to openfl-workspace/torch/histology_fedcurv/README.md index de6f18e42d..1ebcd40329 100644 --- a/openfl-workspace/torch_cnn_histology_fedcurv/README.md +++ b/openfl-workspace/torch/histology_fedcurv/README.md @@ -4,7 +4,7 @@ It uses the Pytorch framework and OpenFL's TaskTunner API. The federation aggregates intermediate models using the [Fedcurv](https://arxiv.org/pdf/1910.07796) aggregation algorithm, which performs well (Compared to [FedAvg](https://arxiv.org/abs/2104.11375)) when the datasets are not independent and identically distributed (IID) among collaborators. -Note that this example is similar to the one present in the `torch_cnn_histology` directory and is here to demonstrate the usage of a different aggregation algorithm using OpenFL's Taskrunner API. +Note that this example is similar to the one present in the `torch/histology` directory and is here to demonstrate the usage of a different aggregation algorithm using OpenFL's Taskrunner API. The differenece between the two examples lies both in the `PyTorchCNNWithFedCurv` class which is used to define a stateful training method which uses an existing `FedCurv` object, and in the `plan.yaml` file in which the training task is explicitly defined with a non-default aggregation method - `FedCurvWeightedAverage`. @@ -13,7 +13,7 @@ and in the `plan.yaml` file in which the training task is explicitly defined wit The following instructions can be used to run the federation: ``` # Copy the workspace template, create collaborators and aggregator -fx workspace create --template torch_cnn_histology_fedcurv --prefix fedcurv +fx workspace create --template torch/histology_fedcurv --prefix fedcurv cd fedcurv fx workspace certify fx aggregator generate-cert-request fx aggregator certify --silent diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/cols.yaml b/openfl-workspace/torch/histology_fedcurv/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/cols.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/data.yaml b/openfl-workspace/torch/histology_fedcurv/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/data.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/plan.yaml b/openfl-workspace/torch/histology_fedcurv/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/plan.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/requirements.txt b/openfl-workspace/torch/histology_fedcurv/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/requirements.txt rename to openfl-workspace/torch/histology_fedcurv/requirements.txt diff --git a/openfl-workspace/torch/histology_fedcurv/src/__init__.py b/openfl-workspace/torch/histology_fedcurv/src/__init__.py new file mode 100644 index 0000000000..f1410b1298 --- /dev/null +++ b/openfl-workspace/torch/histology_fedcurv/src/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""You may copy this file as the starting point of your own model.""" diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/dataloader.py b/openfl-workspace/torch/histology_fedcurv/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/dataloader.py rename to openfl-workspace/torch/histology_fedcurv/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/taskrunner.py b/openfl-workspace/torch/histology_fedcurv/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/taskrunner.py rename to openfl-workspace/torch/histology_fedcurv/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/.workspace b/openfl-workspace/torch/llm_horovod/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/.workspace rename to openfl-workspace/torch/llm_horovod/.workspace diff --git a/openfl-workspace/torch_llm_horovod/LLM_Horovod.MD b/openfl-workspace/torch/llm_horovod/LLM_Horovod.MD similarity index 92% rename from openfl-workspace/torch_llm_horovod/LLM_Horovod.MD rename to openfl-workspace/torch/llm_horovod/LLM_Horovod.MD index 71b6d05165..63060de6f5 100644 --- a/openfl-workspace/torch_llm_horovod/LLM_Horovod.MD +++ b/openfl-workspace/torch/llm_horovod/LLM_Horovod.MD @@ -9,7 +9,7 @@ Before running the Horovod example, ensure that the following prerequisites are ## Setting up Horovod Dependencies To set up the Horovod dependencies, follow these steps: -1. Run the `setup_env.sh` script located in `openfl-workspace/torch_llm_horovod/setup_env.sh` within your virtual environment (venv). +1. Run the `setup_env.sh` script located in `openfl-workspace/torch/llm_horovod/setup_env.sh` within your virtual environment (venv). 2. Create aggregator and collaborator workspaces. See [Running the experiment](#running-the-experiment) 3. Ensure that the collaborator workspace is present in each node with the same file structure. 4. Make sure the dataset is available in each node. @@ -28,13 +28,13 @@ Set the following environmental variables for Horovod: ## Customizing Data and Models To use your own data and models, follow these steps: -1. Copy the `openfl/openfl-workspace/torch_llm_horovod` directory to `openfl/openfl-workspace/name_of_your_template`. +1. Copy the `openfl/openfl-workspace/torch/llm_horovod` directory to `openfl/openfl-workspace/name_of_your_template`. 2. In the `src/InHorovodrun` file, make the following changes: - Replace `EmotionDataLoader` with your own dataloader. - Replace `LLMTrainer` with your own training/validation scripts. ## Running the Experiment -To run the experiment, follow the instructions provided in the [OpenFL documentation](https://openfl.readthedocs.io/en/latest/running_the_federation.html#bare-metal-approach) using either the `torch_llm_horovod` template or your own template. +To run the experiment, follow the instructions provided in the [OpenFL documentation](https://openfl.readthedocs.io/en/latest/running_the_federation.html#bare-metal-approach) using either the `torch/llm_horovod` template or your own template. That's it! You're now ready to use the Horovod example with your own data and models. Enjoy! diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/cols.yaml b/openfl-workspace/torch/llm_horovod/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/cols.yaml rename to openfl-workspace/torch/llm_horovod/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/data.yaml b/openfl-workspace/torch/llm_horovod/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/data.yaml rename to openfl-workspace/torch/llm_horovod/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/defaults b/openfl-workspace/torch/llm_horovod/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/defaults rename to openfl-workspace/torch/llm_horovod/plan/defaults diff --git a/openfl-workspace/torch_llm_horovod/plan/plan.yaml b/openfl-workspace/torch/llm_horovod/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/plan.yaml rename to openfl-workspace/torch/llm_horovod/plan/plan.yaml diff --git a/openfl-workspace/torch_llm_horovod/requirements.txt b/openfl-workspace/torch/llm_horovod/requirements.txt similarity index 100% rename from openfl-workspace/torch_llm_horovod/requirements.txt rename to openfl-workspace/torch/llm_horovod/requirements.txt diff --git a/openfl-workspace/torch_llm_horovod/setup_env.sh b/openfl-workspace/torch/llm_horovod/setup_env.sh similarity index 72% rename from openfl-workspace/torch_llm_horovod/setup_env.sh rename to openfl-workspace/torch/llm_horovod/setup_env.sh index bb014dbe83..96f2d02133 100755 --- a/openfl-workspace/torch_llm_horovod/setup_env.sh +++ b/openfl-workspace/torch/llm_horovod/setup_env.sh @@ -3,6 +3,6 @@ pip install -U pip --no-cache pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1 export HOROVOD_WITH_PYTORCH=1 export HOROVOD_WITHOUT_MPI=1 -pip install -r openfl-workspace/torch_llm_horovod/requirements.txt --no-cache +pip install -r openfl-workspace/torch/llm_horovod/requirements.txt --no-cache diff --git a/openfl-workspace/torch_llm_horovod/src/InHorovodLLMTrainer.py b/openfl-workspace/torch/llm_horovod/src/InHorovodLLMTrainer.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/InHorovodLLMTrainer.py rename to openfl-workspace/torch/llm_horovod/src/InHorovodLLMTrainer.py diff --git a/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py b/openfl-workspace/torch/llm_horovod/src/InHorovodrun.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/InHorovodrun.py rename to openfl-workspace/torch/llm_horovod/src/InHorovodrun.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/__init__.py b/openfl-workspace/torch/llm_horovod/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/__init__.py rename to openfl-workspace/torch/llm_horovod/src/__init__.py diff --git a/openfl-workspace/torch_llm_horovod/src/emotion_utils.py b/openfl-workspace/torch/llm_horovod/src/emotion_utils.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/emotion_utils.py rename to openfl-workspace/torch/llm_horovod/src/emotion_utils.py diff --git a/openfl-workspace/torch_llm_horovod/src/model_utils.py b/openfl-workspace/torch/llm_horovod/src/model_utils.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/model_utils.py rename to openfl-workspace/torch/llm_horovod/src/model_utils.py diff --git a/openfl-workspace/torch_llm_horovod/src/pt_model.py b/openfl-workspace/torch/llm_horovod/src/pt_model.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/pt_model.py rename to openfl-workspace/torch/llm_horovod/src/pt_model.py diff --git a/openfl-workspace/torch_llm_horovod/src/ptemotion_inmemory.py b/openfl-workspace/torch/llm_horovod/src/ptemotion_inmemory.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/ptemotion_inmemory.py rename to openfl-workspace/torch/llm_horovod/src/ptemotion_inmemory.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/.workspace b/openfl-workspace/torch/mnist/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/.workspace rename to openfl-workspace/torch/mnist/.workspace diff --git a/openfl-workspace/torch_cnn_mnist/README.md b/openfl-workspace/torch/mnist/README.md similarity index 97% rename from openfl-workspace/torch_cnn_mnist/README.md rename to openfl-workspace/torch/mnist/README.md index 17019afa1b..e3666c1763 100644 --- a/openfl-workspace/torch_cnn_mnist/README.md +++ b/openfl-workspace/torch/mnist/README.md @@ -1,5 +1,5 @@ ## Instantiating a Workspace from Torch Template -To instantiate a workspace from the torch_cnn_mnist template, you can use the fx workspace create command. This allows you to quickly set up a new workspace based on a predefined configuration and template. +To instantiate a workspace from the torch/mnist template, you can use the fx workspace create command. This allows you to quickly set up a new workspace based on a predefined configuration and template. 1. Ensure the necessary dependencies are installed. ``` @@ -13,7 +13,7 @@ pip install openfl ``` cd ~/openfl-quickstart -fx workspace create --template torch_cnn_mnist --prefix fl_workspace +fx workspace create --template torch/mnist --prefix fl_workspace cd ~/openfl-quickstart/fl_workspace ``` diff --git a/openfl-workspace/torch_cnn_mnist/plan/cols.yaml b/openfl-workspace/torch/mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/cols.yaml rename to openfl-workspace/torch/mnist/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist/plan/data.yaml b/openfl-workspace/torch/mnist/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/data.yaml rename to openfl-workspace/torch/mnist/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist/plan/defaults b/openfl-workspace/torch/mnist/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/defaults rename to openfl-workspace/torch/mnist/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist/plan/plan.yaml b/openfl-workspace/torch/mnist/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/plan.yaml rename to openfl-workspace/torch/mnist/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist/requirements.txt b/openfl-workspace/torch/mnist/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist/requirements.txt rename to openfl-workspace/torch/mnist/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist/src/__init__.py b/openfl-workspace/torch/mnist/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/__init__.py rename to openfl-workspace/torch/mnist/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist/src/cnn_model.py b/openfl-workspace/torch/mnist/src/cnn_model.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/cnn_model.py rename to openfl-workspace/torch/mnist/src/cnn_model.py diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch/mnist/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/dataloader.py rename to openfl-workspace/torch/mnist/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_mnist/src/taskrunner.py b/openfl-workspace/torch/mnist/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/taskrunner.py rename to openfl-workspace/torch/mnist/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/.workspace b/openfl-workspace/torch/mnist_eden_compression/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/.workspace rename to openfl-workspace/torch/mnist_eden_compression/.workspace diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/cols.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/cols.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/data.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/data.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/defaults b/openfl-workspace/torch/mnist_eden_compression/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/defaults rename to openfl-workspace/torch/mnist_eden_compression/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/plan.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/plan.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt b/openfl-workspace/torch/mnist_eden_compression/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt rename to openfl-workspace/torch/mnist_eden_compression/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/__init__.py b/openfl-workspace/torch/mnist_eden_compression/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/__init__.py rename to openfl-workspace/torch/mnist_eden_compression/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/mnist_utils.py b/openfl-workspace/torch/mnist_eden_compression/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/mnist_utils.py rename to openfl-workspace/torch/mnist_eden_compression/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/pt_cnn.py b/openfl-workspace/torch/mnist_eden_compression/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/pt_cnn.py rename to openfl-workspace/torch/mnist_eden_compression/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_eden_compression/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_eden_compression/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_llm_horovod/.workspace b/openfl-workspace/torch/mnist_fed_eval/.workspace similarity index 100% rename from openfl-workspace/torch_llm_horovod/.workspace rename to openfl-workspace/torch/mnist_fed_eval/.workspace diff --git a/openfl-workspace/torch_llm_horovod/plan/cols.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/cols.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/data.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/data.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/data.yaml diff --git a/openfl-workspace/torch_llm_horovod/plan/defaults b/openfl-workspace/torch/mnist_fed_eval/plan/defaults similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/defaults rename to openfl-workspace/torch/mnist_fed_eval/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt b/openfl-workspace/torch/mnist_fed_eval/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt rename to openfl-workspace/torch/mnist_fed_eval/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/__init__.py b/openfl-workspace/torch/mnist_fed_eval/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/__init__.py rename to openfl-workspace/torch/mnist_fed_eval/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/mnist_utils.py b/openfl-workspace/torch/mnist_fed_eval/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/mnist_utils.py rename to openfl-workspace/torch/mnist_fed_eval/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/pt_cnn.py b/openfl-workspace/torch/mnist_fed_eval/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/pt_cnn.py rename to openfl-workspace/torch/mnist_fed_eval/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_fed_eval/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_fed_eval/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_unet_kvasir/.workspace b/openfl-workspace/torch/mnist_straggler_check/.workspace similarity index 100% rename from openfl-workspace/torch_unet_kvasir/.workspace rename to openfl-workspace/torch/mnist_straggler_check/.workspace diff --git a/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml new file mode 100644 index 0000000000..95307de3bc --- /dev/null +++ b/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml @@ -0,0 +1,5 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +collaborators: + \ No newline at end of file diff --git a/openfl-workspace/torch_llm_horovod/plan/data.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/data.yaml rename to openfl-workspace/torch/mnist_straggler_check/plan/data.yaml diff --git a/openfl-workspace/torch_unet_kvasir/plan/defaults b/openfl-workspace/torch/mnist_straggler_check/plan/defaults similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/defaults rename to openfl-workspace/torch/mnist_straggler_check/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/plan.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/plan.yaml rename to openfl-workspace/torch/mnist_straggler_check/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt b/openfl-workspace/torch/mnist_straggler_check/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt rename to openfl-workspace/torch/mnist_straggler_check/requirements.txt diff --git a/openfl-workspace/torch_llm_horovod/src/__init__.py b/openfl-workspace/torch/mnist_straggler_check/src/__init__.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/__init__.py rename to openfl-workspace/torch/mnist_straggler_check/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/mnist_utils.py b/openfl-workspace/torch/mnist_straggler_check/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/mnist_utils.py rename to openfl-workspace/torch/mnist_straggler_check/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/pt_cnn.py b/openfl-workspace/torch/mnist_straggler_check/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/pt_cnn.py rename to openfl-workspace/torch/mnist_straggler_check/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_straggler_check/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_straggler_check/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_template/.workspace b/openfl-workspace/torch/template/.workspace similarity index 100% rename from openfl-workspace/torch_template/.workspace rename to openfl-workspace/torch/template/.workspace diff --git a/openfl-workspace/torch_template/plan/cols.yaml b/openfl-workspace/torch/template/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/cols.yaml rename to openfl-workspace/torch/template/plan/cols.yaml diff --git a/openfl-workspace/torch_template/plan/data.yaml b/openfl-workspace/torch/template/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/data.yaml rename to openfl-workspace/torch/template/plan/data.yaml diff --git a/openfl-workspace/torch_template/plan/defaults b/openfl-workspace/torch/template/plan/defaults similarity index 100% rename from openfl-workspace/torch_template/plan/defaults rename to openfl-workspace/torch/template/plan/defaults diff --git a/openfl-workspace/torch_template/plan/plan.yaml b/openfl-workspace/torch/template/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/plan.yaml rename to openfl-workspace/torch/template/plan/plan.yaml diff --git a/openfl-workspace/torch_template/requirements.txt b/openfl-workspace/torch/template/requirements.txt similarity index 100% rename from openfl-workspace/torch_template/requirements.txt rename to openfl-workspace/torch/template/requirements.txt diff --git a/openfl-workspace/torch_template/src/__init__.py b/openfl-workspace/torch/template/src/__init__.py similarity index 100% rename from openfl-workspace/torch_template/src/__init__.py rename to openfl-workspace/torch/template/src/__init__.py diff --git a/openfl-workspace/torch_template/src/dataloader.py b/openfl-workspace/torch/template/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_template/src/dataloader.py rename to openfl-workspace/torch/template/src/dataloader.py diff --git a/openfl-workspace/torch_template/src/taskrunner.py b/openfl-workspace/torch/template/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_template/src/taskrunner.py rename to openfl-workspace/torch/template/src/taskrunner.py diff --git a/openfl-workspace/torch/unet_kvasir/.workspace b/openfl-workspace/torch/unet_kvasir/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch/unet_kvasir/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_unet_kvasir/plan/cols.yaml b/openfl-workspace/torch/unet_kvasir/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/cols.yaml rename to openfl-workspace/torch/unet_kvasir/plan/cols.yaml diff --git a/openfl-workspace/torch_unet_kvasir/plan/data.yaml b/openfl-workspace/torch/unet_kvasir/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/data.yaml rename to openfl-workspace/torch/unet_kvasir/plan/data.yaml diff --git a/openfl-workspace/torch/unet_kvasir/plan/defaults b/openfl-workspace/torch/unet_kvasir/plan/defaults new file mode 100644 index 0000000000..fb82f9c5b6 --- /dev/null +++ b/openfl-workspace/torch/unet_kvasir/plan/defaults @@ -0,0 +1,2 @@ +../../workspace/plan/defaults + diff --git a/openfl-workspace/torch_unet_kvasir/plan/plan.yaml b/openfl-workspace/torch/unet_kvasir/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/plan.yaml rename to openfl-workspace/torch/unet_kvasir/plan/plan.yaml diff --git a/openfl-workspace/torch_unet_kvasir/requirements.txt b/openfl-workspace/torch/unet_kvasir/requirements.txt similarity index 100% rename from openfl-workspace/torch_unet_kvasir/requirements.txt rename to openfl-workspace/torch/unet_kvasir/requirements.txt diff --git a/openfl-workspace/torch_unet_kvasir/src/__init__.py b/openfl-workspace/torch/unet_kvasir/src/__init__.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/__init__.py rename to openfl-workspace/torch/unet_kvasir/src/__init__.py diff --git a/openfl-workspace/torch_unet_kvasir/src/dataloader.py b/openfl-workspace/torch/unet_kvasir/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/dataloader.py rename to openfl-workspace/torch/unet_kvasir/src/dataloader.py diff --git a/openfl-workspace/torch_unet_kvasir/src/pt_unet_parts.py b/openfl-workspace/torch/unet_kvasir/src/pt_unet_parts.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/pt_unet_parts.py rename to openfl-workspace/torch/unet_kvasir/src/pt_unet_parts.py diff --git a/openfl-workspace/torch_unet_kvasir/src/taskrunner.py b/openfl-workspace/torch/unet_kvasir/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/taskrunner.py rename to openfl-workspace/torch/unet_kvasir/src/taskrunner.py diff --git a/openfl/federated/data/loader_jax.py b/openfl/federated/data/loader_jax.py new file mode 100644 index 0000000000..bf5a460186 --- /dev/null +++ b/openfl/federated/data/loader_jax.py @@ -0,0 +1,138 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +"""KerasDataLoader module.""" + +import numpy as np + +from openfl.federated.data.loader import DataLoader + + +class JAXDataLoader(DataLoader): + """A class used to represent a Federation Data Loader for Keras models. + + Attributes: + batch_size (int): Size of batches used for all data loaders. + X_train (np.array): Training features. + y_train (np.array): Training labels. + X_valid (np.array): Validation features. + y_valid (np.array): Validation labels. + """ + + def __init__(self, batch_size, **kwargs): + """Initializes the KerasDataLoader object. + + Args: + batch_size (int): The size of batches used for all data loaders. + kwargs: Additional arguments to pass to the function. + """ + self.batch_size = batch_size + self.X_train = None + self.y_train = None + self.X_valid = None + self.y_valid = None + + # Child classes should have init signature: + # (self, batch_size, **kwargs), should call this __init__ and then + # define self.X_train, self.y_train, self.X_valid, and self.y_valid + + def get_feature_shape(self): + """Returns the shape of an example feature array. + + Returns: + tuple: The shape of an example feature array. + """ + return self.X_train[0].shape + + def get_train_loader(self, batch_size=None, num_batches=None): + """Returns the data loader for the training data. + + Args: + batch_size (int, optional): The batch size for the data loader + (default is None). + num_batches (int, optional): The number of batches for the data + loader (default is None). + + Returns: + DataLoader: The DataLoader object for the training data. + """ + return self._get_batch_generator( + X=self.X_train, + y=self.y_train, + batch_size=batch_size, + num_batches=num_batches, + ) + + def get_valid_loader(self, batch_size=None): + """Returns the data loader for the validation data. + + Args: + batch_size (int, optional): The batch size for the data loader + (default is None). + + Returns: + DataLoader: The DataLoader object for the validation data. + """ + return self._get_batch_generator(X=self.X_valid, y=self.y_valid, batch_size=batch_size) + + def get_train_data_size(self): + """Returns the total number of training samples. + + Returns: + int: The total number of training samples. + """ + return self.X_train.shape[0] + + def get_valid_data_size(self): + """Returns the total number of validation samples. + + Returns: + int: The total number of validation samples. + """ + return self.X_valid.shape[0] + + @staticmethod + def _batch_generator(X, y, idxs, batch_size, num_batches): + """Generates batches of data. + + Args: + X (np.array): The input data. + y (np.array): The label data. + idxs (np.array): The index of the dataset. + batch_size (int): The batch size for the data loader. + num_batches (int): The number of batches. + + Yields: + tuple: The input data and label data for each batch. + """ + for i in range(num_batches): + a = i * batch_size + b = a + batch_size + yield X[idxs[a:b]], y[idxs[a:b]] + + def _get_batch_generator(self, X, y, batch_size, num_batches=None): + """Returns the dataset generator. + + Args: + X (np.array): The input data. + y (np.array): The label data. + batch_size (int): The batch size for the data loader. + num_batches (int, optional): The number of batches (default is + None). + + Returns: + generator: The dataset generator. + """ + if batch_size is None: + batch_size = self.batch_size + + # shuffle data indices + idxs = np.random.permutation(np.arange(X.shape[0])) + + if num_batches is None: + # compute the number of batches + num_batches = int(np.ceil(X.shape[0] / batch_size)) + + # build the generator and return it + return self._batch_generator(X, y, idxs, batch_size, num_batches) diff --git a/openfl/federated/task/runner_jax.py b/openfl/federated/task/runner_jax.py new file mode 100644 index 0000000000..5b1641ce93 --- /dev/null +++ b/openfl/federated/task/runner_jax.py @@ -0,0 +1,489 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +""" +Base classes for developing a keras.Model() Federated Learning model. + +You may copy this file as the starting point of your own keras model. +""" + +import copy +from warnings import catch_warnings, simplefilter + +import numpy as np + +from openfl.federated.task.runner import TaskRunner +from openfl.utilities import Metric, TensorKey, change_tags +from openfl.utilities.split import split_tensor_dict_for_holdouts + +with catch_warnings(): + simplefilter(action="ignore") + import keras + + +class JAXTaskRunner(TaskRunner): + """The base model for Keras models in the federation. + + Attributes: + model (keras.Model): The Keras model. + model_tensor_names (list): List of model tensor names. + required_tensorkeys_for_function (dict): A map of all of the required + tensors for each of the public functions in KerasTaskRunner. + """ + + def __init__(self, **kwargs): + """Initializes the KerasTaskRunner instance. + + Args: + **kwargs: Additional parameters to pass to the function + """ + super().__init__(**kwargs) + + self.model = keras.models.Model() + + self.model_tensor_names = [] + + # this is a map of all of the required tensors for each of the public + # functions in KerasTaskRunner + self.required_tensorkeys_for_function = {} + + def rebuild_model(self, round_num, input_tensor_dict, validation=False): + """Parse tensor names and update weights of model. Handles the + optimizer treatment. + + Args: + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. + validation (bool, optional): If True, validate the model. Defaults + to False. + """ + if self.opt_treatment == "RESET": + self.reset_opt_vars() + self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) + elif round_num > 0 and self.opt_treatment == "CONTINUE_GLOBAL" and not validation: + self.set_tensor_dict(input_tensor_dict, with_opt_vars=True) + else: + self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) + + def train_task( + self, + col_name, + round_num, + input_tensor_dict, + metrics, + epochs=1, + batch_size=1, + **kwargs, + ): + """ + Perform the training. Is expected to perform draws randomly, without + replacement until data is exausted. Then data is replaced and shuffled + and draws continue. + + Args: + col_name (str): The collaborator name. + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. + metrics (list): List of metrics. + epochs (int, optional): Number of epochs to train. Defaults to 1. + batch_size (int, optional): Batch size. Defaults to 1. + **kwargs: Additional parameters. + + Returns: + global_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + local_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + """ + if metrics is None: + raise KeyError("metrics must be defined") + + # rebuild model with updated weights + self.rebuild_model(round_num, input_tensor_dict) + for epoch in range(epochs): + self.logger.info("Run %s epoch of %s round", epoch, round_num) + results = self.train_( + self.data_loader.get_train_loader(batch_size), + metrics=metrics, + **kwargs, + ) + + # output metric tensors (scalar) + origin = col_name + tags = ("trained",) + output_metric_dict = { + TensorKey(metric_name, origin, round_num, True, ("metric",)): metric_value + for (metric_name, metric_value) in results + } + + # output model tensors (Doesn't include TensorKey) + output_model_dict = self.get_tensor_dict(with_opt_vars=True) + global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( + self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs + ) + + # create global tensorkeys + global_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num, False, tags): nparray + for tensor_name, nparray in global_model_dict.items() + } + # create tensorkeys that should stay local + local_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num, False, tags): nparray + for tensor_name, nparray in local_model_dict.items() + } + # the train/validate aggregated function of the next round will look + # for the updated model parameters. + # this ensures they will be resolved locally + next_local_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num + 1, False, ("model",)): nparray + for tensor_name, nparray in local_model_dict.items() + } + + global_tensor_dict = { + **output_metric_dict, + **global_tensorkey_model_dict, + } + local_tensor_dict = { + **local_tensorkey_model_dict, + **next_local_tensorkey_model_dict, + } + + # update the required tensors if they need to be pulled from the + # aggregator + # TODO this logic can break if different collaborators have different + # roles between rounds. + # for example, if a collaborator only performs validation in the first + # round but training in the second, it has no way of knowing the + # optimizer state tensor names to request from the aggregator because + # these are only created after training occurs. A work around could + # involve doing a single epoch of training on random data to get the + # optimizer names, and then throwing away the model. + if self.opt_treatment == "CONTINUE_GLOBAL": + self.initialize_tensorkeys_for_functions(with_opt_vars=True) + + self.update_tensorkeys_for_functions() + return global_tensor_dict, local_tensor_dict + + def train_(self, batch_generator, metrics: list = None, **kwargs): + """Train single epoch. Override this function for custom training. + + Args: + batch_generator (generator): Generator of training batches. + metrics (list, optional): Names of metrics to save. Defaults to + None. + **kwargs: Additional parameters. + + Returns: + results (list): List of Metric objects. + """ + if metrics is None: + metrics = [] + # TODO Currently assuming that all metrics are defined at + # initialization (build_model). + # If metrics are added (i.e. not a subset of what was originally + # defined) then the model must be recompiled. + try: + results = self.model.get_metrics_result() + except ValueError: + if "batch_size" in kwargs: + batch_size = kwargs["batch_size"] + else: + batch_size = 1 + # evaluation needed before metrics can be resolved + self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) + results = self.model.get_metrics_result() + + # TODO if there are new metrics in the flplan that were not included + # in the originally + # compiled model, that behavior is not currently handled. + for param in metrics: + if param not in results: + raise ValueError( + f"KerasTaskRunner does not support specifying new metrics. " + f"Param_metrics = {metrics}" + ) + + history = self.model.fit(batch_generator, verbose=2, **kwargs) + results = [] + for metric in metrics: + value = np.mean([history.history[metric]]) + results.append(Metric(name=metric, value=np.array(value))) + return results + + def validate_task(self, col_name, round_num, input_tensor_dict, **kwargs): + """Run the trained model on validation data; report results. + + Args: + col_name (str): The collaborator name. + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. Either the + last aggregated or locally trained model + **kwargs: Additional parameters. + + Returns: + output_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + These correspond to acc, precision, f1_score, etc. + dict: Empty dictionary. + """ + if "batch_size" in kwargs: + batch_size = kwargs["batch_size"] + else: + batch_size = 1 + + self.rebuild_model(round_num, input_tensor_dict, validation=True) + param_metrics = kwargs["metrics"] + + self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) + results = self.model.get_metrics_result() + + # TODO if there are new metrics in the flplan that were not included in + # the originally compiled model, that behavior is not currently + # handled. + for param in param_metrics: + if param not in results: + raise ValueError( + f"KerasTaskRunner does not support specifying new metrics. " + f"Param_metrics = {param_metrics}" + ) + + origin = col_name + suffix = "validate" + if kwargs["apply"] == "local": + suffix += "_local" + else: + suffix += "_agg" + tags = ("metric",) + tags = change_tags(tags, add_field=suffix) + output_tensor_dict = { + TensorKey(metric, origin, round_num, True, tags): np.array(results[metric]) + for metric in param_metrics + } + + return output_tensor_dict, {} + + def save_native(self, filepath): + """Save model. + + Args: + filepath (str): The file path to save the model. + """ + self.model.export(filepath) + + def load_native(self, filepath): + """Load model. + + Args: + filepath (str): The file path to load the model. + """ + self.model = keras.models.load_model(filepath) + + @staticmethod + def _get_weights_names(obj): + """Get the list of weight names. + + Args: + obj (Model or Optimizer): The target object that we want to get + the weights. + + Returns: + weight_names (list): The weight name list. + """ + if isinstance(obj, keras.optimizers.Optimizer): + weight_names = [weight.name for weight in obj.variables] + else: + weight_names = [ + layer.name + "/" + weight.name for layer in obj.layers for weight in layer.weights + ] + return weight_names + + @staticmethod + def _get_weights_dict(obj, suffix=""): + """ + Get the dictionary of weights. + + Args: + obj (Model or Optimizer): The target object that we want to get + the weights. + suffix (str, optional): Suffix for weight names. Defaults to ''. + + Returns: + weights_dict (dict): The weight dictionary. + """ + weights_dict = {} + weight_names = KerasTaskRunner._get_weights_names(obj) + if isinstance(obj, keras.optimizers.Optimizer): + weights_dict = { + weight_names[i] + suffix: weight.numpy() + for i, weight in enumerate(copy.deepcopy(obj.variables)) + } + else: + weight_name_index = 0 + for layer in obj.layers: + if weight_name_index < len(weight_names) and len(layer.get_weights()) > 0: + for weight in layer.get_weights(): + weights_dict[weight_names[weight_name_index] + suffix] = weight + weight_name_index += 1 + return weights_dict + + @staticmethod + def _set_weights_dict(obj, weights_dict): + """Set the object weights with a dictionary. + + Args: + obj (Model or Optimizer): The target object that we want to set + the weights. + weights_dict (dict): The weight dictionary. + """ + weight_names = KerasTaskRunner._get_weights_names(obj) + weight_values = [weights_dict[name] for name in weight_names] + obj.set_weights(weight_values) + + def get_tensor_dict(self, with_opt_vars, suffix=""): + """ + Get the model weights as a tensor dictionary. + + Args: + with_opt_vars (bool): If we should include the optimizer's status. + suffix (str): Universally. + + Returns: + model_weights (dict): The tensor dictionary. + """ + model_weights = self._get_weights_dict(self.model, suffix) + + if with_opt_vars: + opt_weights = self._get_weights_dict(self.model.optimizer, suffix) + + model_weights.update(opt_weights) + if len(opt_weights) == 0: + self.logger.debug("WARNING: We didn't find variables for the optimizer.") + return model_weights + + def set_tensor_dict(self, tensor_dict, with_opt_vars): + """Set the model weights with a tensor dictionary. + + Args: + tensor_dict (dict): The tensor dictionary. + with_opt_vars (bool): True = include the optimizer's status. + """ + if with_opt_vars is False: + # It is possible to pass in opt variables from the input tensor + # dict. This will make sure that the correct layers are updated + model_weight_names = self._get_weights_names(self.model) + model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} + self._set_weights_dict(self.model, model_weights_dict) + else: + model_weight_names = self._get_weights_names(self.model) + model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} + opt_weight_names = self._get_weights_names(self.model.optimizer) + opt_weights_dict = {name: tensor_dict[name] for name in opt_weight_names} + self._set_weights_dict(self.model, model_weights_dict) + self._set_weights_dict(self.model.optimizer, opt_weights_dict) + + def reset_opt_vars(self): + """Resets the optimizer variables.""" + pass + + def get_required_tensorkeys_for_function(self, func_name, **kwargs): + """Get the required tensors for specified function that could be called + as part of a task. + + By default, this is just all of the layers and optimizer of the model. + + Args: + func_name (str): The function name. + **kwargs: Any function arguments. + + Returns: + list: List of TensorKey objects. + """ + if func_name == "validate_task": + local_model = "apply=" + str(kwargs["apply"]) + return self.required_tensorkeys_for_function[func_name][local_model] + else: + return self.required_tensorkeys_for_function[func_name] + + def update_tensorkeys_for_functions(self): + """Update the required tensors for all publicly accessible methods that + could be called as part of a task. + + By default, this is just all of the layers and optimizer of the model. + Custom tensors should be added to this function + """ + # TODO complete this function. It is only needed for opt_treatment, + # and making the model stateless + + # Minimal required tensors for train function + model_layer_names = self._get_weights_names(self.model) + opt_names = self._get_weights_names(self.model.optimizer) + tensor_names = model_layer_names + opt_names + self.logger.debug("Updating model tensor names: %s", tensor_names) + self.required_tensorkeys_for_function["train_task"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names + ] + + # Validation may be performed on local or aggregated (global) model, + # so there is an extra lookup dimension for kwargs + self.required_tensorkeys_for_function["validate_task"] = {} + self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ + TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) for tensor_name in tensor_names + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names + ] + + def initialize_tensorkeys_for_functions(self, with_opt_vars=False): + """Set the required tensors for all publicly accessible methods that + could be called as part of a task. + + By default, this is just all of the layers and optimizer of the model. + Custom tensors should be added to this function + + Args: + with_opt_vars (bool, optional): If True, include the optimizer's + status. Defaults to False. + """ + # TODO there should be a way to programmatically iterate through all + # of the methods in the class and declare the tensors. + # For now this is done manually + + output_model_dict = self.get_tensor_dict(with_opt_vars=with_opt_vars) + global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( + self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs + ) + if not with_opt_vars: + global_model_dict_val = global_model_dict + local_model_dict_val = local_model_dict + else: + output_model_dict = self.get_tensor_dict(with_opt_vars=False) + global_model_dict_val, local_model_dict_val = split_tensor_dict_for_holdouts( + self.logger, + output_model_dict, + **self.tensor_dict_split_fn_kwargs, + ) + + self.required_tensorkeys_for_function["train_task"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) + for tensor_name in global_model_dict + ] + self.required_tensorkeys_for_function["train_task"] += [ + TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) + for tensor_name in local_model_dict + ] + + # Validation may be performed on local or aggregated (global) model, + # so there is an extra lookup dimension for kwargs + self.required_tensorkeys_for_function["validate_task"] = {} + # TODO This is not stateless. The optimizer will not be + self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ + TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) + for tensor_name in {**global_model_dict_val, **local_model_dict_val} + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) + for tensor_name in global_model_dict_val + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] += [ + TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) + for tensor_name in local_model_dict_val + ] diff --git a/openfl/native/native.py b/openfl/native/native.py index c1b3813f92..5a5153b8a2 100644 --- a/openfl/native/native.py +++ b/openfl/native/native.py @@ -216,8 +216,8 @@ def init( workspace_template (str): The template that should be used as the basis for the experiment. Defaults to 'default'. Other options include are any of the template names - [keras/cnn_mnist, tf_2dunet, tf_cnn_histology, - mtorch_cnn_histology, torch_cnn_mnist]. + [keras/mnist, tf_2dunet, tf_cnn_histology, + mtorch_cnn_histology, torch/mnist]. log_level (str): Log level for logging. METRIC level is available. Defaults to 'INFO'. log_file (str): Name of the file in which the log will be duplicated. diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 0495629c1e..cd6ada7227 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -49,16 +49,16 @@ Below parameters are available for modification: 4. --disable_tls - to disable TLS communication (by default it is enabled) 5. --disable_client_auth - to disable the client authentication (by default it is enabled) -For example, to run Task runner (bare metal approach) with - torch_cnn_mnist model, 3 collaborators, 5 rounds and non-TLS scenario: +For example, to run Task runner (bare metal approach) with - torch/mnist model, 3 collaborators, 5 rounds and non-TLS scenario: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch/mnist --disable_tls ``` -And, to run Task runner (via dockerized workspace) with keras/cnn_mnist, 2 collaborators, 3 rounds: +And, to run Task runner (via dockerized workspace) with keras/mnist, 2 collaborators, 3 rounds: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/cnn_mnist +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/mnist ``` ### Fixture and marker mapping: diff --git a/tests/end_to_end/utils/conftest_helper.py b/tests/end_to_end/utils/conftest_helper.py index d623c21201..af912dd2c6 100644 --- a/tests/end_to_end/utils/conftest_helper.py +++ b/tests/end_to_end/utils/conftest_helper.py @@ -17,7 +17,7 @@ def parse_arguments(): - results_dir (str, optional): Directory to store the results - num_collaborators (int, default=2): Number of collaborators - num_rounds (int, default=5): Number of rounds to train - - model_name (str, default="torch_cnn_mnist"): Model name + - model_name (str, default="torch/mnist"): Model name - disable_client_auth (bool): Disable client authentication - disable_tls (bool): Disable TLS for communication - log_memory_usage (bool): Enable Memory leak logs diff --git a/tests/end_to_end/utils/constants.py b/tests/end_to_end/utils/constants.py index 971174c381..e370093f5f 100644 --- a/tests/end_to_end/utils/constants.py +++ b/tests/end_to_end/utils/constants.py @@ -10,15 +10,15 @@ class ModelName(Enum): """ # IMP - The model name must be same (and in uppercase) as the model value. # This is used to identify the model in the tests. - TORCH_CNN_MNIST = "torch_cnn_mnist" - KERAS_CNN_MNIST = "keras/cnn_mnist" - TORCH_CNN_HISTOLOGY = "torch_cnn_histology" + TORCH_CNN_MNIST = "torch/mnist" + KERAS_CNN_MNIST = "keras/mnist" + TORCH_CNN_HISTOLOGY = "torch/histology" XGB_HIGGS = "xgb_higgs" NUM_COLLABORATORS = 2 NUM_ROUNDS = 5 WORKSPACE_NAME = "my_federation" -DEFAULT_MODEL_NAME = "torch_cnn_mnist" +DEFAULT_MODEL_NAME = "torch/mnist" SUCCESS_MARKER = "✔️ OK" # Docker specific constants diff --git a/tests/github/test_double_ws_export.py b/tests/github/test_double_ws_export.py index 7e9b42bec3..07d2714ce8 100644 --- a/tests/github/test_double_ws_export.py +++ b/tests/github/test_double_ws_export.py @@ -22,7 +22,7 @@ def main(): for entry in iterator: if entry.name not in ['__init__.py', 'workspace', 'default']: workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') diff --git a/tests/github/test_gandlf.py b/tests/github/test_gandlf.py index 08e80e2118..a8696be541 100644 --- a/tests/github/test_gandlf.py +++ b/tests/github/test_gandlf.py @@ -21,7 +21,7 @@ def exec(command, directory): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--template', default='keras/cnn_mnist') + parser.add_argument('--template', default='keras/mnist') parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one') parser.add_argument('--col2', default='two') diff --git a/tests/github/test_hello_federation.py b/tests/github/test_hello_federation.py index a3ef7296a8..a769ffcb58 100644 --- a/tests/github/test_hello_federation.py +++ b/tests/github/test_hello_federation.py @@ -24,7 +24,7 @@ def main(): dir_path = dir_path.replace(os.sep, '/') if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): workspace_choice.append(dir_path) - parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') From a7b65bdfd6a5b78b9bbd01cd7e3e2c7455416ae3 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:21:07 -0800 Subject: [PATCH 05/15] code changes Signed-off-by: yes --- Jenkinsfile | 2 +- openfl-workspace/JAX/.workspace | 2 - openfl-workspace/JAX/plan/cols.yaml | 5 - openfl-workspace/JAX/plan/data.yaml | 7 - openfl-workspace/JAX/plan/defaults | 2 - openfl-workspace/JAX/plan/plan.yaml | 46 --- openfl-workspace/JAX/requirements.txt | 3 - openfl-workspace/JAX/src/__init__.py | 3 - openfl-workspace/JAX/src/dataloader.py | 47 --- openfl-workspace/JAX/src/mnist_utils.py | 118 ------ openfl-workspace/JAX/src/taskrunner.py | 78 ---- openfl/federated/data/loader_jax.py | 138 ------- openfl/federated/task/runner_jax.py | 489 ------------------------ 13 files changed, 1 insertion(+), 939 deletions(-) delete mode 100644 openfl-workspace/JAX/.workspace delete mode 100644 openfl-workspace/JAX/plan/cols.yaml delete mode 100644 openfl-workspace/JAX/plan/data.yaml delete mode 100644 openfl-workspace/JAX/plan/defaults delete mode 100644 openfl-workspace/JAX/plan/plan.yaml delete mode 100644 openfl-workspace/JAX/requirements.txt delete mode 100644 openfl-workspace/JAX/src/__init__.py delete mode 100644 openfl-workspace/JAX/src/dataloader.py delete mode 100644 openfl-workspace/JAX/src/mnist_utils.py delete mode 100644 openfl-workspace/JAX/src/taskrunner.py delete mode 100644 openfl/federated/data/loader_jax.py delete mode 100644 openfl/federated/task/runner_jax.py diff --git a/Jenkinsfile b/Jenkinsfile index bd97ab7f2b..c1c9a374fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ def snykData = [ 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch/histology/src/requirements.txt', 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', - 'openfl-workspace_torch/unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', + 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', diff --git a/openfl-workspace/JAX/.workspace b/openfl-workspace/JAX/.workspace deleted file mode 100644 index 3c2c5d08b4..0000000000 --- a/openfl-workspace/JAX/.workspace +++ /dev/null @@ -1,2 +0,0 @@ -current_plan_name: default - diff --git a/openfl-workspace/JAX/plan/cols.yaml b/openfl-workspace/JAX/plan/cols.yaml deleted file mode 100644 index 95307de3bc..0000000000 --- a/openfl-workspace/JAX/plan/cols.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -collaborators: - \ No newline at end of file diff --git a/openfl-workspace/JAX/plan/data.yaml b/openfl-workspace/JAX/plan/data.yaml deleted file mode 100644 index 257c7825fe..0000000000 --- a/openfl-workspace/JAX/plan/data.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -# collaborator_name,data_directory_path -one,1 - - diff --git a/openfl-workspace/JAX/plan/defaults b/openfl-workspace/JAX/plan/defaults deleted file mode 100644 index fb82f9c5b6..0000000000 --- a/openfl-workspace/JAX/plan/defaults +++ /dev/null @@ -1,2 +0,0 @@ -../../workspace/plan/defaults - diff --git a/openfl-workspace/JAX/plan/plan.yaml b/openfl-workspace/JAX/plan/plan.yaml deleted file mode 100644 index 54867f4578..0000000000 --- a/openfl-workspace/JAX/plan/plan.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -aggregator : - defaults : plan/defaults/aggregator.yaml - template : openfl.component.Aggregator - settings : - init_state_path : save/init.pbuf - best_state_path : save/best.pbuf - last_state_path : save/last.pbuf - rounds_to_train : 10 - -collaborator : - defaults : plan/defaults/collaborator.yaml - template : openfl.component.Collaborator - settings : - delta_updates : false - opt_treatment : RESET - -data_loader : - defaults : plan/defaults/data_loader.yaml - template : src.dataloader.KerasMNISTInMemory - settings : - collaborator_count : 2 - data_group_name : mnist - batch_size : 256 - -task_runner : - defaults : plan/defaults/task_runner.yaml - template : src.taskrunner.KerasCNN - -network : - defaults : plan/defaults/network.yaml - -assigner : - defaults : plan/defaults/assigner.yaml - -tasks : - defaults : plan/defaults/tasks_keras.yaml - -compression_pipeline : - defaults : plan/defaults/compression_pipeline.yaml - # To use different Compression Pipeline, uncomment the following lines - # template : openfl.pipelines.KCPipeline - # settings : - # n_clusters : 6 diff --git a/openfl-workspace/JAX/requirements.txt b/openfl-workspace/JAX/requirements.txt deleted file mode 100644 index 858a7dc3c8..0000000000 --- a/openfl-workspace/JAX/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -keras==3.8.0 -tensorflow==2.18.0 - diff --git a/openfl-workspace/JAX/src/__init__.py b/openfl-workspace/JAX/src/__init__.py deleted file mode 100644 index f1410b1298..0000000000 --- a/openfl-workspace/JAX/src/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -"""You may copy this file as the starting point of your own model.""" diff --git a/openfl-workspace/JAX/src/dataloader.py b/openfl-workspace/JAX/src/dataloader.py deleted file mode 100644 index 040e8091c9..0000000000 --- a/openfl-workspace/JAX/src/dataloader.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from openfl.federated import KerasDataLoader -from .mnist_utils import load_mnist_shard - - -class KerasMNISTInMemory(KerasDataLoader): - """Data Loader for MNIST Dataset.""" - - def __init__(self, data_path, batch_size, **kwargs): - """ - Initialize. - - Args: - data_path: File path for the dataset - batch_size (int): The batch size for the data loader - **kwargs: Additional arguments, passed to super init and load_mnist_shard - """ - super().__init__(batch_size, **kwargs) - - # TODO: We should be downloading the dataset shard into a directory - # TODO: There needs to be a method to ask how many collaborators and - # what index/rank is this collaborator. - # Then we have a way to automatically shard based on rank and size of - # collaborator list. - try: - int(data_path) - except: - raise ValueError( - "Expected `%s` to be representable as `int`, as it refers to the data shard " + - "number used by the collaborator.", - data_path - ) - - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - - self.X_train = X_train - self.y_train = y_train - self.X_valid = X_valid - self.y_valid = y_valid - - self.num_classes = num_classes diff --git a/openfl-workspace/JAX/src/mnist_utils.py b/openfl-workspace/JAX/src/mnist_utils.py deleted file mode 100644 index d19e13d9dd..0000000000 --- a/openfl-workspace/JAX/src/mnist_utils.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from logging import getLogger - -import numpy as np -from tensorflow.python.keras.utils.data_utils import get_file - -logger = getLogger(__name__) - - -def one_hot(labels, classes): - """ - One Hot encode a vector. - - Args: - labels (list): List of labels to onehot encode - classes (int): Total number of categorical classes - - Returns: - np.array: Matrix of one-hot encoded labels - """ - return np.eye(classes)[labels] - - -def _load_raw_datashards(shard_num, collaborator_count): - """ - Load the raw data by shard. - - Returns tuples of the dataset shard divided into training and validation. - - Args: - shard_num (int): The shard number to use - collaborator_count (int): The number of collaborators in the federation - - Returns: - 2 tuples: (image, label) of the training, validation dataset - """ - origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' - path = get_file('mnist.npz', - origin=origin_folder + 'mnist.npz', - file_hash='731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1') - - with np.load(path) as f: - # get all of mnist - X_train_tot = f['x_train'] - y_train_tot = f['y_train'] - - X_valid_tot = f['x_test'] - y_valid_tot = f['y_test'] - - # create the shards - shard_num = int(shard_num) - X_train = X_train_tot[shard_num::collaborator_count] - y_train = y_train_tot[shard_num::collaborator_count] - - X_valid = X_valid_tot[shard_num::collaborator_count] - y_valid = y_valid_tot[shard_num::collaborator_count] - - return (X_train, y_train), (X_valid, y_valid) - - -def load_mnist_shard(shard_num, collaborator_count, categorical=True, - channels_last=True, **kwargs): - """ - Load the MNIST dataset. - - Args: - shard_num (int): The shard to use from the dataset - collaborator_count (int): The number of collaborators in the federation - categorical (bool): True = convert the labels to one-hot encoded - vectors (Default = True) - channels_last (bool): True = The input images have the channels - last (Default = True) - **kwargs: Additional parameters to pass to the function - - Returns: - list: The input shape - int: The number of classes - numpy.ndarray: The training data - numpy.ndarray: The training labels - numpy.ndarray: The validation data - numpy.ndarray: The validation labels - """ - img_rows, img_cols = 28, 28 - num_classes = 10 - - (X_train, y_train), (X_valid, y_valid) = _load_raw_datashards( - shard_num, collaborator_count - ) - - if channels_last: - X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) - X_valid = X_valid.reshape(X_valid.shape[0], img_rows, img_cols, 1) - input_shape = (img_rows, img_cols, 1) - else: - X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) - X_valid = X_valid.reshape(X_valid.shape[0], 1, img_rows, img_cols) - input_shape = (1, img_rows, img_cols) - - X_train = X_train.astype('float32') - X_valid = X_valid.astype('float32') - X_train /= 255 - X_valid /= 255 - - logger.info(f'MNIST > X_train Shape : {X_train.shape}') - logger.info(f'MNIST > y_train Shape : {y_train.shape}') - logger.info(f'MNIST > Train Samples : {X_train.shape[0]}') - logger.info(f'MNIST > Valid Samples : {X_valid.shape[0]}') - - if categorical: - # convert class vectors to binary class matrices - y_train = one_hot(y_train, num_classes) - y_valid = one_hot(y_valid, num_classes) - - return input_shape, num_classes, X_train, y_train, X_valid, y_valid diff --git a/openfl-workspace/JAX/src/taskrunner.py b/openfl-workspace/JAX/src/taskrunner.py deleted file mode 100644 index 165861033c..0000000000 --- a/openfl-workspace/JAX/src/taskrunner.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (C) 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from keras.models import Sequential -from keras.layers import Conv2D -from keras.layers import Dense -from keras.layers import Flatten - -from openfl.federated import KerasTaskRunner - - -class KerasCNN(KerasTaskRunner): - """A basic convolutional neural network model.""" - - def __init__(self, **kwargs): - """ - Initialize. - - Args: - **kwargs: Additional parameters to pass to the function - """ - super().__init__(**kwargs) - - self.model = self.build_model(self.feature_shape, self.data_loader.num_classes, **kwargs) - - self.initialize_tensorkeys_for_functions() - - self.model.summary(print_fn=self.logger.info) - - self.logger.info(f'Train Set Size : {self.get_train_data_size()}') - self.logger.info(f'Valid Set Size : {self.get_valid_data_size()}') - - def build_model(self, - input_shape, - num_classes, - conv_kernel_size=(4, 4), - conv_strides=(2, 2), - conv1_channels_out=16, - conv2_channels_out=32, - final_dense_inputsize=100, - **kwargs): - """ - Define the model architecture. - - Args: - input_shape (numpy.ndarray): The shape of the data - num_classes (int): The number of classes of the dataset - - Returns: - keras.models.Sequential: The model defined in Keras - - """ - model = Sequential() - - model.add(Conv2D(conv1_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu', - input_shape=input_shape)) - - model.add(Conv2D(conv2_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu')) - - model.add(Flatten()) - - model.add(Dense(final_dense_inputsize, activation='relu')) - - model.add(Dense(num_classes, activation='softmax')) - - model.compile(loss="categorical_crossentropy", - optimizer="adam", - metrics=["accuracy"]) - - return model diff --git a/openfl/federated/data/loader_jax.py b/openfl/federated/data/loader_jax.py deleted file mode 100644 index bf5a460186..0000000000 --- a/openfl/federated/data/loader_jax.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -"""KerasDataLoader module.""" - -import numpy as np - -from openfl.federated.data.loader import DataLoader - - -class JAXDataLoader(DataLoader): - """A class used to represent a Federation Data Loader for Keras models. - - Attributes: - batch_size (int): Size of batches used for all data loaders. - X_train (np.array): Training features. - y_train (np.array): Training labels. - X_valid (np.array): Validation features. - y_valid (np.array): Validation labels. - """ - - def __init__(self, batch_size, **kwargs): - """Initializes the KerasDataLoader object. - - Args: - batch_size (int): The size of batches used for all data loaders. - kwargs: Additional arguments to pass to the function. - """ - self.batch_size = batch_size - self.X_train = None - self.y_train = None - self.X_valid = None - self.y_valid = None - - # Child classes should have init signature: - # (self, batch_size, **kwargs), should call this __init__ and then - # define self.X_train, self.y_train, self.X_valid, and self.y_valid - - def get_feature_shape(self): - """Returns the shape of an example feature array. - - Returns: - tuple: The shape of an example feature array. - """ - return self.X_train[0].shape - - def get_train_loader(self, batch_size=None, num_batches=None): - """Returns the data loader for the training data. - - Args: - batch_size (int, optional): The batch size for the data loader - (default is None). - num_batches (int, optional): The number of batches for the data - loader (default is None). - - Returns: - DataLoader: The DataLoader object for the training data. - """ - return self._get_batch_generator( - X=self.X_train, - y=self.y_train, - batch_size=batch_size, - num_batches=num_batches, - ) - - def get_valid_loader(self, batch_size=None): - """Returns the data loader for the validation data. - - Args: - batch_size (int, optional): The batch size for the data loader - (default is None). - - Returns: - DataLoader: The DataLoader object for the validation data. - """ - return self._get_batch_generator(X=self.X_valid, y=self.y_valid, batch_size=batch_size) - - def get_train_data_size(self): - """Returns the total number of training samples. - - Returns: - int: The total number of training samples. - """ - return self.X_train.shape[0] - - def get_valid_data_size(self): - """Returns the total number of validation samples. - - Returns: - int: The total number of validation samples. - """ - return self.X_valid.shape[0] - - @staticmethod - def _batch_generator(X, y, idxs, batch_size, num_batches): - """Generates batches of data. - - Args: - X (np.array): The input data. - y (np.array): The label data. - idxs (np.array): The index of the dataset. - batch_size (int): The batch size for the data loader. - num_batches (int): The number of batches. - - Yields: - tuple: The input data and label data for each batch. - """ - for i in range(num_batches): - a = i * batch_size - b = a + batch_size - yield X[idxs[a:b]], y[idxs[a:b]] - - def _get_batch_generator(self, X, y, batch_size, num_batches=None): - """Returns the dataset generator. - - Args: - X (np.array): The input data. - y (np.array): The label data. - batch_size (int): The batch size for the data loader. - num_batches (int, optional): The number of batches (default is - None). - - Returns: - generator: The dataset generator. - """ - if batch_size is None: - batch_size = self.batch_size - - # shuffle data indices - idxs = np.random.permutation(np.arange(X.shape[0])) - - if num_batches is None: - # compute the number of batches - num_batches = int(np.ceil(X.shape[0] / batch_size)) - - # build the generator and return it - return self._batch_generator(X, y, idxs, batch_size, num_batches) diff --git a/openfl/federated/task/runner_jax.py b/openfl/federated/task/runner_jax.py deleted file mode 100644 index 5b1641ce93..0000000000 --- a/openfl/federated/task/runner_jax.py +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -""" -Base classes for developing a keras.Model() Federated Learning model. - -You may copy this file as the starting point of your own keras model. -""" - -import copy -from warnings import catch_warnings, simplefilter - -import numpy as np - -from openfl.federated.task.runner import TaskRunner -from openfl.utilities import Metric, TensorKey, change_tags -from openfl.utilities.split import split_tensor_dict_for_holdouts - -with catch_warnings(): - simplefilter(action="ignore") - import keras - - -class JAXTaskRunner(TaskRunner): - """The base model for Keras models in the federation. - - Attributes: - model (keras.Model): The Keras model. - model_tensor_names (list): List of model tensor names. - required_tensorkeys_for_function (dict): A map of all of the required - tensors for each of the public functions in KerasTaskRunner. - """ - - def __init__(self, **kwargs): - """Initializes the KerasTaskRunner instance. - - Args: - **kwargs: Additional parameters to pass to the function - """ - super().__init__(**kwargs) - - self.model = keras.models.Model() - - self.model_tensor_names = [] - - # this is a map of all of the required tensors for each of the public - # functions in KerasTaskRunner - self.required_tensorkeys_for_function = {} - - def rebuild_model(self, round_num, input_tensor_dict, validation=False): - """Parse tensor names and update weights of model. Handles the - optimizer treatment. - - Args: - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. - validation (bool, optional): If True, validate the model. Defaults - to False. - """ - if self.opt_treatment == "RESET": - self.reset_opt_vars() - self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) - elif round_num > 0 and self.opt_treatment == "CONTINUE_GLOBAL" and not validation: - self.set_tensor_dict(input_tensor_dict, with_opt_vars=True) - else: - self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) - - def train_task( - self, - col_name, - round_num, - input_tensor_dict, - metrics, - epochs=1, - batch_size=1, - **kwargs, - ): - """ - Perform the training. Is expected to perform draws randomly, without - replacement until data is exausted. Then data is replaced and shuffled - and draws continue. - - Args: - col_name (str): The collaborator name. - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. - metrics (list): List of metrics. - epochs (int, optional): Number of epochs to train. Defaults to 1. - batch_size (int, optional): Batch size. Defaults to 1. - **kwargs: Additional parameters. - - Returns: - global_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - local_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - """ - if metrics is None: - raise KeyError("metrics must be defined") - - # rebuild model with updated weights - self.rebuild_model(round_num, input_tensor_dict) - for epoch in range(epochs): - self.logger.info("Run %s epoch of %s round", epoch, round_num) - results = self.train_( - self.data_loader.get_train_loader(batch_size), - metrics=metrics, - **kwargs, - ) - - # output metric tensors (scalar) - origin = col_name - tags = ("trained",) - output_metric_dict = { - TensorKey(metric_name, origin, round_num, True, ("metric",)): metric_value - for (metric_name, metric_value) in results - } - - # output model tensors (Doesn't include TensorKey) - output_model_dict = self.get_tensor_dict(with_opt_vars=True) - global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( - self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs - ) - - # create global tensorkeys - global_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num, False, tags): nparray - for tensor_name, nparray in global_model_dict.items() - } - # create tensorkeys that should stay local - local_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num, False, tags): nparray - for tensor_name, nparray in local_model_dict.items() - } - # the train/validate aggregated function of the next round will look - # for the updated model parameters. - # this ensures they will be resolved locally - next_local_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num + 1, False, ("model",)): nparray - for tensor_name, nparray in local_model_dict.items() - } - - global_tensor_dict = { - **output_metric_dict, - **global_tensorkey_model_dict, - } - local_tensor_dict = { - **local_tensorkey_model_dict, - **next_local_tensorkey_model_dict, - } - - # update the required tensors if they need to be pulled from the - # aggregator - # TODO this logic can break if different collaborators have different - # roles between rounds. - # for example, if a collaborator only performs validation in the first - # round but training in the second, it has no way of knowing the - # optimizer state tensor names to request from the aggregator because - # these are only created after training occurs. A work around could - # involve doing a single epoch of training on random data to get the - # optimizer names, and then throwing away the model. - if self.opt_treatment == "CONTINUE_GLOBAL": - self.initialize_tensorkeys_for_functions(with_opt_vars=True) - - self.update_tensorkeys_for_functions() - return global_tensor_dict, local_tensor_dict - - def train_(self, batch_generator, metrics: list = None, **kwargs): - """Train single epoch. Override this function for custom training. - - Args: - batch_generator (generator): Generator of training batches. - metrics (list, optional): Names of metrics to save. Defaults to - None. - **kwargs: Additional parameters. - - Returns: - results (list): List of Metric objects. - """ - if metrics is None: - metrics = [] - # TODO Currently assuming that all metrics are defined at - # initialization (build_model). - # If metrics are added (i.e. not a subset of what was originally - # defined) then the model must be recompiled. - try: - results = self.model.get_metrics_result() - except ValueError: - if "batch_size" in kwargs: - batch_size = kwargs["batch_size"] - else: - batch_size = 1 - # evaluation needed before metrics can be resolved - self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) - results = self.model.get_metrics_result() - - # TODO if there are new metrics in the flplan that were not included - # in the originally - # compiled model, that behavior is not currently handled. - for param in metrics: - if param not in results: - raise ValueError( - f"KerasTaskRunner does not support specifying new metrics. " - f"Param_metrics = {metrics}" - ) - - history = self.model.fit(batch_generator, verbose=2, **kwargs) - results = [] - for metric in metrics: - value = np.mean([history.history[metric]]) - results.append(Metric(name=metric, value=np.array(value))) - return results - - def validate_task(self, col_name, round_num, input_tensor_dict, **kwargs): - """Run the trained model on validation data; report results. - - Args: - col_name (str): The collaborator name. - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. Either the - last aggregated or locally trained model - **kwargs: Additional parameters. - - Returns: - output_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - These correspond to acc, precision, f1_score, etc. - dict: Empty dictionary. - """ - if "batch_size" in kwargs: - batch_size = kwargs["batch_size"] - else: - batch_size = 1 - - self.rebuild_model(round_num, input_tensor_dict, validation=True) - param_metrics = kwargs["metrics"] - - self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) - results = self.model.get_metrics_result() - - # TODO if there are new metrics in the flplan that were not included in - # the originally compiled model, that behavior is not currently - # handled. - for param in param_metrics: - if param not in results: - raise ValueError( - f"KerasTaskRunner does not support specifying new metrics. " - f"Param_metrics = {param_metrics}" - ) - - origin = col_name - suffix = "validate" - if kwargs["apply"] == "local": - suffix += "_local" - else: - suffix += "_agg" - tags = ("metric",) - tags = change_tags(tags, add_field=suffix) - output_tensor_dict = { - TensorKey(metric, origin, round_num, True, tags): np.array(results[metric]) - for metric in param_metrics - } - - return output_tensor_dict, {} - - def save_native(self, filepath): - """Save model. - - Args: - filepath (str): The file path to save the model. - """ - self.model.export(filepath) - - def load_native(self, filepath): - """Load model. - - Args: - filepath (str): The file path to load the model. - """ - self.model = keras.models.load_model(filepath) - - @staticmethod - def _get_weights_names(obj): - """Get the list of weight names. - - Args: - obj (Model or Optimizer): The target object that we want to get - the weights. - - Returns: - weight_names (list): The weight name list. - """ - if isinstance(obj, keras.optimizers.Optimizer): - weight_names = [weight.name for weight in obj.variables] - else: - weight_names = [ - layer.name + "/" + weight.name for layer in obj.layers for weight in layer.weights - ] - return weight_names - - @staticmethod - def _get_weights_dict(obj, suffix=""): - """ - Get the dictionary of weights. - - Args: - obj (Model or Optimizer): The target object that we want to get - the weights. - suffix (str, optional): Suffix for weight names. Defaults to ''. - - Returns: - weights_dict (dict): The weight dictionary. - """ - weights_dict = {} - weight_names = KerasTaskRunner._get_weights_names(obj) - if isinstance(obj, keras.optimizers.Optimizer): - weights_dict = { - weight_names[i] + suffix: weight.numpy() - for i, weight in enumerate(copy.deepcopy(obj.variables)) - } - else: - weight_name_index = 0 - for layer in obj.layers: - if weight_name_index < len(weight_names) and len(layer.get_weights()) > 0: - for weight in layer.get_weights(): - weights_dict[weight_names[weight_name_index] + suffix] = weight - weight_name_index += 1 - return weights_dict - - @staticmethod - def _set_weights_dict(obj, weights_dict): - """Set the object weights with a dictionary. - - Args: - obj (Model or Optimizer): The target object that we want to set - the weights. - weights_dict (dict): The weight dictionary. - """ - weight_names = KerasTaskRunner._get_weights_names(obj) - weight_values = [weights_dict[name] for name in weight_names] - obj.set_weights(weight_values) - - def get_tensor_dict(self, with_opt_vars, suffix=""): - """ - Get the model weights as a tensor dictionary. - - Args: - with_opt_vars (bool): If we should include the optimizer's status. - suffix (str): Universally. - - Returns: - model_weights (dict): The tensor dictionary. - """ - model_weights = self._get_weights_dict(self.model, suffix) - - if with_opt_vars: - opt_weights = self._get_weights_dict(self.model.optimizer, suffix) - - model_weights.update(opt_weights) - if len(opt_weights) == 0: - self.logger.debug("WARNING: We didn't find variables for the optimizer.") - return model_weights - - def set_tensor_dict(self, tensor_dict, with_opt_vars): - """Set the model weights with a tensor dictionary. - - Args: - tensor_dict (dict): The tensor dictionary. - with_opt_vars (bool): True = include the optimizer's status. - """ - if with_opt_vars is False: - # It is possible to pass in opt variables from the input tensor - # dict. This will make sure that the correct layers are updated - model_weight_names = self._get_weights_names(self.model) - model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} - self._set_weights_dict(self.model, model_weights_dict) - else: - model_weight_names = self._get_weights_names(self.model) - model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} - opt_weight_names = self._get_weights_names(self.model.optimizer) - opt_weights_dict = {name: tensor_dict[name] for name in opt_weight_names} - self._set_weights_dict(self.model, model_weights_dict) - self._set_weights_dict(self.model.optimizer, opt_weights_dict) - - def reset_opt_vars(self): - """Resets the optimizer variables.""" - pass - - def get_required_tensorkeys_for_function(self, func_name, **kwargs): - """Get the required tensors for specified function that could be called - as part of a task. - - By default, this is just all of the layers and optimizer of the model. - - Args: - func_name (str): The function name. - **kwargs: Any function arguments. - - Returns: - list: List of TensorKey objects. - """ - if func_name == "validate_task": - local_model = "apply=" + str(kwargs["apply"]) - return self.required_tensorkeys_for_function[func_name][local_model] - else: - return self.required_tensorkeys_for_function[func_name] - - def update_tensorkeys_for_functions(self): - """Update the required tensors for all publicly accessible methods that - could be called as part of a task. - - By default, this is just all of the layers and optimizer of the model. - Custom tensors should be added to this function - """ - # TODO complete this function. It is only needed for opt_treatment, - # and making the model stateless - - # Minimal required tensors for train function - model_layer_names = self._get_weights_names(self.model) - opt_names = self._get_weights_names(self.model.optimizer) - tensor_names = model_layer_names + opt_names - self.logger.debug("Updating model tensor names: %s", tensor_names) - self.required_tensorkeys_for_function["train_task"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names - ] - - # Validation may be performed on local or aggregated (global) model, - # so there is an extra lookup dimension for kwargs - self.required_tensorkeys_for_function["validate_task"] = {} - self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ - TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) for tensor_name in tensor_names - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names - ] - - def initialize_tensorkeys_for_functions(self, with_opt_vars=False): - """Set the required tensors for all publicly accessible methods that - could be called as part of a task. - - By default, this is just all of the layers and optimizer of the model. - Custom tensors should be added to this function - - Args: - with_opt_vars (bool, optional): If True, include the optimizer's - status. Defaults to False. - """ - # TODO there should be a way to programmatically iterate through all - # of the methods in the class and declare the tensors. - # For now this is done manually - - output_model_dict = self.get_tensor_dict(with_opt_vars=with_opt_vars) - global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( - self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs - ) - if not with_opt_vars: - global_model_dict_val = global_model_dict - local_model_dict_val = local_model_dict - else: - output_model_dict = self.get_tensor_dict(with_opt_vars=False) - global_model_dict_val, local_model_dict_val = split_tensor_dict_for_holdouts( - self.logger, - output_model_dict, - **self.tensor_dict_split_fn_kwargs, - ) - - self.required_tensorkeys_for_function["train_task"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) - for tensor_name in global_model_dict - ] - self.required_tensorkeys_for_function["train_task"] += [ - TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) - for tensor_name in local_model_dict - ] - - # Validation may be performed on local or aggregated (global) model, - # so there is an extra lookup dimension for kwargs - self.required_tensorkeys_for_function["validate_task"] = {} - # TODO This is not stateless. The optimizer will not be - self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ - TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) - for tensor_name in {**global_model_dict_val, **local_model_dict_val} - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) - for tensor_name in global_model_dict_val - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] += [ - TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) - for tensor_name in local_model_dict_val - ] From 9faa4d33616a822ee1a58e0a8a3070d6d831a69a Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:24:18 -0800 Subject: [PATCH 06/15] code changes Signed-off-by: yes --- Jenkinsfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c1c9a374fb..088b3199f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ def snykData = [ 'openfl-docker': 'openfl-docker/Dockerfile.base', 'openfl': 'setup.py', - 'openfl-workspace_tf_2dunet': 'openfl-workspace/tf_2dunet/requirements.txt', + 'openfl-workspace_keras_2dunet': 'openfl-workspace/keras/2dunet/requirements.txt', 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch/mnist_straggler_check/requirements.txt', // CN-14619 snyk test CLI does not support -f in requirements.txt file // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch/histology/requirements.txt', @@ -9,9 +9,6 @@ def snykData = [ 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', - 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', - 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', - 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt', From a52e2144158d7cc8c75398c9a6fe12c9b1f318d5 Mon Sep 17 00:00:00 2001 From: refai06 <149057514+refai06@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:13:20 +0530 Subject: [PATCH 07/15] Harmonize Envoy CLI and Configuration Files with Director Component for Consistency (#1284) * Clean envoy CLI & config Signed-off-by: refai06 * Cleanup federatedruntime testcase-config Signed-off-by: refai06 * Update files Signed-off-by: refai06 * Envoy execption-handling Signed-off-by: refai06 * Update error messages Signed-off-by: refai06 * Documentation update for recent change Signed-off-by: refai06 * Fix code format Signed-off-by: refai06 * Document Update Signed-off-by: refai06 --------- Signed-off-by: refai06 --- .../features_index/workflowinterface.rst | 20 ++++--- .../101_MNIST/Portland/Portland_config.yaml | 4 ++ .../101_MNIST/Portland/start_envoy.sh | 2 +- .../101_MNIST/Seattle/Seattle_config.yaml | 4 ++ .../101_MNIST/Seattle/start_envoy.sh | 2 +- .../Bangalore/Bangalore_config.yaml | 4 ++ .../Bangalore/start_envoy.sh | 2 +- .../Chandler/Chandler_config.yaml | 4 ++ .../Chandler/start_envoy.sh | 2 +- .../workflow/interface/cli/envoy.py | 59 +++++++++++-------- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 +- .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 3 + .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 3 + .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 3 + .../envoy_one/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 +- .../envoy_two/start_envoy.sh | 2 +- .../envoy_four/envoy_config.yaml | 4 ++ .../envoy_four/start_envoy.sh | 2 +- .../envoy_one/envoy_config.yaml | 4 ++ .../envoy_one/start_envoy.sh | 2 +- .../envoy_three/envoy_config.yaml | 4 ++ .../envoy_three/start_envoy.sh | 2 +- .../envoy_two/envoy_config.yaml | 4 ++ .../envoy_two/start_envoy.sh | 2 +- 50 files changed, 161 insertions(+), 59 deletions(-) diff --git a/docs/about/features_index/workflowinterface.rst b/docs/about/features_index/workflowinterface.rst index 6dc9527ba1..8ef440f4ce 100644 --- a/docs/about/features_index/workflowinterface.rst +++ b/docs/about/features_index/workflowinterface.rst @@ -342,37 +342,43 @@ An example configuration file `director_config.yaml` is shown below: **Envoy: Participating nodes in the Federation** -The `fx envoy start` command is used to start the Envoy. You can run it with or without TLS, depending on your setup. +The :code:`fx envoy start` command is used to start the Envoy. You can run it with or without TLS, depending on your setup. **With TLS:** Use the following command: .. code-block:: shell - $ fx envoy start -n -ec -dh -dp -rc -pk -oc + $ fx envoy start -n -c -rc -pk -oc **Without TLS:** Use the following command: .. code-block:: shell - $ fx envoy start -n --disable-tls -ec + $ fx envoy start -n --disable-tls -c **Explanation of Command Options** - `-n `: Specifies the name of the Envoy. -- `-ec `: Path to the Envoy's configuration file. -- `-dh `: Hostname or IP address of the Director. -- `-dp `: Port on which the Director is running. +- `-c `: Path to the Envoy's configuration file. - `-rc `: Path to the root certificate (used with TLS). - `-pk `: Path to the private key file (used with TLS). - `-oc `: Path to the API certificate file (used with TLS). - `--disable-tls`: Disables TLS encryption. -The Envoy configuration file includes details about the private attributes. An example configuration file :code:`envoy_config.yaml` for :code:`envoy_one` is shown below: +The Envoy configuration file includes details of director_host, director_port and private attributes. An example configuration file :code:`envoy_config.yaml` for :code:`settings` and :code:`envoy_one` is shown below: + +- Hostname (`director_host`) +- Port (`director_port`) +- Private attributes for envoy_one .. code-block:: yaml + settings: + director_host: localhost + director_port: 50050 + envoy_one: private_attributes: private_attributes.envoy_one_attrs diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/Portland_config.yaml b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/Portland_config.yaml index 318e22d71d..a37e5010bd 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/Portland_config.yaml +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/Portland_config.yaml @@ -1,2 +1,6 @@ +settings: + director_host: localhost + director_port: 50050 + Portland: private_attributes: private_attributes.portland_attrs \ No newline at end of file diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/start_envoy.sh b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/start_envoy.sh +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Portland/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/Seattle_config.yaml b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/Seattle_config.yaml index b95c8242ef..07a819115c 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/Seattle_config.yaml +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/Seattle_config.yaml @@ -1,2 +1,6 @@ +settings: + director_host: localhost + director_port: 50050 + Seattle: private_attributes: private_attributes.seattle_attrs \ No newline at end of file diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/start_envoy.sh b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/start_envoy.sh +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/101_MNIST/Seattle/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/Bangalore_config.yaml b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/Bangalore_config.yaml index ec4a088af7..51366acea8 100644 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/Bangalore_config.yaml +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/Bangalore_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + Bangalore: callable_func: settings: diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/start_envoy.sh b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/start_envoy.sh +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/Chandler_config.yaml b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/Chandler_config.yaml index 60f763ee55..ac0cffda09 100644 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/Chandler_config.yaml +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/Chandler_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + Chandler: callable_func: settings: diff --git a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/start_envoy.sh b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/start_envoy.sh +++ b/openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/openfl/experimental/workflow/interface/cli/envoy.py b/openfl/experimental/workflow/interface/cli/envoy.py index dbb5d57ce9..1c16ec9990 100644 --- a/openfl/experimental/workflow/interface/cli/envoy.py +++ b/openfl/experimental/workflow/interface/cli/envoy.py @@ -12,7 +12,7 @@ from dynaconf import Validator from openfl.experimental.workflow.component.envoy import Envoy -from openfl.utilities import click_types, merge_configs +from openfl.utilities import is_fqdn, merge_configs from openfl.utilities.path_check import is_directory_traversal logger = logging.getLogger(__name__) @@ -32,18 +32,11 @@ def envoy(context): @envoy.command(name="start") @option("-n", "--envoy_name", required=True, help="Current shard name") @option( - "-dh", - "--director-host", - required=True, - help="The FQDN of the federation director", - type=click_types.FQDN, -) -@option( - "-dp", - "--director-port", - required=True, - help="The federation director port", - type=click.IntRange(1, 65535), + "-c", + "--envoy-config-path", + default="envoy_config.yaml", + help="The envoy config path", + type=ClickPath(exists=True), ) @option( "--tls/--disable-tls", @@ -51,13 +44,6 @@ def envoy(context): is_flag=True, help="Use TLS or not (By default TLS is enabled)", ) -@option( - "-ec", - "--envoy-config-path", - default="envoy_config.yaml", - help="The envoy config path", - type=ClickPath(exists=True), -) @option( "-rc", "--root-cert-path", @@ -84,8 +70,6 @@ def envoy(context): ) def start_( envoy_name, - director_host, - director_port, tls, envoy_config_path, root_certificate, @@ -96,8 +80,6 @@ def start_( Args: envoy_name (str): Name of the Envoy. - director_host (str): The FQDN of the federation director. - director_port (int): The federation director port. tls (bool): Use TLS or not. envoy_config_path (str): The envoy config path. root_certificate (str): Path to a root CA cert. @@ -118,6 +100,31 @@ def start_( "certificate": certificate, }, validators=[ + Validator( + "settings", + must_exist=True, + messages={"must_exist_true": "Missing 'settings' in Envoy Configuration file."}, + ), + Validator( + "settings.director_host", + must_exist=True, + condition=lambda x: bool(x) and is_fqdn(x), + messages={ + "must_exist_true": "Missing 'director_host' in Envoy Configuration file", + "condition": "Invalid 'director_host' in Envoy Configuration file." + "Must be a valid FQDN", + }, + ), + Validator( + "settings.director_port", + must_exist=True, + condition=lambda value: isinstance(value, int) and 1024 <= value <= 65535, + messages={ + "must_exist_true": "Missing 'director_port' in Envoy Configuration file", + "condition": "Invalid 'director_port' in Envoy Configuration file" + "Must be an integer between 1024 & 65535", + }, + ), Validator("params.install_requirements", default=True), ], ) @@ -138,8 +145,8 @@ def start_( envoy = Envoy( envoy_name=envoy_name, - director_host=director_host, - director_port=director_port, + director_host=config.settings.director_host, + director_port=config.settings.director_port, envoy_config=Path(envoy_config_path).absolute(), root_certificate=config.root_certificate, private_key=config.private_key, diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/envoy_config.yaml index f6f96d3d6b..a7ec1ff3e5 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/envoy_config.yaml index eaf34da1df..53af1eff87 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_datastore_cli/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/envoy_config.yaml index 3d8b4e2882..d52281fda0 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/envoy_config.yaml @@ -1,2 +1,4 @@ - +settings: + director_host: localhost + director_port: 50050 \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/start_envoy.sh index bfc8af2959..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/envoy_config.yaml index e69de29bb2..d52281fda0 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/envoy_config.yaml @@ -0,0 +1,4 @@ +settings: + director_host: localhost + director_port: 50050 + \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/start_envoy.sh index bfc8af2959..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_include_exclude/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/envoy_config.yaml index e69de29bb2..6d54e2b64a 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/envoy_config.yaml @@ -0,0 +1,3 @@ +settings: + director_host: localhost + director_port: 50050 \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/envoy_config.yaml index e69de29bb2..6d54e2b64a 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/envoy_config.yaml @@ -0,0 +1,3 @@ +settings: + director_host: localhost + director_port: 50050 \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_internalloop/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/envoy_config.yaml index 73a1ce09f4..2db6f82dee 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/envoy_config.yaml index ec859a79cd..060b7dbd37 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/envoy_config.yaml index afce54bcd2..4d4591cef9 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/envoy_config.yaml index 51c4e7be6e..629fa0b14a 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_with_both_options/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/envoy_config.yaml index a4a2ecb2d8..5ab81a1b49 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/envoy_config.yaml @@ -1,2 +1,6 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: private_attributes: private_attributes.collaborator_private_attrs.collaborator_private_attributes diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/envoy_config.yaml index 9bce3671b7..3827c74155 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/envoy_config.yaml @@ -1,2 +1,6 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: private_attributes: private_attributes.collaborator_private_attrs.collaborator_private_attributes \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_private_attributes_initialization_without_callable/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/envoy_config.yaml index bb76d4845e..8370aa8985 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/envoy_config.yaml index 2c44953a5e..ca7e8b60ab 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/envoy_config.yaml index e69de29bb2..6d54e2b64a 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/envoy_config.yaml @@ -0,0 +1,3 @@ +settings: + director_host: localhost + director_port: 50050 \ No newline at end of file diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/envoy_config.yaml index 8b13789179..9aee5ed92b 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/envoy_config.yaml @@ -1 +1,3 @@ - +settings: + director_host: localhost + director_port: 50050 diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_reference_with_include_exclude/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/envoy_config.yaml index d5c49635f3..ad23030ec3 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_four: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_four/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/envoy_config.yaml index 04e7ad4fd8..1eb2d73be3 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_one: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_one/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/envoy_config.yaml index f78af08e58..9ee2ad5aa1 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_three: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_three/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/envoy_config.yaml b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/envoy_config.yaml index 6149ff9f1d..4b2a040a1f 100644 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/envoy_config.yaml +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/envoy_config.yaml @@ -1,3 +1,7 @@ +settings: + director_host: localhost + director_port: 50050 + envoy_two: callable_func: settings: diff --git a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/start_envoy.sh b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/start_envoy.sh index 4da07821af..5c2d296cda 100755 --- a/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/start_envoy.sh +++ b/tests/github/experimental/workflow/FederatedRuntime/testcase_subset_of_collaborators/envoy_two/start_envoy.sh @@ -3,4 +3,4 @@ set -e ENVOY_NAME=$1 ENVOY_CONF=$2 -fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$ENVOY_CONF" -dh localhost -dp 50050 +fx envoy start -n "$ENVOY_NAME" --disable-tls -c "$ENVOY_CONF" From 7f90f42ca16ae4636d560d131abf76ee0fc0bb2e Mon Sep 17 00:00:00 2001 From: Yehonatan Buchnik Date: Tue, 28 Jan 2025 12:51:55 +0200 Subject: [PATCH 08/15] A callback for re-fetching the root ca in the aggregator (#1315) * dynamically let the aggregator learn the clients certificates on every TLS handshake Signed-off-by: Buchnik, Yehonatan * formatting Signed-off-by: Buchnik, Yehonatan * formatting Signed-off-by: Buchnik, Yehonatan --------- Signed-off-by: Buchnik, Yehonatan --- openfl/transport/grpc/aggregator_server.py | 31 +++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/openfl/transport/grpc/aggregator_server.py b/openfl/transport/grpc/aggregator_server.py index 12f658a0aa..3aa7977ebb 100644 --- a/openfl/transport/grpc/aggregator_server.py +++ b/openfl/transport/grpc/aggregator_server.py @@ -10,7 +10,12 @@ from random import random from time import sleep -from grpc import StatusCode, server, ssl_server_credentials +from grpc import ( + StatusCode, + dynamic_ssl_server_credentials, + server, + ssl_server_certificate_configuration, +) from openfl.protocols import aggregator_pb2, aggregator_pb2_grpc, utils from openfl.transport.grpc.grpc_channel_options import channel_options @@ -50,6 +55,7 @@ def __init__( root_certificate=None, certificate=None, private_key=None, + root_certificate_refresher_cb=None, **kwargs, ): """ @@ -68,6 +74,8 @@ def __init__( TLS connection. private_key (str): The path to the server's private key for the TLS connection. + root_certificate_refresher_cb (Callable): A callback function + that receive no arguments and return the current root certificate. **kwargs: Additional keyword arguments. """ self.aggregator = aggregator @@ -81,6 +89,7 @@ def __init__( self.server_credentials = None self.logger = logging.getLogger(__name__) + self.root_certificate_refresher_cb = root_certificate_refresher_cb def validate_collaborator(self, request, context): """Validate the collaborator. @@ -325,13 +334,23 @@ def get_server(self): if not self.require_client_auth: self.logger.warning("Client-side authentication is disabled.") - - self.server_credentials = ssl_server_credentials( - ((private_key_b, certificate_b),), - root_certificates=root_certificate_b, - require_client_auth=self.require_client_auth, + cert_config = ssl_server_certificate_configuration( + ((private_key_b, certificate_b),), root_certificates=root_certificate_b ) + def certificate_configuration_fetcher(): + root_cert = root_certificate_b + if self.root_certificate_refresher_cb is not None: + root_cert = self.root_certificate_refresher_cb() + return ssl_server_certificate_configuration( + ((private_key_b, certificate_b),), root_certificates=root_cert + ) + + self.server_credentials = dynamic_ssl_server_credentials( + cert_config, + certificate_configuration_fetcher, + require_client_authentication=self.require_client_auth, + ) self.server.add_secure_port(self.uri, self.server_credentials) return self.server From d9642e851f55cf09c0d9b96cc6cf1a6714611f0c Mon Sep 17 00:00:00 2001 From: Tarunkumar Banda Date: Tue, 28 Jan 2025 21:44:59 +0530 Subject: [PATCH 09/15] updated requirements file and readme of Privacy Meter workflow (#1295) Signed-off-by: Tarunkumar Banda --- .../workflow/Privacy_Meter/readme.md | 35 ++++++++++++++++--- .../requirements_privacy_meter.txt | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/openfl-tutorials/experimental/workflow/Privacy_Meter/readme.md b/openfl-tutorials/experimental/workflow/Privacy_Meter/readme.md index f62e8662ea..33148914f4 100644 --- a/openfl-tutorials/experimental/workflow/Privacy_Meter/readme.md +++ b/openfl-tutorials/experimental/workflow/Privacy_Meter/readme.md @@ -29,11 +29,32 @@ To measure the success of the attack (privacy loss), we generate the ROC of the ## Examples Here, we give a few commands and the results for each of them. +## Running the cifar10_PM script +The script requires a dedicated allocation of atleast 18GB of RAM to run without issues. +1) Create a Python virtual environment for better isolation +```shell +python -m venv venv +source venv/bin/activate +``` +2) Install OpenFL from the latest sources +```shell +git clone https://github.com/securefederatedai/openfl.git && cd openfl +pip install -e . +``` +3) Install the requirements for Privacy Meter Workflow API +```shell +cd openfl-tutorials/experimental/workflow/ +pip install -r workflow_interface_requirements.txt +cd Privacy_Meter/ +pip install -r requirements_privacy_meter.txt +``` ### Auditing the privacy loss based on the model loss, logits, and gradient norm (the 10th layer of the representation), where the model is trained using SGD. - -**Command:** -> `python cifar10_PM.py --audit_dataset_ratio 0.2 --test_dataset_ratio 0.4 --train_dataset_ratio 0.4 --signals loss logits gradient_norm --fpr_tolerance 0.1 0.2 0.3 --log_dir test_sgd --comm_round 30 --optimizer_type SGD --is_feature True --layer_number 10` +4) Start the training script with SGB optimizer
+Note that the number of training rounds can be adjusted via the `--comm_round` parameter: +```shell +python cifar10_PM.py --audit_dataset_ratio 0.2 --test_dataset_ratio 0.4 --train_dataset_ratio 0.4 --signals loss logits gradient_norm --fpr_tolerance 0.1 0.2 0.3 --log_dir test_sgd --comm_round 30 --optimizer_type SGD --is_feature True --layer_number 10 +``` **Results:** The performance of the target model is as follows: @@ -60,9 +81,13 @@ Portland: ### Auditing the privacy loss based on the model loss, logits, and gradient norm (the 10th layer of the representation), where the model is trained using Adam. -**Command:** -> `python cifar10_PM.py --audit_dataset_ratio 0.2 --test_dataset_ratio 0.4 --train_dataset_ratio 0.4 --signals loss logits gradient_norm --fpr_tolerance 0.1 0.2 0.3 --log_dir test_adam --comm_round 30 --optimizer_type Adam --is_feature True --layer_number 10` +4) Start the training script with Adam optimizer
+Note that the number of training rounds can be adjusted via the `--comm_round` parameter: +```shell +python cifar10_PM.py --audit_dataset_ratio 0.2 --test_dataset_ratio 0.4 --train_dataset_ratio 0.4 --signals loss logits gradient_norm --fpr_tolerance 0.1 0.2 0.3 --log_dir test_adam --comm_round 30 --optimizer_type Adam --is_feature True --layer_number 10 +``` +**Results:** The performance of the target model is as follows: ``` Average aggregated model validation values = 0.6075416505336761 diff --git a/openfl-tutorials/experimental/workflow/Privacy_Meter/requirements_privacy_meter.txt b/openfl-tutorials/experimental/workflow/Privacy_Meter/requirements_privacy_meter.txt index ce4435ea21..c617e3a732 100644 --- a/openfl-tutorials/experimental/workflow/Privacy_Meter/requirements_privacy_meter.txt +++ b/openfl-tutorials/experimental/workflow/Privacy_Meter/requirements_privacy_meter.txt @@ -5,4 +5,4 @@ opacus==1.5.2 pillow scikit-learn torch==2.3.1 -torchvision==0.18.1 +torchvision==0.18.1 \ No newline at end of file From 0aff46d893fa986c91e02d98e34c587fdeeb2416 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:28:56 -0800 Subject: [PATCH 10/15] code changes Signed-off-by: yes --- .github/workflows/task_runner_basic_e2e.yml | 10 +++++----- .../workflows/task_runner_dockerized_ws_e2e.yml | 8 ++++---- .github/workflows/tr_docker_gramine_direct.yml | 2 +- .github/workflows/tr_docker_native.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- Jenkinsfile | 4 ++-- docs/about/features_index/taskrunner.rst | 8 ++++---- docs/releases.md | 2 +- docs/tutorials/taskrunner.ipynb | 2 +- openfl/interface/workspace.py | 14 +++++++++----- openfl/native/native.py | 2 +- tests/end_to_end/README.md | 4 ++-- tests/end_to_end/utils/constants.py | 2 +- tests/github/test_double_ws_export.py | 2 +- tests/github/test_gandlf.py | 2 +- tests/github/test_hello_federation.py | 13 ++++++++----- 17 files changed, 44 insertions(+), 37 deletions(-) diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index 5b64de09df..caeaf23088 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -26,7 +26,7 @@ on: options: - all - torch_cnn_mnist - - keras_cnn_mnist + - keras/cnn_mnist python_version: description: "Python version" required: false @@ -88,17 +88,17 @@ jobs: # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. # Default combination if no input is provided (i.e. 'all' is selected). - # * TLS - models [torch_cnn_mnist, keras_cnn_mnist] and python versions [3.10, 3.11, 3.12] + # * TLS - models [torch_cnn_mnist, keras/cnn_mnist] and python versions [3.10, 3.11, 3.12] # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] - # * No client auth - models [keras_cnn_mnist] and python version [3.10] + # * No client auth - models [keras/cnn_mnist] and python version [3.10] # * Memory logs - models [torch_cnn_mnist] and python version [3.10] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" if [ "${{ env.MODEL_NAME }}" == "all" ]; then - echo "models_for_tls=[\"torch_cnn_mnist\", \"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"torch_cnn_mnist\", \"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"keras_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" else echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index ce677c5789..896b87400d 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10", "3.11", "3.12"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -73,7 +73,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -114,7 +114,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -155,7 +155,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras_cnn_mnist"] + model_name: ["keras/cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 169189a42d..9ecbe90819 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras_cnn_mnist + fx workspace create --prefix example_workspace --template keras/cnn_mnist cd example_workspace fx plan initialize -a localhost diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index 9fcb9b8759..b80e5ca161 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras_cnn_mnist + fx workspace create --prefix example_workspace --template keras/cnn_mnist cd example_workspace fx plan initialize -a localhost fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 617c004751..ebaa206867 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -53,4 +53,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model + python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5f3ffa1220..5bf4fb2bf4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,4 +52,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 73f919c844..fd7db70d21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,13 +6,13 @@ def snykData = [ // CN-14619 snyk test CLI does not support -f in requirements.txt file // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch_cnn_histology/requirements.txt', 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch_cnn_histology/src/requirements.txt', - 'openfl-workspace_keras_nlp': 'openfl-workspace/keras_nlp/requirements.txt', + 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch_cnn_mnist/requirements.txt', 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch_unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', - 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras_cnn_mnist/requirements.txt', + 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/cnn_mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_histology_envoy': 'openfl-tutorials/interactive_api/PyTorch_Histology/envoy/requirements.txt', diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index f8730f463b..255a63d904 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -150,18 +150,18 @@ STEP 1: Create a Workspace $ fx -2. This example uses the :code:`keras_cnn_mnist` template. +2. This example uses the :code:`keras/cnn_mnist` template. - Set the environment variables to use the :code:`keras_cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. + Set the environment variables to use the :code:`keras/cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. .. code-block:: shell - $ export WORKSPACE_TEMPLATE=keras_cnn_mnist + $ export WORKSPACE_TEMPLATE=keras/cnn_mnist $ export WORKSPACE_PATH=${HOME}/my_federation 3. Decide a workspace template, which are end-to-end federated learning training demonstrations. The following is a sample of available templates: - - :code:`keras_cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`keras/cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. - :code:`tf_2dunet`: a workspace with a simple `TensorFlow `__ CNN model that will use the `BraTS `_ dataset and train in a federation. - :code:`tf_cnn_histology`: a workspace with a simple `TensorFlow `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - :code:`torch_cnn_histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. diff --git a/docs/releases.md b/docs/releases.md index 30ed029ea6..539336ceb1 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -133,7 +133,7 @@ The OpenFL v1.2 release contains the following updates: - New [Interactive Python API](https://github.com/securefederatedai/openfl/tree/main/openfl-tutorials/deprecated/interactive_api) (experimental) - Example FedProx algorithm implementation for PyTorch and Tensorflow - `AggregationFunctionInterface` for custom aggregation functions -- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_nlp) +- Adds a [Keras-based NLP Example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras/nlp) - Fixed lossy compression pipelines and added an [example](https://github.com/intel/openfl/tree/develop/openfl-workspace/keras_cnn_with_compression) for usage - Bug fixes and documentation improvements diff --git a/docs/tutorials/taskrunner.ipynb b/docs/tutorials/taskrunner.ipynb index d19fcdc6d0..a95236e17b 100644 --- a/docs/tutorials/taskrunner.ipynb +++ b/docs/tutorials/taskrunner.ipynb @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fx workspace create --prefix ./mnist_example --template keras_cnn_mnist\n", + "!fx workspace create --prefix ./mnist_example --template keras/cnn_mnist\n", "%cd ./mnist_example" ] }, diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 522ff99b5f..8881ec9d8f 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -97,11 +97,15 @@ def get_templates(): list: A list of default templates. """ - return [ - d.name - for d in WORKSPACE.glob("*") - if d.is_dir() and d.name not in ["__pycache__", "workspace", "experimental"] - ] + templates = [] + excluded_dirs = ["workspace", "experimental"] + for root, _, files in os.walk(WORKSPACE): + if any(file.endswith(".workspace") for file in files): + dir_path = os.path.relpath(root, WORKSPACE) + dir_path = dir_path.replace(os.sep, "/") + if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): + templates.append(dir_path) + return templates @workspace.command(name="create") diff --git a/openfl/native/native.py b/openfl/native/native.py index 117058abbb..c1b3813f92 100644 --- a/openfl/native/native.py +++ b/openfl/native/native.py @@ -216,7 +216,7 @@ def init( workspace_template (str): The template that should be used as the basis for the experiment. Defaults to 'default'. Other options include are any of the template names - [keras_cnn_mnist, tf_2dunet, tf_cnn_histology, + [keras/cnn_mnist, tf_2dunet, tf_cnn_histology, mtorch_cnn_histology, torch_cnn_mnist]. log_level (str): Log level for logging. METRIC level is available. Defaults to 'INFO'. diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 191cfd0db4..0495629c1e 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -55,10 +55,10 @@ For example, to run Task runner (bare metal approach) with - torch_cnn_mnist mod python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls ``` -And, to run Task runner (via dockerized workspace) with keras_cnn_mnist, 2 collaborators, 3 rounds: +And, to run Task runner (via dockerized workspace) with keras/cnn_mnist, 2 collaborators, 3 rounds: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras_cnn_mnist +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/cnn_mnist ``` ### Fixture and marker mapping: diff --git a/tests/end_to_end/utils/constants.py b/tests/end_to_end/utils/constants.py index a9f66e47ef..971174c381 100644 --- a/tests/end_to_end/utils/constants.py +++ b/tests/end_to_end/utils/constants.py @@ -11,7 +11,7 @@ class ModelName(Enum): # IMP - The model name must be same (and in uppercase) as the model value. # This is used to identify the model in the tests. TORCH_CNN_MNIST = "torch_cnn_mnist" - KERAS_CNN_MNIST = "keras_cnn_mnist" + KERAS_CNN_MNIST = "keras/cnn_mnist" TORCH_CNN_HISTOLOGY = "torch_cnn_histology" XGB_HIGGS = "xgb_higgs" diff --git a/tests/github/test_double_ws_export.py b/tests/github/test_double_ws_export.py index 95c9440b31..7e9b42bec3 100644 --- a/tests/github/test_double_ws_export.py +++ b/tests/github/test_double_ws_export.py @@ -22,7 +22,7 @@ def main(): for entry in iterator: if entry.name not in ['__init__.py', 'workspace', 'default']: workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') diff --git a/tests/github/test_gandlf.py b/tests/github/test_gandlf.py index a57f9f53a0..08e80e2118 100644 --- a/tests/github/test_gandlf.py +++ b/tests/github/test_gandlf.py @@ -21,7 +21,7 @@ def exec(command, directory): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--template', default='keras_cnn_mnist') + parser.add_argument('--template', default='keras/cnn_mnist') parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one') parser.add_argument('--col2', default='two') diff --git a/tests/github/test_hello_federation.py b/tests/github/test_hello_federation.py index e6b84b8de2..a3ef7296a8 100644 --- a/tests/github/test_hello_federation.py +++ b/tests/github/test_hello_federation.py @@ -17,11 +17,14 @@ def main(): # Test the pipeline parser = argparse.ArgumentParser() workspace_choice = [] - with os.scandir('openfl-workspace') as iterator: - for entry in iterator: - if entry.name not in ['__init__.py', 'workspace', 'default']: - workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras_cnn_mnist', choices=workspace_choice) + excluded_dirs = ['workspace', 'default', "experimental"] + for root, _, files in os.walk('openfl-workspace'): + if any(file.endswith(".workspace") for file in files): + dir_path = os.path.relpath(root, 'openfl-workspace') + dir_path = dir_path.replace(os.sep, '/') + if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): + workspace_choice.append(dir_path) + parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') From aa826f2d4830c31c8485c73f8b19254946a5141f Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:32:13 -0800 Subject: [PATCH 11/15] code changes Signed-off-by: yes --- openfl-workspace/{keras_2dunet => keras/2dunet}/.workspace | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/README.md | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/cols.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/data.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/defaults | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/plan/plan.yaml | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/requirements.txt | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/__init__.py | 0 .../{keras_2dunet => keras/2dunet}/src/brats_utils.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/dataloader.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/nii_reader.py | 0 openfl-workspace/{keras_2dunet => keras/2dunet}/src/taskrunner.py | 0 openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/.workspace | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/cols.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/data.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/defaults | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/plan/plan.yaml | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/requirements.txt | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/__init__.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/dataloader.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/mnist_utils.py | 0 .../{keras_cnn_mnist => keras/cnn_mnist}/src/taskrunner.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/.workspace | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/cols.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/data.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/plan/plan.yaml | 0 openfl-workspace/{keras_nlp => keras/nlp}/requirements.txt | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/__init__.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader_utils.py | 0 openfl-workspace/{keras_nlp => keras/nlp}/src/taskrunner.py | 0 31 files changed, 0 insertions(+), 0 deletions(-) rename openfl-workspace/{keras_2dunet => keras/2dunet}/.workspace (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/README.md (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/cols.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/data.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/defaults (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/plan/plan.yaml (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/requirements.txt (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/__init__.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/brats_utils.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/dataloader.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/nii_reader.py (100%) rename openfl-workspace/{keras_2dunet => keras/2dunet}/src/taskrunner.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/.workspace (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/cols.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/data.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/defaults (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/plan/plan.yaml (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/requirements.txt (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/__init__.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/dataloader.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/mnist_utils.py (100%) rename openfl-workspace/{keras_cnn_mnist => keras/cnn_mnist}/src/taskrunner.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/.workspace (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/cols.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/data.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/plan/plan.yaml (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/requirements.txt (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/__init__.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/dataloader_utils.py (100%) rename openfl-workspace/{keras_nlp => keras/nlp}/src/taskrunner.py (100%) diff --git a/openfl-workspace/keras_2dunet/.workspace b/openfl-workspace/keras/2dunet/.workspace similarity index 100% rename from openfl-workspace/keras_2dunet/.workspace rename to openfl-workspace/keras/2dunet/.workspace diff --git a/openfl-workspace/keras_2dunet/README.md b/openfl-workspace/keras/2dunet/README.md similarity index 100% rename from openfl-workspace/keras_2dunet/README.md rename to openfl-workspace/keras/2dunet/README.md diff --git a/openfl-workspace/keras_2dunet/plan/cols.yaml b/openfl-workspace/keras/2dunet/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/cols.yaml rename to openfl-workspace/keras/2dunet/plan/cols.yaml diff --git a/openfl-workspace/keras_2dunet/plan/data.yaml b/openfl-workspace/keras/2dunet/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/data.yaml rename to openfl-workspace/keras/2dunet/plan/data.yaml diff --git a/openfl-workspace/keras_2dunet/plan/defaults b/openfl-workspace/keras/2dunet/plan/defaults similarity index 100% rename from openfl-workspace/keras_2dunet/plan/defaults rename to openfl-workspace/keras/2dunet/plan/defaults diff --git a/openfl-workspace/keras_2dunet/plan/plan.yaml b/openfl-workspace/keras/2dunet/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_2dunet/plan/plan.yaml rename to openfl-workspace/keras/2dunet/plan/plan.yaml diff --git a/openfl-workspace/keras_2dunet/requirements.txt b/openfl-workspace/keras/2dunet/requirements.txt similarity index 100% rename from openfl-workspace/keras_2dunet/requirements.txt rename to openfl-workspace/keras/2dunet/requirements.txt diff --git a/openfl-workspace/keras_2dunet/src/__init__.py b/openfl-workspace/keras/2dunet/src/__init__.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/__init__.py rename to openfl-workspace/keras/2dunet/src/__init__.py diff --git a/openfl-workspace/keras_2dunet/src/brats_utils.py b/openfl-workspace/keras/2dunet/src/brats_utils.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/brats_utils.py rename to openfl-workspace/keras/2dunet/src/brats_utils.py diff --git a/openfl-workspace/keras_2dunet/src/dataloader.py b/openfl-workspace/keras/2dunet/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/dataloader.py rename to openfl-workspace/keras/2dunet/src/dataloader.py diff --git a/openfl-workspace/keras_2dunet/src/nii_reader.py b/openfl-workspace/keras/2dunet/src/nii_reader.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/nii_reader.py rename to openfl-workspace/keras/2dunet/src/nii_reader.py diff --git a/openfl-workspace/keras_2dunet/src/taskrunner.py b/openfl-workspace/keras/2dunet/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_2dunet/src/taskrunner.py rename to openfl-workspace/keras/2dunet/src/taskrunner.py diff --git a/openfl-workspace/keras_cnn_mnist/.workspace b/openfl-workspace/keras/cnn_mnist/.workspace similarity index 100% rename from openfl-workspace/keras_cnn_mnist/.workspace rename to openfl-workspace/keras/cnn_mnist/.workspace diff --git a/openfl-workspace/keras_cnn_mnist/plan/cols.yaml b/openfl-workspace/keras/cnn_mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/cols.yaml rename to openfl-workspace/keras/cnn_mnist/plan/cols.yaml diff --git a/openfl-workspace/keras_cnn_mnist/plan/data.yaml b/openfl-workspace/keras/cnn_mnist/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/data.yaml rename to openfl-workspace/keras/cnn_mnist/plan/data.yaml diff --git a/openfl-workspace/keras_cnn_mnist/plan/defaults b/openfl-workspace/keras/cnn_mnist/plan/defaults similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/defaults rename to openfl-workspace/keras/cnn_mnist/plan/defaults diff --git a/openfl-workspace/keras_cnn_mnist/plan/plan.yaml b/openfl-workspace/keras/cnn_mnist/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_cnn_mnist/plan/plan.yaml rename to openfl-workspace/keras/cnn_mnist/plan/plan.yaml diff --git a/openfl-workspace/keras_cnn_mnist/requirements.txt b/openfl-workspace/keras/cnn_mnist/requirements.txt similarity index 100% rename from openfl-workspace/keras_cnn_mnist/requirements.txt rename to openfl-workspace/keras/cnn_mnist/requirements.txt diff --git a/openfl-workspace/keras_cnn_mnist/src/__init__.py b/openfl-workspace/keras/cnn_mnist/src/__init__.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/__init__.py rename to openfl-workspace/keras/cnn_mnist/src/__init__.py diff --git a/openfl-workspace/keras_cnn_mnist/src/dataloader.py b/openfl-workspace/keras/cnn_mnist/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/dataloader.py rename to openfl-workspace/keras/cnn_mnist/src/dataloader.py diff --git a/openfl-workspace/keras_cnn_mnist/src/mnist_utils.py b/openfl-workspace/keras/cnn_mnist/src/mnist_utils.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/mnist_utils.py rename to openfl-workspace/keras/cnn_mnist/src/mnist_utils.py diff --git a/openfl-workspace/keras_cnn_mnist/src/taskrunner.py b/openfl-workspace/keras/cnn_mnist/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_cnn_mnist/src/taskrunner.py rename to openfl-workspace/keras/cnn_mnist/src/taskrunner.py diff --git a/openfl-workspace/keras_nlp/.workspace b/openfl-workspace/keras/nlp/.workspace similarity index 100% rename from openfl-workspace/keras_nlp/.workspace rename to openfl-workspace/keras/nlp/.workspace diff --git a/openfl-workspace/keras_nlp/plan/cols.yaml b/openfl-workspace/keras/nlp/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/cols.yaml rename to openfl-workspace/keras/nlp/plan/cols.yaml diff --git a/openfl-workspace/keras_nlp/plan/data.yaml b/openfl-workspace/keras/nlp/plan/data.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/data.yaml rename to openfl-workspace/keras/nlp/plan/data.yaml diff --git a/openfl-workspace/keras_nlp/plan/plan.yaml b/openfl-workspace/keras/nlp/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras_nlp/plan/plan.yaml rename to openfl-workspace/keras/nlp/plan/plan.yaml diff --git a/openfl-workspace/keras_nlp/requirements.txt b/openfl-workspace/keras/nlp/requirements.txt similarity index 100% rename from openfl-workspace/keras_nlp/requirements.txt rename to openfl-workspace/keras/nlp/requirements.txt diff --git a/openfl-workspace/keras_nlp/src/__init__.py b/openfl-workspace/keras/nlp/src/__init__.py similarity index 100% rename from openfl-workspace/keras_nlp/src/__init__.py rename to openfl-workspace/keras/nlp/src/__init__.py diff --git a/openfl-workspace/keras_nlp/src/dataloader.py b/openfl-workspace/keras/nlp/src/dataloader.py similarity index 100% rename from openfl-workspace/keras_nlp/src/dataloader.py rename to openfl-workspace/keras/nlp/src/dataloader.py diff --git a/openfl-workspace/keras_nlp/src/dataloader_utils.py b/openfl-workspace/keras/nlp/src/dataloader_utils.py similarity index 100% rename from openfl-workspace/keras_nlp/src/dataloader_utils.py rename to openfl-workspace/keras/nlp/src/dataloader_utils.py diff --git a/openfl-workspace/keras_nlp/src/taskrunner.py b/openfl-workspace/keras/nlp/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras_nlp/src/taskrunner.py rename to openfl-workspace/keras/nlp/src/taskrunner.py From c87fb6732d6c95a93bb0d3b779426bc8eae0e6db Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 01:38:37 -0800 Subject: [PATCH 12/15] code changes Signed-off-by: yes --- openfl-workspace/gandlf_seg_test/.workspace | 2 ++ openfl-workspace/torch_cnn_histology/.workspace | 2 ++ openfl-workspace/torch_cnn_histology_fedcurv/.workspace | 2 ++ openfl-workspace/torch_cnn_mnist/.workspace | 2 ++ openfl-workspace/xgb_higgs/.workspace | 2 ++ 5 files changed, 10 insertions(+) create mode 100644 openfl-workspace/gandlf_seg_test/.workspace create mode 100644 openfl-workspace/torch_cnn_histology/.workspace create mode 100644 openfl-workspace/torch_cnn_histology_fedcurv/.workspace create mode 100644 openfl-workspace/torch_cnn_mnist/.workspace create mode 100644 openfl-workspace/xgb_higgs/.workspace diff --git a/openfl-workspace/gandlf_seg_test/.workspace b/openfl-workspace/gandlf_seg_test/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/gandlf_seg_test/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_histology/.workspace b/openfl-workspace/torch_cnn_histology/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_histology/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/.workspace b/openfl-workspace/torch_cnn_histology_fedcurv/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_histology_fedcurv/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_cnn_mnist/.workspace b/openfl-workspace/torch_cnn_mnist/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch_cnn_mnist/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/xgb_higgs/.workspace b/openfl-workspace/xgb_higgs/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/xgb_higgs/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + From 511dd1af865ad3704931510935f1b419034a2ef3 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:14:12 -0800 Subject: [PATCH 13/15] code changes Signed-off-by: yes --- .github/workflows/straggler-handling.yml | 2 +- .github/workflows/task_runner_basic_e2e.yml | 22 +- .../task_runner_dockerized_ws_e2e.yml | 8 +- .github/workflows/task_runner_fedeval_e2e.yml | 8 +- .github/workflows/taskrunner.yml | 2 +- .../workflows/taskrunner_eden_pipeline.yml | 2 +- .../workflows/tr_docker_gramine_direct.yml | 2 +- .github/workflows/tr_docker_native.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- Jenkinsfile | 12 +- docs/about/features_index/fed_eval.rst | 8 +- docs/about/features_index/taskrunner.rst | 14 +- .../advanced_topics/log_metric_callback.rst | 2 +- .../straggler_handling_algorithms.rst | 2 +- docs/releases.md | 4 +- docs/tutorials/taskrunner.ipynb | 2 +- .../{keras/cnn_mnist => JAX}/.workspace | 0 .../{keras/cnn_mnist => JAX}/plan/cols.yaml | 0 .../{keras/cnn_mnist => JAX}/plan/data.yaml | 0 .../{keras/cnn_mnist => JAX}/plan/defaults | 0 .../{keras/cnn_mnist => JAX}/plan/plan.yaml | 0 .../{keras/cnn_mnist => JAX}/requirements.txt | 0 .../{keras/cnn_mnist => JAX}/src/__init__.py | 0 .../cnn_mnist => JAX}/src/dataloader.py | 0 .../cnn_mnist => JAX}/src/mnist_utils.py | 0 .../cnn_mnist => JAX}/src/taskrunner.py | 0 .../mnist}/.workspace | 0 .../mnist}/plan/cols.yaml | 0 openfl-workspace/keras/mnist/plan/data.yaml | 7 + .../mnist}/plan/defaults | 0 openfl-workspace/keras/mnist/plan/plan.yaml | 46 ++ openfl-workspace/keras/mnist/requirements.txt | 3 + .../mnist}/src/__init__.py | 0 .../keras/mnist/src/dataloader.py | 47 ++ .../keras/mnist/src/mnist_utils.py | 118 +++++ .../keras/mnist/src/taskrunner.py | 78 +++ .../histology}/.workspace | 0 .../histology}/plan/cols.yaml | 0 .../histology}/plan/data.yaml | 0 .../histology}/plan/plan.yaml | 0 .../histology}/requirements.txt | 0 .../histology}/src/__init__.py | 0 .../histology}/src/dataloader.py | 0 .../histology}/src/taskrunner.py | 0 .../histology_fedcurv}/.workspace | 0 .../histology_fedcurv}/README.md | 4 +- .../histology_fedcurv}/plan/cols.yaml | 0 .../histology_fedcurv}/plan/data.yaml | 0 .../histology_fedcurv}/plan/plan.yaml | 0 .../histology_fedcurv}/requirements.txt | 0 .../torch/histology_fedcurv/src/__init__.py | 3 + .../histology_fedcurv}/src/dataloader.py | 0 .../histology_fedcurv}/src/taskrunner.py | 0 .../llm_horovod}/.workspace | 0 .../llm_horovod}/LLM_Horovod.MD | 6 +- .../llm_horovod}/plan/cols.yaml | 0 .../llm_horovod}/plan/data.yaml | 0 .../llm_horovod}/plan/defaults | 0 .../llm_horovod}/plan/plan.yaml | 0 .../llm_horovod}/requirements.txt | 0 .../llm_horovod}/setup_env.sh | 2 +- .../llm_horovod}/src/InHorovodLLMTrainer.py | 0 .../llm_horovod}/src/InHorovodrun.py | 0 .../llm_horovod}/src/__init__.py | 0 .../llm_horovod}/src/emotion_utils.py | 0 .../llm_horovod}/src/model_utils.py | 0 .../llm_horovod}/src/pt_model.py | 0 .../llm_horovod}/src/ptemotion_inmemory.py | 0 .../mnist}/.workspace | 0 .../mnist}/README.md | 4 +- .../mnist}/plan/cols.yaml | 0 .../mnist}/plan/data.yaml | 0 .../mnist}/plan/defaults | 0 .../mnist}/plan/plan.yaml | 0 .../mnist}/requirements.txt | 0 .../mnist}/src/__init__.py | 0 .../mnist}/src/cnn_model.py | 0 .../mnist}/src/dataloader.py | 0 .../mnist}/src/taskrunner.py | 0 .../mnist_eden_compression}/.workspace | 0 .../mnist_eden_compression}/plan/cols.yaml | 0 .../mnist_eden_compression}/plan/data.yaml | 0 .../mnist_eden_compression}/plan/defaults | 0 .../mnist_eden_compression}/plan/plan.yaml | 0 .../mnist_eden_compression}/requirements.txt | 0 .../mnist_eden_compression}/src/__init__.py | 0 .../src/mnist_utils.py | 0 .../mnist_eden_compression}/src/pt_cnn.py | 0 .../src/ptmnist_inmemory.py | 0 .../mnist_fed_eval}/.workspace | 0 .../mnist_fed_eval}/plan/cols.yaml | 0 .../mnist_fed_eval}/plan/data.yaml | 0 .../mnist_fed_eval}/plan/defaults | 0 .../mnist_fed_eval}/plan/plan.yaml | 0 .../mnist_fed_eval}/requirements.txt | 0 .../mnist_fed_eval}/src/__init__.py | 0 .../mnist_fed_eval}/src/mnist_utils.py | 0 .../mnist_fed_eval}/src/pt_cnn.py | 0 .../mnist_fed_eval}/src/ptmnist_inmemory.py | 0 .../mnist_straggler_check}/.workspace | 0 .../mnist_straggler_check/plan/cols.yaml | 5 + .../mnist_straggler_check}/plan/data.yaml | 0 .../mnist_straggler_check}/plan/defaults | 0 .../mnist_straggler_check}/plan/plan.yaml | 0 .../mnist_straggler_check}/requirements.txt | 0 .../mnist_straggler_check}/src/__init__.py | 0 .../mnist_straggler_check}/src/mnist_utils.py | 0 .../mnist_straggler_check}/src/pt_cnn.py | 0 .../src/ptmnist_inmemory.py | 0 .../template}/.workspace | 0 .../template}/plan/cols.yaml | 0 .../template}/plan/data.yaml | 0 .../template}/plan/defaults | 0 .../template}/plan/plan.yaml | 0 .../template}/requirements.txt | 0 .../template}/src/__init__.py | 0 .../template}/src/dataloader.py | 0 .../template}/src/taskrunner.py | 0 openfl-workspace/torch/unet_kvasir/.workspace | 2 + .../unet_kvasir}/plan/cols.yaml | 0 .../unet_kvasir}/plan/data.yaml | 0 .../torch/unet_kvasir/plan/defaults | 2 + .../unet_kvasir}/plan/plan.yaml | 0 .../unet_kvasir}/requirements.txt | 0 .../unet_kvasir}/src/__init__.py | 0 .../unet_kvasir}/src/dataloader.py | 0 .../unet_kvasir}/src/pt_unet_parts.py | 0 .../unet_kvasir}/src/taskrunner.py | 0 openfl/federated/data/loader_jax.py | 138 +++++ openfl/federated/task/runner_jax.py | 489 ++++++++++++++++++ openfl/native/native.py | 4 +- tests/end_to_end/README.md | 8 +- tests/end_to_end/utils/conftest_helper.py | 2 +- tests/end_to_end/utils/constants.py | 8 +- tests/github/test_double_ws_export.py | 2 +- tests/github/test_gandlf.py | 2 +- tests/github/test_hello_federation.py | 2 +- 138 files changed, 1008 insertions(+), 70 deletions(-) rename openfl-workspace/{keras/cnn_mnist => JAX}/.workspace (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/cols.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/data.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/defaults (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/plan/plan.yaml (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/requirements.txt (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/__init__.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/dataloader.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/mnist_utils.py (100%) rename openfl-workspace/{keras/cnn_mnist => JAX}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_histology => keras/mnist}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => keras/mnist}/plan/cols.yaml (100%) create mode 100644 openfl-workspace/keras/mnist/plan/data.yaml rename openfl-workspace/{torch_cnn_mnist_eden_compression => keras/mnist}/plan/defaults (100%) create mode 100644 openfl-workspace/keras/mnist/plan/plan.yaml create mode 100644 openfl-workspace/keras/mnist/requirements.txt rename openfl-workspace/{torch_cnn_histology => keras/mnist}/src/__init__.py (100%) create mode 100644 openfl-workspace/keras/mnist/src/dataloader.py create mode 100644 openfl-workspace/keras/mnist/src/mnist_utils.py create mode 100644 openfl-workspace/keras/mnist/src/taskrunner.py rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology}/.workspace (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_histology => torch/histology}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/histology_fedcurv}/.workspace (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/README.md (90%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/requirements.txt (100%) create mode 100644 openfl-workspace/torch/histology_fedcurv/src/__init__.py rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_histology_fedcurv => torch/histology_fedcurv}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/.workspace (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/LLM_Horovod.MD (92%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/llm_horovod}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/llm_horovod}/plan/defaults (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/plan/plan.yaml (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/requirements.txt (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/setup_env.sh (72%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/InHorovodLLMTrainer.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/InHorovodrun.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/llm_horovod}/src/__init__.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/emotion_utils.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/model_utils.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/pt_model.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/llm_horovod}/src/ptemotion_inmemory.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/README.md (97%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/cnn_model.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/dataloader.py (100%) rename openfl-workspace/{torch_cnn_mnist => torch/mnist}/src/taskrunner.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/.workspace (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_eden_compression}/plan/data.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_eden_compression}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_eden_compression}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_eden_compression => torch/mnist_eden_compression}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/.workspace (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/plan/cols.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_fed_eval}/plan/data.yaml (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_fed_eval}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/requirements.txt (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_fed_eval}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_fed_eval => torch/mnist_fed_eval}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/mnist_straggler_check}/.workspace (100%) create mode 100644 openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml rename openfl-workspace/{torch_llm_horovod => torch/mnist_straggler_check}/plan/data.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/mnist_straggler_check}/plan/defaults (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/plan/plan.yaml (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/requirements.txt (100%) rename openfl-workspace/{torch_llm_horovod => torch/mnist_straggler_check}/src/__init__.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/mnist_utils.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/pt_cnn.py (100%) rename openfl-workspace/{torch_cnn_mnist_straggler_check => torch/mnist_straggler_check}/src/ptmnist_inmemory.py (100%) rename openfl-workspace/{torch_template => torch/template}/.workspace (100%) rename openfl-workspace/{torch_template => torch/template}/plan/cols.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/plan/data.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/plan/defaults (100%) rename openfl-workspace/{torch_template => torch/template}/plan/plan.yaml (100%) rename openfl-workspace/{torch_template => torch/template}/requirements.txt (100%) rename openfl-workspace/{torch_template => torch/template}/src/__init__.py (100%) rename openfl-workspace/{torch_template => torch/template}/src/dataloader.py (100%) rename openfl-workspace/{torch_template => torch/template}/src/taskrunner.py (100%) create mode 100644 openfl-workspace/torch/unet_kvasir/.workspace rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/cols.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/data.yaml (100%) create mode 100644 openfl-workspace/torch/unet_kvasir/plan/defaults rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/plan/plan.yaml (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/requirements.txt (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/__init__.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/dataloader.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/pt_unet_parts.py (100%) rename openfl-workspace/{torch_unet_kvasir => torch/unet_kvasir}/src/taskrunner.py (100%) create mode 100644 openfl/federated/data/loader_jax.py create mode 100644 openfl/federated/task/runner_jax.py diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml index b5bb6996ce..5dba77278e 100644 --- a/.github/workflows/straggler-handling.yml +++ b/.github/workflows/straggler-handling.yml @@ -35,4 +35,4 @@ jobs: pip install . - name: Test Straggler Handling Interface run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist_straggler_check --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 + python -m tests.github.test_hello_federation --template torch/mnist_straggler_check --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml index caeaf23088..5430c5695c 100644 --- a/.github/workflows/task_runner_basic_e2e.yml +++ b/.github/workflows/task_runner_basic_e2e.yml @@ -25,8 +25,8 @@ on: type: choice options: - all - - torch_cnn_mnist - - keras/cnn_mnist + - torch/mnist + - keras/mnist python_version: description: "Python version" required: false @@ -85,21 +85,21 @@ jobs: id: input_selection run: | # --------------------------------------------------------------- - # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. + # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. # Default combination if no input is provided (i.e. 'all' is selected). - # * TLS - models [torch_cnn_mnist, keras/cnn_mnist] and python versions [3.10, 3.11, 3.12] - # * Non-TLS - models [torch_cnn_mnist] and python version [3.10] - # * No client auth - models [keras/cnn_mnist] and python version [3.10] - # * Memory logs - models [torch_cnn_mnist] and python version [3.10] + # * TLS - models [torch/mnist, keras/mnist] and python versions [3.10, 3.11, 3.12] + # * Non-TLS - models [torch/mnist] and python version [3.10] + # * No client auth - models [keras/mnist] and python version [3.10] + # * Memory logs - models [torch/mnist] and python version [3.10] # --------------------------------------------------------------- echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" if [ "${{ env.MODEL_NAME }}" == "all" ]; then - echo "models_for_tls=[\"torch_cnn_mnist\", \"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_non_tls=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_no_client_auth=[\"keras/cnn_mnist\"]" >> "$GITHUB_OUTPUT" - echo "models_for_memory_logs=[\"torch_cnn_mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_tls=[\"torch/mnist\", \"keras/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_non_tls=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_no_client_auth=[\"keras/mnist\"]" >> "$GITHUB_OUTPUT" + echo "models_for_memory_logs=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" else echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/task_runner_dockerized_ws_e2e.yml b/.github/workflows/task_runner_dockerized_ws_e2e.yml index 896b87400d..23febe3e64 100644 --- a/.github/workflows/task_runner_dockerized_ws_e2e.yml +++ b/.github/workflows/task_runner_dockerized_ws_e2e.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10", "3.11", "3.12"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -73,7 +73,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -114,7 +114,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -155,7 +155,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - model_name: ["keras/cnn_mnist"] + model_name: ["keras/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/task_runner_fedeval_e2e.yml b/.github/workflows/task_runner_fedeval_e2e.yml index fce6408686..88bd114836 100644 --- a/.github/workflows/task_runner_fedeval_e2e.yml +++ b/.github/workflows/task_runner_fedeval_e2e.yml @@ -34,9 +34,9 @@ jobs: timeout-minutes: 30 strategy: matrix: - # Models like XGBoost (xgb_higgs) and torch_cnn_histology require runners with higher memory and CPU to run. + # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. # Thus these models are excluded from the matrix for now. - model_name: ["torch_cnn_mnist", "keras_cnn_mnist"] + model_name: ["torch/mnist", "keras_cnn_mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail @@ -77,9 +77,9 @@ jobs: timeout-minutes: 30 strategy: matrix: - # Testing this scenario only for torch_cnn_mnist model and python 3.10 + # Testing this scenario only for torch/mnist model and python 3.10 # If required, this can be extended to other models and python versions - model_name: ["torch_cnn_mnist"] + model_name: ["torch/mnist"] python_version: ["3.10"] fail-fast: false # do not immediately fail if one of the combinations fail diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml index 11692a4b5c..1df43888eb 100644 --- a/.github/workflows/taskrunner.yml +++ b/.github/workflows/taskrunner.yml @@ -32,4 +32,4 @@ jobs: pip install . - name: Task Runner API run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist --fed_workspace aggregator --col1 collaborator1 --col2 collaborator2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template torch/mnist --fed_workspace aggregator --col1 collaborator1 --col2 collaborator2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml index 02eea0e2dc..bfacb7a61d 100644 --- a/.github/workflows/taskrunner_eden_pipeline.yml +++ b/.github/workflows/taskrunner_eden_pipeline.yml @@ -31,4 +31,4 @@ jobs: pip install . - name: Test TaskRunner API with Eden Compression run: | - python -m tests.github.test_hello_federation --template torch_cnn_mnist_eden_compression --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 + python -m tests.github.test_hello_federation --template torch/mnist_eden_compression --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 9ecbe90819..ca9155a647 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras/cnn_mnist + fx workspace create --prefix example_workspace --template keras/mnist cd example_workspace fx plan initialize -a localhost diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index b80e5ca161..7824897e03 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -27,7 +27,7 @@ jobs: - name: Create workspace image run: | - fx workspace create --prefix example_workspace --template keras/cnn_mnist + fx workspace create --prefix example_workspace --template keras/mnist cd example_workspace fx plan initialize -a localhost fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index ebaa206867..f82193a433 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -53,4 +53,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model + python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5bf4fb2bf4..e33ca1e09c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -52,4 +52,4 @@ jobs: pip install . - name: Test TaskRunner API run: | - python -m tests.github.test_hello_federation --template keras/cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file + python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index fd7db70d21..bd97ab7f2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,17 +2,17 @@ def snykData = [ 'openfl-docker': 'openfl-docker/Dockerfile.base', 'openfl': 'setup.py', 'openfl-workspace_tf_2dunet': 'openfl-workspace/tf_2dunet/requirements.txt', - 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt', + 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch/mnist_straggler_check/requirements.txt', // CN-14619 snyk test CLI does not support -f in requirements.txt file - // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch_cnn_histology/requirements.txt', - 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch_cnn_histology/src/requirements.txt', + // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch/histology/requirements.txt', + 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch/histology/src/requirements.txt', 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', - 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch_cnn_mnist/requirements.txt', - 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch_unet_kvasir/requirements.txt', + 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', + 'openfl-workspace_torch/unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', - 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/cnn_mnist/requirements.txt', + 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_histology_envoy': 'openfl-tutorials/interactive_api/PyTorch_Histology/envoy/requirements.txt', diff --git a/docs/about/features_index/fed_eval.rst b/docs/about/features_index/fed_eval.rst index 3f8e6b8637..01dca906ba 100644 --- a/docs/about/features_index/fed_eval.rst +++ b/docs/about/features_index/fed_eval.rst @@ -26,7 +26,7 @@ Example Using the Task Runner API (Aggregator-based Workflow) The following steps can be leveraged to achieve practical e2e usage of FedEval -*N.B*: We will be using torch_cnn_mnist plan itself for both training and with some minor changes for evaluation as well +*N.B*: We will be using torch/mnist plan itself for both training and with some minor changes for evaluation as well *Prerequisites*: Please ensure that OpenFL version==1.7 is installed or you can also choose to install latest from source. @@ -48,13 +48,13 @@ With OpenFL version==1.7 aggregator start command is enhanced to have an optiona --help Show this message and exit. 1. **Setup** -We will use the `torch_cnn_mnist` workspace for training +We will use the `torch/mnist` workspace for training Let's first configure a workspace with all necesary certificates .. code-block:: shell - fx workspace create --prefix ./cnn_train_eval --template torch_cnn_mnist + fx workspace create --prefix ./cnn_train_eval --template torch/mnist cd cnn_train_eval fx workspace certify fx aggregator generate-cert-request @@ -416,7 +416,7 @@ The updated plan post initialization with edits to make it ready for evaluation metrics: - loss -We have done following changes to the initialized torch_cnn_mnist plan in the new workspace: +We have done following changes to the initialized torch/mnist plan in the new workspace: - Set the rounds_to_train to 1 as evaluation needs just one round of federation run across the collaborators - Removed all other training related tasks from assigner settings except "aggregated_model_validation" Now let's replace the ``init.pbuf`` with the previously saved ``trained_model.pbuf`` diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst index 255a63d904..0956a2cd8e 100644 --- a/docs/about/features_index/taskrunner.rst +++ b/docs/about/features_index/taskrunner.rst @@ -88,7 +88,7 @@ Each YAML top-level section contains the following subsections: The following is an example of a **plan.yaml**: -.. literalinclude:: ../../../openfl-workspace/torch_cnn_mnist/plan/plan.yaml +.. literalinclude:: ../../../openfl-workspace/torch/mnist/plan/plan.yaml :language: yaml @@ -150,22 +150,22 @@ STEP 1: Create a Workspace $ fx -2. This example uses the :code:`keras/cnn_mnist` template. +2. This example uses the :code:`keras/mnist` template. - Set the environment variables to use the :code:`keras/cnn_mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. + Set the environment variables to use the :code:`keras/mnist` as the template and :code:`${HOME}/my_federation` as the path to the workspace directory. .. code-block:: shell - $ export WORKSPACE_TEMPLATE=keras/cnn_mnist + $ export WORKSPACE_TEMPLATE=keras/mnist $ export WORKSPACE_PATH=${HOME}/my_federation 3. Decide a workspace template, which are end-to-end federated learning training demonstrations. The following is a sample of available templates: - - :code:`keras/cnn_mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`keras/mnist`: a workspace with a simple `Keras `__ CNN model that will download the `MNIST `_ dataset and train in a federation. - :code:`tf_2dunet`: a workspace with a simple `TensorFlow `__ CNN model that will use the `BraTS `_ dataset and train in a federation. - :code:`tf_cnn_histology`: a workspace with a simple `TensorFlow `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - - :code:`torch_cnn_histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. - - :code:`torch_cnn_mnist`: a workspace with a simple `PyTorch `__ CNN model that will download the `MNIST `_ dataset and train in a federation. + - :code:`torch/histology`: a workspace with a simple `PyTorch `__ CNN model that will download the `Colorectal Histology `_ dataset and train in a federation. + - :code:`torch/mnist`: a workspace with a simple `PyTorch `__ CNN model that will download the `MNIST `_ dataset and train in a federation. See the complete list of available templates. diff --git a/docs/developer_guide/advanced_topics/log_metric_callback.rst b/docs/developer_guide/advanced_topics/log_metric_callback.rst index a449feba61..35256da30f 100644 --- a/docs/developer_guide/advanced_topics/log_metric_callback.rst +++ b/docs/developer_guide/advanced_topics/log_metric_callback.rst @@ -83,7 +83,7 @@ For logging through Tensorboard, enable the parameter :code:`write_logs : true` settings : write_logs : true -Follow the steps below to write your custom callback function instead. As an example, a full implementation can be found at `Federated_Pytorch_MNIST_Tutorial.ipynb `_ and in the **torch_cnn_mnist** workspace. +Follow the steps below to write your custom callback function instead. As an example, a full implementation can be found at `Federated_Pytorch_MNIST_Tutorial.ipynb `_ and in the **torch/mnist** workspace. 1. Define the callback function, like how you defined in Python API, in the **src** directory in your workspace. diff --git a/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst b/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst index cdfa2b6a2c..3ec03cc68c 100644 --- a/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst +++ b/docs/developer_guide/advanced_topics/straggler_handling_algorithms.rst @@ -29,7 +29,7 @@ The following are the straggler handling algorithms supported in OpenFL: Demonstration of adding the straggler handling interface ========================================================= -The example template, **torch_cnn_mnist_straggler_check**, uses the ``PercentageBasedStragglerHandling``. To gain a better understanding of how experiments perform, you can modify the **percent_collaborators_needed** or **minimum_reporting** parameter in the template **plan.yaml** or even choose **CutoffTimeBasedStragglerHandling** function instead: +The example template, **torch/mnist_straggler_check**, uses the ``PercentageBasedStragglerHandling``. To gain a better understanding of how experiments perform, you can modify the **percent_collaborators_needed** or **minimum_reporting** parameter in the template **plan.yaml** or even choose **CutoffTimeBasedStragglerHandling** function instead: .. code-block:: yaml diff --git a/docs/releases.md b/docs/releases.md index 539336ceb1..bcd0f19a39 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -15,7 +15,7 @@ - **FL Workspace Dockerization**: Revised Task Runner API workspace dockerization process, with TEE-ready containers (using Gramine and Intel® Software Guard Extensions). Follow the [updated instructions](https://github.com/securefederatedai/openfl/blob/develop/openfl-docker/README.md) to enhance the privacy and security of your FL experiments. -- **Federated Evaluation via TaskRunner API**: OpenFL 1.7 further simplifies the creation of Federated Evaluation experiments via the TaskRunner API (see the example [FedEval workspace](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_cnn_mnist_fed_eval)). +- **Federated Evaluation via TaskRunner API**: OpenFL 1.7 further simplifies the creation of Federated Evaluation experiments via the TaskRunner API (see the example [FedEval workspace](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch/mnist_fed_eval)). - **Keras 3 API**: Upgrading the base TaskRunner classes and example workspaces to Keras 3 for building state-of-the-art FL experiments with TensorFlow (more backends to be included in the upcoming OpenFL releases). @@ -28,7 +28,7 @@ ### New Features and APIs: - **Federated LLM fine-tuning**: - - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters + - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch/llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters - **Neuralchat-7b fine-tuning**: Learn how to fine-tune [neuralchat-7b](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/LLM/neuralchat) using the Intel® Extension for Transformers and the workflow interface. - **Workflow API enhancements**: Introducing an experimental [Workspace Export](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/workflow/1001_Workspace_Creation_from_JupyterNotebook.ipynb) feature that can be used to transform a Workflow API-based FL experiment into the TaskRunner API format for running in a distributed deployment. There is also groundwork laid for a future FederatedRuntime implementation for Workflow API, in addition to the currently supported LocalRuntime. diff --git a/docs/tutorials/taskrunner.ipynb b/docs/tutorials/taskrunner.ipynb index a95236e17b..a1874eaf5a 100644 --- a/docs/tutorials/taskrunner.ipynb +++ b/docs/tutorials/taskrunner.ipynb @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "!fx workspace create --prefix ./mnist_example --template keras/cnn_mnist\n", + "!fx workspace create --prefix ./mnist_example --template keras/mnist\n", "%cd ./mnist_example" ] }, diff --git a/openfl-workspace/keras/cnn_mnist/.workspace b/openfl-workspace/JAX/.workspace similarity index 100% rename from openfl-workspace/keras/cnn_mnist/.workspace rename to openfl-workspace/JAX/.workspace diff --git a/openfl-workspace/keras/cnn_mnist/plan/cols.yaml b/openfl-workspace/JAX/plan/cols.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/cols.yaml rename to openfl-workspace/JAX/plan/cols.yaml diff --git a/openfl-workspace/keras/cnn_mnist/plan/data.yaml b/openfl-workspace/JAX/plan/data.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/data.yaml rename to openfl-workspace/JAX/plan/data.yaml diff --git a/openfl-workspace/keras/cnn_mnist/plan/defaults b/openfl-workspace/JAX/plan/defaults similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/defaults rename to openfl-workspace/JAX/plan/defaults diff --git a/openfl-workspace/keras/cnn_mnist/plan/plan.yaml b/openfl-workspace/JAX/plan/plan.yaml similarity index 100% rename from openfl-workspace/keras/cnn_mnist/plan/plan.yaml rename to openfl-workspace/JAX/plan/plan.yaml diff --git a/openfl-workspace/keras/cnn_mnist/requirements.txt b/openfl-workspace/JAX/requirements.txt similarity index 100% rename from openfl-workspace/keras/cnn_mnist/requirements.txt rename to openfl-workspace/JAX/requirements.txt diff --git a/openfl-workspace/keras/cnn_mnist/src/__init__.py b/openfl-workspace/JAX/src/__init__.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/__init__.py rename to openfl-workspace/JAX/src/__init__.py diff --git a/openfl-workspace/keras/cnn_mnist/src/dataloader.py b/openfl-workspace/JAX/src/dataloader.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/dataloader.py rename to openfl-workspace/JAX/src/dataloader.py diff --git a/openfl-workspace/keras/cnn_mnist/src/mnist_utils.py b/openfl-workspace/JAX/src/mnist_utils.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/mnist_utils.py rename to openfl-workspace/JAX/src/mnist_utils.py diff --git a/openfl-workspace/keras/cnn_mnist/src/taskrunner.py b/openfl-workspace/JAX/src/taskrunner.py similarity index 100% rename from openfl-workspace/keras/cnn_mnist/src/taskrunner.py rename to openfl-workspace/JAX/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_histology/.workspace b/openfl-workspace/keras/mnist/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_histology/.workspace rename to openfl-workspace/keras/mnist/.workspace diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/cols.yaml b/openfl-workspace/keras/mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/cols.yaml rename to openfl-workspace/keras/mnist/plan/cols.yaml diff --git a/openfl-workspace/keras/mnist/plan/data.yaml b/openfl-workspace/keras/mnist/plan/data.yaml new file mode 100644 index 0000000000..257c7825fe --- /dev/null +++ b/openfl-workspace/keras/mnist/plan/data.yaml @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +# collaborator_name,data_directory_path +one,1 + + diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/defaults b/openfl-workspace/keras/mnist/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/defaults rename to openfl-workspace/keras/mnist/plan/defaults diff --git a/openfl-workspace/keras/mnist/plan/plan.yaml b/openfl-workspace/keras/mnist/plan/plan.yaml new file mode 100644 index 0000000000..54867f4578 --- /dev/null +++ b/openfl-workspace/keras/mnist/plan/plan.yaml @@ -0,0 +1,46 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +aggregator : + defaults : plan/defaults/aggregator.yaml + template : openfl.component.Aggregator + settings : + init_state_path : save/init.pbuf + best_state_path : save/best.pbuf + last_state_path : save/last.pbuf + rounds_to_train : 10 + +collaborator : + defaults : plan/defaults/collaborator.yaml + template : openfl.component.Collaborator + settings : + delta_updates : false + opt_treatment : RESET + +data_loader : + defaults : plan/defaults/data_loader.yaml + template : src.dataloader.KerasMNISTInMemory + settings : + collaborator_count : 2 + data_group_name : mnist + batch_size : 256 + +task_runner : + defaults : plan/defaults/task_runner.yaml + template : src.taskrunner.KerasCNN + +network : + defaults : plan/defaults/network.yaml + +assigner : + defaults : plan/defaults/assigner.yaml + +tasks : + defaults : plan/defaults/tasks_keras.yaml + +compression_pipeline : + defaults : plan/defaults/compression_pipeline.yaml + # To use different Compression Pipeline, uncomment the following lines + # template : openfl.pipelines.KCPipeline + # settings : + # n_clusters : 6 diff --git a/openfl-workspace/keras/mnist/requirements.txt b/openfl-workspace/keras/mnist/requirements.txt new file mode 100644 index 0000000000..858a7dc3c8 --- /dev/null +++ b/openfl-workspace/keras/mnist/requirements.txt @@ -0,0 +1,3 @@ +keras==3.8.0 +tensorflow==2.18.0 + diff --git a/openfl-workspace/torch_cnn_histology/src/__init__.py b/openfl-workspace/keras/mnist/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/__init__.py rename to openfl-workspace/keras/mnist/src/__init__.py diff --git a/openfl-workspace/keras/mnist/src/dataloader.py b/openfl-workspace/keras/mnist/src/dataloader.py new file mode 100644 index 0000000000..040e8091c9 --- /dev/null +++ b/openfl-workspace/keras/mnist/src/dataloader.py @@ -0,0 +1,47 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from openfl.federated import KerasDataLoader +from .mnist_utils import load_mnist_shard + + +class KerasMNISTInMemory(KerasDataLoader): + """Data Loader for MNIST Dataset.""" + + def __init__(self, data_path, batch_size, **kwargs): + """ + Initialize. + + Args: + data_path: File path for the dataset + batch_size (int): The batch size for the data loader + **kwargs: Additional arguments, passed to super init and load_mnist_shard + """ + super().__init__(batch_size, **kwargs) + + # TODO: We should be downloading the dataset shard into a directory + # TODO: There needs to be a method to ask how many collaborators and + # what index/rank is this collaborator. + # Then we have a way to automatically shard based on rank and size of + # collaborator list. + try: + int(data_path) + except: + raise ValueError( + "Expected `%s` to be representable as `int`, as it refers to the data shard " + + "number used by the collaborator.", + data_path + ) + + _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( + shard_num=int(data_path), **kwargs + ) + + self.X_train = X_train + self.y_train = y_train + self.X_valid = X_valid + self.y_valid = y_valid + + self.num_classes = num_classes diff --git a/openfl-workspace/keras/mnist/src/mnist_utils.py b/openfl-workspace/keras/mnist/src/mnist_utils.py new file mode 100644 index 0000000000..d19e13d9dd --- /dev/null +++ b/openfl-workspace/keras/mnist/src/mnist_utils.py @@ -0,0 +1,118 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from logging import getLogger + +import numpy as np +from tensorflow.python.keras.utils.data_utils import get_file + +logger = getLogger(__name__) + + +def one_hot(labels, classes): + """ + One Hot encode a vector. + + Args: + labels (list): List of labels to onehot encode + classes (int): Total number of categorical classes + + Returns: + np.array: Matrix of one-hot encoded labels + """ + return np.eye(classes)[labels] + + +def _load_raw_datashards(shard_num, collaborator_count): + """ + Load the raw data by shard. + + Returns tuples of the dataset shard divided into training and validation. + + Args: + shard_num (int): The shard number to use + collaborator_count (int): The number of collaborators in the federation + + Returns: + 2 tuples: (image, label) of the training, validation dataset + """ + origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' + path = get_file('mnist.npz', + origin=origin_folder + 'mnist.npz', + file_hash='731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1') + + with np.load(path) as f: + # get all of mnist + X_train_tot = f['x_train'] + y_train_tot = f['y_train'] + + X_valid_tot = f['x_test'] + y_valid_tot = f['y_test'] + + # create the shards + shard_num = int(shard_num) + X_train = X_train_tot[shard_num::collaborator_count] + y_train = y_train_tot[shard_num::collaborator_count] + + X_valid = X_valid_tot[shard_num::collaborator_count] + y_valid = y_valid_tot[shard_num::collaborator_count] + + return (X_train, y_train), (X_valid, y_valid) + + +def load_mnist_shard(shard_num, collaborator_count, categorical=True, + channels_last=True, **kwargs): + """ + Load the MNIST dataset. + + Args: + shard_num (int): The shard to use from the dataset + collaborator_count (int): The number of collaborators in the federation + categorical (bool): True = convert the labels to one-hot encoded + vectors (Default = True) + channels_last (bool): True = The input images have the channels + last (Default = True) + **kwargs: Additional parameters to pass to the function + + Returns: + list: The input shape + int: The number of classes + numpy.ndarray: The training data + numpy.ndarray: The training labels + numpy.ndarray: The validation data + numpy.ndarray: The validation labels + """ + img_rows, img_cols = 28, 28 + num_classes = 10 + + (X_train, y_train), (X_valid, y_valid) = _load_raw_datashards( + shard_num, collaborator_count + ) + + if channels_last: + X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) + X_valid = X_valid.reshape(X_valid.shape[0], img_rows, img_cols, 1) + input_shape = (img_rows, img_cols, 1) + else: + X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) + X_valid = X_valid.reshape(X_valid.shape[0], 1, img_rows, img_cols) + input_shape = (1, img_rows, img_cols) + + X_train = X_train.astype('float32') + X_valid = X_valid.astype('float32') + X_train /= 255 + X_valid /= 255 + + logger.info(f'MNIST > X_train Shape : {X_train.shape}') + logger.info(f'MNIST > y_train Shape : {y_train.shape}') + logger.info(f'MNIST > Train Samples : {X_train.shape[0]}') + logger.info(f'MNIST > Valid Samples : {X_valid.shape[0]}') + + if categorical: + # convert class vectors to binary class matrices + y_train = one_hot(y_train, num_classes) + y_valid = one_hot(y_valid, num_classes) + + return input_shape, num_classes, X_train, y_train, X_valid, y_valid diff --git a/openfl-workspace/keras/mnist/src/taskrunner.py b/openfl-workspace/keras/mnist/src/taskrunner.py new file mode 100644 index 0000000000..165861033c --- /dev/null +++ b/openfl-workspace/keras/mnist/src/taskrunner.py @@ -0,0 +1,78 @@ +# Copyright (C) 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""You may copy this file as the starting point of your own model.""" + +from keras.models import Sequential +from keras.layers import Conv2D +from keras.layers import Dense +from keras.layers import Flatten + +from openfl.federated import KerasTaskRunner + + +class KerasCNN(KerasTaskRunner): + """A basic convolutional neural network model.""" + + def __init__(self, **kwargs): + """ + Initialize. + + Args: + **kwargs: Additional parameters to pass to the function + """ + super().__init__(**kwargs) + + self.model = self.build_model(self.feature_shape, self.data_loader.num_classes, **kwargs) + + self.initialize_tensorkeys_for_functions() + + self.model.summary(print_fn=self.logger.info) + + self.logger.info(f'Train Set Size : {self.get_train_data_size()}') + self.logger.info(f'Valid Set Size : {self.get_valid_data_size()}') + + def build_model(self, + input_shape, + num_classes, + conv_kernel_size=(4, 4), + conv_strides=(2, 2), + conv1_channels_out=16, + conv2_channels_out=32, + final_dense_inputsize=100, + **kwargs): + """ + Define the model architecture. + + Args: + input_shape (numpy.ndarray): The shape of the data + num_classes (int): The number of classes of the dataset + + Returns: + keras.models.Sequential: The model defined in Keras + + """ + model = Sequential() + + model.add(Conv2D(conv1_channels_out, + kernel_size=conv_kernel_size, + strides=conv_strides, + activation='relu', + input_shape=input_shape)) + + model.add(Conv2D(conv2_channels_out, + kernel_size=conv_kernel_size, + strides=conv_strides, + activation='relu')) + + model.add(Flatten()) + + model.add(Dense(final_dense_inputsize, activation='relu')) + + model.add(Dense(num_classes, activation='softmax')) + + model.compile(loss="categorical_crossentropy", + optimizer="adam", + metrics=["accuracy"]) + + return model diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/.workspace b/openfl-workspace/torch/histology/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/.workspace rename to openfl-workspace/torch/histology/.workspace diff --git a/openfl-workspace/torch_cnn_histology/plan/cols.yaml b/openfl-workspace/torch/histology/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/cols.yaml rename to openfl-workspace/torch/histology/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_histology/plan/data.yaml b/openfl-workspace/torch/histology/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/data.yaml rename to openfl-workspace/torch/histology/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_histology/plan/plan.yaml b/openfl-workspace/torch/histology/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology/plan/plan.yaml rename to openfl-workspace/torch/histology/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_histology/requirements.txt b/openfl-workspace/torch/histology/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_histology/requirements.txt rename to openfl-workspace/torch/histology/requirements.txt diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/__init__.py b/openfl-workspace/torch/histology/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/__init__.py rename to openfl-workspace/torch/histology/src/__init__.py diff --git a/openfl-workspace/torch_cnn_histology/src/dataloader.py b/openfl-workspace/torch/histology/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/dataloader.py rename to openfl-workspace/torch/histology/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_histology/src/taskrunner.py b/openfl-workspace/torch/histology/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_histology/src/taskrunner.py rename to openfl-workspace/torch/histology/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist/.workspace b/openfl-workspace/torch/histology_fedcurv/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist/.workspace rename to openfl-workspace/torch/histology_fedcurv/.workspace diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/README.md b/openfl-workspace/torch/histology_fedcurv/README.md similarity index 90% rename from openfl-workspace/torch_cnn_histology_fedcurv/README.md rename to openfl-workspace/torch/histology_fedcurv/README.md index de6f18e42d..1ebcd40329 100644 --- a/openfl-workspace/torch_cnn_histology_fedcurv/README.md +++ b/openfl-workspace/torch/histology_fedcurv/README.md @@ -4,7 +4,7 @@ It uses the Pytorch framework and OpenFL's TaskTunner API. The federation aggregates intermediate models using the [Fedcurv](https://arxiv.org/pdf/1910.07796) aggregation algorithm, which performs well (Compared to [FedAvg](https://arxiv.org/abs/2104.11375)) when the datasets are not independent and identically distributed (IID) among collaborators. -Note that this example is similar to the one present in the `torch_cnn_histology` directory and is here to demonstrate the usage of a different aggregation algorithm using OpenFL's Taskrunner API. +Note that this example is similar to the one present in the `torch/histology` directory and is here to demonstrate the usage of a different aggregation algorithm using OpenFL's Taskrunner API. The differenece between the two examples lies both in the `PyTorchCNNWithFedCurv` class which is used to define a stateful training method which uses an existing `FedCurv` object, and in the `plan.yaml` file in which the training task is explicitly defined with a non-default aggregation method - `FedCurvWeightedAverage`. @@ -13,7 +13,7 @@ and in the `plan.yaml` file in which the training task is explicitly defined wit The following instructions can be used to run the federation: ``` # Copy the workspace template, create collaborators and aggregator -fx workspace create --template torch_cnn_histology_fedcurv --prefix fedcurv +fx workspace create --template torch/histology_fedcurv --prefix fedcurv cd fedcurv fx workspace certify fx aggregator generate-cert-request fx aggregator certify --silent diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/cols.yaml b/openfl-workspace/torch/histology_fedcurv/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/cols.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/data.yaml b/openfl-workspace/torch/histology_fedcurv/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/data.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/plan/plan.yaml b/openfl-workspace/torch/histology_fedcurv/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/plan/plan.yaml rename to openfl-workspace/torch/histology_fedcurv/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/requirements.txt b/openfl-workspace/torch/histology_fedcurv/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/requirements.txt rename to openfl-workspace/torch/histology_fedcurv/requirements.txt diff --git a/openfl-workspace/torch/histology_fedcurv/src/__init__.py b/openfl-workspace/torch/histology_fedcurv/src/__init__.py new file mode 100644 index 0000000000..f1410b1298 --- /dev/null +++ b/openfl-workspace/torch/histology_fedcurv/src/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2020-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""You may copy this file as the starting point of your own model.""" diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/dataloader.py b/openfl-workspace/torch/histology_fedcurv/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/dataloader.py rename to openfl-workspace/torch/histology_fedcurv/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_histology_fedcurv/src/taskrunner.py b/openfl-workspace/torch/histology_fedcurv/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_histology_fedcurv/src/taskrunner.py rename to openfl-workspace/torch/histology_fedcurv/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/.workspace b/openfl-workspace/torch/llm_horovod/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/.workspace rename to openfl-workspace/torch/llm_horovod/.workspace diff --git a/openfl-workspace/torch_llm_horovod/LLM_Horovod.MD b/openfl-workspace/torch/llm_horovod/LLM_Horovod.MD similarity index 92% rename from openfl-workspace/torch_llm_horovod/LLM_Horovod.MD rename to openfl-workspace/torch/llm_horovod/LLM_Horovod.MD index 71b6d05165..63060de6f5 100644 --- a/openfl-workspace/torch_llm_horovod/LLM_Horovod.MD +++ b/openfl-workspace/torch/llm_horovod/LLM_Horovod.MD @@ -9,7 +9,7 @@ Before running the Horovod example, ensure that the following prerequisites are ## Setting up Horovod Dependencies To set up the Horovod dependencies, follow these steps: -1. Run the `setup_env.sh` script located in `openfl-workspace/torch_llm_horovod/setup_env.sh` within your virtual environment (venv). +1. Run the `setup_env.sh` script located in `openfl-workspace/torch/llm_horovod/setup_env.sh` within your virtual environment (venv). 2. Create aggregator and collaborator workspaces. See [Running the experiment](#running-the-experiment) 3. Ensure that the collaborator workspace is present in each node with the same file structure. 4. Make sure the dataset is available in each node. @@ -28,13 +28,13 @@ Set the following environmental variables for Horovod: ## Customizing Data and Models To use your own data and models, follow these steps: -1. Copy the `openfl/openfl-workspace/torch_llm_horovod` directory to `openfl/openfl-workspace/name_of_your_template`. +1. Copy the `openfl/openfl-workspace/torch/llm_horovod` directory to `openfl/openfl-workspace/name_of_your_template`. 2. In the `src/InHorovodrun` file, make the following changes: - Replace `EmotionDataLoader` with your own dataloader. - Replace `LLMTrainer` with your own training/validation scripts. ## Running the Experiment -To run the experiment, follow the instructions provided in the [OpenFL documentation](https://openfl.readthedocs.io/en/latest/running_the_federation.html#bare-metal-approach) using either the `torch_llm_horovod` template or your own template. +To run the experiment, follow the instructions provided in the [OpenFL documentation](https://openfl.readthedocs.io/en/latest/running_the_federation.html#bare-metal-approach) using either the `torch/llm_horovod` template or your own template. That's it! You're now ready to use the Horovod example with your own data and models. Enjoy! diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/cols.yaml b/openfl-workspace/torch/llm_horovod/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/cols.yaml rename to openfl-workspace/torch/llm_horovod/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/data.yaml b/openfl-workspace/torch/llm_horovod/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/data.yaml rename to openfl-workspace/torch/llm_horovod/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/defaults b/openfl-workspace/torch/llm_horovod/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/defaults rename to openfl-workspace/torch/llm_horovod/plan/defaults diff --git a/openfl-workspace/torch_llm_horovod/plan/plan.yaml b/openfl-workspace/torch/llm_horovod/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/plan.yaml rename to openfl-workspace/torch/llm_horovod/plan/plan.yaml diff --git a/openfl-workspace/torch_llm_horovod/requirements.txt b/openfl-workspace/torch/llm_horovod/requirements.txt similarity index 100% rename from openfl-workspace/torch_llm_horovod/requirements.txt rename to openfl-workspace/torch/llm_horovod/requirements.txt diff --git a/openfl-workspace/torch_llm_horovod/setup_env.sh b/openfl-workspace/torch/llm_horovod/setup_env.sh similarity index 72% rename from openfl-workspace/torch_llm_horovod/setup_env.sh rename to openfl-workspace/torch/llm_horovod/setup_env.sh index bb014dbe83..96f2d02133 100755 --- a/openfl-workspace/torch_llm_horovod/setup_env.sh +++ b/openfl-workspace/torch/llm_horovod/setup_env.sh @@ -3,6 +3,6 @@ pip install -U pip --no-cache pip install torch==2.0.0 torchvision==0.15.1 torchaudio==2.0.1 export HOROVOD_WITH_PYTORCH=1 export HOROVOD_WITHOUT_MPI=1 -pip install -r openfl-workspace/torch_llm_horovod/requirements.txt --no-cache +pip install -r openfl-workspace/torch/llm_horovod/requirements.txt --no-cache diff --git a/openfl-workspace/torch_llm_horovod/src/InHorovodLLMTrainer.py b/openfl-workspace/torch/llm_horovod/src/InHorovodLLMTrainer.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/InHorovodLLMTrainer.py rename to openfl-workspace/torch/llm_horovod/src/InHorovodLLMTrainer.py diff --git a/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py b/openfl-workspace/torch/llm_horovod/src/InHorovodrun.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/InHorovodrun.py rename to openfl-workspace/torch/llm_horovod/src/InHorovodrun.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/__init__.py b/openfl-workspace/torch/llm_horovod/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/__init__.py rename to openfl-workspace/torch/llm_horovod/src/__init__.py diff --git a/openfl-workspace/torch_llm_horovod/src/emotion_utils.py b/openfl-workspace/torch/llm_horovod/src/emotion_utils.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/emotion_utils.py rename to openfl-workspace/torch/llm_horovod/src/emotion_utils.py diff --git a/openfl-workspace/torch_llm_horovod/src/model_utils.py b/openfl-workspace/torch/llm_horovod/src/model_utils.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/model_utils.py rename to openfl-workspace/torch/llm_horovod/src/model_utils.py diff --git a/openfl-workspace/torch_llm_horovod/src/pt_model.py b/openfl-workspace/torch/llm_horovod/src/pt_model.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/pt_model.py rename to openfl-workspace/torch/llm_horovod/src/pt_model.py diff --git a/openfl-workspace/torch_llm_horovod/src/ptemotion_inmemory.py b/openfl-workspace/torch/llm_horovod/src/ptemotion_inmemory.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/ptemotion_inmemory.py rename to openfl-workspace/torch/llm_horovod/src/ptemotion_inmemory.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/.workspace b/openfl-workspace/torch/mnist/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/.workspace rename to openfl-workspace/torch/mnist/.workspace diff --git a/openfl-workspace/torch_cnn_mnist/README.md b/openfl-workspace/torch/mnist/README.md similarity index 97% rename from openfl-workspace/torch_cnn_mnist/README.md rename to openfl-workspace/torch/mnist/README.md index 17019afa1b..e3666c1763 100644 --- a/openfl-workspace/torch_cnn_mnist/README.md +++ b/openfl-workspace/torch/mnist/README.md @@ -1,5 +1,5 @@ ## Instantiating a Workspace from Torch Template -To instantiate a workspace from the torch_cnn_mnist template, you can use the fx workspace create command. This allows you to quickly set up a new workspace based on a predefined configuration and template. +To instantiate a workspace from the torch/mnist template, you can use the fx workspace create command. This allows you to quickly set up a new workspace based on a predefined configuration and template. 1. Ensure the necessary dependencies are installed. ``` @@ -13,7 +13,7 @@ pip install openfl ``` cd ~/openfl-quickstart -fx workspace create --template torch_cnn_mnist --prefix fl_workspace +fx workspace create --template torch/mnist --prefix fl_workspace cd ~/openfl-quickstart/fl_workspace ``` diff --git a/openfl-workspace/torch_cnn_mnist/plan/cols.yaml b/openfl-workspace/torch/mnist/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/cols.yaml rename to openfl-workspace/torch/mnist/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist/plan/data.yaml b/openfl-workspace/torch/mnist/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/data.yaml rename to openfl-workspace/torch/mnist/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist/plan/defaults b/openfl-workspace/torch/mnist/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/defaults rename to openfl-workspace/torch/mnist/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist/plan/plan.yaml b/openfl-workspace/torch/mnist/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist/plan/plan.yaml rename to openfl-workspace/torch/mnist/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist/requirements.txt b/openfl-workspace/torch/mnist/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist/requirements.txt rename to openfl-workspace/torch/mnist/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist/src/__init__.py b/openfl-workspace/torch/mnist/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/__init__.py rename to openfl-workspace/torch/mnist/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist/src/cnn_model.py b/openfl-workspace/torch/mnist/src/cnn_model.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/cnn_model.py rename to openfl-workspace/torch/mnist/src/cnn_model.py diff --git a/openfl-workspace/torch_cnn_mnist/src/dataloader.py b/openfl-workspace/torch/mnist/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/dataloader.py rename to openfl-workspace/torch/mnist/src/dataloader.py diff --git a/openfl-workspace/torch_cnn_mnist/src/taskrunner.py b/openfl-workspace/torch/mnist/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist/src/taskrunner.py rename to openfl-workspace/torch/mnist/src/taskrunner.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/.workspace b/openfl-workspace/torch/mnist_eden_compression/.workspace similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/.workspace rename to openfl-workspace/torch/mnist_eden_compression/.workspace diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/cols.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/cols.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/data.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/data.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/data.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/defaults b/openfl-workspace/torch/mnist_eden_compression/plan/defaults similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/defaults rename to openfl-workspace/torch/mnist_eden_compression/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/plan/plan.yaml b/openfl-workspace/torch/mnist_eden_compression/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/plan/plan.yaml rename to openfl-workspace/torch/mnist_eden_compression/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt b/openfl-workspace/torch/mnist_eden_compression/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/requirements.txt rename to openfl-workspace/torch/mnist_eden_compression/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/__init__.py b/openfl-workspace/torch/mnist_eden_compression/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/__init__.py rename to openfl-workspace/torch/mnist_eden_compression/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/mnist_utils.py b/openfl-workspace/torch/mnist_eden_compression/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/mnist_utils.py rename to openfl-workspace/torch/mnist_eden_compression/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/pt_cnn.py b/openfl-workspace/torch/mnist_eden_compression/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/pt_cnn.py rename to openfl-workspace/torch/mnist_eden_compression/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_eden_compression/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_eden_compression/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_eden_compression/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_llm_horovod/.workspace b/openfl-workspace/torch/mnist_fed_eval/.workspace similarity index 100% rename from openfl-workspace/torch_llm_horovod/.workspace rename to openfl-workspace/torch/mnist_fed_eval/.workspace diff --git a/openfl-workspace/torch_llm_horovod/plan/cols.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/cols.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/cols.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/data.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/data.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/data.yaml diff --git a/openfl-workspace/torch_llm_horovod/plan/defaults b/openfl-workspace/torch/mnist_fed_eval/plan/defaults similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/defaults rename to openfl-workspace/torch/mnist_fed_eval/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml b/openfl-workspace/torch/mnist_fed_eval/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/plan/plan.yaml rename to openfl-workspace/torch/mnist_fed_eval/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt b/openfl-workspace/torch/mnist_fed_eval/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/requirements.txt rename to openfl-workspace/torch/mnist_fed_eval/requirements.txt diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/__init__.py b/openfl-workspace/torch/mnist_fed_eval/src/__init__.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/__init__.py rename to openfl-workspace/torch/mnist_fed_eval/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/mnist_utils.py b/openfl-workspace/torch/mnist_fed_eval/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/mnist_utils.py rename to openfl-workspace/torch/mnist_fed_eval/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/pt_cnn.py b/openfl-workspace/torch/mnist_fed_eval/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/pt_cnn.py rename to openfl-workspace/torch/mnist_fed_eval/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_fed_eval/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_fed_eval/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_fed_eval/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_unet_kvasir/.workspace b/openfl-workspace/torch/mnist_straggler_check/.workspace similarity index 100% rename from openfl-workspace/torch_unet_kvasir/.workspace rename to openfl-workspace/torch/mnist_straggler_check/.workspace diff --git a/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml new file mode 100644 index 0000000000..95307de3bc --- /dev/null +++ b/openfl-workspace/torch/mnist_straggler_check/plan/cols.yaml @@ -0,0 +1,5 @@ +# Copyright (C) 2020-2021 Intel Corporation +# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. + +collaborators: + \ No newline at end of file diff --git a/openfl-workspace/torch_llm_horovod/plan/data.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_llm_horovod/plan/data.yaml rename to openfl-workspace/torch/mnist_straggler_check/plan/data.yaml diff --git a/openfl-workspace/torch_unet_kvasir/plan/defaults b/openfl-workspace/torch/mnist_straggler_check/plan/defaults similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/defaults rename to openfl-workspace/torch/mnist_straggler_check/plan/defaults diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/plan/plan.yaml b/openfl-workspace/torch/mnist_straggler_check/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/plan/plan.yaml rename to openfl-workspace/torch/mnist_straggler_check/plan/plan.yaml diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt b/openfl-workspace/torch/mnist_straggler_check/requirements.txt similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/requirements.txt rename to openfl-workspace/torch/mnist_straggler_check/requirements.txt diff --git a/openfl-workspace/torch_llm_horovod/src/__init__.py b/openfl-workspace/torch/mnist_straggler_check/src/__init__.py similarity index 100% rename from openfl-workspace/torch_llm_horovod/src/__init__.py rename to openfl-workspace/torch/mnist_straggler_check/src/__init__.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/mnist_utils.py b/openfl-workspace/torch/mnist_straggler_check/src/mnist_utils.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/mnist_utils.py rename to openfl-workspace/torch/mnist_straggler_check/src/mnist_utils.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/pt_cnn.py b/openfl-workspace/torch/mnist_straggler_check/src/pt_cnn.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/pt_cnn.py rename to openfl-workspace/torch/mnist_straggler_check/src/pt_cnn.py diff --git a/openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py b/openfl-workspace/torch/mnist_straggler_check/src/ptmnist_inmemory.py similarity index 100% rename from openfl-workspace/torch_cnn_mnist_straggler_check/src/ptmnist_inmemory.py rename to openfl-workspace/torch/mnist_straggler_check/src/ptmnist_inmemory.py diff --git a/openfl-workspace/torch_template/.workspace b/openfl-workspace/torch/template/.workspace similarity index 100% rename from openfl-workspace/torch_template/.workspace rename to openfl-workspace/torch/template/.workspace diff --git a/openfl-workspace/torch_template/plan/cols.yaml b/openfl-workspace/torch/template/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/cols.yaml rename to openfl-workspace/torch/template/plan/cols.yaml diff --git a/openfl-workspace/torch_template/plan/data.yaml b/openfl-workspace/torch/template/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/data.yaml rename to openfl-workspace/torch/template/plan/data.yaml diff --git a/openfl-workspace/torch_template/plan/defaults b/openfl-workspace/torch/template/plan/defaults similarity index 100% rename from openfl-workspace/torch_template/plan/defaults rename to openfl-workspace/torch/template/plan/defaults diff --git a/openfl-workspace/torch_template/plan/plan.yaml b/openfl-workspace/torch/template/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_template/plan/plan.yaml rename to openfl-workspace/torch/template/plan/plan.yaml diff --git a/openfl-workspace/torch_template/requirements.txt b/openfl-workspace/torch/template/requirements.txt similarity index 100% rename from openfl-workspace/torch_template/requirements.txt rename to openfl-workspace/torch/template/requirements.txt diff --git a/openfl-workspace/torch_template/src/__init__.py b/openfl-workspace/torch/template/src/__init__.py similarity index 100% rename from openfl-workspace/torch_template/src/__init__.py rename to openfl-workspace/torch/template/src/__init__.py diff --git a/openfl-workspace/torch_template/src/dataloader.py b/openfl-workspace/torch/template/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_template/src/dataloader.py rename to openfl-workspace/torch/template/src/dataloader.py diff --git a/openfl-workspace/torch_template/src/taskrunner.py b/openfl-workspace/torch/template/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_template/src/taskrunner.py rename to openfl-workspace/torch/template/src/taskrunner.py diff --git a/openfl-workspace/torch/unet_kvasir/.workspace b/openfl-workspace/torch/unet_kvasir/.workspace new file mode 100644 index 0000000000..3c2c5d08b4 --- /dev/null +++ b/openfl-workspace/torch/unet_kvasir/.workspace @@ -0,0 +1,2 @@ +current_plan_name: default + diff --git a/openfl-workspace/torch_unet_kvasir/plan/cols.yaml b/openfl-workspace/torch/unet_kvasir/plan/cols.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/cols.yaml rename to openfl-workspace/torch/unet_kvasir/plan/cols.yaml diff --git a/openfl-workspace/torch_unet_kvasir/plan/data.yaml b/openfl-workspace/torch/unet_kvasir/plan/data.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/data.yaml rename to openfl-workspace/torch/unet_kvasir/plan/data.yaml diff --git a/openfl-workspace/torch/unet_kvasir/plan/defaults b/openfl-workspace/torch/unet_kvasir/plan/defaults new file mode 100644 index 0000000000..fb82f9c5b6 --- /dev/null +++ b/openfl-workspace/torch/unet_kvasir/plan/defaults @@ -0,0 +1,2 @@ +../../workspace/plan/defaults + diff --git a/openfl-workspace/torch_unet_kvasir/plan/plan.yaml b/openfl-workspace/torch/unet_kvasir/plan/plan.yaml similarity index 100% rename from openfl-workspace/torch_unet_kvasir/plan/plan.yaml rename to openfl-workspace/torch/unet_kvasir/plan/plan.yaml diff --git a/openfl-workspace/torch_unet_kvasir/requirements.txt b/openfl-workspace/torch/unet_kvasir/requirements.txt similarity index 100% rename from openfl-workspace/torch_unet_kvasir/requirements.txt rename to openfl-workspace/torch/unet_kvasir/requirements.txt diff --git a/openfl-workspace/torch_unet_kvasir/src/__init__.py b/openfl-workspace/torch/unet_kvasir/src/__init__.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/__init__.py rename to openfl-workspace/torch/unet_kvasir/src/__init__.py diff --git a/openfl-workspace/torch_unet_kvasir/src/dataloader.py b/openfl-workspace/torch/unet_kvasir/src/dataloader.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/dataloader.py rename to openfl-workspace/torch/unet_kvasir/src/dataloader.py diff --git a/openfl-workspace/torch_unet_kvasir/src/pt_unet_parts.py b/openfl-workspace/torch/unet_kvasir/src/pt_unet_parts.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/pt_unet_parts.py rename to openfl-workspace/torch/unet_kvasir/src/pt_unet_parts.py diff --git a/openfl-workspace/torch_unet_kvasir/src/taskrunner.py b/openfl-workspace/torch/unet_kvasir/src/taskrunner.py similarity index 100% rename from openfl-workspace/torch_unet_kvasir/src/taskrunner.py rename to openfl-workspace/torch/unet_kvasir/src/taskrunner.py diff --git a/openfl/federated/data/loader_jax.py b/openfl/federated/data/loader_jax.py new file mode 100644 index 0000000000..bf5a460186 --- /dev/null +++ b/openfl/federated/data/loader_jax.py @@ -0,0 +1,138 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +"""KerasDataLoader module.""" + +import numpy as np + +from openfl.federated.data.loader import DataLoader + + +class JAXDataLoader(DataLoader): + """A class used to represent a Federation Data Loader for Keras models. + + Attributes: + batch_size (int): Size of batches used for all data loaders. + X_train (np.array): Training features. + y_train (np.array): Training labels. + X_valid (np.array): Validation features. + y_valid (np.array): Validation labels. + """ + + def __init__(self, batch_size, **kwargs): + """Initializes the KerasDataLoader object. + + Args: + batch_size (int): The size of batches used for all data loaders. + kwargs: Additional arguments to pass to the function. + """ + self.batch_size = batch_size + self.X_train = None + self.y_train = None + self.X_valid = None + self.y_valid = None + + # Child classes should have init signature: + # (self, batch_size, **kwargs), should call this __init__ and then + # define self.X_train, self.y_train, self.X_valid, and self.y_valid + + def get_feature_shape(self): + """Returns the shape of an example feature array. + + Returns: + tuple: The shape of an example feature array. + """ + return self.X_train[0].shape + + def get_train_loader(self, batch_size=None, num_batches=None): + """Returns the data loader for the training data. + + Args: + batch_size (int, optional): The batch size for the data loader + (default is None). + num_batches (int, optional): The number of batches for the data + loader (default is None). + + Returns: + DataLoader: The DataLoader object for the training data. + """ + return self._get_batch_generator( + X=self.X_train, + y=self.y_train, + batch_size=batch_size, + num_batches=num_batches, + ) + + def get_valid_loader(self, batch_size=None): + """Returns the data loader for the validation data. + + Args: + batch_size (int, optional): The batch size for the data loader + (default is None). + + Returns: + DataLoader: The DataLoader object for the validation data. + """ + return self._get_batch_generator(X=self.X_valid, y=self.y_valid, batch_size=batch_size) + + def get_train_data_size(self): + """Returns the total number of training samples. + + Returns: + int: The total number of training samples. + """ + return self.X_train.shape[0] + + def get_valid_data_size(self): + """Returns the total number of validation samples. + + Returns: + int: The total number of validation samples. + """ + return self.X_valid.shape[0] + + @staticmethod + def _batch_generator(X, y, idxs, batch_size, num_batches): + """Generates batches of data. + + Args: + X (np.array): The input data. + y (np.array): The label data. + idxs (np.array): The index of the dataset. + batch_size (int): The batch size for the data loader. + num_batches (int): The number of batches. + + Yields: + tuple: The input data and label data for each batch. + """ + for i in range(num_batches): + a = i * batch_size + b = a + batch_size + yield X[idxs[a:b]], y[idxs[a:b]] + + def _get_batch_generator(self, X, y, batch_size, num_batches=None): + """Returns the dataset generator. + + Args: + X (np.array): The input data. + y (np.array): The label data. + batch_size (int): The batch size for the data loader. + num_batches (int, optional): The number of batches (default is + None). + + Returns: + generator: The dataset generator. + """ + if batch_size is None: + batch_size = self.batch_size + + # shuffle data indices + idxs = np.random.permutation(np.arange(X.shape[0])) + + if num_batches is None: + # compute the number of batches + num_batches = int(np.ceil(X.shape[0] / batch_size)) + + # build the generator and return it + return self._batch_generator(X, y, idxs, batch_size, num_batches) diff --git a/openfl/federated/task/runner_jax.py b/openfl/federated/task/runner_jax.py new file mode 100644 index 0000000000..5b1641ce93 --- /dev/null +++ b/openfl/federated/task/runner_jax.py @@ -0,0 +1,489 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +""" +Base classes for developing a keras.Model() Federated Learning model. + +You may copy this file as the starting point of your own keras model. +""" + +import copy +from warnings import catch_warnings, simplefilter + +import numpy as np + +from openfl.federated.task.runner import TaskRunner +from openfl.utilities import Metric, TensorKey, change_tags +from openfl.utilities.split import split_tensor_dict_for_holdouts + +with catch_warnings(): + simplefilter(action="ignore") + import keras + + +class JAXTaskRunner(TaskRunner): + """The base model for Keras models in the federation. + + Attributes: + model (keras.Model): The Keras model. + model_tensor_names (list): List of model tensor names. + required_tensorkeys_for_function (dict): A map of all of the required + tensors for each of the public functions in KerasTaskRunner. + """ + + def __init__(self, **kwargs): + """Initializes the KerasTaskRunner instance. + + Args: + **kwargs: Additional parameters to pass to the function + """ + super().__init__(**kwargs) + + self.model = keras.models.Model() + + self.model_tensor_names = [] + + # this is a map of all of the required tensors for each of the public + # functions in KerasTaskRunner + self.required_tensorkeys_for_function = {} + + def rebuild_model(self, round_num, input_tensor_dict, validation=False): + """Parse tensor names and update weights of model. Handles the + optimizer treatment. + + Args: + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. + validation (bool, optional): If True, validate the model. Defaults + to False. + """ + if self.opt_treatment == "RESET": + self.reset_opt_vars() + self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) + elif round_num > 0 and self.opt_treatment == "CONTINUE_GLOBAL" and not validation: + self.set_tensor_dict(input_tensor_dict, with_opt_vars=True) + else: + self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) + + def train_task( + self, + col_name, + round_num, + input_tensor_dict, + metrics, + epochs=1, + batch_size=1, + **kwargs, + ): + """ + Perform the training. Is expected to perform draws randomly, without + replacement until data is exausted. Then data is replaced and shuffled + and draws continue. + + Args: + col_name (str): The collaborator name. + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. + metrics (list): List of metrics. + epochs (int, optional): Number of epochs to train. Defaults to 1. + batch_size (int, optional): Batch size. Defaults to 1. + **kwargs: Additional parameters. + + Returns: + global_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + local_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + """ + if metrics is None: + raise KeyError("metrics must be defined") + + # rebuild model with updated weights + self.rebuild_model(round_num, input_tensor_dict) + for epoch in range(epochs): + self.logger.info("Run %s epoch of %s round", epoch, round_num) + results = self.train_( + self.data_loader.get_train_loader(batch_size), + metrics=metrics, + **kwargs, + ) + + # output metric tensors (scalar) + origin = col_name + tags = ("trained",) + output_metric_dict = { + TensorKey(metric_name, origin, round_num, True, ("metric",)): metric_value + for (metric_name, metric_value) in results + } + + # output model tensors (Doesn't include TensorKey) + output_model_dict = self.get_tensor_dict(with_opt_vars=True) + global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( + self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs + ) + + # create global tensorkeys + global_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num, False, tags): nparray + for tensor_name, nparray in global_model_dict.items() + } + # create tensorkeys that should stay local + local_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num, False, tags): nparray + for tensor_name, nparray in local_model_dict.items() + } + # the train/validate aggregated function of the next round will look + # for the updated model parameters. + # this ensures they will be resolved locally + next_local_tensorkey_model_dict = { + TensorKey(tensor_name, origin, round_num + 1, False, ("model",)): nparray + for tensor_name, nparray in local_model_dict.items() + } + + global_tensor_dict = { + **output_metric_dict, + **global_tensorkey_model_dict, + } + local_tensor_dict = { + **local_tensorkey_model_dict, + **next_local_tensorkey_model_dict, + } + + # update the required tensors if they need to be pulled from the + # aggregator + # TODO this logic can break if different collaborators have different + # roles between rounds. + # for example, if a collaborator only performs validation in the first + # round but training in the second, it has no way of knowing the + # optimizer state tensor names to request from the aggregator because + # these are only created after training occurs. A work around could + # involve doing a single epoch of training on random data to get the + # optimizer names, and then throwing away the model. + if self.opt_treatment == "CONTINUE_GLOBAL": + self.initialize_tensorkeys_for_functions(with_opt_vars=True) + + self.update_tensorkeys_for_functions() + return global_tensor_dict, local_tensor_dict + + def train_(self, batch_generator, metrics: list = None, **kwargs): + """Train single epoch. Override this function for custom training. + + Args: + batch_generator (generator): Generator of training batches. + metrics (list, optional): Names of metrics to save. Defaults to + None. + **kwargs: Additional parameters. + + Returns: + results (list): List of Metric objects. + """ + if metrics is None: + metrics = [] + # TODO Currently assuming that all metrics are defined at + # initialization (build_model). + # If metrics are added (i.e. not a subset of what was originally + # defined) then the model must be recompiled. + try: + results = self.model.get_metrics_result() + except ValueError: + if "batch_size" in kwargs: + batch_size = kwargs["batch_size"] + else: + batch_size = 1 + # evaluation needed before metrics can be resolved + self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) + results = self.model.get_metrics_result() + + # TODO if there are new metrics in the flplan that were not included + # in the originally + # compiled model, that behavior is not currently handled. + for param in metrics: + if param not in results: + raise ValueError( + f"KerasTaskRunner does not support specifying new metrics. " + f"Param_metrics = {metrics}" + ) + + history = self.model.fit(batch_generator, verbose=2, **kwargs) + results = [] + for metric in metrics: + value = np.mean([history.history[metric]]) + results.append(Metric(name=metric, value=np.array(value))) + return results + + def validate_task(self, col_name, round_num, input_tensor_dict, **kwargs): + """Run the trained model on validation data; report results. + + Args: + col_name (str): The collaborator name. + round_num (int): The round number. + input_tensor_dict (dict): The input tensor dictionary. Either the + last aggregated or locally trained model + **kwargs: Additional parameters. + + Returns: + output_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. + These correspond to acc, precision, f1_score, etc. + dict: Empty dictionary. + """ + if "batch_size" in kwargs: + batch_size = kwargs["batch_size"] + else: + batch_size = 1 + + self.rebuild_model(round_num, input_tensor_dict, validation=True) + param_metrics = kwargs["metrics"] + + self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) + results = self.model.get_metrics_result() + + # TODO if there are new metrics in the flplan that were not included in + # the originally compiled model, that behavior is not currently + # handled. + for param in param_metrics: + if param not in results: + raise ValueError( + f"KerasTaskRunner does not support specifying new metrics. " + f"Param_metrics = {param_metrics}" + ) + + origin = col_name + suffix = "validate" + if kwargs["apply"] == "local": + suffix += "_local" + else: + suffix += "_agg" + tags = ("metric",) + tags = change_tags(tags, add_field=suffix) + output_tensor_dict = { + TensorKey(metric, origin, round_num, True, tags): np.array(results[metric]) + for metric in param_metrics + } + + return output_tensor_dict, {} + + def save_native(self, filepath): + """Save model. + + Args: + filepath (str): The file path to save the model. + """ + self.model.export(filepath) + + def load_native(self, filepath): + """Load model. + + Args: + filepath (str): The file path to load the model. + """ + self.model = keras.models.load_model(filepath) + + @staticmethod + def _get_weights_names(obj): + """Get the list of weight names. + + Args: + obj (Model or Optimizer): The target object that we want to get + the weights. + + Returns: + weight_names (list): The weight name list. + """ + if isinstance(obj, keras.optimizers.Optimizer): + weight_names = [weight.name for weight in obj.variables] + else: + weight_names = [ + layer.name + "/" + weight.name for layer in obj.layers for weight in layer.weights + ] + return weight_names + + @staticmethod + def _get_weights_dict(obj, suffix=""): + """ + Get the dictionary of weights. + + Args: + obj (Model or Optimizer): The target object that we want to get + the weights. + suffix (str, optional): Suffix for weight names. Defaults to ''. + + Returns: + weights_dict (dict): The weight dictionary. + """ + weights_dict = {} + weight_names = KerasTaskRunner._get_weights_names(obj) + if isinstance(obj, keras.optimizers.Optimizer): + weights_dict = { + weight_names[i] + suffix: weight.numpy() + for i, weight in enumerate(copy.deepcopy(obj.variables)) + } + else: + weight_name_index = 0 + for layer in obj.layers: + if weight_name_index < len(weight_names) and len(layer.get_weights()) > 0: + for weight in layer.get_weights(): + weights_dict[weight_names[weight_name_index] + suffix] = weight + weight_name_index += 1 + return weights_dict + + @staticmethod + def _set_weights_dict(obj, weights_dict): + """Set the object weights with a dictionary. + + Args: + obj (Model or Optimizer): The target object that we want to set + the weights. + weights_dict (dict): The weight dictionary. + """ + weight_names = KerasTaskRunner._get_weights_names(obj) + weight_values = [weights_dict[name] for name in weight_names] + obj.set_weights(weight_values) + + def get_tensor_dict(self, with_opt_vars, suffix=""): + """ + Get the model weights as a tensor dictionary. + + Args: + with_opt_vars (bool): If we should include the optimizer's status. + suffix (str): Universally. + + Returns: + model_weights (dict): The tensor dictionary. + """ + model_weights = self._get_weights_dict(self.model, suffix) + + if with_opt_vars: + opt_weights = self._get_weights_dict(self.model.optimizer, suffix) + + model_weights.update(opt_weights) + if len(opt_weights) == 0: + self.logger.debug("WARNING: We didn't find variables for the optimizer.") + return model_weights + + def set_tensor_dict(self, tensor_dict, with_opt_vars): + """Set the model weights with a tensor dictionary. + + Args: + tensor_dict (dict): The tensor dictionary. + with_opt_vars (bool): True = include the optimizer's status. + """ + if with_opt_vars is False: + # It is possible to pass in opt variables from the input tensor + # dict. This will make sure that the correct layers are updated + model_weight_names = self._get_weights_names(self.model) + model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} + self._set_weights_dict(self.model, model_weights_dict) + else: + model_weight_names = self._get_weights_names(self.model) + model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} + opt_weight_names = self._get_weights_names(self.model.optimizer) + opt_weights_dict = {name: tensor_dict[name] for name in opt_weight_names} + self._set_weights_dict(self.model, model_weights_dict) + self._set_weights_dict(self.model.optimizer, opt_weights_dict) + + def reset_opt_vars(self): + """Resets the optimizer variables.""" + pass + + def get_required_tensorkeys_for_function(self, func_name, **kwargs): + """Get the required tensors for specified function that could be called + as part of a task. + + By default, this is just all of the layers and optimizer of the model. + + Args: + func_name (str): The function name. + **kwargs: Any function arguments. + + Returns: + list: List of TensorKey objects. + """ + if func_name == "validate_task": + local_model = "apply=" + str(kwargs["apply"]) + return self.required_tensorkeys_for_function[func_name][local_model] + else: + return self.required_tensorkeys_for_function[func_name] + + def update_tensorkeys_for_functions(self): + """Update the required tensors for all publicly accessible methods that + could be called as part of a task. + + By default, this is just all of the layers and optimizer of the model. + Custom tensors should be added to this function + """ + # TODO complete this function. It is only needed for opt_treatment, + # and making the model stateless + + # Minimal required tensors for train function + model_layer_names = self._get_weights_names(self.model) + opt_names = self._get_weights_names(self.model.optimizer) + tensor_names = model_layer_names + opt_names + self.logger.debug("Updating model tensor names: %s", tensor_names) + self.required_tensorkeys_for_function["train_task"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names + ] + + # Validation may be performed on local or aggregated (global) model, + # so there is an extra lookup dimension for kwargs + self.required_tensorkeys_for_function["validate_task"] = {} + self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ + TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) for tensor_name in tensor_names + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names + ] + + def initialize_tensorkeys_for_functions(self, with_opt_vars=False): + """Set the required tensors for all publicly accessible methods that + could be called as part of a task. + + By default, this is just all of the layers and optimizer of the model. + Custom tensors should be added to this function + + Args: + with_opt_vars (bool, optional): If True, include the optimizer's + status. Defaults to False. + """ + # TODO there should be a way to programmatically iterate through all + # of the methods in the class and declare the tensors. + # For now this is done manually + + output_model_dict = self.get_tensor_dict(with_opt_vars=with_opt_vars) + global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( + self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs + ) + if not with_opt_vars: + global_model_dict_val = global_model_dict + local_model_dict_val = local_model_dict + else: + output_model_dict = self.get_tensor_dict(with_opt_vars=False) + global_model_dict_val, local_model_dict_val = split_tensor_dict_for_holdouts( + self.logger, + output_model_dict, + **self.tensor_dict_split_fn_kwargs, + ) + + self.required_tensorkeys_for_function["train_task"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) + for tensor_name in global_model_dict + ] + self.required_tensorkeys_for_function["train_task"] += [ + TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) + for tensor_name in local_model_dict + ] + + # Validation may be performed on local or aggregated (global) model, + # so there is an extra lookup dimension for kwargs + self.required_tensorkeys_for_function["validate_task"] = {} + # TODO This is not stateless. The optimizer will not be + self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ + TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) + for tensor_name in {**global_model_dict_val, **local_model_dict_val} + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ + TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) + for tensor_name in global_model_dict_val + ] + self.required_tensorkeys_for_function["validate_task"]["apply=global"] += [ + TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) + for tensor_name in local_model_dict_val + ] diff --git a/openfl/native/native.py b/openfl/native/native.py index c1b3813f92..5a5153b8a2 100644 --- a/openfl/native/native.py +++ b/openfl/native/native.py @@ -216,8 +216,8 @@ def init( workspace_template (str): The template that should be used as the basis for the experiment. Defaults to 'default'. Other options include are any of the template names - [keras/cnn_mnist, tf_2dunet, tf_cnn_histology, - mtorch_cnn_histology, torch_cnn_mnist]. + [keras/mnist, tf_2dunet, tf_cnn_histology, + mtorch_cnn_histology, torch/mnist]. log_level (str): Log level for logging. METRIC level is available. Defaults to 'INFO'. log_file (str): Name of the file in which the log will be duplicated. diff --git a/tests/end_to_end/README.md b/tests/end_to_end/README.md index 0495629c1e..cd6ada7227 100644 --- a/tests/end_to_end/README.md +++ b/tests/end_to_end/README.md @@ -49,16 +49,16 @@ Below parameters are available for modification: 4. --disable_tls - to disable TLS communication (by default it is enabled) 5. --disable_client_auth - to disable the client authentication (by default it is enabled) -For example, to run Task runner (bare metal approach) with - torch_cnn_mnist model, 3 collaborators, 5 rounds and non-TLS scenario: +For example, to run Task runner (bare metal approach) with - torch/mnist model, 3 collaborators, 5 rounds and non-TLS scenario: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_basic --num_rounds 5 --num_collaborators 3 --model_name torch/mnist --disable_tls ``` -And, to run Task runner (via dockerized workspace) with keras/cnn_mnist, 2 collaborators, 3 rounds: +And, to run Task runner (via dockerized workspace) with keras/mnist, 2 collaborators, 3 rounds: ```sh -python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/cnn_mnist +python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 3 --num_collaborators 2 --model_name keras/mnist ``` ### Fixture and marker mapping: diff --git a/tests/end_to_end/utils/conftest_helper.py b/tests/end_to_end/utils/conftest_helper.py index d623c21201..af912dd2c6 100644 --- a/tests/end_to_end/utils/conftest_helper.py +++ b/tests/end_to_end/utils/conftest_helper.py @@ -17,7 +17,7 @@ def parse_arguments(): - results_dir (str, optional): Directory to store the results - num_collaborators (int, default=2): Number of collaborators - num_rounds (int, default=5): Number of rounds to train - - model_name (str, default="torch_cnn_mnist"): Model name + - model_name (str, default="torch/mnist"): Model name - disable_client_auth (bool): Disable client authentication - disable_tls (bool): Disable TLS for communication - log_memory_usage (bool): Enable Memory leak logs diff --git a/tests/end_to_end/utils/constants.py b/tests/end_to_end/utils/constants.py index 971174c381..e370093f5f 100644 --- a/tests/end_to_end/utils/constants.py +++ b/tests/end_to_end/utils/constants.py @@ -10,15 +10,15 @@ class ModelName(Enum): """ # IMP - The model name must be same (and in uppercase) as the model value. # This is used to identify the model in the tests. - TORCH_CNN_MNIST = "torch_cnn_mnist" - KERAS_CNN_MNIST = "keras/cnn_mnist" - TORCH_CNN_HISTOLOGY = "torch_cnn_histology" + TORCH_CNN_MNIST = "torch/mnist" + KERAS_CNN_MNIST = "keras/mnist" + TORCH_CNN_HISTOLOGY = "torch/histology" XGB_HIGGS = "xgb_higgs" NUM_COLLABORATORS = 2 NUM_ROUNDS = 5 WORKSPACE_NAME = "my_federation" -DEFAULT_MODEL_NAME = "torch_cnn_mnist" +DEFAULT_MODEL_NAME = "torch/mnist" SUCCESS_MARKER = "✔️ OK" # Docker specific constants diff --git a/tests/github/test_double_ws_export.py b/tests/github/test_double_ws_export.py index 7e9b42bec3..07d2714ce8 100644 --- a/tests/github/test_double_ws_export.py +++ b/tests/github/test_double_ws_export.py @@ -22,7 +22,7 @@ def main(): for entry in iterator: if entry.name not in ['__init__.py', 'workspace', 'default']: workspace_choice.append(entry.name) - parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') diff --git a/tests/github/test_gandlf.py b/tests/github/test_gandlf.py index 08e80e2118..a8696be541 100644 --- a/tests/github/test_gandlf.py +++ b/tests/github/test_gandlf.py @@ -21,7 +21,7 @@ def exec(command, directory): def main(): parser = argparse.ArgumentParser() - parser.add_argument('--template', default='keras/cnn_mnist') + parser.add_argument('--template', default='keras/mnist') parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one') parser.add_argument('--col2', default='two') diff --git a/tests/github/test_hello_federation.py b/tests/github/test_hello_federation.py index a3ef7296a8..a769ffcb58 100644 --- a/tests/github/test_hello_federation.py +++ b/tests/github/test_hello_federation.py @@ -24,7 +24,7 @@ def main(): dir_path = dir_path.replace(os.sep, '/') if dir_path and not any(dir_path.startswith(prefix) for prefix in excluded_dirs): workspace_choice.append(dir_path) - parser.add_argument('--template', default='keras/cnn_mnist', choices=workspace_choice) + parser.add_argument('--template', default='keras/mnist', choices=workspace_choice) parser.add_argument('--fed_workspace', default='fed_work12345alpha81671') parser.add_argument('--col1', default='one123dragons') parser.add_argument('--col2', default='beta34unicorns') From ef297f4bd629fd8190128619f0b59ffe89f5b4d0 Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:21:07 -0800 Subject: [PATCH 14/15] code changes Signed-off-by: yes --- Jenkinsfile | 2 +- openfl-workspace/JAX/.workspace | 2 - openfl-workspace/JAX/plan/cols.yaml | 5 - openfl-workspace/JAX/plan/data.yaml | 7 - openfl-workspace/JAX/plan/defaults | 2 - openfl-workspace/JAX/plan/plan.yaml | 46 --- openfl-workspace/JAX/requirements.txt | 3 - openfl-workspace/JAX/src/__init__.py | 3 - openfl-workspace/JAX/src/dataloader.py | 47 --- openfl-workspace/JAX/src/mnist_utils.py | 118 ------ openfl-workspace/JAX/src/taskrunner.py | 78 ---- openfl/federated/data/loader_jax.py | 138 ------- openfl/federated/task/runner_jax.py | 489 ------------------------ 13 files changed, 1 insertion(+), 939 deletions(-) delete mode 100644 openfl-workspace/JAX/.workspace delete mode 100644 openfl-workspace/JAX/plan/cols.yaml delete mode 100644 openfl-workspace/JAX/plan/data.yaml delete mode 100644 openfl-workspace/JAX/plan/defaults delete mode 100644 openfl-workspace/JAX/plan/plan.yaml delete mode 100644 openfl-workspace/JAX/requirements.txt delete mode 100644 openfl-workspace/JAX/src/__init__.py delete mode 100644 openfl-workspace/JAX/src/dataloader.py delete mode 100644 openfl-workspace/JAX/src/mnist_utils.py delete mode 100644 openfl-workspace/JAX/src/taskrunner.py delete mode 100644 openfl/federated/data/loader_jax.py delete mode 100644 openfl/federated/task/runner_jax.py diff --git a/Jenkinsfile b/Jenkinsfile index bd97ab7f2b..c1c9a374fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,7 @@ def snykData = [ 'openfl-workspace_torch_cnn_histology_src': 'openfl-workspace/torch/histology/src/requirements.txt', 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', - 'openfl-workspace_torch/unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', + 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', diff --git a/openfl-workspace/JAX/.workspace b/openfl-workspace/JAX/.workspace deleted file mode 100644 index 3c2c5d08b4..0000000000 --- a/openfl-workspace/JAX/.workspace +++ /dev/null @@ -1,2 +0,0 @@ -current_plan_name: default - diff --git a/openfl-workspace/JAX/plan/cols.yaml b/openfl-workspace/JAX/plan/cols.yaml deleted file mode 100644 index 95307de3bc..0000000000 --- a/openfl-workspace/JAX/plan/cols.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -collaborators: - \ No newline at end of file diff --git a/openfl-workspace/JAX/plan/data.yaml b/openfl-workspace/JAX/plan/data.yaml deleted file mode 100644 index 257c7825fe..0000000000 --- a/openfl-workspace/JAX/plan/data.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -# collaborator_name,data_directory_path -one,1 - - diff --git a/openfl-workspace/JAX/plan/defaults b/openfl-workspace/JAX/plan/defaults deleted file mode 100644 index fb82f9c5b6..0000000000 --- a/openfl-workspace/JAX/plan/defaults +++ /dev/null @@ -1,2 +0,0 @@ -../../workspace/plan/defaults - diff --git a/openfl-workspace/JAX/plan/plan.yaml b/openfl-workspace/JAX/plan/plan.yaml deleted file mode 100644 index 54867f4578..0000000000 --- a/openfl-workspace/JAX/plan/plan.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# Licensed subject to the terms of the separately executed evaluation license agreement between Intel Corporation and you. - -aggregator : - defaults : plan/defaults/aggregator.yaml - template : openfl.component.Aggregator - settings : - init_state_path : save/init.pbuf - best_state_path : save/best.pbuf - last_state_path : save/last.pbuf - rounds_to_train : 10 - -collaborator : - defaults : plan/defaults/collaborator.yaml - template : openfl.component.Collaborator - settings : - delta_updates : false - opt_treatment : RESET - -data_loader : - defaults : plan/defaults/data_loader.yaml - template : src.dataloader.KerasMNISTInMemory - settings : - collaborator_count : 2 - data_group_name : mnist - batch_size : 256 - -task_runner : - defaults : plan/defaults/task_runner.yaml - template : src.taskrunner.KerasCNN - -network : - defaults : plan/defaults/network.yaml - -assigner : - defaults : plan/defaults/assigner.yaml - -tasks : - defaults : plan/defaults/tasks_keras.yaml - -compression_pipeline : - defaults : plan/defaults/compression_pipeline.yaml - # To use different Compression Pipeline, uncomment the following lines - # template : openfl.pipelines.KCPipeline - # settings : - # n_clusters : 6 diff --git a/openfl-workspace/JAX/requirements.txt b/openfl-workspace/JAX/requirements.txt deleted file mode 100644 index 858a7dc3c8..0000000000 --- a/openfl-workspace/JAX/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -keras==3.8.0 -tensorflow==2.18.0 - diff --git a/openfl-workspace/JAX/src/__init__.py b/openfl-workspace/JAX/src/__init__.py deleted file mode 100644 index f1410b1298..0000000000 --- a/openfl-workspace/JAX/src/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -"""You may copy this file as the starting point of your own model.""" diff --git a/openfl-workspace/JAX/src/dataloader.py b/openfl-workspace/JAX/src/dataloader.py deleted file mode 100644 index 040e8091c9..0000000000 --- a/openfl-workspace/JAX/src/dataloader.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from openfl.federated import KerasDataLoader -from .mnist_utils import load_mnist_shard - - -class KerasMNISTInMemory(KerasDataLoader): - """Data Loader for MNIST Dataset.""" - - def __init__(self, data_path, batch_size, **kwargs): - """ - Initialize. - - Args: - data_path: File path for the dataset - batch_size (int): The batch size for the data loader - **kwargs: Additional arguments, passed to super init and load_mnist_shard - """ - super().__init__(batch_size, **kwargs) - - # TODO: We should be downloading the dataset shard into a directory - # TODO: There needs to be a method to ask how many collaborators and - # what index/rank is this collaborator. - # Then we have a way to automatically shard based on rank and size of - # collaborator list. - try: - int(data_path) - except: - raise ValueError( - "Expected `%s` to be representable as `int`, as it refers to the data shard " + - "number used by the collaborator.", - data_path - ) - - _, num_classes, X_train, y_train, X_valid, y_valid = load_mnist_shard( - shard_num=int(data_path), **kwargs - ) - - self.X_train = X_train - self.y_train = y_train - self.X_valid = X_valid - self.y_valid = y_valid - - self.num_classes = num_classes diff --git a/openfl-workspace/JAX/src/mnist_utils.py b/openfl-workspace/JAX/src/mnist_utils.py deleted file mode 100644 index d19e13d9dd..0000000000 --- a/openfl-workspace/JAX/src/mnist_utils.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright (C) 2020-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from logging import getLogger - -import numpy as np -from tensorflow.python.keras.utils.data_utils import get_file - -logger = getLogger(__name__) - - -def one_hot(labels, classes): - """ - One Hot encode a vector. - - Args: - labels (list): List of labels to onehot encode - classes (int): Total number of categorical classes - - Returns: - np.array: Matrix of one-hot encoded labels - """ - return np.eye(classes)[labels] - - -def _load_raw_datashards(shard_num, collaborator_count): - """ - Load the raw data by shard. - - Returns tuples of the dataset shard divided into training and validation. - - Args: - shard_num (int): The shard number to use - collaborator_count (int): The number of collaborators in the federation - - Returns: - 2 tuples: (image, label) of the training, validation dataset - """ - origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' - path = get_file('mnist.npz', - origin=origin_folder + 'mnist.npz', - file_hash='731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1') - - with np.load(path) as f: - # get all of mnist - X_train_tot = f['x_train'] - y_train_tot = f['y_train'] - - X_valid_tot = f['x_test'] - y_valid_tot = f['y_test'] - - # create the shards - shard_num = int(shard_num) - X_train = X_train_tot[shard_num::collaborator_count] - y_train = y_train_tot[shard_num::collaborator_count] - - X_valid = X_valid_tot[shard_num::collaborator_count] - y_valid = y_valid_tot[shard_num::collaborator_count] - - return (X_train, y_train), (X_valid, y_valid) - - -def load_mnist_shard(shard_num, collaborator_count, categorical=True, - channels_last=True, **kwargs): - """ - Load the MNIST dataset. - - Args: - shard_num (int): The shard to use from the dataset - collaborator_count (int): The number of collaborators in the federation - categorical (bool): True = convert the labels to one-hot encoded - vectors (Default = True) - channels_last (bool): True = The input images have the channels - last (Default = True) - **kwargs: Additional parameters to pass to the function - - Returns: - list: The input shape - int: The number of classes - numpy.ndarray: The training data - numpy.ndarray: The training labels - numpy.ndarray: The validation data - numpy.ndarray: The validation labels - """ - img_rows, img_cols = 28, 28 - num_classes = 10 - - (X_train, y_train), (X_valid, y_valid) = _load_raw_datashards( - shard_num, collaborator_count - ) - - if channels_last: - X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) - X_valid = X_valid.reshape(X_valid.shape[0], img_rows, img_cols, 1) - input_shape = (img_rows, img_cols, 1) - else: - X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols) - X_valid = X_valid.reshape(X_valid.shape[0], 1, img_rows, img_cols) - input_shape = (1, img_rows, img_cols) - - X_train = X_train.astype('float32') - X_valid = X_valid.astype('float32') - X_train /= 255 - X_valid /= 255 - - logger.info(f'MNIST > X_train Shape : {X_train.shape}') - logger.info(f'MNIST > y_train Shape : {y_train.shape}') - logger.info(f'MNIST > Train Samples : {X_train.shape[0]}') - logger.info(f'MNIST > Valid Samples : {X_valid.shape[0]}') - - if categorical: - # convert class vectors to binary class matrices - y_train = one_hot(y_train, num_classes) - y_valid = one_hot(y_valid, num_classes) - - return input_shape, num_classes, X_train, y_train, X_valid, y_valid diff --git a/openfl-workspace/JAX/src/taskrunner.py b/openfl-workspace/JAX/src/taskrunner.py deleted file mode 100644 index 165861033c..0000000000 --- a/openfl-workspace/JAX/src/taskrunner.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (C) 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""You may copy this file as the starting point of your own model.""" - -from keras.models import Sequential -from keras.layers import Conv2D -from keras.layers import Dense -from keras.layers import Flatten - -from openfl.federated import KerasTaskRunner - - -class KerasCNN(KerasTaskRunner): - """A basic convolutional neural network model.""" - - def __init__(self, **kwargs): - """ - Initialize. - - Args: - **kwargs: Additional parameters to pass to the function - """ - super().__init__(**kwargs) - - self.model = self.build_model(self.feature_shape, self.data_loader.num_classes, **kwargs) - - self.initialize_tensorkeys_for_functions() - - self.model.summary(print_fn=self.logger.info) - - self.logger.info(f'Train Set Size : {self.get_train_data_size()}') - self.logger.info(f'Valid Set Size : {self.get_valid_data_size()}') - - def build_model(self, - input_shape, - num_classes, - conv_kernel_size=(4, 4), - conv_strides=(2, 2), - conv1_channels_out=16, - conv2_channels_out=32, - final_dense_inputsize=100, - **kwargs): - """ - Define the model architecture. - - Args: - input_shape (numpy.ndarray): The shape of the data - num_classes (int): The number of classes of the dataset - - Returns: - keras.models.Sequential: The model defined in Keras - - """ - model = Sequential() - - model.add(Conv2D(conv1_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu', - input_shape=input_shape)) - - model.add(Conv2D(conv2_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu')) - - model.add(Flatten()) - - model.add(Dense(final_dense_inputsize, activation='relu')) - - model.add(Dense(num_classes, activation='softmax')) - - model.compile(loss="categorical_crossentropy", - optimizer="adam", - metrics=["accuracy"]) - - return model diff --git a/openfl/federated/data/loader_jax.py b/openfl/federated/data/loader_jax.py deleted file mode 100644 index bf5a460186..0000000000 --- a/openfl/federated/data/loader_jax.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -"""KerasDataLoader module.""" - -import numpy as np - -from openfl.federated.data.loader import DataLoader - - -class JAXDataLoader(DataLoader): - """A class used to represent a Federation Data Loader for Keras models. - - Attributes: - batch_size (int): Size of batches used for all data loaders. - X_train (np.array): Training features. - y_train (np.array): Training labels. - X_valid (np.array): Validation features. - y_valid (np.array): Validation labels. - """ - - def __init__(self, batch_size, **kwargs): - """Initializes the KerasDataLoader object. - - Args: - batch_size (int): The size of batches used for all data loaders. - kwargs: Additional arguments to pass to the function. - """ - self.batch_size = batch_size - self.X_train = None - self.y_train = None - self.X_valid = None - self.y_valid = None - - # Child classes should have init signature: - # (self, batch_size, **kwargs), should call this __init__ and then - # define self.X_train, self.y_train, self.X_valid, and self.y_valid - - def get_feature_shape(self): - """Returns the shape of an example feature array. - - Returns: - tuple: The shape of an example feature array. - """ - return self.X_train[0].shape - - def get_train_loader(self, batch_size=None, num_batches=None): - """Returns the data loader for the training data. - - Args: - batch_size (int, optional): The batch size for the data loader - (default is None). - num_batches (int, optional): The number of batches for the data - loader (default is None). - - Returns: - DataLoader: The DataLoader object for the training data. - """ - return self._get_batch_generator( - X=self.X_train, - y=self.y_train, - batch_size=batch_size, - num_batches=num_batches, - ) - - def get_valid_loader(self, batch_size=None): - """Returns the data loader for the validation data. - - Args: - batch_size (int, optional): The batch size for the data loader - (default is None). - - Returns: - DataLoader: The DataLoader object for the validation data. - """ - return self._get_batch_generator(X=self.X_valid, y=self.y_valid, batch_size=batch_size) - - def get_train_data_size(self): - """Returns the total number of training samples. - - Returns: - int: The total number of training samples. - """ - return self.X_train.shape[0] - - def get_valid_data_size(self): - """Returns the total number of validation samples. - - Returns: - int: The total number of validation samples. - """ - return self.X_valid.shape[0] - - @staticmethod - def _batch_generator(X, y, idxs, batch_size, num_batches): - """Generates batches of data. - - Args: - X (np.array): The input data. - y (np.array): The label data. - idxs (np.array): The index of the dataset. - batch_size (int): The batch size for the data loader. - num_batches (int): The number of batches. - - Yields: - tuple: The input data and label data for each batch. - """ - for i in range(num_batches): - a = i * batch_size - b = a + batch_size - yield X[idxs[a:b]], y[idxs[a:b]] - - def _get_batch_generator(self, X, y, batch_size, num_batches=None): - """Returns the dataset generator. - - Args: - X (np.array): The input data. - y (np.array): The label data. - batch_size (int): The batch size for the data loader. - num_batches (int, optional): The number of batches (default is - None). - - Returns: - generator: The dataset generator. - """ - if batch_size is None: - batch_size = self.batch_size - - # shuffle data indices - idxs = np.random.permutation(np.arange(X.shape[0])) - - if num_batches is None: - # compute the number of batches - num_batches = int(np.ceil(X.shape[0] / batch_size)) - - # build the generator and return it - return self._batch_generator(X, y, idxs, batch_size, num_batches) diff --git a/openfl/federated/task/runner_jax.py b/openfl/federated/task/runner_jax.py deleted file mode 100644 index 5b1641ce93..0000000000 --- a/openfl/federated/task/runner_jax.py +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -""" -Base classes for developing a keras.Model() Federated Learning model. - -You may copy this file as the starting point of your own keras model. -""" - -import copy -from warnings import catch_warnings, simplefilter - -import numpy as np - -from openfl.federated.task.runner import TaskRunner -from openfl.utilities import Metric, TensorKey, change_tags -from openfl.utilities.split import split_tensor_dict_for_holdouts - -with catch_warnings(): - simplefilter(action="ignore") - import keras - - -class JAXTaskRunner(TaskRunner): - """The base model for Keras models in the federation. - - Attributes: - model (keras.Model): The Keras model. - model_tensor_names (list): List of model tensor names. - required_tensorkeys_for_function (dict): A map of all of the required - tensors for each of the public functions in KerasTaskRunner. - """ - - def __init__(self, **kwargs): - """Initializes the KerasTaskRunner instance. - - Args: - **kwargs: Additional parameters to pass to the function - """ - super().__init__(**kwargs) - - self.model = keras.models.Model() - - self.model_tensor_names = [] - - # this is a map of all of the required tensors for each of the public - # functions in KerasTaskRunner - self.required_tensorkeys_for_function = {} - - def rebuild_model(self, round_num, input_tensor_dict, validation=False): - """Parse tensor names and update weights of model. Handles the - optimizer treatment. - - Args: - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. - validation (bool, optional): If True, validate the model. Defaults - to False. - """ - if self.opt_treatment == "RESET": - self.reset_opt_vars() - self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) - elif round_num > 0 and self.opt_treatment == "CONTINUE_GLOBAL" and not validation: - self.set_tensor_dict(input_tensor_dict, with_opt_vars=True) - else: - self.set_tensor_dict(input_tensor_dict, with_opt_vars=False) - - def train_task( - self, - col_name, - round_num, - input_tensor_dict, - metrics, - epochs=1, - batch_size=1, - **kwargs, - ): - """ - Perform the training. Is expected to perform draws randomly, without - replacement until data is exausted. Then data is replaced and shuffled - and draws continue. - - Args: - col_name (str): The collaborator name. - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. - metrics (list): List of metrics. - epochs (int, optional): Number of epochs to train. Defaults to 1. - batch_size (int, optional): Batch size. Defaults to 1. - **kwargs: Additional parameters. - - Returns: - global_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - local_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - """ - if metrics is None: - raise KeyError("metrics must be defined") - - # rebuild model with updated weights - self.rebuild_model(round_num, input_tensor_dict) - for epoch in range(epochs): - self.logger.info("Run %s epoch of %s round", epoch, round_num) - results = self.train_( - self.data_loader.get_train_loader(batch_size), - metrics=metrics, - **kwargs, - ) - - # output metric tensors (scalar) - origin = col_name - tags = ("trained",) - output_metric_dict = { - TensorKey(metric_name, origin, round_num, True, ("metric",)): metric_value - for (metric_name, metric_value) in results - } - - # output model tensors (Doesn't include TensorKey) - output_model_dict = self.get_tensor_dict(with_opt_vars=True) - global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( - self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs - ) - - # create global tensorkeys - global_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num, False, tags): nparray - for tensor_name, nparray in global_model_dict.items() - } - # create tensorkeys that should stay local - local_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num, False, tags): nparray - for tensor_name, nparray in local_model_dict.items() - } - # the train/validate aggregated function of the next round will look - # for the updated model parameters. - # this ensures they will be resolved locally - next_local_tensorkey_model_dict = { - TensorKey(tensor_name, origin, round_num + 1, False, ("model",)): nparray - for tensor_name, nparray in local_model_dict.items() - } - - global_tensor_dict = { - **output_metric_dict, - **global_tensorkey_model_dict, - } - local_tensor_dict = { - **local_tensorkey_model_dict, - **next_local_tensorkey_model_dict, - } - - # update the required tensors if they need to be pulled from the - # aggregator - # TODO this logic can break if different collaborators have different - # roles between rounds. - # for example, if a collaborator only performs validation in the first - # round but training in the second, it has no way of knowing the - # optimizer state tensor names to request from the aggregator because - # these are only created after training occurs. A work around could - # involve doing a single epoch of training on random data to get the - # optimizer names, and then throwing away the model. - if self.opt_treatment == "CONTINUE_GLOBAL": - self.initialize_tensorkeys_for_functions(with_opt_vars=True) - - self.update_tensorkeys_for_functions() - return global_tensor_dict, local_tensor_dict - - def train_(self, batch_generator, metrics: list = None, **kwargs): - """Train single epoch. Override this function for custom training. - - Args: - batch_generator (generator): Generator of training batches. - metrics (list, optional): Names of metrics to save. Defaults to - None. - **kwargs: Additional parameters. - - Returns: - results (list): List of Metric objects. - """ - if metrics is None: - metrics = [] - # TODO Currently assuming that all metrics are defined at - # initialization (build_model). - # If metrics are added (i.e. not a subset of what was originally - # defined) then the model must be recompiled. - try: - results = self.model.get_metrics_result() - except ValueError: - if "batch_size" in kwargs: - batch_size = kwargs["batch_size"] - else: - batch_size = 1 - # evaluation needed before metrics can be resolved - self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) - results = self.model.get_metrics_result() - - # TODO if there are new metrics in the flplan that were not included - # in the originally - # compiled model, that behavior is not currently handled. - for param in metrics: - if param not in results: - raise ValueError( - f"KerasTaskRunner does not support specifying new metrics. " - f"Param_metrics = {metrics}" - ) - - history = self.model.fit(batch_generator, verbose=2, **kwargs) - results = [] - for metric in metrics: - value = np.mean([history.history[metric]]) - results.append(Metric(name=metric, value=np.array(value))) - return results - - def validate_task(self, col_name, round_num, input_tensor_dict, **kwargs): - """Run the trained model on validation data; report results. - - Args: - col_name (str): The collaborator name. - round_num (int): The round number. - input_tensor_dict (dict): The input tensor dictionary. Either the - last aggregated or locally trained model - **kwargs: Additional parameters. - - Returns: - output_tensor_dict (dict): Dictionary of 'TensorKey: nparray'. - These correspond to acc, precision, f1_score, etc. - dict: Empty dictionary. - """ - if "batch_size" in kwargs: - batch_size = kwargs["batch_size"] - else: - batch_size = 1 - - self.rebuild_model(round_num, input_tensor_dict, validation=True) - param_metrics = kwargs["metrics"] - - self.model.evaluate(self.data_loader.get_valid_loader(batch_size), verbose=1) - results = self.model.get_metrics_result() - - # TODO if there are new metrics in the flplan that were not included in - # the originally compiled model, that behavior is not currently - # handled. - for param in param_metrics: - if param not in results: - raise ValueError( - f"KerasTaskRunner does not support specifying new metrics. " - f"Param_metrics = {param_metrics}" - ) - - origin = col_name - suffix = "validate" - if kwargs["apply"] == "local": - suffix += "_local" - else: - suffix += "_agg" - tags = ("metric",) - tags = change_tags(tags, add_field=suffix) - output_tensor_dict = { - TensorKey(metric, origin, round_num, True, tags): np.array(results[metric]) - for metric in param_metrics - } - - return output_tensor_dict, {} - - def save_native(self, filepath): - """Save model. - - Args: - filepath (str): The file path to save the model. - """ - self.model.export(filepath) - - def load_native(self, filepath): - """Load model. - - Args: - filepath (str): The file path to load the model. - """ - self.model = keras.models.load_model(filepath) - - @staticmethod - def _get_weights_names(obj): - """Get the list of weight names. - - Args: - obj (Model or Optimizer): The target object that we want to get - the weights. - - Returns: - weight_names (list): The weight name list. - """ - if isinstance(obj, keras.optimizers.Optimizer): - weight_names = [weight.name for weight in obj.variables] - else: - weight_names = [ - layer.name + "/" + weight.name for layer in obj.layers for weight in layer.weights - ] - return weight_names - - @staticmethod - def _get_weights_dict(obj, suffix=""): - """ - Get the dictionary of weights. - - Args: - obj (Model or Optimizer): The target object that we want to get - the weights. - suffix (str, optional): Suffix for weight names. Defaults to ''. - - Returns: - weights_dict (dict): The weight dictionary. - """ - weights_dict = {} - weight_names = KerasTaskRunner._get_weights_names(obj) - if isinstance(obj, keras.optimizers.Optimizer): - weights_dict = { - weight_names[i] + suffix: weight.numpy() - for i, weight in enumerate(copy.deepcopy(obj.variables)) - } - else: - weight_name_index = 0 - for layer in obj.layers: - if weight_name_index < len(weight_names) and len(layer.get_weights()) > 0: - for weight in layer.get_weights(): - weights_dict[weight_names[weight_name_index] + suffix] = weight - weight_name_index += 1 - return weights_dict - - @staticmethod - def _set_weights_dict(obj, weights_dict): - """Set the object weights with a dictionary. - - Args: - obj (Model or Optimizer): The target object that we want to set - the weights. - weights_dict (dict): The weight dictionary. - """ - weight_names = KerasTaskRunner._get_weights_names(obj) - weight_values = [weights_dict[name] for name in weight_names] - obj.set_weights(weight_values) - - def get_tensor_dict(self, with_opt_vars, suffix=""): - """ - Get the model weights as a tensor dictionary. - - Args: - with_opt_vars (bool): If we should include the optimizer's status. - suffix (str): Universally. - - Returns: - model_weights (dict): The tensor dictionary. - """ - model_weights = self._get_weights_dict(self.model, suffix) - - if with_opt_vars: - opt_weights = self._get_weights_dict(self.model.optimizer, suffix) - - model_weights.update(opt_weights) - if len(opt_weights) == 0: - self.logger.debug("WARNING: We didn't find variables for the optimizer.") - return model_weights - - def set_tensor_dict(self, tensor_dict, with_opt_vars): - """Set the model weights with a tensor dictionary. - - Args: - tensor_dict (dict): The tensor dictionary. - with_opt_vars (bool): True = include the optimizer's status. - """ - if with_opt_vars is False: - # It is possible to pass in opt variables from the input tensor - # dict. This will make sure that the correct layers are updated - model_weight_names = self._get_weights_names(self.model) - model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} - self._set_weights_dict(self.model, model_weights_dict) - else: - model_weight_names = self._get_weights_names(self.model) - model_weights_dict = {name: tensor_dict[name] for name in model_weight_names} - opt_weight_names = self._get_weights_names(self.model.optimizer) - opt_weights_dict = {name: tensor_dict[name] for name in opt_weight_names} - self._set_weights_dict(self.model, model_weights_dict) - self._set_weights_dict(self.model.optimizer, opt_weights_dict) - - def reset_opt_vars(self): - """Resets the optimizer variables.""" - pass - - def get_required_tensorkeys_for_function(self, func_name, **kwargs): - """Get the required tensors for specified function that could be called - as part of a task. - - By default, this is just all of the layers and optimizer of the model. - - Args: - func_name (str): The function name. - **kwargs: Any function arguments. - - Returns: - list: List of TensorKey objects. - """ - if func_name == "validate_task": - local_model = "apply=" + str(kwargs["apply"]) - return self.required_tensorkeys_for_function[func_name][local_model] - else: - return self.required_tensorkeys_for_function[func_name] - - def update_tensorkeys_for_functions(self): - """Update the required tensors for all publicly accessible methods that - could be called as part of a task. - - By default, this is just all of the layers and optimizer of the model. - Custom tensors should be added to this function - """ - # TODO complete this function. It is only needed for opt_treatment, - # and making the model stateless - - # Minimal required tensors for train function - model_layer_names = self._get_weights_names(self.model) - opt_names = self._get_weights_names(self.model.optimizer) - tensor_names = model_layer_names + opt_names - self.logger.debug("Updating model tensor names: %s", tensor_names) - self.required_tensorkeys_for_function["train_task"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names - ] - - # Validation may be performed on local or aggregated (global) model, - # so there is an extra lookup dimension for kwargs - self.required_tensorkeys_for_function["validate_task"] = {} - self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ - TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) for tensor_name in tensor_names - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) for tensor_name in tensor_names - ] - - def initialize_tensorkeys_for_functions(self, with_opt_vars=False): - """Set the required tensors for all publicly accessible methods that - could be called as part of a task. - - By default, this is just all of the layers and optimizer of the model. - Custom tensors should be added to this function - - Args: - with_opt_vars (bool, optional): If True, include the optimizer's - status. Defaults to False. - """ - # TODO there should be a way to programmatically iterate through all - # of the methods in the class and declare the tensors. - # For now this is done manually - - output_model_dict = self.get_tensor_dict(with_opt_vars=with_opt_vars) - global_model_dict, local_model_dict = split_tensor_dict_for_holdouts( - self.logger, output_model_dict, **self.tensor_dict_split_fn_kwargs - ) - if not with_opt_vars: - global_model_dict_val = global_model_dict - local_model_dict_val = local_model_dict - else: - output_model_dict = self.get_tensor_dict(with_opt_vars=False) - global_model_dict_val, local_model_dict_val = split_tensor_dict_for_holdouts( - self.logger, - output_model_dict, - **self.tensor_dict_split_fn_kwargs, - ) - - self.required_tensorkeys_for_function["train_task"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) - for tensor_name in global_model_dict - ] - self.required_tensorkeys_for_function["train_task"] += [ - TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) - for tensor_name in local_model_dict - ] - - # Validation may be performed on local or aggregated (global) model, - # so there is an extra lookup dimension for kwargs - self.required_tensorkeys_for_function["validate_task"] = {} - # TODO This is not stateless. The optimizer will not be - self.required_tensorkeys_for_function["validate_task"]["apply=local"] = [ - TensorKey(tensor_name, "LOCAL", 0, False, ("trained",)) - for tensor_name in {**global_model_dict_val, **local_model_dict_val} - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] = [ - TensorKey(tensor_name, "GLOBAL", 0, False, ("model",)) - for tensor_name in global_model_dict_val - ] - self.required_tensorkeys_for_function["validate_task"]["apply=global"] += [ - TensorKey(tensor_name, "LOCAL", 0, False, ("model",)) - for tensor_name in local_model_dict_val - ] From ea62674503ec6edf718846373667caf9aec57daf Mon Sep 17 00:00:00 2001 From: yes Date: Mon, 27 Jan 2025 21:24:18 -0800 Subject: [PATCH 15/15] code changes Signed-off-by: yes --- Jenkinsfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c1c9a374fb..088b3199f4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ def snykData = [ 'openfl-docker': 'openfl-docker/Dockerfile.base', 'openfl': 'setup.py', - 'openfl-workspace_tf_2dunet': 'openfl-workspace/tf_2dunet/requirements.txt', + 'openfl-workspace_keras_2dunet': 'openfl-workspace/keras/2dunet/requirements.txt', 'openfl-workspace_torch_cnn_mnist_straggler_check': 'openfl-workspace/torch/mnist_straggler_check/requirements.txt', // CN-14619 snyk test CLI does not support -f in requirements.txt file // 'openfl-workspace_torch_cnn_histology': 'openfl-workspace/torch/histology/requirements.txt', @@ -9,9 +9,6 @@ def snykData = [ 'openfl-workspace_keras_nlp': 'openfl-workspace/keras/nlp/requirements.txt', 'openfl-workspace_torch_cnn_mnist': 'openfl-workspace/torch/mnist/requirements.txt', 'openfl-workspace_torch_unet_kvasir': 'openfl-workspace/torch/unet_kvasir/requirements.txt', - 'openfl-workspace_tf_cnn_histology': 'openfl-workspace/tf_cnn_histology/requirements.txt', - 'openfl-workspace_tf_3dunet_brats': 'openfl-workspace/tf_3dunet_brats/requirements.txt', - 'openfl-workspace_keras_cnn_with_compression': 'openfl-workspace/keras_cnn_with_compression/requirements.txt', 'openfl-workspace_keras_cnn_mnist': 'openfl-workspace/keras/mnist/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_medmnist_2d_envoy': 'openfl-tutorials/interactive_api/PyTorch_MedMNIST_2D/envoy/requirements.txt', 'openfl-tutorials_interactive_api_pytorch_dogscats_vit_workspace': 'openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/workspace/requirements.txt',