Adding updates that are compatible with fedn v0.9.0

Power-consumption-keras/.dockerignore

@@ -0,0 +1,4 @@
+data
+seed.npz
+*.tgz
+*.tar.gz

Power-consumption-keras/.gitignore

@@ -0,0 +1,6 @@
+data
+*.npz
+*.tgz
+*.tar.gz
+.mnist-keras
+client.yaml

Power-consumption-keras/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.10.6-slim as base
+LABEL maintainer="[email protected]"
+WORKDIR /app
+COPY requirements.txt .
+RUN apt-get update \ 
+    && apt-get install --no-install-recommends -y git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \ 
+    && pip install git+https://github.com/scaleoutsystems/fedn.git@master#egg=fedn\&subdirectory=fedn \
+    && pip install --no-cache-dir -r requirements.txt
+
+
+FROM python:3.10.6-slim as build
+COPY --from=base /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
+COPY --from=base /usr/local/bin/fedn /usr/local/bin/
+WORKDIR /app

Power-consumption-keras/client/data.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+from math import floor
+
+import numpy as np
+dir_path = os.path.dirname(os.path.realpath(__file__))
+abs_path = os.path.abspath(dir_path)
+
+def copy_files(source_dir, destination_dir):
+    # Get a list of all files in the source directory
+    files = os.listdir(source_dir)
+
+    # Iterate through the files and copy them to the destination directory
+    for file_name in files:
+        source_file = os.path.join(source_dir, file_name)
+        destination_file = os.path.join(destination_dir, file_name)
+        shutil.copy2(source_file, destination_file)
+        print(f"Copied {file_name} to {destination_file}")
+
+def get_data(out_dir='data'):
+    # Make dir if necessary
+    if not os.path.exists(out_dir):
+        os.mkdir(out_dir)
+
+    print('dir_path: ', dir_path)
+    parent_dir = os.path.abspath(os.path.join(dir_path, os.pardir))
+    print('parent_dir: ', parent_dir)
+
+    source_dir = parent_dir+'/data'
+    destination_dir = dir_path+'/data'
+
+    copy_files(source_dir, destination_dir)
+
+def load_data(data_path=None, is_train=True):
+    """ Load data from disk.
+
+    :param data_path: Path to data file.
+    :type data_path: str
+    :param is_train: Whether to load training or test data.
+    :type is_train: bool
+    :return: Tuple of data and labels.
+    :rtype: tuple
+    """
+    if data_path is None:
+        data_path = os.environ.get("FEDN_DATA_PATH", abs_path+'/data/power.npz')
+
+    data = np.load(data_path)
+
+    if is_train:
+        X = data['x_train']
+        y = data['y_train']
+    else:
+        X = data['x_test']
+        y = data['y_test']
+
+    # Normalize
+    X = X / 255
+
+    return X, y
+
+if __name__ == '__main__':
+    # Prepare data if not already done
+    if not os.path.exists(abs_path+'/data'):
+        print('Note: The data directory does not exist. Loading the data..')
+        get_data()
+    else:
+        print('Good to go, The data directory exist.')
+

Power-consumption-keras/client/fedn.yaml

@@ -0,0 +1,10 @@
+python_env: python_env.yaml
+entry_points:
+  build:
+    command: python model.py
+  startup:
+    command: python data.py
+  train:
+    command: python train.py
+  validate:
+    command: python validate.py

Power-consumption-keras/client/model.py

@@ -0,0 +1,36 @@
+
+import json
+import os
+
+import numpy as np
+import tensorflow as tf
+
+
+from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics
+
+HELPER_MODULE = 'numpyhelper'
+helper = get_helper(HELPER_MODULE)
+
+
+def compile_model(img_rows=28, img_cols=28):
+    # Set input shape
+    #input_shape = (img_rows, img_cols, 1)
+
+    # Define model
+    opt = tf.keras.optimizers.SGD(lr=0.0001)
+    model = tf.keras.models.Sequential()
+    model.add(tf.keras.layers.Dense(64, input_dim=4, activation="relu"))
+    model.add(tf.keras.layers.Dense(32, activation="relu"))
+    model.add(tf.keras.layers.Dense(1, activation="linear"))
+    #model.summary()
+    model.compile(loss = "mse", optimizer = opt,metrics=['mae'])
+
+    return model
+
+def init_seed(out_path='seed.npz'):
+
+    weights = compile_model().get_weights()
+    helper.save(weights, out_path)
+
+if __name__ == "__main__":
+    init_seed('../seed.npz')

Power-consumption-keras/client/python_env.yaml

@@ -0,0 +1,9 @@
+name: power-consumption-keras
+build_dependencies:
+  - pip
+  - setuptools
+  - wheel==0.37.1
+dependencies:
+  - tensorflow==2.13.1
+  - fire==0.3.1
+  - fedn==0.9.0

Power-consumption-keras/client/python_env_macosx.yaml

@@ -0,0 +1,10 @@
+name: mnist-keras
+build_dependencies:
+  - pip
+  - setuptools
+  - wheel==0.37.1
+dependencies:
+  - tensorflow-macos
+  - tensorflow-metal 
+  - fire==0.3.1
+  - fedn==0.9.0b2

Power-consumption-keras/client/train.py

@@ -0,0 +1,64 @@
+import json
+import os
+import sys
+import fire
+import numpy as np
+import tensorflow as tf
+
+from data import load_data
+from model import compile_model
+
+from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics
+
+HELPER_MODULE = 'numpyhelper'
+helper = get_helper(HELPER_MODULE)
+
+def train(in_model_path, out_model_path, data_path=None, batch_size=32, epochs=1):
+    """ Complete a model update.
+
+    Load model paramters from in_model_path (managed by the FEDn client),
+    perform a model update, and write updated paramters
+    to out_model_path (picked up by the FEDn client).
+
+    :param in_model_path: The path to the input model.
+    :type in_model_path: str
+    :param out_model_path: The path to save the output model to.
+    :type out_model_path: str
+    :param data_path: The path to the data file.
+    :type data_path: str
+    :param batch_size: The batch size to use.
+    :type batch_size: int
+    :param epochs: The number of epochs to train.
+    :type epochs: int
+    """
+    # Load data
+    x_train, y_train = load_data(data_path)
+
+    # Load model
+    model = compile_model()
+    #weights = load_parameters(in_model_path)
+    weights = helper.load(in_model_path)
+    model.set_weights(weights)
+
+    # Train
+    model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
+
+    # Metadata needed for aggregation server side
+    metadata = {
+        # num_examples are mandatory
+        'num_examples': len(x_train),
+        'batch_size': batch_size,
+        'epochs': epochs,
+    }
+
+    # Save JSON metadata file (mandatory)
+    save_metadata(metadata, out_model_path)
+
+    # Save model update (mandatory)
+    weights = model.get_weights()
+    helper.save(weights, out_model_path)
+
+
+if __name__ == "__main__":
+    train(sys.argv[1], sys.argv[2])
+

Power-consumption-keras/client/validate.py

@@ -0,0 +1,59 @@
+import json
+import os
+import sys
+import fire
+import numpy as np
+import tensorflow as tf
+
+from data import load_data
+from model import compile_model
+
+from fedn.utils.helpers.helpers import get_helper, save_metadata, save_metrics
+
+HELPER_MODULE = 'numpyhelper'
+helper = get_helper(HELPER_MODULE)
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+abs_path = os.path.abspath(dir_path)
+
+
+def validate(in_model_path, out_json_path, data_path=None):
+    """ Validate model.
+
+    :param in_model_path: The path to the input model.
+    :type in_model_path: str
+    :param out_json_path: The path to save the output JSON to.
+    :type out_json_path: str
+    :param data_path: The path to the data file.
+    :type data_path: str
+    """
+
+    # Load data
+    x_train, y_train = load_data(data_path)
+    x_test, y_test = load_data(data_path, is_train=False)
+
+    # Load model
+    model = compile_model()
+    helper = get_helper(HELPER_MODULE)
+    weights = helper.load(in_model_path)
+    model.set_weights(weights)
+
+    # Evaluate
+    model_score = model.evaluate(x_train, y_train)
+    model_score_test = model.evaluate(x_test, y_test)
+    y_pred = model.predict(x_test)
+    y_pred = np.argmax(y_pred, axis=1)
+
+    # JSON schema
+    report = {
+        "training_loss": model_score[0],
+        "training_accuracy": model_score[1],
+        "test_loss": model_score_test[0],
+        "test_accuracy": model_score_test[1],
+    }
+
+    # Save JSON
+    save_metrics(report, out_json_path)
+
+if __name__ == "__main__":
+    validate(sys.argv[1], sys.argv[2])

Power-consumption-keras/requirements-macos.txt

@@ -0,0 +1,4 @@
+tensorflow-macos
+tensorflow-metal 
+fire==0.3.1
+docker==5.0.2

Power-consumption-keras/requirements.txt

@@ -0,0 +1,4 @@
+tensorflow==2.13.1
+fire==0.3.1
+docker==6.1.1
+fedn==0.9.0

Power-consumption-pytorch/.dockerignore

@@ -0,0 +1,4 @@
+data
+seed.npz
+*.tgz
+*.tar.gz

Power-consumption-pytorch/.gitignore

@@ -0,0 +1,6 @@
+data
+*.npz
+*.tgz
+*.tar.gz
+.power-consumption-pytorch
+client.yaml

Power-consumption-pytorch/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.10.6-slim as base
+LABEL maintainer="[email protected]"
+WORKDIR /app
+COPY requirements.txt .
+RUN apt-get update \ 
+    && apt-get install --no-install-recommends -y git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \ 
+    && pip install git+https://github.com/scaleoutsystems/fedn.git@master#egg=fedn\&subdirectory=fedn \
+    && pip install --no-cache-dir -r requirements.txt
+
+
+FROM python:3.10.6-slim as build
+COPY --from=base /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
+COPY --from=base /usr/local/bin/fedn /usr/local/bin/
+WORKDIR /app