Merge branch 'master' into fl-kasun-merge

miyurud · Sep 29, 2023 · bfd75b3 · bfd75b3
2 parents 7bf3c93 + cf48fcc
commit bfd75b3
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 5 deletions.
diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml
@@ -0,0 +1,35 @@
+name: Style Check
+
+on:
+    push:
+      branches: [master]
+    pull_request:
+      branches: [master]
+
+jobs:
+    style-check:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Install tools
+              run: |
+                sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt
+                sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint
+                pip install install pylint
+
+            - name: cpplint check
+              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}}
+              run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --exclude=./src/util/sqlite3/ --recursive .
+
+            - name: pylint check
+              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}}
+              run: pylint --recursive=y .
+
+            - name: hadolint check
+              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}}
+              run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 -t warning
+
+            - name: shfmt check
+              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}}
+              run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci
diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp
@@ -4277,15 +4277,17 @@ void JasmineGraphInstanceService::initServer(string trainData){
     std::vector<char *> vc;
     std::transform(trainargs.begin(), trainargs.end(), std::back_inserter(vc), converter);
 
+    std::string log_file = "/tmp/jasminegraph/fl_server_" + partitionID + ".log";
     std::string path = "cd " + utils.getJasmineGraphProperty("org.jasminegraph.fl.location") + " && ";
     std::string command = path + "python3.8 fl_server.py "+ utils.getJasmineGraphProperty("org.jasminegraph.fl.weights") + " "
                                 + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir")
                                 + " " + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir")+ " "+ graphID + " 0 "
                                 + utils.getJasmineGraphProperty("org.jasminegraph.fl_clients")
-                                + " " + utils.getJasmineGraphProperty("org.jasminegraph.fl.epochs") +" localhost 5000 > "
-                                + "/home/ubuntu/software/jasminegraph/logs/server_logs-" + Utils::getCurrentTimestamp() + ".txt";
+                                + " " + utils.getJasmineGraphProperty("org.jasminegraph.fl.epochs") +" localhost 5000" 
+                                + " >>" + log_file + " 2>&1";
     instance_logger.log("Executing : " + command, "info");
     int exit_status = system(command.c_str());
+    chmod(log_file.c_str(), 0666);
     if (exit_status == -1) {
         instance_logger.error("Failed executing python server for query");
     }
@@ -4365,16 +4367,18 @@ void JasmineGraphInstanceService::initClient(string trainData){
     std::vector<char *> vc;
     std::transform(trainargs.begin(), trainargs.end(), std::back_inserter(vc), converter);
 
+    std::string log_file = "/tmp/jasminegraph/fl_client_" + partitionID + ".log";
     std::string path = "cd " + utils.getJasmineGraphProperty("org.jasminegraph.fl.location") + " && ";
     std::string command = path + "python3.8 fl_client.py "+ utils.getJasmineGraphProperty("org.jasminegraph.fl.weights") + " "
                                 + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir")
                                 + " " + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir")+ " "+ graphID + " " + partitionID + " "
                                 + utils.getJasmineGraphProperty("org.jasminegraph.fl.epochs")
                                 + " localhost " + utils.getJasmineGraphProperty("org.jasminegraph.fl.org.port")
-                                + " > /home/ubuntu/software/jasminegraph/logs/client_logs_" + partitionID + "-" + Utils::getCurrentTimestamp() + ".txt";
+                                + " >>" + log_file + " 2>&1";
 
     instance_logger.log("Executing : " + command, "info");
     int exit_status = system(command.c_str());
+    chmod(log_file.c_str(), 0666);
     if (exit_status == -1) {
         instance_logger.error("Could not start python client");
     }
@@ -4388,14 +4392,16 @@ void JasmineGraphInstanceService::mergeFiles(string trainData){
     string partitionID = trainargs[2];
     int exit_status;
 
+    std::string log_file = "/tmp/jasminegraph/merge_" + partitionID + ".log";
     std::string path = "cd " + utils.getJasmineGraphProperty("org.jasminegraph.fl.location") + " && ";
     std::string command = path + "python3.8 merge.py "+ utils.getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")+ " "
                                 + utils.getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder") + " "
-                                + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir") + " " + graphID + " " + partitionID
-                                + " > /home/ubuntu/software/jasminegraph/logs/merge_logs" + partitionID + "-" + Utils::getCurrentTimestamp() + ".txt";
+                                + utils.getJasmineGraphProperty("org.jasminegraph.fl.dataDir") + " " + graphID + " " + partitionID 
+                                + " >>" + log_file + " 2>&1";
 
     instance_logger.log("Executing : " + command, "info");
     exit_status = system(command.c_str());
+    chmod(log_file.c_str(), 0666);
     if (exit_status == -1) {
         instance_logger.error("Merge Command Execution Failed for Graph ID - Patition ID: " + graphID + " - " + partitionID + "; Error : " + strerror(errno));
     }

diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp
@@ -16,6 +16,7 @@ limitations under the License.
 #include <stdio.h>
 #include <stdlib.h>
 #include <string>
+#include <sys/stat.h>
 #include "JasmineGraphServer.h"
 #include "JasmineGraphInstance.h"
 #include "../util/Utils.h"