Fix issues of merging kasundharmadasa's changes

miyurud · Sep 29, 2023 · 7bf3c93 · 7bf3c93
1 parent 8c89673
commit 7bf3c93
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 140 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM miyurud/jasminegraph
+FROM miyurud/jasminegraph:20230909T110050
 ENV HOME="/home/ubuntu"
 ENV JASMINEGRAPH_HOME="/home/ubuntu/software/jasminegraph"
 
@@ -7,14 +7,15 @@ WORKDIR /home/ubuntu/software/jasminegraph
 COPY ./GraphSAGE ./GraphSAGE
 
 COPY ./build.sh ./build.sh
-COPY ./run-docker.sh ./run-docker.sh
 COPY ./CMakeLists.txt ./CMakeLists.txt
-COPY ./src_python ./src_python
 COPY ./main.h ./main.h
 COPY ./main.cpp ./main.cpp
 COPY ./src ./src
 
 RUN sh build.sh
+
+COPY ./run-docker.sh ./run-docker.sh
+COPY ./src_python ./src_python
 COPY ./conf ./conf
 
 ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]

diff --git a/run-docker.sh b/run-docker.sh
@@ -1,78 +1,71 @@
 #!/usr/bin/env bash
 
-
 MODE=${MODE}
 MASTERIP=${MASTERIP}
 WORKERS=${WORKERS}
 WORKERIP=${WORKERIP}
 HOST_NAME=${HOST_NAME}
-SERVER_PORT=${SERVER_PORT};
-SERVER_DATA_PORT=${SERVER_DATA_PORT};
+SERVER_PORT=${SERVER_PORT}
+SERVER_DATA_PORT=${SERVER_DATA_PORT}
 ENABLE_NMON=${ENABLE_NMON}
 
 while [ $# -gt 0 ]; do
 
-   if [[ $1 == *"--"* ]]; then
+    if [[ $1 == *"--"* ]]; then
         param="${1/--/}"
         declare $param="$2"
         echo $1 $2 // Optional to see the parameter:value result
-   fi
+    fi
 
-  shift
+    shift
 done
 
-if [ -z "$MODE" ] ;
-then
+if [ -z "$MODE" ]; then
     echo "MODE OF OPERATION SHOULD BE SPECIFIED"
     echo "Use argument 1 <MASTERIP> <NUMBER OF WORKERS> <WORKER IPS> to start JasmineGraph in Master mode."
     echo "Use 2 <hostName> <serverPort> <serverDataPort> to start in Worker mode."
     exit 1
 fi
 
-if [ $MODE -eq 1 ] ;
-then
-    if [ -z "$MASTERIP" ] ;
-        then
-            echo "MASTER IP SHOULD BE SPECIFIED"
-            exit 1
+if [ $MODE -eq 1 ]; then
+    if [ -z "$MASTERIP" ]; then
+        echo "MASTER IP SHOULD BE SPECIFIED"
+        exit 1
     fi
 
-    if [ -z "$WORKERS" ] ;
-        then
-            echo "Number of workers SHOULD BE SPECIFIED"
-            exit 1
+    if [ -z "$WORKERS" ]; then
+        echo "Number of workers SHOULD BE SPECIFIED"
+        exit 1
     fi
 
-    if [ -z "$WORKERIP" ] ;
-        then
-            echo "Worker IPs SHOULD BE SPECIFIED"
-            exit 1
+    if [ -z "$WORKERIP" ]; then
+        echo "Worker IPs SHOULD BE SPECIFIED"
+        exit 1
     fi
 else
-
-    if [ -z "$MASTERIP" ] ;
-         then
-             echo "MASTER IP SHOULD BE SPECIFIED"
-             exit 1
+    if [ -z "$MASTERIP" ]; then
+        echo "MASTER IP SHOULD BE SPECIFIED"
+        exit 1
     fi
 
-    if [ -z "$SERVER_PORT" ] ;
-        then
-            echo "SERVER PORT SHOULD BE SPECIFIED"
-            exit 1
+    if [ -z "$SERVER_PORT" ]; then
+        echo "SERVER PORT SHOULD BE SPECIFIED"
+        exit 1
     fi
 
-    if [ -z "$SERVER_DATA_PORT" ] ;
-        then
-            echo "SERVER DATA PORT SHOULD BE SPECIFIED"
-            exit 1
+    if [ -z "$SERVER_DATA_PORT" ]; then
+        echo "SERVER DATA PORT SHOULD BE SPECIFIED"
+        exit 1
     fi
 fi
 
 export LD_LIBRARY_PATH=/usr/local/lib
-if [ $MODE -eq 1 ] ;
-then
+if [ $MODE -eq 1 ]; then
     ./JasmineGraph "docker" $MODE $MASTERIP $WORKERS $WORKERIP $ENABLE_NMON
 else
     ./JasmineGraph "docker" $MODE $HOST_NAME $MASTERIP $SERVER_PORT $SERVER_DATA_PORT $ENABLE_NMON
-fi
+fi
+
+if [ "$TESTING" = "true" ]; then
+    chmod -R go+w /tmp/jasminegraph
+fi
diff --git a/run.sh b/run.sh
@@ -1,19 +1,17 @@
 #!/usr/bin/env bash
 
-JASMINEGRAPH_DIR="`dirname \"$0\"`"
-PROFILE=$1;
-MODE=$2;
-
-JASMINEGRAPH_DIR="`( cd \"$JASMINEGRAPH_DIR\" && pwd )`"
-if [ -z "$JASMINEGRAPH_DIR" ] ;
-then
-    exit 1  # fail
+JASMINEGRAPH_DIR="$(dirname \"$0\")"
+PROFILE=$1
+MODE=$2
+
+JASMINEGRAPH_DIR="$(cd \"$JASMINEGRAPH_DIR\" && pwd)"
+if [ -z "$JASMINEGRAPH_DIR" ]; then
+    exit 1 # fail
 fi
 
 export JASMINEGRAPH_HOME="$JASMINEGRAPH_DIR"
 
-if [ -z "$MODE" ] ;
-then
+if [ -z "$MODE" ]; then
     echo "MODE OF OPERATION SHOULD BE SPECIFIED"
     echo "Use argument 1 to start JasmineGraph in Master mode."
     echo "Use 2 <hostName> <serverPort> <serverDataPort> to start in Worker mode."
@@ -22,33 +20,29 @@ fi
 
 cd $JASMINEGRAPH_DIR
 
-if [ $MODE -eq 1 ] ;
-then
+if [ $MODE -eq 1 ]; then
     echo "STARTING MASTER MODE"
 
-    MASTER_HOST_NAME=$3;
-    NUMBER_OF_WORKERS=$4;
-    WORKER_IPS=$5;
+    MASTER_HOST_NAME=$3
+    NUMBER_OF_WORKERS=$4
+    WORKER_IPS=$5
 
     ./JasmineGraph "native" $MODE $MASTER_HOST_NAME $NUMBER_OF_WORKERS $WORKER_IPS "false"
 else
-    WORKER_HOST=$3;
-    MASTER_HOST_NAME=$4;
-    SERVER_PORT=$5;
-    SERVER_DATA_PORT=$6;
+    WORKER_HOST=$3
+    MASTER_HOST_NAME=$4
+    SERVER_PORT=$5
+    SERVER_DATA_PORT=$6
 
-    if [ -z "$SERVER_PORT" ] ;
-    then
+    if [ -z "$SERVER_PORT" ]; then
         echo "SERVER PORT SHOULD BE SPECIFIED"
         exit 1
     fi
 
-    if [ -z "$SERVER_DATA_PORT" ] ;
-        then
-            echo "SERVER DATA PORT SHOULD BE SPECIFIED"
-            exit 1
+    if [ -z "$SERVER_DATA_PORT" ]; then
+        echo "SERVER DATA PORT SHOULD BE SPECIFIED"
+        exit 1
     fi
 
     ./JasmineGraph "native" $MODE $WORKER_HOST $MASTER_HOST_NAME $SERVER_PORT $SERVER_DATA_PORT "false"
 fi
-
diff --git a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp
@@ -24,30 +24,28 @@ Logger trainScheduler_logger;
 
 map<string, std::map<int, int>> JasmineGraphTrainingSchedular::schedulePartitionTraining(std::string graphID) {
 
-    map<string, std::map<int, int>> scheduleForEachWorker;
+    map<string, std::map<int, int>> scheduleForEachHost;
     vector<pair<string, string>> hostData;
     SQLiteDBInterface refToSqlite = *new SQLiteDBInterface();
     refToSqlite.init();
 
-    string sqlStatement = "SELECT worker_idworker, name FROM worker_has_partition INNER JOIN worker ON worker_idworker = "
-                          "idworker WHERE partition_graph_idgraph = " + graphID;
-
+    string sqlStatement = "SELECT host_idhost, name FROM worker_has_partition INNER JOIN worker ON worker_idworker = "
+                          "idworker WHERE partition_graph_idgraph = " + graphID + " group by host_idhost";
     std::vector<vector<pair<string, string>>> result = refToSqlite.runSelect(sqlStatement);
     for (vector<vector<pair<string, string>>>::iterator i = result.begin(); i != result.end(); ++i) {
         int count = 0;
-        string workerID;
+        string hostID;
         string hostname;
         for (std::vector<pair<string, string>>::iterator j = (i->begin()); j != i->end(); ++j) {
             if (count == 0) {
-                workerID = j->second;
+                hostID = j->second;
             } else {
                 hostname = j->second;
-                hostData.push_back(pair<string, string>(workerID, hostname));
+                hostData.push_back(pair<string, string>(hostID, hostname));
             }
             count++;
         }
     }
-
     string id_partition = "";
     string vertexCount = "";
     string centralvertexCount = "";
@@ -57,8 +55,9 @@ map<string, std::map<int, int>> JasmineGraphTrainingSchedular::schedulePartition
     trainScheduler_logger.log("Scheduling training order for each worker", "info");
     for (std::vector<pair<string, string>>::iterator j = (hostData.begin()); j != hostData.end(); ++j) {
         sqlStatement = "SELECT idpartition, vertexcount, central_vertexcount, graph_idgraph FROM partition INNER JOIN "
-                       "worker_has_partition w ON w.partition_idpartition = partition.idpartition"
-                       " WHERE partition.graph_idgraph =  "+ graphID +" AND w.worker_idworker = " + j->first;
+                       "(SELECT host_idhost, partition_idpartition, partition_graph_idgraph, worker_idworker FROM "
+                       "worker_has_partition INNER JOIN worker ON worker_idworker = idworker) AS a ON partition.idpartition = "
+                       "a.partition_idpartition WHERE partition.graph_idgraph =  "+ graphID +" AND a.host_idhost = " + j->first;
         std::vector<vector<pair<string, string>>> results = refToSqlite.runSelect(sqlStatement);
         std::map<int, int> vertexCountToPartitionId;
         for (std::vector<vector<pair<string, string>>>::iterator i = results.begin(); i != results.end(); ++i) {
@@ -89,9 +88,9 @@ map<string, std::map<int, int>> JasmineGraphTrainingSchedular::schedulePartition
         }
         std::map<int, int> scheduledPartitionSets = packPartitionsToMemory(memoryEstimationForEachPartition,
                                                                            availableMemory);
-        scheduleForEachWorker.insert(make_pair(j->first, scheduledPartitionSets));
+        scheduleForEachHost.insert(make_pair(j->second, scheduledPartitionSets));
     }
-    return scheduleForEachWorker;
+    return scheduleForEachHost;
 }
 
 long JasmineGraphTrainingSchedular::estimateMemory(int vertexCount, string graph_id) {