Log with worker and thread id. Improve getJasmineGraphProperty(). Remove unnecessary docker mounts when testing.

Dockerfile

@@ -7,7 +7,6 @@ WORKDIR /home/ubuntu/software/jasminegraph
 COPY ./GraphSAGE ./GraphSAGE
 RUN pip install -r ./GraphSAGE/requirements
 
-COPY ./conf ./conf
 COPY ./build.sh ./build.sh
 COPY ./run-docker.sh ./run-docker.sh
 COPY ./CMakeLists.txt ./CMakeLists.txt
@@ -17,5 +16,7 @@ COPY ./main.cpp ./main.cpp
 COPY ./src ./src
 
 RUN sh build.sh
+COPY ./conf ./conf
+
 ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
 CMD ["bash"]

src/server/JasmineGraphServer.cpp

@@ -326,40 +326,67 @@ void JasmineGraphServer::startRemoteWorkers(std::vector<int> workerPortsVector,
             popen(serverStartScript.c_str(),"r");
         }
     } else if (profile == "docker") {
-        std::string image_tag;
         char *env_testing = getenv("TESTING");
-        if (env_testing == NULL || strcasecmp(env_testing, "true") != 0) {
-            image_tag = "latest";
-        } else {
-            image_tag = "test";
-        }
+        bool is_testing = (env_testing != NULL && strcasecmp(env_testing, "true") == 0);
         for (int i =0 ; i < workerPortsVector.size() ; i++) {
             if (masterHost == host || host == "localhost") {
-                serverStartScript = "docker run -v " + instanceDataFolder + ":" + instanceDataFolder +
-                                    " -v " + aggregateDataFolder + ":" + aggregateDataFolder +
-                                    " -v " + nmonFileLocation + ":" + nmonFileLocation +
-                                    " -v " + graphsagelocation + ":" + graphsagelocation +
-                                    " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " +
-                                    std::to_string(workerPortsVector.at(i)) + ":" +
-                                    std::to_string(workerPortsVector.at(i)) + " -p " +
-                                    std::to_string(workerDataPortsVector.at(i)) + ":" +
-                                    std::to_string(workerDataPortsVector.at(i)) + " jasminegraph:" + image_tag + " --MODE 2 --HOST_NAME " + host +
-                                    " --MASTERIP " + masterHost + " --SERVER_PORT " +
-                                    std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
-                                    std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon;
+                if (is_testing) {
+                    serverStartScript = "docker run -p " +
+                                        std::to_string(workerPortsVector.at(i)) + ":" +
+                                        std::to_string(workerPortsVector.at(i)) + " -p " +
+                                        std::to_string(workerDataPortsVector.at(i)) + ":" +
+                                        std::to_string(workerDataPortsVector.at(i)) +
+                                        " -e WORKER_ID=" + to_string(i) +
+                                        " jasminegraph:test --MODE 2 --HOST_NAME " + host +
+                                        " --MASTERIP " + masterHost + " --SERVER_PORT " +
+                                        std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
+                                        std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon +
+                                        " >/tmp/worker_logs/worker_" + to_string(i) + ".log 2>&1";
+                } else {
+                    serverStartScript = "docker run -v " + instanceDataFolder + ":" + instanceDataFolder +
+                                        " -v " + aggregateDataFolder + ":" + aggregateDataFolder +
+                                        " -v " + nmonFileLocation + ":" + nmonFileLocation +
+                                        " -v " + graphsagelocation + ":" + graphsagelocation +
+                                        " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " +
+                                        std::to_string(workerPortsVector.at(i)) + ":" +
+                                        std::to_string(workerPortsVector.at(i)) + " -p " +
+                                        std::to_string(workerDataPortsVector.at(i)) + ":" +
+                                        std::to_string(workerDataPortsVector.at(i)) +
+                                        " -e WORKER_ID=" + to_string(i) +
+                                        " jasminegraph:latest --MODE 2 --HOST_NAME " + host +
+                                        " --MASTERIP " + masterHost + " --SERVER_PORT " +
+                                        std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
+                                        std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon;
+                }
             } else {
-                serverStartScript = "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder +
-                                    " -v " + aggregateDataFolder + ":" + aggregateDataFolder +
-                                    " -v " + nmonFileLocation + ":" + nmonFileLocation +
-                                    " -v " + graphsagelocation + ":" + graphsagelocation +
-                                    " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " +
-                                    std::to_string(workerPortsVector.at(i)) + ":" +
-                                    std::to_string(workerPortsVector.at(i)) + " -p " +
-                                    std::to_string(workerDataPortsVector.at(i)) + ":" +
-                                    std::to_string(workerDataPortsVector.at(i)) + " jasminegraph:" + image_tag + " --MODE 2 --HOST_NAME " + host +
-                                    " --MASTERIP " + masterHost + " --SERVER_PORT " +
-                                    std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
-                                    std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon;
+                if (is_testing) {
+                    serverStartScript = "docker -H ssh://" + host + " run -p " +
+                                        std::to_string(workerPortsVector.at(i)) + ":" +
+                                        std::to_string(workerPortsVector.at(i)) + " -p " +
+                                        std::to_string(workerDataPortsVector.at(i)) + ":" +
+                                        std::to_string(workerDataPortsVector.at(i)) +
+                                        " -e WORKER_ID=" + to_string(i) +
+                                        " jasminegraph:test --MODE 2 --HOST_NAME " + host +
+                                        " --MASTERIP " + masterHost + " --SERVER_PORT " +
+                                        std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
+                                        std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon +
+                                        " >/tmp/worker_logs/worker_" + to_string(i) + ".log 2>&1";
+                } else {
+                    serverStartScript = "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder +
+                                        " -v " + aggregateDataFolder + ":" + aggregateDataFolder +
+                                        " -v " + nmonFileLocation + ":" + nmonFileLocation +
+                                        " -v " + graphsagelocation + ":" + graphsagelocation +
+                                        " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " +
+                                        std::to_string(workerPortsVector.at(i)) + ":" +
+                                        std::to_string(workerPortsVector.at(i)) + " -p " +
+                                        std::to_string(workerDataPortsVector.at(i)) + ":" +
+                                        std::to_string(workerDataPortsVector.at(i)) +
+                                        " -e WORKER_ID=" + to_string(i) +
+                                        " jasminegraph:latest --MODE 2 --HOST_NAME " + host +
+                                        " --MASTERIP " + masterHost + " --SERVER_PORT " +
+                                        std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " +
+                                        std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon;
+                }
             }
             server_logger.log(serverStartScript, "info");
             popen(serverStartScript.c_str(),"r");

src/util/Utils.cpp

@@ -24,6 +24,8 @@ limitations under the License.
 using namespace std;
 Logger util_logger;
 
+unordered_map<std::string, std::string> Utils::propertiesMap;
+
 map<std::string, std::string> Utils::getBatchUploadFileList(std::string file) {
     std::vector<std::string> batchUploadFileContent = getFileContent(file);
     std::vector<std::string>::iterator iterator1 = batchUploadFileContent.begin();
@@ -81,24 +83,28 @@ std::vector<std::string> Utils::getFileContent(std::string file) {
 };
 
 std::string Utils::getJasmineGraphProperty(std::string key) {
-    std::vector<std::string>::iterator it;
-    vector<std::string> vec = getFileContent("conf/jasminegraph-server.properties");
-    it = vec.begin();
-
-    for (it = vec.begin(); it < vec.end(); it++) {
-        std::string item = *it;
-        if (item.length() > 0 && !(item.rfind("#", 0) == 0)) {
-            std::vector<std::string> vec2 = split(item, '=');
-            if (vec2.at(0).compare(key) == 0) {
-                if (item.substr(item.length() - 1, item.length()).compare("=") != 0) {
-                    return vec2.at(1);
+    if (Utils::propertiesMap.empty()) {
+        std::vector<std::string>::iterator it;
+        vector<std::string> vec = getFileContent("conf/jasminegraph-server.properties");
+        it = vec.begin();
+
+        for (it = vec.begin(); it < vec.end(); it++) {
+            std::string item = *it;
+            if (item.length() > 0 && !(item.rfind("#", 0) == 0)) {
+                std::vector<std::string> vec2 = split(item, '=');
+                if (vec2.size() == 2){
+                    Utils::propertiesMap[vec2.at(0)] = vec2.at(1);
                 } else {
-                    return " ";
+                    Utils::propertiesMap[vec2.at(0)] =  string(" ");
                 }
             }
         }
     }
-
+    unordered_map<std::string,std::string>::iterator it = Utils::propertiesMap.find(key);
+    if(it != Utils::propertiesMap.end())
+    {
+        return it->second;
+    }
     return NULL;
 }
 

src/util/Utils.h

@@ -16,19 +16,25 @@ limitations under the License.
 #include <fstream>
 #include <iostream>
 #include <map>
+#include <unordered_map>
 #include <vector>
 //#include "../frontend/JasmineGraphFrontEnd.h"
-#include <string.h>
 #include <arpa/inet.h>
+#include <string.h>
+
 #include <algorithm>
 
 #include "../metadb/SQLiteDBInterface.h"
 #include "../performancedb/PerformanceSQLiteDBInterface.h"
 #include "Conts.h"
 
 using std::map;
+using std::unordered_map;
 
 class Utils {
+   private:
+    static unordered_map<std::string, std::string> propertiesMap;
+
    public:
     struct worker {
         std::string workerID;

src/util/logger/Logger.cpp

@@ -13,14 +13,33 @@ limitations under the License.
 
 #include "Logger.h"
 
+#include <pthread.h>
 #include <spdlog/sinks/daily_file_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 #include <spdlog/spdlog.h>
+#include <stdlib.h>
+
+#include <iostream>
+
+using namespace std;
+
+static string get_worker_name();
 
 auto logger = spdlog::stdout_color_mt("logger");
 auto daily_logger = spdlog::daily_logger_mt("JasmineGraph", "logs/server_logs.log", 00, 01);
+string worker_name = get_worker_name();
+
+static string get_worker_name() {
+    char *worker_id = getenv("WORKER_ID");
+    if (worker_id) {
+        return string("WORKER ") + string(worker_id);
+    }
+    return string("MASTER");
+}
 
 void Logger::log(std::string message, const std::string log_type) {
+    pthread_t tid = pthread_self();
+    message = "[" + worker_name + " : " + to_string(tid) + "] " + message;
     if (log_type.compare("info") == 0) {
         daily_logger->info(message);
         logger->info(message);

test-docker.sh

@@ -1,14 +1,24 @@
 #!/bin/bash
 set -ex
 PROJECT_ROOT="$(pwd)"
+TEST_ROOT="${PROJECT_ROOT}/tests/integration"
 TIMEOUT_SECONDS=600
 RUN_ID="$(date +%y%m%d_%H%M%S)"
+LOG_DIR="${PROJECT_ROOT}/logs/${RUN_ID}"
+while [ -d "$LOG_DIR"]; do
+    tmp_id="$((${tmp_id}+1))"
+    new_run="${RUN_ID}_${tmp_id}"
+    LOG_DIR="${PROJECT_ROOT}/logs/${RUN_ID}"
+done
+RUN_ID=''
+
+mkdir -p "${PROJECT_ROOT}/logs"
+mkdir "$LOG_DIR"
 
-BUILD_LOG="logs/${RUN_ID}_build.txt"
-RUN_LOG="logs/${RUN_ID}_run.txt"
-TEST_LOG="logs/${RUN_ID}_test.txt"
+BUILD_LOG="${LOG_DIR}/build.log"
+RUN_LOG="${LOG_DIR}/run_master.log"
+TEST_LOG="${LOG_DIR}/test.log"
 
-mkdir -p logs/
 
 stop_and_remove_containers () {
     if [ "$(docker ps -q)" ]; then
@@ -24,18 +34,19 @@ build_and_run_docker () {
     docker build -t jasminegraph:test . |& tee "$BUILD_LOG"
     build_status="${PIPESTATUS[0]}"
     if [ "$build_status" != '0' ]; then
-        set +e
+        set +ex
         echo
         echo -e '\e[31;1mERROR: Build failed\e[0m'
-        rm -rf tests/integration/env
+        rm -rf "${TEST_ROOT}/env"
         exit "$build_status"
     fi
-    docker compose -f tests/integration/docker-compose.yml up |& tee "$RUN_LOG" &>/dev/null &
+    docker compose -f "${TEST_ROOT}/docker-compose.yml" up |& tee "$RUN_LOG" &>/dev/null &
 }
 
-cd tests/integration
+cd "$TEST_ROOT"
 rm -rf env
 cp -r env_init env
+mkdir -p env/logs
 cd "$PROJECT_ROOT"
 build_and_run_docker
 
@@ -44,30 +55,46 @@ cur_timestamp="$(date +%s)"
 end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))"
 while ! nc -zvn 127.0.0.1 7777 &>/dev/null; do
     if [ "$(date +%s)" -gt "$end_timestamp" ]; then
-        set +e
+        set +ex
         echo "JasmineGraph is not listening"
         echo "Build log:"
         cat "$BUILD_LOG"
         echo "Build log:"
         cat "$RUN_LOG"
-        rm -rf tests/integration/env
+        rm -rf "${TEST_ROOT}/env"
         stop_and_remove_containers
         exit 1
     fi
     sleep .5
 done
 
-timeout "$TIMEOUT_SECONDS" python3 -u tests/integration/test.py |& tee "$TEST_LOG"
+timeout "$TIMEOUT_SECONDS" python3 -u "${TEST_ROOT}/test.py" |& tee "$TEST_LOG"
 exit_code="${PIPESTATUS[0]}"
-set +e
+set +ex
 if [ "$exit_code" == '124' ]; then
     echo
     echo -e '\e[31;1mERROR: Test Timeout\e[0m'
     echo
 fi
+
+cd "$TEST_ROOT"
+for f in env/logs/*; do
+    fname="$(basename ${f})"
+    cp "$f" "${LOG_DIR}/run_${fname}"
+done
+cd "$LOG_DIR"
 if [ "$exit_code" != '0' ]; then
+    echo
+    echo -e '\e[33;1mMaster log:\e[0m'
     cat "$RUN_LOG"
+
+    for f in run_worker_*; do
+        echo
+        echo -e '\e[33;1m'"${f:4:-4}"' log:\e[0m'
+        cat "$f"
+    done
 fi
-rm -rf tests/integration/env
+
+rm -rf "${TEST_ROOT}/env"
 stop_and_remove_containers
 exit "$exit_code"

tests/integration/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - './env/databases/metadb:/home/ubuntu/software/jasminegraph/metadb'
       - './env/databases/performancedb:/home/ubuntu/software/jasminegraph/performancedb'
       - './env/data:/var/tmp/data'
+      - './env/logs:/tmp/worker_logs'
     environment:
       - TESTING=true
     networks:

tests/integration/test.py

@@ -121,3 +121,4 @@ def send_and_expect_response(sock, testName, send, expected, exitOnFail=False):
         print()
         logging.critical('Failed some tests')
         print(*failedTests, sep='\n', file=sys.stderr)
+        sys.exit(1)