From 409ede87829539382c5b23cb575e5345ccc02aff Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 28 Aug 2023 19:45:35 +0530 Subject: [PATCH 1/3] Prevent mounting /var/tmp to workers when running tests --- src/server/JasmineGraphServer.cpp | 85 ++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index ffed6248c..15cf64c43 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -326,40 +326,65 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, popen(serverStartScript.c_str(),"r"); } } else if (profile == "docker") { - std::string image_tag; char *env_testing = getenv("TESTING"); - if (env_testing == NULL || strcasecmp(env_testing, "true") != 0) { - image_tag = "latest"; - } else { - image_tag = "test"; - } + bool is_testing = (env_testing != NULL && strcasecmp(env_testing, "true") == 0); for (int i =0 ; i < workerPortsVector.size() ; i++) { if (masterHost == host || host == "localhost") { - serverStartScript = "docker run -v " + instanceDataFolder + ":" + instanceDataFolder + - " -v " + aggregateDataFolder + ":" + aggregateDataFolder + - " -v " + nmonFileLocation + ":" + nmonFileLocation + - " -v " + graphsagelocation + ":" + graphsagelocation + - " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " + - std::to_string(workerPortsVector.at(i)) + ":" + - std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " jasminegraph:" + image_tag + " --MODE 2 --HOST_NAME " + host + - " --MASTERIP " + masterHost + " --SERVER_PORT " + - std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + - std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + if (is_testing) { + serverStartScript = "docker run -p " + + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + + std::to_string(workerDataPortsVector.at(i)) + ":" + + std::to_string(workerDataPortsVector.at(i)) + + " -e WORKER_ID=" + to_string(i) + + " jasminegraph:test --MODE 2 --HOST_NAME " + host + + " --MASTERIP " + masterHost + " --SERVER_PORT " + + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + } else { + serverStartScript = "docker run -v " + instanceDataFolder + ":" + instanceDataFolder + + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + + " -v " + nmonFileLocation + ":" + nmonFileLocation + + " -v " + graphsagelocation + ":" + graphsagelocation + + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " + + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + + std::to_string(workerDataPortsVector.at(i)) + ":" + + std::to_string(workerDataPortsVector.at(i)) + + " -e WORKER_ID=" + to_string(i) + + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + + " --MASTERIP " + masterHost + " --SERVER_PORT " + + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + } } else { - serverStartScript = "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder + - " -v " + aggregateDataFolder + ":" + aggregateDataFolder + - " -v " + nmonFileLocation + ":" + nmonFileLocation + - " -v " + graphsagelocation + ":" + graphsagelocation + - " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " + - std::to_string(workerPortsVector.at(i)) + ":" + - std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " jasminegraph:" + image_tag + " --MODE 2 --HOST_NAME " + host + - " --MASTERIP " + masterHost + " --SERVER_PORT " + - std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + - std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + if (is_testing) { + serverStartScript = "docker -H ssh://" + host + " run -p " + + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + + std::to_string(workerDataPortsVector.at(i)) + ":" + + std::to_string(workerDataPortsVector.at(i)) + + " -e WORKER_ID=" + to_string(i) + + " jasminegraph:test --MODE 2 --HOST_NAME " + host + + " --MASTERIP " + masterHost + " --SERVER_PORT " + + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + } else { + serverStartScript = "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder + + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + + " -v " + nmonFileLocation + ":" + nmonFileLocation + + " -v " + graphsagelocation + ":" + graphsagelocation + + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + "/home/ubuntu/software/jasminegraph/logs" + " -p " + + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + + std::to_string(workerDataPortsVector.at(i)) + ":" + + std::to_string(workerDataPortsVector.at(i)) + + " -e WORKER_ID=" + to_string(i) + + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + + " --MASTERIP " + masterHost + " --SERVER_PORT " + + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; + } } server_logger.log(serverStartScript, "info"); popen(serverStartScript.c_str(),"r"); From a869a858df55818494f8b126561635feeb54824d Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 28 Aug 2023 19:46:42 +0530 Subject: [PATCH 2/3] Show worker_id/master and thread_id in log entries --- src/util/logger/Logger.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index ffed4dddc..149c3ffac 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -13,14 +13,33 @@ limitations under the License. #include "Logger.h" +#include #include #include #include +#include + +#include + +using namespace std; + +static string get_worker_name(); auto logger = spdlog::stdout_color_mt("logger"); auto daily_logger = spdlog::daily_logger_mt("JasmineGraph", "logs/server_logs.log", 00, 01); +string worker_name = get_worker_name(); + +static string get_worker_name() { + char *worker_id = getenv("WORKER_ID"); + if (worker_id) { + return string("WORKER ") + string(worker_id); + } + return string("MASTER"); +} void Logger::log(std::string message, const std::string log_type) { + pthread_t tid = pthread_self(); + message = "[" + worker_name + " : " + to_string(tid) + "] " + message; if (log_type.compare("info") == 0) { daily_logger->info(message); logger->info(message); From 314105b9b8d3ea91e083d170d3829aa3da8239dc Mon Sep 17 00:00:00 2001 From: thevindu-w Date: Mon, 28 Aug 2023 20:56:30 +0530 Subject: [PATCH 3/3] Improved efficiency of Utils::getJasmineGraphProperty() using an unordered_map to cache property values --- src/util/Utils.cpp | 32 +++++++++++++++++++------------- src/util/Utils.h | 8 +++++++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index 55f0a5c98..0a4b89a0b 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -24,6 +24,8 @@ limitations under the License. using namespace std; Logger util_logger; +unordered_map Utils::propertiesMap; + map Utils::getBatchUploadFileList(std::string file) { std::vector batchUploadFileContent = getFileContent(file); std::vector::iterator iterator1 = batchUploadFileContent.begin(); @@ -81,24 +83,28 @@ std::vector Utils::getFileContent(std::string file) { }; std::string Utils::getJasmineGraphProperty(std::string key) { - std::vector::iterator it; - vector vec = getFileContent("conf/jasminegraph-server.properties"); - it = vec.begin(); - - for (it = vec.begin(); it < vec.end(); it++) { - std::string item = *it; - if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { - std::vector vec2 = split(item, '='); - if (vec2.at(0).compare(key) == 0) { - if (item.substr(item.length() - 1, item.length()).compare("=") != 0) { - return vec2.at(1); + if (Utils::propertiesMap.empty()) { + std::vector::iterator it; + vector vec = getFileContent("conf/jasminegraph-server.properties"); + it = vec.begin(); + + for (it = vec.begin(); it < vec.end(); it++) { + std::string item = *it; + if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { + std::vector vec2 = split(item, '='); + if (vec2.size() == 2){ + Utils::propertiesMap[vec2.at(0)] = vec2.at(1); } else { - return " "; + Utils::propertiesMap[vec2.at(0)] = string(" "); } } } } - + unordered_map::iterator it = Utils::propertiesMap.find(key); + if(it != Utils::propertiesMap.end()) + { + return it->second; + } return NULL; } diff --git a/src/util/Utils.h b/src/util/Utils.h index 8de6432a9..475fb0c40 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -16,10 +16,12 @@ limitations under the License. #include #include #include +#include #include //#include "../frontend/JasmineGraphFrontEnd.h" -#include #include +#include + #include #include "../metadb/SQLiteDBInterface.h" @@ -27,8 +29,12 @@ limitations under the License. #include "Conts.h" using std::map; +using std::unordered_map; class Utils { + private: + static unordered_map propertiesMap; + public: struct worker { std::string workerID;