Merge pull request #171 from FYP-Auto-Scale-JasmineGraph/master

Add tests for lst, adgr, and some other commads

.dockerignore

@@ -0,0 +1,14 @@
+tests
+metadb
+performancedb
+docs
+logs
+.git
+.github
+.vscode
+.idea
+.dockerignore
+.gitignore
+test-docker.sh
+README.md
+LICENSE

.github/workflows/build.yml

@@ -22,4 +22,8 @@ jobs:
         repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
     - name: Build
       # Build JasmineGraph docker image
-      run: docker build -t jasminegraph .
+      run: docker build -t jasminegraph .
+    - name: Run Integration Tests
+      run: |
+        chmod +x test-docker.sh
+        ./test-docker.sh

.gitignore

@@ -1,3 +1,5 @@
+tests/env/
+
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
@@ -69,4 +71,5 @@ fabric.properties
 build/
 
 # Log files
-logs/
+logs/
+.trash/

CMakeLists.txt

@@ -48,26 +48,14 @@ if(CMAKE_ENABLE_DEBUG)
     target_compile_options(JasmineGraph PRIVATE -g)
 endif()
 
-target_link_libraries(JasmineGraph /usr/local/lib/libmetis.so)
-target_link_libraries(JasmineGraph /usr/local/lib/libxerces-c-3.2.so)
 #TODO: Need to install the library separately
 target_link_libraries(JasmineGraph sqlite3)
 target_link_libraries(JasmineGraph pthread)
 target_link_libraries(JasmineGraph rdkafka)
-#target_link_libraries(JasmineGraph cppkafka)
 target_link_libraries(JasmineGraph Threads::Threads)
-include_directories("$ENV{HOME}/software/flatbuffers/include")
-include_directories("$ENV{HOME}/software/xerces/include")
-include_directories("$ENV{HOME}/software/spdlog/include")
-include_directories("$ENV{HOME}/software/jsoncpp/include")
-include_directories("$ENV{HOME}/software/cppkafka/include")
-include_directories("$ENV{HOME}/software/nlohmann_json/include")
-
-#target_link_libraries(JasmineGraph /usr/local/lib/libmetis.a)
-#target_link_libraries(JasmineGraph $ENV{HOME}/software/xerces-c-3.2.2/lib/libxerces-c.so)
-target_link_libraries(JasmineGraph $ENV{HOME}/software/flatbuffers/libflatbuffers.a)
-include_directories("$ENV{HOME}/software/xerces-c-3.2.2/include")
-target_link_libraries(JasmineGraph $ENV{HOME}/software/jsoncpp/build/debug/src/lib_json/libjsoncpp.a)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libxerces-c.so)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libflatbuffers.a)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libjsoncpp.a)
 target_link_libraries(JasmineGraph /usr/local/lib/libcppkafka.so)
 
 set(PYTHON_EXECUTABLE "/usr/bin/python3.5")

Dockerfile

@@ -1,25 +1,46 @@
-FROM miyurud/jasminegraph
+FROM ubuntu:focal
+WORKDIR /home/ubuntu
+RUN mkdir software
 WORKDIR /home/ubuntu/software
+
 RUN apt-get update
-COPY . jasminegraph/
+RUN apt-get install --no-install-recommends -y apt-transport-https
+RUN apt-get update
+RUN apt-get install --no-install-recommends -y curl gnupg2 ca-certificates software-properties-common nlohmann-json3-dev
+RUN apt-get install --no-install-recommends -y git cmake build-essential sqlite3 libsqlite3-dev libssl-dev librdkafka-dev libboost-all-dev libtool libxerces-c-dev libflatbuffers-dev python3-pip
+RUN add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get install --no-install-recommends -y python3.5-dev
+RUN apt-get install --no-install-recommends -y libjsoncpp-dev libspdlog-dev pigz
+
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+RUN apt-get update
+RUN apt-get install --no-install-recommends -y docker-ce-cli
+
+RUN git clone --single-branch --depth 1 https://github.com/mfontanini/cppkafka.git
+
+WORKDIR /home/ubuntu/software
+RUN git clone --single-branch --depth 1 --branch v5.1.1-DistDGL-v0.5 https://github.com/KarypisLab/METIS.git
+WORKDIR /home/ubuntu/software/METIS
+RUN git submodule update --init
+RUN make config shared=1 cc=gcc prefix=/usr/local
+RUN make install
+
+RUN mkdir /home/ubuntu/software/cppkafka/build
+WORKDIR /home/ubuntu/software/cppkafka/build
+RUN cmake ..
+RUN make -j4
+RUN make install
+
 ENV HOME="/home/ubuntu"
-RUN mkdir -p /var/tmp/nmon
-WORKDIR /home/ubuntu/software/jasminegraph
-RUN apt-get update && \
-    sed -i '/target_link_libraries(JasmineGraph \/usr\/local\/lib\/libmetis.so)/c\target_link_libraries(JasmineGraph \/usr\/local\/lib\/libmetis.so)' CMakeLists.txt && \
-    sed -i '/target_link_libraries(JasmineGraph \/usr\/local\/lib\/libxerces-c-3.2.so)/c\target_link_libraries(JasmineGraph \/usr\/local\/lib\/libxerces-c-3.2.so)' CMakeLists.txt && \
-    sed -i '/target_link_libraries(JasmineGraph \/usr\/local\/lib\/libmetis.a)/c\#target_link_libraries(JasmineGraph \/usr\/local\/lib\/libmetis.a)' CMakeLists.txt && \
-    sed -i '/target_link_libraries(JasmineGraph $ENV{HOME}\/software\/xerces-c-3.2.2\/lib\/libxerces-c.so)/c\#target_link_libraries(JasmineGraph $ENV{HOME}\/software\/xerces-c-3.2.2\/lib\/libxerces-c.so)' CMakeLists.txt && \
-    sed -i '/target_link_libraries(JasmineGraph \/opt\/lib\/libxerces-c.a)/c\#target_link_libraries(JasmineGraph \/opt\/lib\/libxerces-c.a)' CMakeLists.txt && \
-    sed -i '/target_link_libraries(JasmineGraph $ENV{HOME}\/software\/cppkafka\/build\/usr\/local\/lib\/libcppkafka.so)/c\target_link_libraries(JasmineGraph \/usr\/local\/lib\/libcppkafka.so)' CMakeLists.txt && \
-    sed -i '/namespace JasminGraph.Edgestore;/c\namespace JasmineGraph.Edgestore;' src/util/dbutil/edgestore.fbs && \
-    sed -i '/org.jasminegraph.partitioner.metis.bin=home\/ubuntu\/software\/metis-5.1.0\/bin/c\org.jasminegraph.partitioner.metis.bin=\/home\/ubuntu\/software\/metis\/metis-5.1.0\/bin' conf/jasminegraph-server.properties
-WORKDIR /home/ubuntu/software/flatbuffers
-RUN ./flatc --cpp -o /home/ubuntu/software/jasminegraph/src/util/dbutil /home/ubuntu/software/jasminegraph/src/util/dbutil/edgestore.fbs && \
-    ./flatc --cpp -o /home/ubuntu/software/jasminegraph/src/util/dbutil /home/ubuntu/software/jasminegraph/src/util/dbutil/attributestore.fbs && \
-    ./flatc --cpp -o /home/ubuntu/software/jasminegraph/src/util/dbutil /home/ubuntu/software/jasminegraph/src/util/dbutil/partedgemapstore.fbs
-WORKDIR /home/ubuntu/software/jasminegraph
 ENV JASMINEGRAPH_HOME="/home/ubuntu/software/jasminegraph"
+RUN mkdir /home/ubuntu/software/jasminegraph
+WORKDIR /home/ubuntu/software/jasminegraph
+RUN pip install tensorflow==2.5.3
+RUN pip install -U scikit-learn
+COPY . .
+RUN pip install -r ./GraphSAGE/requirements
+
 RUN sh build.sh
 ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
-CMD ["bash"]
+CMD ["bash"]

GraphSAGE/requirements

@@ -1,25 +1,5 @@
-absl-py==0.2.2
-astor==0.6.2
-backports.weakref==1.0.post1
-bleach==1.5.0
-decorator==4.3.0
-enum34==1.1.6
-funcsigs==1.0.2
-futures==3.2.0
-gast==0.2.0
-grpcio==1.12.1
-html5lib==0.9999999
-Markdown==2.6.11
-mock==2.0.0
 networkx==1.11
-numpy==1.14.5
-pbr==4.0.4
-protobuf==3.6.0
-scikit-learn==0.19.1
-scipy==1.1.0
-six==1.11.0
-sklearn==0.0
-tensorboard==1.8.0
-tensorflow==1.8.0
-termcolor==1.1.0
-Werkzeug==0.14.1
+numpy==1.17.3
+scipy==1.4.1
+joblib==1.1.1
+threadpoolctl==2.0.0

README.md

@@ -84,7 +84,6 @@ JasmineGraph can be run by executing the run.sh script. This will start master o
 ## 4. Building and Running JasmineGraph on Docker
 JasmineGraph can be run inside a docker image. After cloning the project, build the image as follows:
 
-    cd docker
     docker build -t jasminegraph .
 
 

build.sh

@@ -4,4 +4,4 @@ if [ "$1" == "--debug" ]; then
 else
     cmake clean .
 fi
-cmake --build . --target JasmineGraph -- -j 2
+cmake --build . --target JasmineGraph -- -j 4

conf/jasminegraph-server.properties

@@ -22,7 +22,7 @@ org.jasminegraph.server.host=localhost
 #Number of workers JasmineGraph cluster should use if it was not specified as a console argument
 org.jasminegraph.server.nworkers=2
 #org.jasminegraph.server.npartitions is the number of partitions into which the graph should be partitioned
-org.jasminegraph.server.npartitions=1
+org.jasminegraph.server.npartitions=2
 org.jasminegraph.server.streaming.kafka.host=127.0.0.1:9092
 #org.jasminegraph.server.runtime.location is the location where temporary files of JasmineGraph master is kept
 org.jasminegraph.server.runtime.location=./tmp
@@ -32,7 +32,7 @@ org.jasminegraph.worker.path=/var/tmp/jasminegraph/
 #if it is empty please leave a space, else it will cause an issue
 org.jasminegraph.artifact.path=
 #org.jasminegraph.partitioner.metis.bin is the location where the METIS graph partitioner's gpmetis executable is installed
-org.jasminegraph.partitioner.metis.bin=/home/ubuntu/software/metis/metis-5.1.0/bin
+org.jasminegraph.partitioner.metis.bin=/usr/local/bin
 #The following folder is the location where workers keep their data.
 #This is the location where the actual data storage takes place in JasmineGraph.
 org.jasminegraph.server.instance.datafolder=/var/tmp/jasminegraph-localstore

run-docker.sh

@@ -69,6 +69,7 @@ else
     fi
 fi
 
+export LD_LIBRARY_PATH=/home/ubuntu/software/METIS/build/libmetis
 if [ $MODE -eq 1 ] ;
 then
     ./JasmineGraph "docker" $MODE $MASTERIP $WORKERS $WORKERIP $ENABLE_NMON

src/backend/JasmineGraphBackend.cpp

@@ -27,7 +27,7 @@ void *backendservicesesion(void *dummyPt) {
     int connFd = sessionargs->connFd;
     SQLiteDBInterface sqLiteDbInterface = sessionargs->sqlite;
     backend_logger.log("Thread No: " + to_string(pthread_self()), "info");
-    char data[300];
+    char data[301];
     bzero(data, 301);
     bool loop = false;
     while (!loop) {
@@ -46,7 +46,7 @@ void *backendservicesesion(void *dummyPt) {
             write(connFd, HANDSHAKE_OK.c_str(), HANDSHAKE_OK.size());
             write(connFd, "\r\n", 2);
 
-            char host[300];
+            char host[301];
             bzero(host, 301);
             read(connFd, host, 300);
             string hostname(host);
@@ -66,7 +66,7 @@ void *backendservicesesion(void *dummyPt) {
             }
 
             // We get the name and the path to graph as a pair separated by |.
-            char worker_info_data[300];
+            char worker_info_data[301];
             bzero(worker_info_data, 301);
             string name = "";
 
@@ -92,6 +92,7 @@ void *backendservicesesion(void *dummyPt) {
     }
     backend_logger.log("Closing thread " + to_string(pthread_self()) + " and connection", "info");
     close(connFd);
+    return NULL;
 }
 
 JasmineGraphBackend::JasmineGraphBackend(SQLiteDBInterface db, int numberOfWorkers) {
@@ -170,4 +171,5 @@ int JasmineGraphBackend::run() {
     for (int i = 0; i < noThread; i++) {
         pthread_join(threadA[i], NULL);
     }
+    return 1;
 }