Merge branch 'master' into fl-kasun-merge

.dockerignore

@@ -0,0 +1,14 @@
+tests
+metadb
+performancedb
+docs
+logs
+.git
+.github
+.vscode
+.idea
+.dockerignore
+.gitignore
+test-docker.sh
+README.md
+LICENSE

.github/workflows/build.yml

@@ -1,6 +1,10 @@
 name: Build
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 env:
   # Customize the build type here (Release, Debug, RelWithDebInfo, etc.)
@@ -11,10 +15,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{github.head_ref}}
+        fetch-depth: 0
+        repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
     - name: Build
       # Build JasmineGraph docker image
+      run: docker build -t jasminegraph .
+    - name: Run Integration Tests
       run: |
-        cd docker
-        docker build -t jasminegraph .
+        chmod +x test-docker.sh
+        ./test-docker.sh

.gitignore

@@ -1,3 +1,5 @@
+tests/integration/env/
+
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
@@ -69,4 +71,5 @@ fabric.properties
 build/
 
 # Log files
-logs/
+logs/
+.trash/

CMakeLists.txt

@@ -41,38 +41,27 @@ add_executable(JasmineGraph main.cpp src/server/JasmineGraphServer.cpp src/serve
         src/frontend/core/factory/ExecutorFactory.cpp src/frontend/core/factory/ExecutorFactory.h src/frontend/core/CoreConstants.cpp
         src/frontend/core/CoreConstants.h)
 
-target_compile_options(JasmineGraph PRIVATE -std=c++11 -I/usr/include/python3.5)
+target_compile_options(JasmineGraph PRIVATE -std=c++11 -I/usr/include/python3.11)
 
 if(CMAKE_ENABLE_DEBUG)
     message(STATUS "DEBUG enabled")
     target_compile_options(JasmineGraph PRIVATE -g)
 endif()
 
-target_link_libraries(JasmineGraph /usr/local/lib/libmetis.so)
-target_link_libraries(JasmineGraph /usr/local/lib/libxerces-c-3.2.so)
 #TODO: Need to install the library separately
 target_link_libraries(JasmineGraph sqlite3)
 target_link_libraries(JasmineGraph pthread)
 target_link_libraries(JasmineGraph rdkafka)
-#target_link_libraries(JasmineGraph cppkafka)
 target_link_libraries(JasmineGraph Threads::Threads)
-include_directories("$ENV{HOME}/software/flatbuffers/include")
-include_directories("$ENV{HOME}/software/xerces/include")
-include_directories("$ENV{HOME}/software/spdlog/include")
-include_directories("$ENV{HOME}/software/jsoncpp/include")
-include_directories("$ENV{HOME}/software/cppkafka/include")
-include_directories("$ENV{HOME}/software/nlohmann_json/include")
-
-#target_link_libraries(JasmineGraph /usr/local/lib/libmetis.a)
-#target_link_libraries(JasmineGraph $ENV{HOME}/software/xerces-c-3.2.2/lib/libxerces-c.so)
-target_link_libraries(JasmineGraph $ENV{HOME}/software/flatbuffers/libflatbuffers.a)
-include_directories("$ENV{HOME}/software/xerces-c-3.2.2/include")
-target_link_libraries(JasmineGraph $ENV{HOME}/software/jsoncpp/build/debug/src/lib_json/libjsoncpp.a)
+target_link_libraries(JasmineGraph fmt)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libxerces-c.so)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libflatbuffers.a)
+target_link_libraries(JasmineGraph /usr/lib/x86_64-linux-gnu/libjsoncpp.so)
 target_link_libraries(JasmineGraph /usr/local/lib/libcppkafka.so)
 
-set(PYTHON_EXECUTABLE "/usr/bin/python3.7")
-set(PYTHON_INCLUDE_DIR "/usr/include/python3.7m")
-set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.7m.so")
+set(PYTHON_EXECUTABLE "/usr/bin/python3.11")
+set(PYTHON_INCLUDE_DIR "/usr/include/python3.11m")
+set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.11.so")
 
 include_directories(${PYTHON_INCLUDE_DIRS})
 include_directories(${PYTHON_DIRECTORIES})

Dockerfile

@@ -0,0 +1,22 @@
+FROM miyurud/jasminegraph
+ENV HOME="/home/ubuntu"
+ENV JASMINEGRAPH_HOME="/home/ubuntu/software/jasminegraph"
+
+WORKDIR /home/ubuntu/software/jasminegraph
+
+COPY ./GraphSAGE ./GraphSAGE
+RUN pip install -r ./GraphSAGE/requirements
+
+COPY ./build.sh ./build.sh
+COPY ./run-docker.sh ./run-docker.sh
+COPY ./CMakeLists.txt ./CMakeLists.txt
+COPY ./src_python ./src_python
+COPY ./main.h ./main.h
+COPY ./main.cpp ./main.cpp
+COPY ./src ./src
+
+RUN sh build.sh
+COPY ./conf ./conf
+
+ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
+CMD ["bash"]

GraphSAGE/graphsage/aggregators.py

@@ -1,8 +1,10 @@
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from layers import Layer, Dense
 from inits import glorot, zeros
 
+tf.disable_v2_behavior()
+
 class MeanAggregator(Layer):
     """
     Aggregates via mean followed by matmul and non-linearity.

GraphSAGE/graphsage/inits.py

@@ -1,6 +1,8 @@
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 import numpy as np
 
+tf.disable_v2_behavior()
+
 # DISCLAIMER:
 # Parts of this code file are derived from
 # https://github.com/tkipf/gcn

GraphSAGE/graphsage/layers.py

@@ -1,10 +1,11 @@
 from __future__ import division
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from inits import zeros
 
+tf.disable_v2_behavior()
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 

GraphSAGE/graphsage/metrics.py

@@ -1,4 +1,5 @@
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 
 # DISCLAIMER:
 # Parts of this code file were originally forked from

GraphSAGE/graphsage/minibatch.py

@@ -25,16 +25,16 @@ def __init__(self, G, G_local,id2idx,
             **kwargs):
 
         self.G = G
-        self.nodes = G.nodes()
+        self.nodes = list(G.nodes())
         self.local_nodes = G_local.nodes()
-        self.central_nodes = list(set(G.nodes())-set(G_local.nodes()))
+        self.central_nodes = list(set(list(G.nodes()))-set(G_local.nodes()))
         self.id2idx = id2idx
         self.placeholders = placeholders
         self.batch_size = batch_size
         self.max_degree = max_degree
         self.batch_num = 0
 
-        self.nodes = np.random.permutation(G.nodes())
+        self.nodes = np.random.permutation(list(G.nodes()))
         self.adj, self.deg = self.construct_adj()
         self.test_adj = self.construct_test_adj()
         if context_pairs is None:
@@ -52,12 +52,12 @@ def __init__(self, G, G_local,id2idx,
             else:
                 self.train_edges = self.val_edges = self.edges
 
-        print(len([n for n in G.nodes() if not G.node[n]['test'] and not G.node[n]['val']]), 'train nodes')
-        print(len([n for n in G.nodes() if G.node[n]['test'] or G.node[n]['val']]), 'test nodes')
+        print(len([n for n in list(G.nodes()) if not G.nodes[n]['test'] and not G.nodes[n]['val']]), 'train nodes')
+        print(len([n for n in list(G.nodes()) if G.nodes[n]['test'] or G.nodes[n]['val']]), 'test nodes')
         self.val_set_size = len(self.val_edges)
 
     def _n2v_prune(self, edges):
-        is_val = lambda n : self.G.node[n]["val"] or self.G.node[n]["test"]
+        is_val = lambda n : self.G.nodes[n]["val"] or self.G.nodes[n]["test"]
         return [e for e in edges if not is_val(e[1])]
 
     def _remove_isolated(self, edge_list):
@@ -67,11 +67,11 @@ def _remove_isolated(self, edge_list):
         count = 0
         for n1, n2 in edge_list:
             count +=1
-            if not n1 in self.G.node or not n2 in self.G.node:
+            if not n1 in list(self.G.nodes()) or not n2 in list(self.G.nodes()):
                 missing += 1
                 continue
-            if (self.G.node[n1]['test'] and self.G.node[n2]['test']) \
-                    or (self.G.node[n1]['val'] and self.G.node[n2]['val']):
+            if (self.G.nodes[n1]['test'] and self.G.nodes[n2]['test']) \
+                    or (self.G.nodes[n1]['val'] and self.G.nodes[n2]['val']):
                 continue
             else:
                 new_edge_list.append((n1,n2))
@@ -82,8 +82,8 @@ def construct_adj(self):
         adj = len(self.id2idx)*np.ones((len(self.id2idx)+1, self.max_degree))
         deg = np.zeros((len(self.id2idx),))
 
-        for nodeid in self.G.nodes():
-            if self.G.node[nodeid]['test'] or self.G.node[nodeid]['val']:
+        for nodeid in list(self.G.nodes()):
+            if self.G.nodes[nodeid]['test'] or self.G.nodes[nodeid]['val']:
                 continue
             neighbors = np.array([self.id2idx[neighbor] 
                 for neighbor in self.G.neighbors(nodeid)
@@ -100,7 +100,7 @@ def construct_adj(self):
 
     def construct_test_adj(self):
         adj = len(self.id2idx)*np.ones((len(self.id2idx)+1, self.max_degree))
-        for nodeid in self.G.nodes():
+        for nodeid in list(self.G.nodes()):
             neighbors = np.array([self.id2idx[neighbor] 
                 for neighbor in self.G.neighbors(nodeid)])
             if len(neighbors) == 0:
@@ -183,8 +183,8 @@ def label_val(self):
         train_edges = []
         val_edges = []
         for n1, n2 in self.G.edges():
-            if (self.G.node[n1]['val'] or self.G.node[n1]['test'] 
-                    or self.G.node[n2]['val'] or self.G.node[n2]['test']):
+            if (self.G.nodes[n1]['val'] or self.G.nodes[n1]['test'] 
+                    or self.G.nodes[n2]['val'] or self.G.nodes[n2]['test']):
                 val_edges.append((n1,n2))
             else:
                 train_edges.append((n1,n2))
@@ -217,7 +217,7 @@ def __init__(self, G, id2idx,
             **kwargs):
 
         self.G = G
-        self.nodes = G.nodes()
+        self.nodes = list(G.nodes())
         self.id2idx = id2idx
         self.placeholders = placeholders
         self.batch_size = batch_size
@@ -229,11 +229,11 @@ def __init__(self, G, id2idx,
         self.adj, self.deg = self.construct_adj()
         self.test_adj = self.construct_test_adj()
 
-        self.val_nodes = [n for n in self.G.nodes() if self.G.node[n]['val']]
-        self.test_nodes = [n for n in self.G.nodes() if self.G.node[n]['test']]
+        self.val_nodes = [n for n in list(self.G.nodes()) if self.G.nodes[n]['val']]
+        self.test_nodes = [n for n in list(self.G.nodes()) if self.G.nodes[n]['test']]
 
         self.no_train_nodes_set = set(self.val_nodes + self.test_nodes)
-        self.train_nodes = set(G.nodes()).difference(self.no_train_nodes_set)
+        self.train_nodes = set(list(G.nodes())).difference(self.no_train_nodes_set)
         # don't train on nodes that only have edges to test set
         self.train_nodes = [n for n in self.train_nodes if self.deg[id2idx[n]] > 0]
 
@@ -251,8 +251,8 @@ def construct_adj(self):
         adj = len(self.id2idx)*np.ones((len(self.id2idx)+1, self.max_degree))
         deg = np.zeros((len(self.id2idx),))
 
-        for nodeid in self.G.nodes():
-            if self.G.node[nodeid]['test'] or self.G.node[nodeid]['val']:
+        for nodeid in list(self.G.nodes()):
+            if self.G.nodes[nodeid]['test'] or self.G.nodes[nodeid]['val']:
                 continue
             neighbors = np.array([self.id2idx[neighbor]
                 for neighbor in self.G.neighbors(nodeid)
@@ -269,7 +269,7 @@ def construct_adj(self):
 
     def construct_test_adj(self):
         adj = len(self.id2idx)*np.ones((len(self.id2idx)+1, self.max_degree))
-        for nodeid in self.G.nodes():
+        for nodeid in list(self.G.nodes()):
             neighbors = np.array([self.id2idx[neighbor] 
                 for neighbor in self.G.neighbors(nodeid)])
             if len(neighbors) == 0:

GraphSAGE/graphsage/models.py

@@ -1,6 +1,6 @@
 from collections import namedtuple
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 import math
 
 import layers as layers
@@ -9,6 +9,7 @@
 from prediction import BipartiteEdgePredLayer
 from aggregators import MeanAggregator, MaxPoolingAggregator, MeanPoolingAggregator, SeqAggregator, GCNAggregator
 
+tf.disable_v2_behavior()
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 

GraphSAGE/graphsage/neigh_samplers.py

@@ -3,7 +3,8 @@
 
 from layers import Layer
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 

GraphSAGE/graphsage/prediction.py

@@ -3,8 +3,9 @@
 
 from inits import zeros
 from layers import Layer
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
+tf.disable_v2_behavior()
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 

GraphSAGE/graphsage/supervised_models.py

@@ -1,9 +1,10 @@
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 import models as models
 import layers as layers
 from aggregators import MeanAggregator, MaxPoolingAggregator, MeanPoolingAggregator, SeqAggregator, GCNAggregator
 
+tf.disable_v2_behavior()
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 

GraphSAGE/graphsage/supervised_train.py

@@ -3,7 +3,7 @@
 
 import os
 import time
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 import numpy as np
 import sklearn
 from sklearn import metrics
@@ -16,6 +16,7 @@
 
 os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
 
+tf.disable_v2_behavior()
 # Set random seed
 seed = 123
 np.random.seed(seed)
@@ -25,7 +26,7 @@
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 
-tf.app.flags.DEFINE_boolean('log_device_placement', False,
+flags.DEFINE_boolean('log_device_placement', False,
                             """Whether to log device placement.""")
 #core params..
 flags.DEFINE_string('model', 'graphsage_mean', 'model names. See README for possible values.')