code to calculate displacement factor for every test case in predict option

CMakeLists.txt

@@ -31,14 +31,15 @@ if(SERVER)
     set(COMPILE_FLAGS "${COMPILE_FLAGS} -DSERVER=1")
     message("SERVER forces the enabling of SPARQL")
     set(SPARQL "1")
+    set(ML "1")
 ENDIF()
 IF(SPARQL)
     set(COMPILE_FLAGS "${COMPILE_FLAGS} -DSPARQL=1")
 ENDIF()
 IF(PYTHON)
     message("PYTHON forces the enabling of ML and ANALYTICS")
     set(ML "1")
-    set(ANALYTICS "1")
+    set(ANALYTICS "0")
 ENDIF()
 if(ANALYTICS)
     set(COMPILE_FLAGS "${COMPILE_FLAGS} -DANALYTICS=1")
@@ -52,9 +53,9 @@ ENDIF()
 
 #Set compiler options
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3")
-set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -DDEBUG")
-set(COMPILE_FLAGS "-c -fPIC -std=c++11 -Wall -Wno-sign-compare ${COMPILE_FLAGS}")
+    set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG -O3")
+    set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -DDEBUG")
+    set(COMPILE_FLAGS "-c -fPIC -std=c++11 -Wall -Wno-sign-compare ${COMPILE_FLAGS}")
 endif()
 
 #Set to release mode if nothing is specified
@@ -183,9 +184,37 @@ IF(ML)
         "src/trident/mining/*.cpp"
         "src/trident/ml/*.cpp"
         src/trident/mining/miner.cpp)
+
+    # Install FAISS
+    set(CMAKE_PREFIX_PATH /cm/shared/apps/open64/4.5.2.1/lib)
+    #find_library(omp omp)
+    find_package(OpenMP QUIET)
+    if (OPENMP_FOUND)
+        message("OpenMP found")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    endif()
+    find_library(openblas openblas PATHS /cm/images/MLNX-centos74-MIC-image/cm/shared/apps/openblas/0.2.18/lib)
+    ExternalProject_Add(git-faiss
+        DOWNLOAD_COMMAND git clone https://github.com/facebookresearch/faiss.git
+        DOWNLOAD_DIR .
+        SOURCE_DIR faiss/
+        CONFIGURE_COMMAND ""
+        BUILD_IN_SOURCE 1
+        BUILD_COMMAND "export" "CXX=/cm/shared/package/gcc/4.9.3/bin/g++ -std=c++11" && "export" "CPUFLAGS=-msse4 -mpopcnt" && "export" "" && "export" "LDFLAGS=-fPIC -fopenmp " && "export" "CPPFLAGS=-fPIC -m64 -Wall -g -O3 -Wno-sign-compare -DFINTEGER=int -fopenmp " && make -j libfaiss.a
+        INSTALL_COMMAND ""
+        )
+    ExternalProject_Get_Property(git-faiss SOURCE_DIR)
+    include_directories(${SOURCE_DIR}/..)
+    add_library(faiss STATIC IMPORTED)
+    add_dependencies(faiss git-faiss)
+    set(FAISS_LIB "${CMAKE_BINARY_DIR}/faiss")
+    set_property(TARGET faiss PROPERTY IMPORTED_LOCATION "${FAISS_LIB}/libfaiss.a")
+
+    set(COMPILE_FLAGS "-DFINTEGER=int -m64 -msse4 ${COMPILE_FLAGS}")
     add_library(trident-ml SHARED ${ml_SRC})
     set_target_properties(trident-ml PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS}")
-    TARGET_LINK_LIBRARIES(trident-ml kognac-core trident-core)
+    TARGET_LINK_LIBRARIES(trident-ml kognac-core trident-core faiss ${openblas})
     TARGET_LINK_LIBRARIES(trident trident-ml)
 ENDIF()
 
@@ -242,7 +271,7 @@ IF (SERVER)
 
     add_library(trident-web SHARED ${webserver_SRC})
     set_target_properties(trident-web PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS}")
-    TARGET_LINK_LIBRARIES(trident-web kognac-core trident-core trident-sparql)
+    TARGET_LINK_LIBRARIES(trident-web kognac-core trident-core trident-sparql trident-ml)
     TARGET_LINK_LIBRARIES(trident trident-web)
 ENDIF()
 
@@ -251,9 +280,11 @@ IF(PYTHON)
     include(FindPkgConfig)
     set(Python_ADDITIONAL_VERSIONS 3.5)
     pkg_check_modules(PY3 REQUIRED QUIET python3)
+    find_package(PythonLibs REQUIRED)
     IF(PY3_FOUND)
-        INCLUDE_DIRECTORIES(${PY3_INCLUDE_DIRS})
-        LINK_DIRECTORIES(${PY3_LIBRARY_DIRS})
+        INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIRS})
+        LINK_DIRECTORIES(${PYTHON_LIBRARY})
+        message("Python include path = ${PYTHON_INCLUDE_DIRS}")
     ENDIF()
     #NumPY
     if(NOT PYTHON_EXECUTABLE)
@@ -270,6 +301,7 @@ IF(PYTHON)
             COMMAND "${PYTHON_EXECUTABLE}" -c
             "from __future__ import print_function\ntry: import numpy; print(numpy.get_include(), end='')\nexcept:pass\n"
             OUTPUT_VARIABLE __numpy_path)
+        message("numpy path : ${__numpy_path}")
         # And the version
         execute_process(
             COMMAND "${PYTHON_EXECUTABLE}" -c
@@ -294,7 +326,7 @@ IF(PYTHON)
     add_library(tridentp SHARED ${tridentp_SRC}) #Python library
     set_target_properties(tridentp PROPERTIES OUTPUT_NAME "trident" PREFIX "" SUFFIX ".so")
     set_target_properties(tridentp PROPERTIES COMPILE_FLAGS "${COMPILE_FLAGS}")
-    TARGET_LINK_LIBRARIES(tridentp trident-core trident-ml trident-ana ${PY3_LIBRARIES})
+    TARGET_LINK_LIBRARIES(tridentp trident-core trident-ml ${PY3_LIBRARIES})
 ENDIF()
 
 if(JAVA)

README.md

@@ -135,16 +135,22 @@ problem. To check SNAP has been correctly compiled, make sure that the file
 
 ## License
 
-Trident is released under Apache 2 license. In the source directory
+Trident is released under Apache 2 license.
+
+In the source directory
 there is one directory called "rdf3x". This directory
 contains a modified version of the RDF3X engine written by <a
 href="https://db.in.tum.de/~neumann/">Thomas Neumann</a>. The source code of
 RDF3X was released on  google code but unfortunately that repository is no
 longer available. The original version of RDF3X is released with Creative
 Commons. In order to comply with the original license, also this modified
-version should be considered released with the same license. There is another directory, called "snap", which contains a modified version of the SNAP library developed at
+version should be considered released with the same license.
+
+There is another directory, called "snap", which contains a modified version of the SNAP library developed at
 Stanford (<a href="http://snap.stanford.edu/snap/index.html">Here</a> is the
 link to the homepage of the project). I modified the code of SNAP in order to
 make it more efficient and to be able to scale up to larger graphs. My changes are contained in the "snap", "src/snap" and "include/snap" directories. Since the original version of SNAP is
-released with the BSD license, then also my modifications are released with
-the same license. This means that everything under these three directories should be considered released under BSD while everything in the directory "rdf3x" should be considered under Creative Commons.
+released with the BSD license, my modifications are released with the same license.
+
+This project contains parts of Kiss FFT (https://github.com/mborgerding/kissfft) which is released with
+the 3-clause BSD license. We only modified it to use double instead of float.

include/layers/TridentLayer.hpp

@@ -169,7 +169,7 @@ class TridentLayer : public DBLayer {
         bifSampl(true), nindices(kb.getNIndices()), supportBuffer(new
                 char[MAX_TERM_SIZE]) { }
 
-		DDLEXPORT bool lookup(const std::string& text,
+        DDLEXPORT bool lookup(const std::string& text,
                 ::Type::ID type,
                 unsigned subType,
                 uint64_t& id);
@@ -178,13 +178,13 @@ class TridentLayer : public DBLayer {
             bifSampl = false;
         }
 
-		DDLEXPORT bool lookupById(uint64_t id,
+        DDLEXPORT bool lookupById(uint64_t id,
                 const char*& start,
                 const char*& stop,
                 ::Type::ID& type,
                 unsigned& subType);
 
-		DDLEXPORT bool lookupById(uint64_t id,
+        DDLEXPORT bool lookupById(uint64_t id,
                 char *output,
                 size_t &length,
                 ::Type::ID& type,
@@ -200,13 +200,13 @@ class TridentLayer : public DBLayer {
                 uint64_t value3,
                 uint64_t value3C);
 
-		DDLEXPORT double getScanCost(DBLayer::DataOrder order,
+        DDLEXPORT double getScanCost(DBLayer::DataOrder order,
                 uint64_t value1,
                 uint64_t value1C,
                 uint64_t value2,
                 uint64_t value2C);
 
-		DDLEXPORT double getJoinSelectivity(bool valueL1,
+        DDLEXPORT double getJoinSelectivity(bool valueL1,
                 uint64_t value1CL,
                 bool value2L,
                 uint64_t value2CL,
@@ -219,7 +219,7 @@ class TridentLayer : public DBLayer {
                 bool value3R,
                 uint64_t value3CR);
 
-		DDLEXPORT double getScanCost(DBLayer::DataOrder order,
+        DDLEXPORT double getScanCost(DBLayer::DataOrder order,
                 uint64_t value1,
                 uint64_t value1C);
 
@@ -233,7 +233,7 @@ class TridentLayer : public DBLayer {
             return kb.getNTerms();
         }
 
-		DDLEXPORT std::unique_ptr<DBLayer::Scan> getScan(const DBLayer::DataOrder order,
+        DDLEXPORT std::unique_ptr<DBLayer::Scan> getScan(const DBLayer::DataOrder order,
                 const DBLayer::Aggr_t,
                 Hint *hint);
 

include/python/trident.h

@@ -23,12 +23,13 @@
 #ifndef _PYTHON_H
 #define _PYTHON_H
 
-//#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-
 #include <Python.h>
 #include <trident/kb/kb.h>
 #include <trident/kb/querier.h>
 #include <trident/ml/batch.h>
+#include <trident/ml/embeddings.h>
+
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 
 typedef struct {
     PyObject_HEAD
@@ -44,8 +45,16 @@ typedef struct {
     char pos;
 } trident_Itr;
 
+typedef struct {
+    PyObject_HEAD
+    std::shared_ptr<Embeddings<double>> E;
+    std::shared_ptr<Embeddings<double>> R;
+} trident_Emb;
+
+
 extern PyTypeObject trident_ItrType;
 extern PyTypeObject trident_DbType;
+extern PyTypeObject trident_EmbType;
 
 typedef struct {
     PyObject_HEAD