diff --git a/CMakeLists.txt b/CMakeLists.txt index 582d056ab2..dc952a6359 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,12 @@ include_directories( # set link path link_directories(${GraphLab_SOURCE_DIR}/deps/local/lib) +if (DEFINED EXTERNAL_VERTEX_ID_TYPE_INCLUDE) + add_definitions(-DEXTERNAL_VERTEX_ID_TYPE_INCLUDE=${EXTERNAL_VERTEX_ID_TYPE_INCLUDE}) +endif() +if (DEFINED EXTERNAL_VERTEX_ID_TYPE) + add_definitions(-DEXTERNAL_VERTEX_ID_TYPE=${EXTERNAL_VERTEX_ID_TYPE}) +endif() add_definitions(-DUSE_DYNAMIC_LOCAL_GRAPH) if(NO_OPENMP) diff --git a/apps/example/hello_world.cpp b/apps/example/hello_world.cpp index bc3292baad..f8acf49dd9 100644 --- a/apps/example/hello_world.cpp +++ b/apps/example/hello_world.cpp @@ -37,7 +37,7 @@ typedef graphlab::distributed_graph graph_type * A simple function used by graph.transform_vertices(init_vertex); * to initialize the vertex data. */ -void init_vertex(graph_type::vertex_type& vertex) { vertex.data() = vertex.id(); } +void init_vertex(graph_type::vertex_type& vertex) { vertex.data() = static_cast(vertex.id()); } struct min_combiner { graphlab::vertex_id_type v; diff --git a/src/graphlab/engine/distributed_chandy_misra.hpp b/src/graphlab/engine/distributed_chandy_misra.hpp index bde8781c11..417a4b29e8 100644 --- a/src/graphlab/engine/distributed_chandy_misra.hpp +++ b/src/graphlab/engine/distributed_chandy_misra.hpp @@ -270,7 +270,7 @@ class distributed_chandy_misra { philosopherset[lvid].lock.unlock(); if (requestor != rmi.procid()) { - unsigned char pkey = rmi.dc().set_sequentialization_key(gvid % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(gvid)); rmi.remote_call(requestor, &dcm_type::rpc_cancellation_accept, gvid, @@ -312,7 +312,7 @@ class distributed_chandy_misra { cancellation_request_unlocked(lvid, rmi.procid(), lockid); } else { - unsigned char pkey = rmi.dc().set_sequentialization_key(lvertex.global_id() % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(lvertex.global_id())); rmi.remote_call(lvertex.owner(), &dcm_type::rpc_cancellation_request, lvertex.global_id(), @@ -523,7 +523,7 @@ class distributed_chandy_misra { signal_ready_unlocked(p_id, philosopherset[p_id].lockid); } else { - unsigned char pkey = rmi.dc().set_sequentialization_key(lvertex.global_id() % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(lvertex.global_id())); if (hors_doeuvre_callback != NULL) hors_doeuvre_callback(p_id); rmi.remote_call(lvertex.owner(), &dcm_type::rpc_signal_ready, @@ -571,7 +571,7 @@ class distributed_chandy_misra { philosopherset[lvid].lock.unlock(); // broadcast EATING local_vertex_type lvertex(graph.l_vertex(lvid)); - unsigned char pkey = rmi.dc().set_sequentialization_key(lvertex.global_id() % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(lvertex.global_id())); rmi.remote_call(lvertex.mirrors().begin(), lvertex.mirrors().end(), &dcm_type::rpc_set_eating, lvertex.global_id(), lockid); set_eating(lvid, lockid); @@ -784,7 +784,7 @@ class distributed_chandy_misra { philosopherset[p_id].lock.unlock(); - unsigned char pkey = rmi.dc().set_sequentialization_key(lvertex.global_id() % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(lvertex.global_id())); rmi.remote_call(lvertex.mirrors().begin(), lvertex.mirrors().end(), &dcm_type::rpc_make_philosopher_hungry, lvertex.global_id(), newlockid); rmi.dc().set_sequentialization_key(pkey); @@ -826,7 +826,7 @@ class distributed_chandy_misra { // ASSERT_EQ(philosopherset[p_id].state, (int)EATING); philosopherset[p_id].counter = 0; philosopherset[p_id].lock.unlock(); - unsigned char pkey = rmi.dc().set_sequentialization_key(lvertex.global_id() % 254 + 1); + unsigned char pkey = rmi.dc().set_sequentialization_key(assess_sequentialization_key(lvertex.global_id())); rmi.remote_call(lvertex.mirrors().begin(), lvertex.mirrors().end(), &dcm_type::rpc_philosopher_stops_eating, lvertex.global_id()); rmi.dc().set_sequentialization_key(pkey); @@ -961,6 +961,10 @@ class distributed_chandy_misra { } } } + + inline unsigned char assess_sequentialization_key(vertex_id_type v_id) { + return static_cast(v_id % 254 + 1); + } }; } diff --git a/src/graphlab/graph/distributed_graph.hpp b/src/graphlab/graph/distributed_graph.hpp index e8fea5a0ee..23b6069579 100644 --- a/src/graphlab/graph/distributed_graph.hpp +++ b/src/graphlab/graph/distributed_graph.hpp @@ -2600,7 +2600,7 @@ namespace graphlab { /// The local vid of this vertex on this proc vertex_id_type gvid; /// The number of in edges - vertex_id_type num_in_edges, num_out_edges; + size_t num_in_edges, num_out_edges; /** The set of proc that mirror this vertex. The owner should NOT be in this set.*/ mirror_type _mirrors; diff --git a/src/graphlab/graph/graph_basic_types.hpp b/src/graphlab/graph/graph_basic_types.hpp index 13ef109794..708cc33972 100644 --- a/src/graphlab/graph/graph_basic_types.hpp +++ b/src/graphlab/graph/graph_basic_types.hpp @@ -26,21 +26,28 @@ #include -namespace graphlab { - - +#ifdef EXTERNAL_VERTEX_ID_TYPE_INCLUDE + #include EXTERNAL_VERTEX_ID_TYPE_INCLUDE +#endif +namespace graphlab { + + #ifdef USE_VID32 + /// Identifier type of a vertex which is globally consistent. Guaranteed to be integral + typedef uint32_t standard_vertex_id_type; + #else + typedef uint64_t standard_vertex_id_type; + #endif -#ifdef USE_VID32 - /// Identifier type of a vertex which is globally consistent. Guaranteed to be integral - typedef uint32_t vertex_id_type; -#else - typedef uint64_t vertex_id_type; -#endif + #ifndef EXTERNAL_VERTEX_ID_TYPE + typedef standard_vertex_id_type vertex_id_type; + #else + typedef EXTERNAL_VERTEX_ID_TYPE vertex_id_type; + #endif /// Identifier type of a vertex which is only locally consistent. Guaranteed to be integral - typedef vertex_id_type lvid_type; + typedef standard_vertex_id_type lvid_type; /** * Identifier type of an edge which is only locally * consistent. Guaranteed to be integral and consecutive. diff --git a/src/graphlab/graph/graph_hash.hpp b/src/graphlab/graph/graph_hash.hpp index 6f64d54f0b..049fbe10fa 100644 --- a/src/graphlab/graph/graph_hash.hpp +++ b/src/graphlab/graph/graph_hash.hpp @@ -30,7 +30,7 @@ namespace graphlab { namespace graph_hash { /** \brief Returns the hashed value of a vertex. */ inline static size_t hash_vertex (const vertex_id_type vid) { - return integer_mix(vid); + return integer_mix(static_cast(vid)); } /** \brief Returns the hashed value of an edge. */ @@ -57,7 +57,7 @@ namespace graphlab { #endif vertex_id_type src = e.first; vertex_id_type dst = e.second; - return (integer_mix(src^a[seed%8]))^(integer_mix(dst^a[(seed+1)%8])); + return (integer_mix(static_cast(src^a[seed%8]))) ^ (integer_mix(static_cast(dst^a[(seed+1)%8]))); } } // end of graph_hash namespace } // end of graphlab namespace diff --git a/src/graphlab/graph/graph_ops.hpp b/src/graphlab/graph/graph_ops.hpp index f9726b13cd..10f1d46c7f 100644 --- a/src/graphlab/graph/graph_ops.hpp +++ b/src/graphlab/graph/graph_ops.hpp @@ -53,7 +53,8 @@ namespace graphlab { * * \param[out] topsort Resultant topological sort of the graph vertices. * - * function will return false if graph is not acyclic. + * function will return false if graph is not acyclic. + * The function requires the vertices of the graph to be numbered from 0 to graph.num_vertices()-1. */ template bool topological_sort(const distributed_graph& graph, @@ -77,8 +78,8 @@ namespace graphlab { topsort.push_back(v); foreach(typename graph_type::edge_type edge, graph.get_out_edges(v)) { vertex_id_type destv = edge.target(); - --indeg[destv]; - if (indeg[destv] == 0) { + --indeg[static_cast(destv)]; + if (indeg[static_cast(destv)] == 0) { q.push(destv); } } diff --git a/src/graphlab/graph/ingress/distributed_ingress_base.hpp b/src/graphlab/graph/ingress/distributed_ingress_base.hpp index ce4fefaff1..20412e58f7 100644 --- a/src/graphlab/graph/ingress/distributed_ingress_base.hpp +++ b/src/graphlab/graph/ingress/distributed_ingress_base.hpp @@ -85,7 +85,7 @@ namespace graphlab { /// Detail vertex record for the second pass coordination. struct vertex_negotiator_record { mirror_type mirrors; - vertex_id_type num_in_edges, num_out_edges; + size_t num_in_edges, num_out_edges; bool has_data; vertex_data_type vdata; vertex_negotiator_record() : num_in_edges(0), num_out_edges(0), has_data(false) { } diff --git a/src/graphlab/util/multiprecision_vertex_id_types.hpp b/src/graphlab/util/multiprecision_vertex_id_types.hpp new file mode 100644 index 0000000000..d36e268cd8 --- /dev/null +++ b/src/graphlab/util/multiprecision_vertex_id_types.hpp @@ -0,0 +1,67 @@ +#include +#include + +#define MULTIPRECISION_VERTEX_ID_SAVE std::stringstream ss;\ + ss << x;\ + std::string s;\ + ss >> s;\ + oarc << s; + +#define MULTIPRECISION_VERTEX_ID_LOAD std::string s;\ + iarc >> s;\ + x = 0;\ + x.assign(s); + +// Integer of 128 bits. + +BEGIN_OUT_OF_PLACE_SAVE(oarc, boost::multiprecision::int128_t , x) + MULTIPRECISION_VERTEX_ID_SAVE +END_OUT_OF_PLACE_SAVE() + +BEGIN_OUT_OF_PLACE_LOAD(iarc, boost::multiprecision::int128_t , x) + MULTIPRECISION_VERTEX_ID_LOAD +END_OUT_OF_PLACE_LOAD() + +// Integer of 256 bits. + +BEGIN_OUT_OF_PLACE_SAVE(oarc, boost::multiprecision::int256_t , x) + MULTIPRECISION_VERTEX_ID_SAVE +END_OUT_OF_PLACE_SAVE() + +BEGIN_OUT_OF_PLACE_LOAD(iarc, boost::multiprecision::int256_t , x) + MULTIPRECISION_VERTEX_ID_LOAD +END_OUT_OF_PLACE_LOAD() + +// Integer of 512 bits. + +BEGIN_OUT_OF_PLACE_SAVE(oarc, boost::multiprecision::int512_t , x) + MULTIPRECISION_VERTEX_ID_SAVE +END_OUT_OF_PLACE_SAVE() + +BEGIN_OUT_OF_PLACE_LOAD(iarc, boost::multiprecision::int512_t , x) + MULTIPRECISION_VERTEX_ID_LOAD +END_OUT_OF_PLACE_LOAD() + +// Integer of 1024 bits. + +BEGIN_OUT_OF_PLACE_SAVE(oarc, boost::multiprecision::int1024_t , x) + MULTIPRECISION_VERTEX_ID_SAVE +END_OUT_OF_PLACE_SAVE() + +BEGIN_OUT_OF_PLACE_LOAD(iarc, boost::multiprecision::int1024_t , x) + MULTIPRECISION_VERTEX_ID_LOAD +END_OUT_OF_PLACE_LOAD() + +namespace boost { + namespace multiprecision { + template + inline size_t hash_value (const T &x) + { + // not a very good hash function, but I just want to get it working first! + return static_cast(x); + } + } +} + + + diff --git a/tests/distributed_chandy_misra_test.cpp b/tests/distributed_chandy_misra_test.cpp index a8666e0995..6c9524133e 100644 --- a/tests/distributed_chandy_misra_test.cpp +++ b/tests/distributed_chandy_misra_test.cpp @@ -87,7 +87,7 @@ void thread_stuff() { deq = locked_elements.dequeue(); if (deq.second == false) break; else { - locks->philosopher_stops_eating(deq.first); + locks->philosopher_stops_eating(static_cast(deq.first)); mt.lock(); current_demand_set[deq.first] = 0; bool getnextlock = nlocks_to_acquire > 0; @@ -115,7 +115,7 @@ void thread_stuff() { } mt.unlock(); } - locks->make_philosopher_hungry(toacquire); + locks->make_philosopher_hungry(static_cast(toacquire)); } } } @@ -253,7 +253,7 @@ int main(int argc, char** argv) { nlocks_to_acquire = INITIAL_NLOCKS_TO_ACQUIRE; dc.full_barrier(); for (graphlab::vertex_id_type v = 0; v < graph.num_local_vertices(); ++v) { - if (graph.l_get_vertex_record(v).owner == dc.procid()) { + if (graph.l_get_vertex_record(static_cast(v)).owner == dc.procid()) { demand_set[v] = 1; current_demand_set[v] = 1; lockable_vertices.push_back(v); @@ -265,9 +265,9 @@ int main(int argc, char** argv) { thrs.launch(thread_stuff); } for (graphlab::vertex_id_type v = 0; v < graph.num_local_vertices(); ++v) { - if (graph.l_get_vertex_record(v).owner == dc.procid()) { + if (graph.l_get_vertex_record(static_cast(v)).owner == dc.procid()) { //std::cout << dc.procid() << ": Lock Req for " << graph.l_get_vertex_record(v).gvid << std::endl; - locks->make_philosopher_hungry(v); + locks->make_philosopher_hungry(static_cast(v)); } } mt.lock(); @@ -282,7 +282,7 @@ int main(int argc, char** argv) { bool bad = (nlocksacquired != INITIAL_NLOCKS_TO_ACQUIRE + lockable_vertices.size()); while (iter != demand_set.end()) { if(locked_set[iter->first] != iter->second) { - std::cout << graph.l_get_vertex_record(iter->first).gvid << " mismatch: " + std::cout << graph.l_get_vertex_record(static_cast(iter->first)).gvid << " mismatch: " << locked_set[iter->first] << ", " << iter->second << "\n"; bad = true; } diff --git a/tests/distributed_graph_test.cpp b/tests/distributed_graph_test.cpp index 1f6a2c922f..9866cd080b 100644 --- a/tests/distributed_graph_test.cpp +++ b/tests/distributed_graph_test.cpp @@ -59,9 +59,9 @@ class distributed_graph_test { }; struct edge_data: public graphlab::IS_POD_TYPE { - int from; - int to; - edge_data (int f = 0, int t = 0) : from(f), to(t) {} + graphlab::vertex_id_type from; + graphlab::vertex_id_type to; + edge_data (graphlab::vertex_id_type f = 0, graphlab::vertex_id_type t = 0) : from(f), to(t) {} bool operator==(const edge_data& other) const { return ((from == other.from) && (to == other.to)); } diff --git a/tests/local_graph_test.cxx b/tests/local_graph_test.cxx index cc09406d0c..d1e56f16ba 100644 --- a/tests/local_graph_test.cxx +++ b/tests/local_graph_test.cxx @@ -43,9 +43,9 @@ class local_graph_test : public CxxTest::TestSuite { }; struct edge_data { - int from; - int to; - edge_data (int f = 0, int t = 0) : from(f), to(t) {} + graphlab::lvid_type from; + graphlab::lvid_type to; + edge_data (graphlab::lvid_type f = 0, graphlab::lvid_type t = 0) : from(f), to(t) {} }; /** @@ -162,15 +162,15 @@ class local_graph_test : public CxxTest::TestSuite { */ template void check_adjacency(Graph& g, - boost::unordered_map >& in_edges, - boost::unordered_map >& out_edges, + boost::unordered_map >& in_edges, + boost::unordered_map >& out_edges, size_t nedges) { typedef typename Graph::edge_list_type edge_list_type; typedef typename Graph::edge_type edge_type; typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; // check size ASSERT_EQ(g.num_edges(), nedges); @@ -214,7 +214,7 @@ template typedef typename Graph::edge_list_type edge_list_type; typedef typename Graph::edge_type edge_type; typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; for (size_t i = 0; i < g.num_vertices(); ++i) { const edge_list_type& in_edges = g.in_edges(i); foreach (const edge_type& e, in_edges) { @@ -231,7 +231,7 @@ template template void test_add_edge_impl(Graph& g, size_t nedges, bool use_dynamic=false) { - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; srand(0); g.clear(); ASSERT_EQ(g.num_edges(), 0); @@ -289,7 +289,7 @@ template typedef typename Graph::edge_list_type edge_list_type; typedef typename Graph::edge_type edge_type; typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; size_t num_v = 10; size_t num_e = 6; @@ -375,7 +375,7 @@ template typedef typename Graph::edge_list_type edge_list_type; typedef typename Graph::edge_type edge_type; typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; g.clear(); if (verbose) @@ -524,7 +524,7 @@ template typedef typename Graph::edge_list_type edge_list_type; typedef typename Graph::edge_type edge_type; typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_id_type vertex_id_type; + typedef typename graphlab::lvid_type vertex_id_type; boost::unordered_map > out_edges; boost::unordered_map > in_edges;