Packed nbr_unit_t to decrease the memory footprint when using int32_t…

… as vid_t (#1715) Fixes #1714 Signed-off-by: Tao He <[email protected]>
v6d-io · Jan 18, 2024 · b0c3129 · b0c3129
1 parent 6bc3c30
commit b0c3129
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 9 deletions.
diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go
@@ -48,7 +48,7 @@ var cmdLong = util.LongDesc(`
 
 var cmd = &cobra.Command{
 	Use:     "vineyardctl [command]",
-	Version: "v0.19.3",
+	Version: "v0.20.0",
 	Short:   "vineyardctl is the command-line tool for interact with the Vineyard Operator.",
 	Long:    cmdLong,
 }

diff --git a/modules/graph/CMakeLists.txt b/modules/graph/CMakeLists.txt
@@ -1,5 +1,18 @@
 include(GNUInstallDirs)
 
+# options to control several default behaviors of vineyard graph
+option(VINEYARD_GRAPH_MAX_LABEL_ID "Maximum label id value of vineyard graphs, defaults to 128" OFF)
+
+option(MY_OPTION "Description of My Option" OFF)
+set(VINEYARD_GRAPH_MAX_LABEL_ID_DEFAULT_VALUE 128 CACHE STRING "Default value for maximum label if for vineyard graphs")
+set_property(CACHE VINEYARD_GRAPH_MAX_LABEL_ID PROPERTY STRINGS "1;2;4;8;16;32;64;128")
+
+if (VINEYARD_GRAPH_MAX_LABEL_ID)
+    message(STATUS "Setting VINEYARD_GRAPH_MAX_LABEL_ID to '${VINEYARD_GRAPH_MAX_LABEL_ID}'")
+else()
+    message(STATUS "Setting VINEYARD_GRAPH_MAX_LABEL_ID to default value: '${VINEYARD_GRAPH_MAX_LABEL_ID_DEFAULT_VALUE}'")
+endif()
+
 # build vineyard-graph
 file(GLOB_RECURSE VINEYARD_MOD_SRCS "${CMAKE_CURRENT_SOURCE_DIR}"
                                     "*.vineyard-mod")
@@ -137,6 +150,10 @@ if(BUILD_VINEYARD_GRAPH_WITH_GAR)
     endif()
 endif()
 
+if (VINEYARD_GRAPH_MAX_LABEL_ID)
+    target_compile_options(vineyard_graph PUBLIC -DVINEYARD_GRAPH_MAX_LABEL_ID=${VINEYARD_GRAPH_MAX_LABEL_ID})
+endif()
+
 target_link_libraries(vineyard_graph PUBLIC vineyard_client
                                             vineyard_basic
                                             vineyard_io

diff --git a/modules/graph/README.rst b/modules/graph/README.rst
@@ -19,6 +19,18 @@ among graph computing engines.
     * `Read Options <#read-options>`_
     * `Global Options <#global-options>`_
 
+CMake configure options
+-----------------------
+
+- :code:`VINEYARD_GRAPH_MAX_LABEL_ID`
+
+  The internal vertex id (aka. :code:`VID`) in vineyard is encoded as fragment id, vertex label id
+  and vertex offset. The option :code:`VINEYARD_GRAPH_MAX_LABEL_ID` decides the bit field width of
+  label id in :code:`VID`. Decreasing this value can be helpful to support larger number of vertices
+  when using :code:`int32_t` as :code:`VID_T`.
+
+  Defaults to `128`, can be `1`, `2`, `4`, `8`, `16`, `32`, `64`, or `128`.
+
 vineyard-graph-loader
 ---------------------
 

diff --git a/modules/graph/fragment/property_graph_types.h b/modules/graph/fragment/property_graph_types.h
@@ -80,8 +80,11 @@ using LABEL_ID_TYPE = int;
 
 }  // namespace property_graph_types
 
-// Hardcoded the max vertex label num to 128
+#if defined(VINEYARD_GRAPH_MAX_LABEL_ID)
+constexpr int MAX_VERTEX_LABEL_NUM = VINEYARD_GRAPH_MAX_LABEL_ID;
+#else
 constexpr int MAX_VERTEX_LABEL_NUM = 128;
+#endif
 
 static inline int num_to_bitwidth(int num) {
   if (num <= 2) {
@@ -104,13 +107,11 @@ static inline int num_to_bitwidth(int num) {
  */
 template <typename ID_TYPE>
 class IdParser {
-  using LabelIDT = int;  // LABEL_ID_TYPE
-
  public:
   IdParser() {}
   ~IdParser() {}
 
-  void Init(fid_t fnum, LabelIDT label_num) {
+  void Init(fid_t fnum, property_graph_types::LABEL_ID_TYPE label_num) {
     CHECK_LE(label_num, MAX_VERTEX_LABEL_NUM);
     int fid_width = num_to_bitwidth(fnum);
     fid_offset_ = (sizeof(ID_TYPE) * 8) - fid_width;
@@ -125,26 +126,30 @@ class IdParser {
 
   fid_t GetFid(ID_TYPE v) const { return (v >> fid_offset_); }
 
-  LabelIDT GetLabelId(ID_TYPE v) const {
+  property_graph_types::LABEL_ID_TYPE GetLabelId(ID_TYPE v) const {
     return (v & label_id_mask_) >> label_id_offset_;
   }
 
   int64_t GetOffset(ID_TYPE v) const { return (v & offset_mask_); }
 
   ID_TYPE GetLid(ID_TYPE v) const { return v & lid_mask_; }
 
+  ID_TYPE GetMaxOffset() const { return offset_mask_; }
+
   /**
    * @brief Generate the LID
    */
-  ID_TYPE GenerateId(LabelIDT label, int64_t offset) const {
+  ID_TYPE GenerateId(property_graph_types::LABEL_ID_TYPE label,
+                     int64_t offset) const {
     return (((ID_TYPE) offset) & offset_mask_) |
            ((((ID_TYPE) label) << label_id_offset_) & label_id_mask_);
   }
 
   /**
    * @brief Generate the GID
    */
-  ID_TYPE GenerateId(fid_t fid, LabelIDT label, int64_t offset) const {
+  ID_TYPE GenerateId(fid_t fid, property_graph_types::LABEL_ID_TYPE label,
+                     int64_t offset) const {
     return (((ID_TYPE) offset) & offset_mask_) |
            ((((ID_TYPE) label) << label_id_offset_) & label_id_mask_) |
            ((((ID_TYPE) fid) << fid_offset_) & fid_mask_);
@@ -170,13 +175,26 @@ struct NbrUnit {
 
   VID_T vid;
   EID_T eid;
+
   NbrUnit() = default;
   NbrUnit(VID_T v, EID_T e) : vid(v), eid(e) {}
 
   grape::Vertex<VID_T> get_neighbor() const {
     return grape::Vertex<VID_T>(vid);
   }
-};
+} __attribute__((packed, aligned(4)));
+
+static_assert(alignof(NbrUnit<int32_t, uint64_t>) == 4,
+              "int32_t vid alignment check");
+static_assert(sizeof(NbrUnit<int32_t, uint64_t>) ==
+                  sizeof(int32_t) + sizeof(uint64_t),
+              "int32_t vid size check");
+
+static_assert(alignof(NbrUnit<int64_t, uint64_t>) == 4,
+              "int64_t vid alignment check");
+static_assert(sizeof(NbrUnit<int64_t, uint64_t>) ==
+                  sizeof(int64_t) + sizeof(uint64_t),
+              "int64_t vid size check");
 
 template <typename VID_T>
 using NbrUnitDefault = NbrUnit<VID_T, property_graph_types::EID_TYPE>;