Merge pull request #137 from DICL/cleaning-logical-blocks-feature

Completes and isolates logical block feature

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,9 @@
 This PR Fixes DICL/EclipseMETA#(Which issue?)
 
+
 ## BRIEF
 
+
 ## STATUS
 - [ ] Its implemented.
 - [ ] It compiles.
@@ -15,4 +17,5 @@ You might want to use the following command:
     $ git rebase -i #hash key of base commit     
 
 ---
-## EXTRA
+
+- This pull request will release: X.Y.Z

.github/make_release.sh

@@ -0,0 +1,7 @@
+release_no=$(grep -oP "release: \K(v\d*.\d*.\d*)" <(git log -1 --pretty=%B))
+
+if [ "$?" = "0" ]; then
+  cmd="git tag -a $release_no -m \"`git log -1 --pretty=%B`\"" 
+  echo "$cmd"
+  eval $cmd
+fi

.travis.yml

@@ -64,4 +64,6 @@ after_success: |
         git add .
         git commit -m "API: $TRAVIS_TAG"
         git push origin gh-pages
+        bash .github/make_release.sh
+        git push --tags
   fi

Makefile.am

@@ -60,14 +60,21 @@ eclipse_node_SOURCES = src/targets/node_main.cc \
                        src/blocknode/block_node_router.cc \
                        src/fileleader/directory.cc \
                        src/fileleader/file_leader.cc \
-                       src/fileleader/file_leader_router.cc \
-                       src/stats/io_monitor_invoker.cc \
+                       src/fileleader/file_leader_router.cc
+
+eclipse_node_LDADD   = $(LDADD) 
+
+if LOGICAL_BLOCKS_FEATURE 
+
+eclipse_node_SOURCES+= src/stats/io_monitor_invoker.cc \
                        src/stats/zk_listener.cc \
                        src/stats/scheduler_factory.cc \
                        src/stats/scheduler_score_based.cc \
                        src/stats/scheduler_simple.cc
 
-eclipse_node_LDADD   = $(LDADD) -lzookeeper_mt
+eclipse_node_LDADD  +=  -lzookeeper_mt
+
+endif
 
 veloxdfs_SOURCES     = src/targets/client.cc \
                        src/client/cli_driver.cc

configure.ac

@@ -21,15 +21,17 @@ AX_CXX_COMPILE_STDCXX([14], [noext], [mandatory])
 AC_ARG_ENABLE([samples],
     [AS_HELP_STRING([--disable-samples], [copy sample of eclipse.json])])
 
-# Configure options: --enable-debug[=no].
 AC_ARG_ENABLE([debug],
   [AS_HELP_STRING([--enable-debug], [enable debug code (default is no)])],
     [debug="$withval"], [debug=no])
 
+AC_ARG_ENABLE([lblocks],
+    [AS_HELP_STRING([--enable-lblocks], [enable logical blocks experimental feature (Needs zookeeper)])])
+
 AM_CONDITIONAL(COPY_SAMPLES, [test "$enable_samples" != no ])
+AM_CONDITIONAL(LOGICAL_BLOCKS_FEATURE, [test "x$enable_lblocks" = xyes])
 AX_BOOST_STATIC_LINKAGE
 
-
 #}}}
 # Dependencies checkings {{{
 # Checks for typedefs, structures, and compiler characteristics.
@@ -63,6 +65,11 @@ AC_DEFINE_UNQUOTED([ECLIPSE_CONF_PATH], ["$sysconfdirfull"])
 
 AH_TEMPLATE([IO_STAT_SCRIPT],[IOSTATS path])
 AC_DEFINE_UNQUOTED([IO_STAT_SCRIPT], ["${bindirfull}/read_io_stats.sh"])
+
+AS_IF([test "x$enable_lblocks" = xyes], [
+      AH_TEMPLATE([LOGICAL_BLOCKS_FEATURE],["Enables logical blocks feature" ]) 
+      AC_DEFINE_UNQUOTED([LOGICAL_BLOCKS_FEATURE], [1])
+     ])
 #}}}
 #OUTPUT{{{
 AC_OUTPUT

m4/AX_BOOST_STATIC_LINKAGE.m4

@@ -7,5 +7,5 @@ AC_DEFUN([AX_BOOST_STATIC_LINKAGE],
 AC_ARG_ENABLE([boost_static],
   [AS_HELP_STRING([--enable-boost-static], [statically link boost])])
 
-AM_CONDITIONAL(BOOST_STATIC, [test x$enable_boost_static = xyes])
+AM_CONDITIONAL(BOOST_STATIC, [test "x$enable_boost_static" = xyes])
 ])

src/client/cli_driver.cc

@@ -140,6 +140,12 @@ void cli_driver::file_show (std::string file) {
 // }}}
 // file_show_optimized {{{
 void cli_driver::file_show_optimized(std::string file) {
+
+#ifndef LOGICAL_BLOCKS_FEATURE
+  cout << "ERROR! LOGICAL BLOCKS FEATURE IS DISABLED IN THIS BUILD" << endl;
+  exit(EXIT_FAILURE);
+#endif
+
   vec_str nodes = GET_VEC_STR("network.nodes");
   Histogram boundaries(nodes.size(), 100);
   boundaries.initialize();

src/fileleader/file_leader.cc

@@ -3,7 +3,11 @@
 #include "../messages/boost_impl.hh"
 #include "../messages/filedescription.hh"
 #include "../common/logical_block_metadata.hh"
+
+#ifdef LOGICAL_BLOCKS_FEATURE
 #include "../stats/logical_blocks_scheduler.hh"
+#endif
+
 #include <set>
 
 using namespace eclipse;
@@ -186,11 +190,17 @@ bool FileLeader::format () {
 // }}}
 // find_best_arrangement {{{
 void FileLeader::find_best_arrangement(messages::FileDescription* file_desc) {
+#ifdef LOGICAL_BLOCKS_FEATURE
   using namespace eclipse::logical_blocks_schedulers;
   auto nodes = context.settings.get<vec_str>("network.nodes");
 
-  auto scheduler = scheduler_factory(GET_STR("addons.block_scheduler"), boundaries.get());
+  map<string, string> opts;
+  opts["alpha"] = GET_STR("addons.alpha");
+  opts["beta"]  = GET_STR("addons.beta");
+
+  auto scheduler = scheduler_factory(GET_STR("addons.block_scheduler"), boundaries.get(), opts);
 
   scheduler->generate(*file_desc, nodes);
+#endif
 }
 // }}}

src/stats/logical_blocks_scheduler.hh

@@ -7,9 +7,11 @@
 #include <vector>
 #include <string>
 #include <memory>
+#include <map>
 
 namespace eclipse {
 namespace logical_blocks_schedulers {
+typedef std::map<std::string, std::string> SCHEDULER_OPTS;
 
 class scheduler;
 
@@ -18,10 +20,12 @@ class scheduler {
     virtual void generate(messages::FileDescription& file_desc, std::vector<std::string> nodes) = 0;
 
     std::shared_ptr<stats_listener> listener;
+    std::map<std::string, std::string> options;
     Histogram* boundaries;
 };
 
-std::shared_ptr<eclipse::logical_blocks_schedulers::scheduler> scheduler_factory(std::string, Histogram*);
+std::shared_ptr<eclipse::logical_blocks_schedulers::scheduler> 
+    scheduler_factory(std::string, Histogram*, SCHEDULER_OPTS = SCHEDULER_OPTS());
 
 class scheduler_simple: public scheduler{
   public:

src/stats/scheduler_factory.cc

@@ -3,12 +3,14 @@
 #include "../common/context_singleton.hh"
 #include "zk_listener.hh"
 #include <memory>
+#include <map>
+
+using namespace std;
 
 namespace eclipse {
 namespace logical_blocks_schedulers {
 
-using namespace std;
-std::shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries) {
+shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries, SCHEDULER_OPTS options) {
   shared_ptr<scheduler> sch;
 
   if (type == "scheduler_simple") {
@@ -23,6 +25,7 @@ std::shared_ptr<scheduler> scheduler_factory(string type, Histogram* boundaries)
 
   sch->boundaries = boundaries;
   sch->listener = make_shared<zk_listener>();
+  sch->options = options;
 
   return sch;
 }

src/stats/scheduler_score_based.cc

@@ -1,7 +1,7 @@
 #include "logical_blocks_scheduler.hh"
 #include "../messages/blockinfo.hh"
-#include "../common/context_singleton.hh"
 
+#include <algorithm>
 #include <set>
 
 using namespace std;
@@ -10,8 +10,8 @@ using namespace eclipse::messages;
 using namespace eclipse::logical_blocks_schedulers;
 
 void scheduler_score_based::generate(FileDescription& file_desc, std::vector<std::string> nodes) {
-  double alpha = atof(GET_STR("addons.alpha").c_str());
-  double beta  = atof(GET_STR("addons.beta").c_str());
+  double alpha = atof(options["alpha"].c_str());
+  double beta  = atof(options["beta"].c_str());
 
   auto io_vec = listener->get_io_stats();
 

src/stats/scheduler_simple.cc

@@ -1,6 +1,6 @@
 #include "logical_blocks_scheduler.hh"
 #include "../messages/blockinfo.hh"
-#include "../common/context_singleton.hh"
+#include <algorithm>
 
 #include <set>
 

src/targets/node_main.cc

@@ -4,7 +4,11 @@
 #include <network/simple_router.hh>
 #include <fileleader/file_leader_router.hh>
 #include <blocknode/block_node_router.hh>
+
+#ifdef LOGICAL_BLOCKS_FEATURE
 #include <src/stats/io_monitor_invoker.hh>
+#endif
+
 #include <memory>
 
 using namespace eclipse;
@@ -26,7 +30,10 @@ int main (int argc, char ** argv) {
 
   external_net->establish();
 
+#ifdef LOGICAL_BLOCKS_FEATURE
   invoke_io_reporter();
+#endif
+
   context.run();
   context.join();
 

tests/Makefile.am

@@ -1,36 +1,59 @@
 AM_CPPFLAGS_T    = $(AM_CPPFLAGS) -include unittest++/UnitTest++.h
-LDADD_T          = $(LDADD) -l:libUnitTest++.a -ldl -lsqlite3
-
-# Input your tests files here
-check_PROGRAMS   = units metadata 
-check_SCRIPTS    = integration
-
-TESTS            = units 
+LDADD_T          = $(LDADD) -l:libUnitTest++.a
 
 check_DATA       = $(top_srcdir)/doc/eclipse.json
 CLEANFILES       = eclipse.json
+check_PROGRAMS   = 
+TESTS            = 
+
+#<<<<<<<<<<<<<<<<Input your tests files here>>>>>>>>>>>>>>>>>>>>
+
+# INTEGRATION TEST
+# ---------------------------------------------------------------------
+check_SCRIPTS     = integration 
+check-integration: integration
+	bash integration.sh
+
+integration:
+	cp $(top_srcdir)/tests/integration_test.sh integration.sh
+	chmod +x integration.sh
 
 
-# Input the dependencies of the test files here
-units_LDADD        = $(LDADD_T)
-units_CPPFLAGS     = $(AM_CPPFLAGS_T)
-units_SOURCES      = $(messages_files) \
+# MESSAGES TEST
+# ---------------------------------------------------------------------
+check_PROGRAMS    += messages
+TESTS             += messages
+messages_LDADD     = $(LDADD_T) -ldl -lsqlite3
+messages_CPPFLAGS  = $(AM_CPPFLAGS_T)
+messages_SOURCES   = $(messages_files) \
                      tests/main.cc \
                      tests/messages_test.cc \
                      src/network/asyncchannel.cc \
                      src/network/server.cc \
                      src/nodes/machine.cc \
-                     src/nodes/node.cc 
+                     src/nodes/node.cc
 
-metadata_LDADD     = $(LDADD_T)
+# METADATA TEST
+# ---------------------------------------------------------------------
+check_PROGRAMS    += metadata 
+metadata_LDADD     = $(LDADD_T) -ldl -lsqlite3  
 metadata_CPPFLAGS  = $(AM_CPPFLAGS_T)
 metadata_SOURCES   = tests/metadata_test.cc \
                      src/fileleader/directory.cc \
                      $(messages_files)
 
-check-integration: integration
-	bash integration.sh
-
-integration:
-	cp $(top_srcdir)/tests/integration_test.sh integration.sh
-	chmod +x integration.sh
+# LOGICAL BLOCKS TEST
+# ---------------------------------------------------------------------
+if LOGICAL_BLOCKS_FEATURE
+check_PROGRAMS   += lblocks
+TESTS            += lblocks
+
+lblocks_LDADD     = $(LDADD_T) -lzookeeper_mt
+lblocks_CPPFLAGS  = $(AM_CPPFLAGS_T)
+lblocks_SOURCES   = tests/main.cc \
+                    tests/stats_scheduler_test.cc \
+                    src/stats/scheduler_factory.cc \
+                    src/stats/scheduler_score_based.cc \
+                    src/stats/scheduler_simple.cc \
+                    src/stats/zk_listener.cc
+endif

tests/stats_scheduler_test.cc

@@ -0,0 +1,69 @@
+#include <settings.hh>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <messages/filedescription.hh>
+#include <common/histogram.hh>
+#include <stats/logical_blocks_scheduler.hh>
+
+using namespace std;
+using namespace eclipse::logical_blocks_schedulers;
+using namespace eclipse::messages;
+
+namespace {
+
+struct listener_1_busy : public eclipse::stats_listener {
+  listener_1_busy() = default;
+  ~listener_1_busy() = default;
+
+  std::vector<double> get_io_stats() {
+    return {0.01, 0.01, 0.01, 0.99};
+  }
+};
+
+struct stats_fixture {
+  std::vector<std::string> nodes = {"1", "2", "3", "4"};
+  map<string, string> opts;
+  FileDescription fd;
+
+  stats_fixture () { 
+    opts["alpha"] = "0.5";
+    opts["beta"]  = "0.5";
+
+    fd.name = "file";
+    fd.blocks = {"file_1", "file_2", "file_3", "file_4", "file_5"};
+    fd.hash_keys = {123123, 124123, 32323, 4242, 424245};
+    fd.block_size = {123123, 124123, 32323, 4242, 424245};
+    fd.block_hosts= {"1", "2", "3", "4", "4"};
+  }
+};
+
+}
+
+SUITE(STATS_TESTS) {
+
+  TEST_FIXTURE(stats_fixture, simple_test) {
+    Histogram boundaries (4, 100);
+    auto scheduler = scheduler_factory("scheduler_simple", &boundaries, opts);
+    scheduler->listener.reset(new listener_1_busy());
+    scheduler->generate(fd, nodes);
+
+    CHECK_EQUAL(fd.logical_blocks.size(), 4);
+    CHECK_EQUAL(fd.logical_blocks[0].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[1].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[2].physical_blocks.size(), 1);
+    CHECK_EQUAL(fd.logical_blocks[3].physical_blocks.size(), 2);
+  }
+
+  TEST_FIXTURE(stats_fixture, score_based_tests) {
+    Histogram boundaries (4, 100);
+    auto scheduler = scheduler_factory("scheduler_score_based", &boundaries, opts);
+    scheduler->listener.reset(new listener_1_busy());
+    scheduler->generate(fd, nodes);
+
+    CHECK_EQUAL(3, fd.logical_blocks.size());
+    CHECK_EQUAL(1, fd.logical_blocks[0].physical_blocks.size());
+    CHECK_EQUAL(2, fd.logical_blocks[1].physical_blocks.size());
+    CHECK_EQUAL(2, fd.logical_blocks[2].physical_blocks.size());
+  }
+}