Simplify failover setup for tests (#1358)

CHANGELOG.rst

@@ -18,13 +18,18 @@ Added
 
 - Update vshard to 0.1.17.
   (`Changelog <https://github.com/tarantool/vshard/releases/tag/0.1.17>`_).
+
 - Update graphql to 0.1.1 which allows to return data and errors together.
 
+- New test helper: ``cartridge.test-helpers.stateboard``.
+
+- New ``failover`` option in the cluster test helper for easier failover setup.
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Fixed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Fix unclear timeout errors in case of ``InitError`` and ``BootError`` states
+- Fix unclear timeout errors in case of ``InitError`` and ``BootError`` states.
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Enhanced in WebUI

cartridge/test-helpers.lua

@@ -20,4 +20,8 @@ helpers.Cluster = require('cartridge.test-helpers.cluster')
 -- @see cartridge.test-helpers.etcd
 helpers.Etcd = require('cartridge.test-helpers.etcd')
 
+--- Class to run and manage stateboard.
+-- @see cartridge.test-helpers.stateboard
+helpers.Stateboard = require('cartridge.test-helpers.stateboard')
+
 return helpers

cartridge/test-helpers/cluster.lua

@@ -10,6 +10,7 @@ local uuid = require('uuid')
 
 local luatest = require('luatest')
 local Server = require('cartridge.test-helpers.server')
+local Stateboard = require('cartridge.test-helpers.stateboard')
 
 -- Defaults.
 local Cluster = {
@@ -19,6 +20,7 @@ local Cluster = {
     cookie = 'test-cluster-cookie',
     base_http_port = 8080,
     base_advertise_port = 13300,
+    failover = 'disabled',
 }
 
 function Cluster:inherit(object)
@@ -35,6 +37,8 @@ end
 -- @int[opt] object.base_advertise_port Value to calculate server's advertise_port.
 -- @bool[opt] object.use_vshard bootstrap vshard after server is started.
 -- @tab object.replicasets Replicasets configuration. List of @{replicaset_config}
+-- @string[opt] object.failover Failover mode: disabled, eventual, or stateful.
+-- @string[opt] object.stateboard_entrypoint Command to run stateboard
 -- @return object
 function Cluster:new(object)
     checks('table', {
@@ -46,6 +50,8 @@ function Cluster:new(object)
         use_vshard = '?boolean',
         replicasets = 'table',
         env = '?table',
+        failover = '?string',
+        stateboard_entrypoint = '?string',
     })
     --- Replicaset config.
     -- @table @replicaset_config
@@ -74,6 +80,31 @@ end
 
 function Cluster:initialize()
     self.servers = {}
+
+    assert(
+        self.failover == 'disabled'
+        or self.failover == 'eventual'
+        or self.failover == 'stateful',
+        "failover must be 'disabled', 'eventual' or 'stateful'"
+    )
+
+    if self.failover == 'stateful' then
+        assert(self.stateboard_entrypoint ~= nil,
+            'stateboard_entrypoint required for stateful failover')
+        self.stateboard = Stateboard:new({
+            workdir = fio.pathjoin(self.datadir, 'stateboard'),
+            command = self.stateboard_entrypoint,
+            net_box_port = 14401,
+            net_box_credentials = {
+                user = 'client',
+                password = self.cookie,
+            },
+            env = {
+                TARANTOOL_PASSWORD = self.cookie,
+            }
+        })
+    end
+
     for _, replicaset_config in ipairs(self.replicasets) do
         replicaset_config.uuid = replicaset_config.uuid or uuid.str()
         if type(replicaset_config.servers) == 'number' then
@@ -91,6 +122,24 @@ function Cluster:initialize()
     end
 end
 
+function Cluster:configure_failover()
+    assert(self.main_server)
+
+    local failover_config = {mode = self.failover}
+    if failover_config.mode == 'stateful' then
+        failover_config.state_provider = 'tarantool'
+        failover_config.tarantool_params = {
+            uri = self.stateboard.net_box_uri,
+            password = self.stateboard.net_box_credentials.password
+        }
+    end
+
+    self.main_server.net_box:call(
+        'package.loaded.cartridge.failover_set_params',
+        {failover_config}
+    )
+end
+
 --- Find server by alias.
 -- @string alias
 -- @return @{cartridge.test-helpers.server}
@@ -159,6 +208,12 @@ function Cluster:start()
     if self.running then
         return
     end
+    if self.failover == 'stateful' then
+        self.stateboard:start()
+        luatest.helpers.retrying({}, function()
+            self.stateboard:connect_net_box()
+        end)
+    end
     for _, server in ipairs(self.servers) do
         server:start()
     end
@@ -170,6 +225,9 @@ function Cluster:start()
         self:bootstrap()
         self.bootstrapped = true
     end
+    if self.failover ~= 'disabled' then
+        self:configure_failover()
+    end
     self.running = true
 end
 
@@ -178,6 +236,9 @@ function Cluster:stop()
     for _, server in ipairs(self.servers) do
         server:stop()
     end
+    if self.stateboard then
+        self.stateboard:stop()
+    end
     self.running = nil
 end
 

cartridge/test-helpers/server.lua

@@ -1,4 +1,4 @@
---- Extended luatest.Server class to run tarantool instance.
+--- Extended luatest.Server class to run a cartridge instance.
 --
 -- @classmod cartridge.test-helpers.server
 

cartridge/test-helpers/stateboard.lua

@@ -0,0 +1,41 @@
+--- Class to run and manage stateboard.
+--
+-- @classmod cartridge.test-helpers.stateboard
+
+local luatest = require('luatest')
+local log = require('log')
+
+--- Build stateboard object.
+-- @function new
+-- @param object
+-- @string object.name Human-readable node name.
+-- @string object.command Command to run stateboard.
+-- @string object.workdir Path to the data directory.
+-- @string object.net_box_port Value to be passed in `TARANTOOL_LISTEN` and used for net_box connection.
+-- @tab[opt] object.net_box_credentials Override default net_box credentials.
+-- @tab[opt] object.env Environment variables passed to the process.
+-- @return object
+local Stateboard = luatest.Server:inherit({})
+
+-- Start stateboard
+function Stateboard:start()
+    getmetatable(getmetatable(self)).start(self)
+end
+
+-- Stop stateboard
+function Stateboard:stop()
+    local process = self.process
+    if process == nil then
+        return
+    end
+    getmetatable(getmetatable(self)).stop(self)
+    luatest.helpers.retrying({}, function()
+        luatest.assert_not(
+            process:is_alive(),
+            string.format('Process %s is still running', self.alias)
+        )
+    end)
+    log.warn('Process %s killed', self.alias)
+end
+
+return Stateboard

config.ld

@@ -18,6 +18,7 @@ file = {
     "cartridge/test-helpers/cluster.lua",
     "cartridge/test-helpers/server.lua",
     "cartridge/test-helpers/etcd.lua",
+    "cartridge/test-helpers/stateboard.lua",
     "cartridge/remote-control.lua",
     "cartridge/service-registry.lua",
     "cartridge/user-defined-role.lua",

test/integration/failover_stateful_test.lua

@@ -72,7 +72,7 @@ g_stateboard.before_all(function()
     g.datadir = fio.tempdir()
 
     g.kvpassword = require('digest').urandom(6):hex()
-    g.state_provider = require('luatest.server'):new({
+    g.state_provider = helpers.Stateboard:new({
         command = helpers.entrypoint('srv_stateboard'),
         workdir = fio.pathjoin(g.datadir, 'stateboard'),
         net_box_port = 14401,

test/integration/fencing_test.lua

@@ -63,7 +63,7 @@ g_stateboard.before_all(function()
 
     fio.mktree(fio.pathjoin(g.datadir, 'stateboard'))
     g.kvpassword = require('digest').urandom(6):hex()
-    g.state_provider = require('luatest.server'):new({
+    g.state_provider = helpers.Stateboard:new({
         command = helpers.entrypoint('srv_stateboard'),
         workdir = fio.pathjoin(g.datadir, 'stateboard'),
         net_box_port = 14401,

test/integration/stateboard_test.lua

@@ -14,7 +14,7 @@ g.before_each(function()
     local password = require('digest').urandom(6):hex()
 
     fio.mktree(fio.pathjoin(g.datadir, 'stateboard'))
-    g.stateboard = require('luatest.server'):new({
+    g.stateboard = helpers.Stateboard:new({
         command = helpers.entrypoint('srv_stateboard'),
         workdir = fio.pathjoin(g.datadir),
         net_box_port = 13301,

test/integration/switchover_test.lua

@@ -49,7 +49,7 @@ g_stateboard.before_all(function()
 
     fio.mktree(fio.pathjoin(g.datadir, 'stateboard'))
     g.kvpassword = require('digest').urandom(6):hex()
-    g.state_provider = require('luatest.server'):new({
+    g.state_provider = helpers.Stateboard:new({
         command = helpers.entrypoint('srv_stateboard'),
         workdir = fio.pathjoin(g.datadir, 'stateboard'),
         net_box_port = 14401,
@@ -65,6 +65,7 @@ g_stateboard.before_all(function()
             ),
         },
     })
+
     g.state_provider:start()
     helpers.retrying({}, function()
         g.state_provider:connect_net_box()

test/unit/helpers_test.lua

@@ -12,13 +12,15 @@ g.before_all = function()
         datadir = fio.tempdir(),
         use_vshard = true,
         server_command = helpers.entrypoint('srv_basic'),
+        failover = 'stateful',
+        stateboard_entrypoint = helpers.entrypoint('srv_stateboard'),
         cookie = require('digest').urandom(6):hex(),
         base_http_port = 8080,
         base_advertise_port = 13300,
         replicasets = {
             {
                 alias = 'vshard',
-                roles = {'vshard-router', 'vshard-storage'},
+                roles = {'vshard-router', 'vshard-storage', 'failover-coordinator'},
                 servers = 1,
             },
             {
@@ -89,6 +91,42 @@ function g.test_replicaset_uuid_generation()
     end
 end
 
+local function get_failover(cluster)
+    return cluster.main_server.net_box:call(
+        'package.loaded.cartridge.failover_get_params')
+end
+
+function g.test_failover()
+    t.assert_equals(get_failover(g.cluster).mode, 'stateful')
+    t.assert(g.cluster.stateboard)
+
+    local coordinator = g.cluster:server('vshard-1')
+    t.assert_equals(g.cluster.stateboard.net_box:call('get_coordinator'),
+        {uri = coordinator.advertise_uri, uuid = coordinator.instance_uuid})
+
+    t.assert_error_msg_contains(
+        "failover must be 'disabled', 'eventual' or 'stateful'",
+        build_cluster, {
+        replicasets = {{roles = {}, servers = 1, alias = ''}},
+        failover = 'thebest'
+    })
+
+    t.assert_error_msg_contains(
+        "stateboard_entrypoint required for stateful failover",
+        build_cluster, {
+        replicasets = {{roles = {}, servers = 1, alias = ''}},
+        failover = 'stateful',
+    })
+
+    -- t.assert_error_msg_contains(
+    --     "fake: no such stateboard_entrypoint",
+    --     build_cluster, {
+    --     replicasets = {{roles = {}, servers = 1, alias = ''}},
+    --     failover = 'stateful',
+    --     stateboard_entrypoint = 'fake'
+    -- })
+end
+
 function g.test_new_with_servers_count()
     local cluster = build_cluster({replicasets = {
         {roles = {}, servers = 2, alias = 'router'},