Merge pull request #2801 from esl/toml-config

TOML config

.gitignore

@@ -60,6 +60,7 @@ priv/mibs/EJABBERD-MIB.bin
 ebin/ejabberd.app
 rel/configure.vars.config
 rel/vars.config
+rel/vars-toml.config
 compile_commands.json
 # Compilation/test ephemera
 *.d

Makefile

@@ -15,6 +15,7 @@ clean:
 	-rm -rf _build
 	-rm rel/configure.vars.config
 	-rm rel/vars.config
+	-rm rel/vars-toml.config
 
 # REBAR_CT_EXTRA_ARGS comes from a test runner
 ct:
@@ -25,7 +26,7 @@ ct:
 eunit:
 	@$(RUN) $(REBAR) eunit
 
-rel: certs configure.out rel/vars.config
+rel: certs configure.out rel/vars.config rel/vars-toml.config
 	. ./configure.out && $(REBAR) as prod release
 
 shell: certs etc/mongooseim.cfg
@@ -41,6 +42,9 @@ rock:
 rel/vars.config: rel/vars.config.in rel/configure.vars.config
 	cat $^ > $@
 
+rel/vars-toml.config: rel/vars-toml.config.in rel/configure.vars.config
+	cat $^ > $@
+
 ## Don't allow these files to go out of sync!
 configure.out rel/configure.vars.config:
 	./tools/configure with-all without-jingle-sip
@@ -54,7 +58,7 @@ devrel: $(DEVNODES)
 print_devnodes:
 	@echo $(DEVNODES)
 
-$(DEVNODES): certs configure.out rel/vars.config
+$(DEVNODES): certs configure.out rel/vars.config rel/vars-toml.config
 	@echo "building $@"
 	(. ./configure.out && \
 	DEVNODE=true $(RUN) $(REBAR) as $@ release)

big_tests/default.spec

@@ -21,6 +21,7 @@
 {suites, "tests", cluster_commands_SUITE}.
 {suites, "tests", component_SUITE}.
 {suites, "tests", conf_reload_SUITE}.
+{suites, "tests", config_format_SUITE}.
 {suites, "tests", connect_SUITE}.
 {suites, "tests", disco_and_caps_SUITE}.
 {suites, "tests", ejabberdctl_SUITE}.

big_tests/run_common_test.erl

@@ -165,8 +165,8 @@ run_test(Test, PresetsToRun, CoverOpts) ->
     prepare_cover(Test, CoverOpts),
     error_logger:info_msg("Presets to run ~p", [PresetsToRun]),
     {ConfigFile, Props} = get_ct_config(Test),
-    case proplists:lookup(ejabberd_presets, Props) of
-        {ejabberd_presets, Presets} ->
+    case get_presets(Props) of
+        {ok, Presets} ->
             Presets1 = case PresetsToRun of
                            all ->
                                Presets;
@@ -186,14 +186,27 @@ run_test(Test, PresetsToRun, CoverOpts) ->
             save_count(Test, Presets1),
             analyze_coverage(Test, CoverOpts),
             R;
-        _ ->
+        {error, not_found} ->
             error_logger:info_msg("Presets were not found in the config file ~ts",
                                   [ConfigFile]),
             R = do_run_quick_test(Test, CoverOpts),
             analyze_coverage(Test, CoverOpts),
             R
     end.
 
+get_presets(Props) ->
+    case proplists:lookup(presets, Props) of
+        {presets, Presets} ->
+            case proplists:lookup(toml, Presets) of
+                {toml, Preset} ->
+                    {ok, Preset};
+                _ ->
+                    {error, not_found}
+            end;
+        _ ->
+            {error, not_found}
+    end.
+
 get_ct_config(Opts) ->
     Spec = proplists:get_value(spec, Opts),
     Props = read_file(Spec),
@@ -263,24 +276,36 @@ is_test_host_enabled(HostName) ->
             lists:member(atom_to_binary(HostName, utf8), BinHosts)
     end.
 
-enable_preset_on_node(Node, PresetVars, HostVars) ->
+enable_preset_on_node(Node, PresetVars, HostVarsFilePrefix) ->
     {ok, Cwd} = call(Node, file, get_cwd, []),
-    Cfg = filename:join([repo_dir(), "rel", "files", "mongooseim.cfg"]),
-    Vars = filename:join([repo_dir(), "rel", HostVars]),
-    CfgFile = filename:join([Cwd, "etc", "mongooseim.cfg"]),
-    {ok, Template} = handle_file_error(Cfg, file:read_file(Cfg)),
-    {ok, Default} = handle_file_error(Vars, file:consult(Vars)),
-    NewVars = lists:foldl(fun ({Var, Val}, Acc) ->
-                              lists:keystore(Var, 1, Acc, {Var, Val})
-                          end, Default, PresetVars),
-    %% Render twice to replace variables in variables
-    Tmp = bbmustache:render(Template, NewVars, [{key_type, atom}]),
-    NewCfgFile = bbmustache:render(Tmp, NewVars, [{key_type, atom}]),
-    ok = call(Node, file, write_file, [CfgFile, NewCfgFile]),
+    TemplatePath = filename:join([repo_dir(), "rel", "files", "mongooseim.toml"]),
+    DefaultVarsPath = filename:join([repo_dir(), "rel", "vars-toml.config"]),
+    NodeVarsPath = filename:join([repo_dir(), "rel", HostVarsFilePrefix ++ ".vars-toml.config"]),
+
+    {ok, Template} = handle_file_error(TemplatePath, file:read_file(TemplatePath)),
+    {ok, DefaultVars} = handle_file_error(DefaultVarsPath, file:consult(DefaultVarsPath)),
+    {ok, NodeVars} = handle_file_error(NodeVarsPath, file:consult(NodeVarsPath)),
+
+    TemplatedConfig = template_config(Template, [DefaultVars, NodeVars, PresetVars]),
+    CfgPath = filename:join([Cwd, "etc", "mongooseim.toml"]),
+    ok = call(Node, file, write_file, [CfgPath, TemplatedConfig]),
     call(Node, application, stop, [mongooseim]),
     call(Node, application, start, [mongooseim]),
     ok.
 
+template_config(Template, Vars) ->
+    MergedVars = merge_vars(Vars),
+    %% Render twice to replace variables in variables
+    Tmp = bbmustache:render(Template, MergedVars, [{key_type, atom}]),
+    bbmustache:render(Tmp, MergedVars, [{key_type, atom}]).
+
+merge_vars([Vars1, Vars2|Rest]) ->
+    Vars = lists:foldl(fun ({Var, Val}, Acc) ->
+                               lists:keystore(Var, 1, Acc, {Var, Val})
+                       end, Vars1, Vars2),
+    merge_vars([Vars|Rest]);
+merge_vars([Vars]) -> Vars.
+
 call(Node, M, F, A) ->
     case rpc:call(Node, M, F, A) of
         {badrpc, Reason} ->

big_tests/test.config

@@ -17,7 +17,7 @@
 %%       so that we rein the "bag of things" approach
 {hosts, [{mim,  [{node, mongooseim@localhost},
                  {domain, <<"localhost">>},
-                 {vars, "mim1.vars.config"},
+                 {vars, "mim1"},
                  {cluster, mim},
                  {secondary_domain, <<"localhost.bis">>},
                  {reloaded_domain, <<"sogndal">>},
@@ -36,28 +36,28 @@
                  {http_notifications_port, 8000}]},
          {mim2, [{node, ejabberd2@localhost},
                  {domain, <<"localhost">>},
-                 {vars, "mim2.vars.config"},
+                 {vars, "mim2"},
                  {cluster, mim},
                  {c2s_tls_port, 5233},
                  {metrics_rest_port, 5289},
                  {gd_endpoint_port, 6666},
                  {service_port, 8899}]},
          {mim3, [{node, mongooseim3@localhost},
                  {domain, <<"localhost">>},
-                 {vars, "mim3.vars.config"},
+                 {vars, "mim3"},
                  {c2s_tls_port, 5263},
                  {cluster, mim}]},
          %% used to test s2s features
          {fed,  [{node, fed1@localhost},
                  {domain, <<"fed1">>},
-                 {vars, "fed1.vars.config"},
+                 {vars, "fed1"},
                  {incoming_s2s_port, 5299},
                  {c2s_port, 5242},
                  {cluster, fed}]},
          %% used to test global distribution features
          {reg,  [{node, reg1@localhost},
                  {domain, <<"reg1">>},
-                 {vars, "reg1.vars.config"},
+                 {vars, "reg1"},
                  {service_port, 9990},
                  {c2s_port, 5252},
                  {gd_endpoint_port, 7777},
@@ -201,7 +201,197 @@
         {auth_method, <<"SASL-ANON">>}]}
                      ]}.
 
-{ejabberd_presets, [
+{presets,
+ [{toml,
+   [
+    {internal_mnesia,
+     %% dbs variable is used by ./tools/test_runner/presets_to_dbs.sh script
+     [{dbs, [redis, minio]},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10"},
+      {mod_offline, "[modules.mod_offline]"}]},
+    {pgsql_mnesia,
+     [{dbs, [redis, pgsql]},
+      {auth_method, "\"rdbms\""},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.rdbms.default]
+  scope = \"global\"
+  workers = 5
+  connection.driver = \"pgsql\"
+  connection.host = \"localhost\"
+  connection.database = \"ejabberd\"
+  connection.username = \"ejabberd\"
+  connection.password = \"mongooseim_secret\"
+  connection.tls.required = true
+  connection.tls.verify_peer = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.server_name_indication = false"},
+      {mod_last, "[modules.mod_last]
+  backend = \"rdbms\""},
+      {mod_privacy, "[modules.mod_privacy]
+  backend = \"rdbms\""},
+      {mod_private, "[modules.mod_private]
+  backend = \"rdbms\""},
+      {mod_offline, "[modules.mod_offline]
+  backend = \"rdbms\""},
+      {mod_vcard, "[modules.mod_vcard]
+  backend = \"rdbms\"
+  host = \"vjud.@HOST@\""},
+      {mod_roster, "[modules.mod_roster]
+  backend = \"rdbms\""}]},
+    {odbc_mssql_mnesia,
+     [{dbs, [redis, mssql]},
+      {auth_method, "\"rdbms\""},
+      {rdbms_server_type, "rdbms_server_type = \"mssql\""},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.rdbms.default]
+  scope = \"global\"
+  workers = 5
+  connection.driver = \"odbc\"
+  connection.settings = \"DSN=mongoose-mssql;UID=sa;PWD=mongooseim_secret+ESL123\""},
+      {mod_last, "[modules.mod_last]
+  backend = \"rdbms\""},
+      {mod_privacy, "[modules.mod_privacy]
+  backend = \"rdbms\""},
+      {mod_private, "[modules.mod_private]
+  backend = \"rdbms\""},
+      {mod_offline, "[modules.mod_offline]
+  backend = \"rdbms\""},
+      {mod_vcard, "[modules.mod_vcard]
+  backend = \"rdbms\"
+  host = \"vjud.@HOST@\""},
+      {mod_roster, "[modules.mod_roster]
+  backend = \"rdbms\""}]},
+    {mysql_redis,
+     [{dbs, [redis, mysql]},
+      {sm_backend, "\"redis\""},
+      {auth_method, "\"rdbms\""},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.redis.default]
+  scope = \"global\"
+  workers = 10
+  strategy = \"random_worker\"
+[outgoing_pools.rdbms.default]
+  scope = \"global\"
+  workers = 5
+  connection.driver = \"mysql\"
+  connection.host = \"localhost\"
+  connection.database = \"ejabberd\"
+  connection.username = \"ejabberd\"
+  connection.password = \"mongooseim_secret\"
+  connection.tls.verify_peer = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.versions = [\"tlsv1.2\"]"},
+      {mod_last, "[modules.mod_last]
+  backend = \"rdbms\""},
+      {mod_privacy, "[modules.mod_privacy]
+  backend = \"rdbms\""},
+      {mod_private, "[modules.mod_private]
+  backend = \"mysql\""},
+      {mod_offline, "[modules.mod_offline]
+  backend = \"rdbms\""},
+      {mod_vcard, "[modules.mod_vcard]
+  backend = \"rdbms\"
+  host = \"vjud.@HOST@\""},
+      {mod_roster, "[modules.mod_roster]
+  backend = \"rdbms\""}]},
+    {ldap_mnesia,
+     [{dbs, [redis, ldap]},
+      {auth_method, "\"ldap\""},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.ldap.default]
+  scope = \"global\"
+  workers = 5
+  connection.port = 3636
+  connection.rootdn = \"cn=admin,dc=esl,dc=com\"
+  connection.password = \"mongooseim_secret\"
+  connection.encrypt = \"tls\"
+  connection.tls.versions = [\"tlsv1.2\"]
+  connection.tls.verify_peer = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.certfile = \"priv/ssl/fake_cert.pem\"
+  connection.tls.keyfile = \"priv/ssl/fake_key.pem\"
+[outgoing_pools.ldap.bind]
+  scope = \"global\"
+  workers = 5
+  connection.port = 3636
+  connection.encrypt = \"tls\"
+  connection.tls.versions = [\"tlsv1.2\"]
+  connection.tls.verify_peer = true
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.certfile = \"priv/ssl/fake_cert.pem\"
+  connection.tls.keyfile = \"priv/ssl/fake_key.pem\""},
+      {mod_offline, "[modules.mod_offline]"},
+      {password_format, "password.format = \"scram\""},
+      {auth_ldap, "ldap.base = \"ou=Users,dc=esl,dc=com\"
+  ldap.filter = \"(objectClass=inetOrgPerson)\""},
+      {mod_vcard, "[modules.mod_vcard]
+  backend = \"ldap\"
+  host = \"vjud.@HOST@\"
+  ldap_base = \"ou=Users,dc=esl,dc=com\"
+  ldap_filter = \"(objectClass=inetOrgPerson)\""}]},
+    {riak_mnesia,
+     [{dbs, [redis, riak]},
+      {auth_method, "\"riak\""},
+      %% Specify a list of ciphers to avoid
+      %% "no function clause matching tls_v1:enum_to_oid(28)" error
+      %% on Riak's side running with Erlang R16.
+      %% https://github.com/basho/riak-erlang-client/issues/232#issuecomment-178612129
+      %% We also set ciphers in tools/setup_riak on the server side.
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.riak.default]
+  scope = \"global\"
+  workers = 5
+  strategy = \"next_worker\"
+  connection.address = \"127.0.0.1\"
+  connection.port = 8087
+  connection.username = \"ejabberd\"
+  connection.password = \"mongooseim_secret\"
+  connection.tls.ciphers = [\"AES256-SHA\", \"DHE-RSA-AES128-SHA256\"]
+  connection.tls.server_name_indication = false
+  connection.cacertfile = \"priv/ssl/cacert.pem\""},
+      {mod_last, "[modules.mod_last]
+  backend = \"riak\""},
+      {mod_privacy, "[modules.mod_privacy]
+  backend = \"riak\""},
+      {mod_private, "[modules.mod_private]
+  backend = \"riak\""},
+      {mod_offline, "[modules.mod_offline]
+  backend = \"riak\""},
+      {mod_vcard, "[modules.mod_vcard]
+  backend = \"riak\"
+  host = \"vjud.@HOST@\""},
+      {mod_roster, "[modules.mod_roster]
+  backend = \"riak\""}
+     ]},
+    {elasticsearch_and_cassandra_mnesia,
+     [{dbs, [redis, elasticsearch, cassandra]},
+      {outgoing_pools, "[outgoing_pools.redis.global_distrib]
+  scope = \"global\"
+  workers = 10
+[outgoing_pools.cassandra.default]
+  scope = \"global\"
+  workers = 20
+  connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
+  connection.tls.verify_peer = true
+[outgoing_pools.elastic.default]
+  scope = \"global\""},
+      {mod_offline, "[modules.mod_offline]"}
+     ]}
+   ]},
+  {cfg, % preset vars for the 'cfg' format - used only by the config equivalence tests
+   [
     {internal_mnesia,
      %% dbs variable is used by ./tools/test_runner/presets_to_dbs.sh script
      [{dbs, [redis, minio]},
@@ -333,7 +523,8 @@
       {auth_method, "internal"},
       {mod_offline, "{mod_offline, []},"}
      ]}
-]}.
+   ]}
+ ]}.
 
 {timetrap,{seconds,30}}.
 {sensible_maximum_repeats, 100}.