Skip to content

Commit

Permalink
tests: don't use system etcd binary
Browse files Browse the repository at this point in the history 
It is more convenient and easier to test using etcd Golang test
packages instead of running a binary from system.
  • Loading branch information
@askalt @psergee
askalt authored and psergee committed Apr 3, 2024
1 parent d535c29 commit 0563aee
Show file tree
Hide file tree
Showing 7 changed files with 1,094 additions and 330 deletions.
241 changes: 115 additions & 126 deletions cli/cluster/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,25 +6,22 @@ import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.etcd.io/etcd/client/pkg/v3/transport"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/tests/v3/integration"

"github.com/tarantool/tt/cli/cluster"
"github.com/tarantool/tt/cli/templates"
libcluster "github.com/tarantool/tt/lib/cluster"
)

const (
baseEndpoint = "127.0.0.1:12379"
httpEndpoint = "http://" + baseEndpoint
httpsEndpoint = "https://" + baseEndpoint
timeout = 5 * time.Second
)
const timeout = 5 * time.Second

type etcdOpts struct {
Username string
Expand All @@ -34,164 +31,149 @@ type etcdOpts struct {
CaFile string
}

type etcdInstance struct {
Cmd *exec.Cmd
Dir string
func doWithCtx(action func(context.Context) error) error {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return action(ctx)
}

func startEtcd(t *testing.T, endpoint string, opts etcdOpts) etcdInstance {
func startEtcd(t *testing.T, opts etcdOpts) integration.LazyCluster {
t.Helper()

mydir, err := os.Getwd()
if err != nil {
t.Fatalf("Failed to get current working directory: %s", err)
}

inst := etcdInstance{}
dir, err := os.MkdirTemp("", "work_dir")
if err != nil {
t.Fatalf("Failed to create a temporary directory: %s", err)
}
inst.Dir = dir
inst.Cmd = exec.Command("etcd")

inst.Cmd.Env = append(
os.Environ(),
fmt.Sprintf("ETCD_LISTEN_CLIENT_URLS=%s", endpoint),
fmt.Sprintf("ETCD_ADVERTISE_CLIENT_URLS=%s", endpoint),
fmt.Sprintf("ETCD_DATA_DIR=%s", inst.Dir),
)
if opts.KeyFile != "" {
keyPath := filepath.Join(mydir, opts.KeyFile)
inst.Cmd.Env = append(inst.Cmd.Env,
fmt.Sprintf("ETCD_KEY_FILE=%s", keyPath))
}
if opts.CertFile != "" {
certPath := filepath.Join(mydir, opts.CertFile)
inst.Cmd.Env = append(inst.Cmd.Env,
fmt.Sprintf("ETCD_CERT_FILE=%s", certPath))
}
if opts.CaFile != "" {
caPath := filepath.Join(mydir, opts.CaFile)
inst.Cmd.Env = append(inst.Cmd.Env,
fmt.Sprintf("ETCD_TRUSTED_CA_FILE=%s", caPath))
}

// Start etcd.
err = inst.Cmd.Start()
if err != nil {
os.RemoveAll(inst.Dir)
t.Fatalf("Failed to start etcd: %s", err)
var tls *transport.TLSInfo
if opts.CaFile != "" || opts.CertFile != "" || opts.KeyFile != "" {
tls = &transport.TLSInfo{}
if opts.CaFile != "" {
caPath := filepath.Join(mydir, opts.CaFile)
tls.TrustedCAFile = caPath
}
if opts.CertFile != "" {
certPath := filepath.Join(mydir, opts.CertFile)
tls.CertFile = certPath
}
if opts.KeyFile != "" {
keyPath := filepath.Join(mydir, opts.KeyFile)
tls.KeyFile = keyPath
}
}
config := integration.ClusterConfig{Size: 1, PeerTLS: tls, UseTCP: true}
inst := integration.NewLazyClusterWithConfig(config)

// Setup user/pass.
if opts.Username != "" {
cmd := exec.Command("etcdctl", "user", "add", opts.Username,
fmt.Sprintf("--new-user-password=%s", opts.Password),
fmt.Sprintf("--endpoints=%s", baseEndpoint))

err := cmd.Run()
if err != nil {
stopEtcd(t, inst)
t.Fatalf("Failed to create user: %s", err)
etcd, err := libcluster.ConnectEtcd(libcluster.EtcdOpts{
Endpoints: inst.EndpointsV3(),
})
require.NoError(t, err)
defer etcd.Close()

if err := doWithCtx(func(ctx context.Context) error {
_, err := etcd.UserAdd(ctx, opts.Username, opts.Password)
return err
}); err != nil {
inst.Terminate()
t.Fatalf("Failed to create user in etcd: %s", err)
}

if opts.Username != "root" {
// We need the root user for auth enable anyway.
cmd := exec.Command("etcdctl", "user", "add", "root",
fmt.Sprintf("--new-user-password=%s", opts.Password),
fmt.Sprintf("--endpoints=%s", baseEndpoint))

err := cmd.Run()
if err != nil {
stopEtcd(t, inst)
t.Fatalf("Failed to create root: %s", err)
if err := doWithCtx(func(ctx context.Context) error {
_, err := etcd.UserAdd(ctx, "root", "")
return err
}); err != nil {
inst.Terminate()
t.Fatalf("Failed to create root in etcd: %s", err)
}

// And additional permissions for a regular user.
cmd = exec.Command("etcdctl", "user", "grant-role", opts.Username,
"root", fmt.Sprintf("--endpoints=%s", baseEndpoint))

err = cmd.Run()
if err != nil {
stopEtcd(t, inst)
t.Fatalf("Failed to grant-role: %s", err)
if err := doWithCtx(func(ctx context.Context) error {
_, err := etcd.UserGrantRole(ctx, "root", "root")
return err
}); err != nil {
inst.Terminate()
t.Fatalf("Failed to grant root in etcd: %s", err)
}
}

cmd = exec.Command("etcdctl", "auth", "enable",
fmt.Sprintf("--user=root:%s", opts.Password),
fmt.Sprintf("--endpoints=%s", baseEndpoint))

err = cmd.Run()
if err != nil {
stopEtcd(t, inst)
t.Fatalf("Failed to enable auth: %s", err)
}
}

return inst
}

func stopEtcd(t *testing.T, inst etcdInstance) {
t.Helper()

if inst.Cmd != nil && inst.Cmd.Process != nil {
if err := inst.Cmd.Process.Kill(); err != nil {
t.Fatalf("Failed to kill etcd (%d) %s", inst.Cmd.Process.Pid, err)
if err := doWithCtx(func(ctx context.Context) error {
_, err := etcd.UserGrantRole(ctx, opts.Username, "root")
return err
}); err != nil {
inst.Terminate()
t.Fatalf("Failed to grant user in etcd: %s", err)
}

// Wait releases any resources associated with the Process.
if _, err := inst.Cmd.Process.Wait(); err != nil {
t.Fatalf("Failed to wait for etcd process to exit, got %s", err)
return
if err := doWithCtx(func(ctx context.Context) error {
_, err = etcd.AuthEnable(ctx)
return err
}); err != nil {
inst.Terminate()
t.Fatalf("Failed to enable auth in etcd: %s", err)
}

inst.Cmd.Process = nil
}

if inst.Dir != "" {
if err := os.RemoveAll(inst.Dir); err != nil {
t.Fatalf("Failed to clean work directory, got %s", err)
}
}
return inst
}

func etcdPut(t *testing.T, etcd *clientv3.Client, key, value string) {
t.Helper()

ctx, cancel := context.WithTimeout(context.Background(), timeout)
presp, err := etcd.Put(ctx, key, value)
cancel()
var (
presp *clientv3.PutResponse
err error
)
doWithCtx(func(ctx context.Context) error {
presp, err = etcd.Put(ctx, key, value)
return nil
})
require.NoError(t, err)
require.NotNil(t, presp)
}

func etcdGet(t *testing.T, etcd *clientv3.Client, key string) ([]byte, int64) {
t.Helper()

ctx, cancel := context.WithTimeout(context.Background(), timeout)
resp, err := etcd.Get(ctx, key)
cancel()

var (
resp *clientv3.GetResponse
err error
)
doWithCtx(func(ctx context.Context) error {
resp, err = etcd.Get(ctx, key)
return nil
})
require.NoError(t, err)
require.NotNil(t, resp)
if len(resp.Kvs) == 0 {
return []byte(""), 0
}

require.Len(t, resp.Kvs, 1)
return resp.Kvs[0].Value, resp.Kvs[0].ModRevision
}

func renderEtcdAppConfig(t *testing.T, endpoint string, src string, dst string) {
engine := templates.NewDefaultEngine()
err := engine.RenderFile(src, dst, map[string]string{
"endpoint": endpoint,
})
require.NoError(t, err)
}

func TestGetClusterConfig_etcd(t *testing.T) {
inst := startEtcd(t, httpEndpoint, etcdOpts{
inst := startEtcd(t, etcdOpts{
Username: "root",
Password: "pass",
})
defer stopEtcd(t, inst)
defer inst.Terminate()
endpoints := inst.EndpointsV3()

tmpDir, err := os.MkdirTemp("", "work_dir")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)

configPath := filepath.Join(tmpDir, "config.yaml")
renderEtcdAppConfig(t, endpoints[0], "testdata/etcdapp/config.yaml.template", configPath)

endpoints := []string{httpEndpoint}
etcd, err := libcluster.ConnectEtcd(libcluster.EtcdOpts{
Endpoints: endpoints,
Username: "root",
Expand All @@ -208,20 +190,20 @@ func TestGetClusterConfig_etcd(t *testing.T) {
os.Setenv("TT_WAL_MODE_DEFAULT", "envmode")
os.Setenv("TT_WAL_MAX_SIZE_DEFAULT", "envsize")
collectors := libcluster.NewCollectorFactory(libcluster.NewDataCollectorFactory())
config, err := cluster.GetClusterConfig(collectors, "testdata/etcdapp/config.yaml")
config, err := cluster.GetClusterConfig(collectors, configPath)
os.Unsetenv("TT_WAL_MODE_DEFAULT")
os.Unsetenv("TT_WAL_MAX_SIZE_DEFAULT")

require.NoError(t, err)
assert.Equal(t, `app:
assert.Equal(t, fmt.Sprintf(`app:
bar: 1
foo: 1
hoo: 1
zoo: 1
config:
etcd:
endpoints:
- http://127.0.0.1:12379
- %s
http:
request:
timeout: 2.5
Expand Down Expand Up @@ -252,7 +234,7 @@ wal:
dir: filedir
max_size: envsize
mode: etcdmode
`, config.RawConfig.String())
`, endpoints[0]), config.RawConfig.String())
}

func TestGetClusterConfig_etcd_connect_from_env(t *testing.T) {
Expand All @@ -262,13 +244,20 @@ func TestGetClusterConfig_etcd_connect_from_env(t *testing.T) {
prefix = "/prefixenv"
)

inst := startEtcd(t, httpEndpoint, etcdOpts{
inst := startEtcd(t, etcdOpts{
Username: user,
Password: pass,
})
defer stopEtcd(t, inst)
defer inst.Terminate()
endpoints := inst.EndpointsV3()

tmpDir, err := os.MkdirTemp("", "work_dir")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)

configPath := filepath.Join(tmpDir, "config.yaml")
renderEtcdAppConfig(t, endpoints[0], "testdata/etcdapp/config.yaml.template", configPath)

endpoints := []string{httpEndpoint}
etcd, err := libcluster.ConnectEtcd(libcluster.EtcdOpts{
Endpoints: endpoints,
Username: user,
Expand All @@ -287,22 +276,22 @@ func TestGetClusterConfig_etcd_connect_from_env(t *testing.T) {
os.Setenv("TT_CONFIG_ETCD_PREFIX", prefix)

collectors := libcluster.NewCollectorFactory(libcluster.NewDataCollectorFactory())
config, err := cluster.GetClusterConfig(collectors, "testdata/etcdapp/config.yaml")
config, err := cluster.GetClusterConfig(collectors, configPath)

os.Unsetenv("TT_CONFIG_ETCD_USERNAME")
os.Unsetenv("TT_CONFIG_ETCD_PASSWORD")
os.Unsetenv("TT_CONFIG_ETCD_PREFIX")

require.NoError(t, err)
assert.Equal(t, `app:
assert.Equal(t, fmt.Sprintf(`app:
bar: 1
foo: 1
hoo: 1
zoo: 1
config:
etcd:
endpoints:
- http://127.0.0.1:12379
- %s
http:
request:
timeout: 2.5
Expand Down Expand Up @@ -332,5 +321,5 @@ groups:
wal:
dir: filedir
mode: etcdmode
`, config.RawConfig.String())
`, endpoints[0]), config.RawConfig.String())
}
2 changes: 1 addition & 1 deletion cli/cluster/testdata/etcdapp/config.yaml → ...ter/testdata/etcdapp/config.yaml.template
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
config:
etcd:
endpoints:
- http://127.0.0.1:12379
- {{.endpoint}}
username: root
password: pass
prefix: "/test"
Expand Down
Loading

0 comments on commit 0563aee

Please sign in to comment.