From 5a70e160d60ecfbe21e393ec845731541d7ca451 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 28 Jan 2024 07:56:48 -0800 Subject: [PATCH] Add host file input (#298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add host file input Allow file as host input for benchmark targets or clients. Example `warp-hosts.txt`: ``` 127.0.0.1:{9001...9004} 127.1.2.3 giga.abc.com ``` Debug printing loaded hosts: ``` λ warp stat --host=file:warp-hosts.txt HOSTS: [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com] λ warp stat --warp-client=file:warp-hosts.txt hosts [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com] ``` Slightly more generic version of https://github.com/minio/warp/pull/297 (and avoids another flag) --- README.md | 4 ++++ cli/benchserver.go | 20 +++++++++++++++++++- cli/client.go | 38 +++++++++++++++++++++++++++++++++----- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 05152df3..85c9dcf7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ Multiple S3 hosts can be specified as comma-separated values, for instance Alternatively numerical ranges can be specified using `--host=10.0.0.{1...10}:9000` which will add `10.0.0.1` through `10.0.0.10`. This syntax can be used for any part of the host name and port. +A file with newline separated hosts can also be specified using `file:` prefix and a file name. +For distributed tests the file will be read locally and sent to each client. + By default a host is chosen between the hosts that have the least number of requests running and with the longest time since the last request finished. This will ensure that in cases where hosts operate at different speeds that the fastest servers will get the most requests. @@ -112,6 +115,7 @@ Each client will also save its own data locally. Enabling server mode is done by adding `--warp-client=client-{1...10}:7761` or a comma separated list of warp client hosts. +Finally, a file with newline separated hosts can also be specified using `file:` prefix and a file name. If no host port is specified the default is added. Example: diff --git a/cli/benchserver.go b/cli/benchserver.go index 066fe67c..a1129c7d 100644 --- a/cli/benchserver.go +++ b/cli/benchserver.go @@ -117,6 +117,20 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { "syncstart": {}, "analyze.out": {}, } + transformFlags := map[string]func(flag cli.Flag) (string, error){ + // Special handling for hosts, we read files and expand it. + "host": func(flag cli.Flag) (string, error) { + hostsIn := flag.String() + if !strings.Contains(hostsIn, "file:") { + return flagToJSON(ctx, flag) + } + // This is a file, we will read it locally and expand. + hosts := parseHosts(hostsIn, false) + // Rejoin + return strings.Join(hosts, ","), nil + }, + } + req := serverRequest{ Operation: serverReqBenchmark, } @@ -130,7 +144,11 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { } if ctx.IsSet(flag.GetName()) { var err error - req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + if t := transformFlags[flag.GetName()]; t != nil { + req.Benchmark.Flags[flag.GetName()], err = t(flag) + } else { + req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + } if err != nil { return true, err } diff --git a/cli/client.go b/cli/client.go index eaa1582f..9210789f 100644 --- a/cli/client.go +++ b/cli/client.go @@ -18,10 +18,11 @@ package cli import ( + "bufio" "crypto/tls" "crypto/x509" "errors" - "log" + "fmt" "math" "math/rand" "net" @@ -227,14 +228,42 @@ func parseHosts(h string, resolveDNS bool) []string { var dst []string for _, host := range hosts { if !ellipses.HasEllipses(host) { - dst = append(dst, host) + if !strings.HasPrefix(host, "file:") { + dst = append(dst, host) + continue + } + // If host starts with file:, then it is a file containing hosts. + f, err := os.Open(strings.TrimPrefix(host, "file:")) + if err != nil { + fatalIf(probe.NewError(err), "Unable to open host file") + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + host := strings.TrimSpace(scanner.Text()) + if len(host) == 0 { + continue + } + if !ellipses.HasEllipses(host) { + dst = append(dst, host) + continue + } + patterns, perr := ellipses.FindEllipsesPatterns(host) + if perr != nil { + fatalIf(probe.NewError(perr), fmt.Sprintf("Unable to parse host parameter: %s", host)) + } + for _, lbls := range patterns.Expand() { + dst = append(dst, strings.Join(lbls, "")) + } + } + if err := scanner.Err(); err != nil { + fatalIf(probe.NewError(err), "Unable to read host file") + } continue } patterns, perr := ellipses.FindEllipsesPatterns(host) if perr != nil { fatalIf(probe.NewError(perr), "Unable to parse host parameter") - - log.Fatal(perr.Error()) } for _, lbls := range patterns.Expand() { dst = append(dst, strings.Join(lbls, "")) @@ -254,7 +283,6 @@ func parseHosts(h string, resolveDNS bool) []string { ips, err := net.LookupIP(host) if err != nil { fatalIf(probe.NewError(err), "Could not get IPs for "+hostport) - log.Fatal(err.Error()) } for _, ip := range ips { if port == "" {