From 9cdfd1344356e8de057dac9bfbacf7437c94c46b Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 26 Jan 2024 11:47:16 +0100 Subject: [PATCH 1/3] Add host file input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow file as host input for benchmark targets or clients. Example `warp-hosts.txt`: ``` 127.0.0.1:{9001...9004} 127.1.2.3 giga.abc.com ``` Debug printing loaded hosts: ``` λ warp stat --host=file:warp-hosts.txt HOSTS: [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com] λ warp stat --warp-client=file:warp-hosts.txt hosts [127.0.0.1:9001 127.0.0.1:9002 127.0.0.1:9003 127.0.0.1:9004 127.1.2.3 giga.abc.com] ``` Slightly more generic version of https://github.com/minio/warp/pull/297 (and avoids another flag) --- README.md | 4 ++++ cli/benchserver.go | 20 +++++++++++++++++++- cli/client.go | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 05152df3..85c9dcf7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ Multiple S3 hosts can be specified as comma-separated values, for instance Alternatively numerical ranges can be specified using `--host=10.0.0.{1...10}:9000` which will add `10.0.0.1` through `10.0.0.10`. This syntax can be used for any part of the host name and port. +A file with newline separated hosts can also be specified using `file:` prefix and a file name. +For distributed tests the file will be read locally and sent to each client. + By default a host is chosen between the hosts that have the least number of requests running and with the longest time since the last request finished. This will ensure that in cases where hosts operate at different speeds that the fastest servers will get the most requests. @@ -112,6 +115,7 @@ Each client will also save its own data locally. Enabling server mode is done by adding `--warp-client=client-{1...10}:7761` or a comma separated list of warp client hosts. +Finally, a file with newline separated hosts can also be specified using `file:` prefix and a file name. If no host port is specified the default is added. Example: diff --git a/cli/benchserver.go b/cli/benchserver.go index 066fe67c..a1129c7d 100644 --- a/cli/benchserver.go +++ b/cli/benchserver.go @@ -117,6 +117,20 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { "syncstart": {}, "analyze.out": {}, } + transformFlags := map[string]func(flag cli.Flag) (string, error){ + // Special handling for hosts, we read files and expand it. + "host": func(flag cli.Flag) (string, error) { + hostsIn := flag.String() + if !strings.Contains(hostsIn, "file:") { + return flagToJSON(ctx, flag) + } + // This is a file, we will read it locally and expand. + hosts := parseHosts(hostsIn, false) + // Rejoin + return strings.Join(hosts, ","), nil + }, + } + req := serverRequest{ Operation: serverReqBenchmark, } @@ -130,7 +144,11 @@ func runServerBenchmark(ctx *cli.Context, b bench.Benchmark) (bool, error) { } if ctx.IsSet(flag.GetName()) { var err error - req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + if t := transformFlags[flag.GetName()]; t != nil { + req.Benchmark.Flags[flag.GetName()], err = t(flag) + } else { + req.Benchmark.Flags[flag.GetName()], err = flagToJSON(ctx, flag) + } if err != nil { return true, err } diff --git a/cli/client.go b/cli/client.go index eaa1582f..23498e0a 100644 --- a/cli/client.go +++ b/cli/client.go @@ -18,9 +18,11 @@ package cli import ( + "bufio" "crypto/tls" "crypto/x509" "errors" + "fmt" "log" "math" "math/rand" @@ -227,7 +229,38 @@ func parseHosts(h string, resolveDNS bool) []string { var dst []string for _, host := range hosts { if !ellipses.HasEllipses(host) { - dst = append(dst, host) + if !strings.HasPrefix(host, "file:") { + dst = append(dst, host) + continue + } + // If host starts with file:, then it is a file containing hosts. + f, err := os.Open(strings.TrimPrefix(host, "file:")) + if err != nil { + fatalIf(probe.NewError(err), "Unable to open host file") + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + host := strings.TrimSpace(scanner.Text()) + if len(host) == 0 { + continue + } + if !ellipses.HasEllipses(host) { + dst = append(dst, host) + continue + } + patterns, perr := ellipses.FindEllipsesPatterns(host) + if perr != nil { + fatalIf(probe.NewError(perr), fmt.Sprintf("Unable to parse host parameter: %s", host)) + log.Fatal(perr.Error()) + } + for _, lbls := range patterns.Expand() { + dst = append(dst, strings.Join(lbls, "")) + } + } + if err := scanner.Err(); err != nil { + fatalIf(probe.NewError(err), "Unable to read host file") + } continue } patterns, perr := ellipses.FindEllipsesPatterns(host) From 5610d6fe579a5cd0bb2a12c26404382084940ca4 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 28 Jan 2024 16:46:10 +0100 Subject: [PATCH 2/3] Remove exit --- cli/client.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/client.go b/cli/client.go index 23498e0a..e0ffbfd4 100644 --- a/cli/client.go +++ b/cli/client.go @@ -252,7 +252,6 @@ func parseHosts(h string, resolveDNS bool) []string { patterns, perr := ellipses.FindEllipsesPatterns(host) if perr != nil { fatalIf(probe.NewError(perr), fmt.Sprintf("Unable to parse host parameter: %s", host)) - log.Fatal(perr.Error()) } for _, lbls := range patterns.Expand() { dst = append(dst, strings.Join(lbls, "")) @@ -266,7 +265,6 @@ func parseHosts(h string, resolveDNS bool) []string { patterns, perr := ellipses.FindEllipsesPatterns(host) if perr != nil { fatalIf(probe.NewError(perr), "Unable to parse host parameter") - log.Fatal(perr.Error()) } for _, lbls := range patterns.Expand() { From 5279034fdc55492d8cc75ddf76cfb51e499d1481 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Sun, 28 Jan 2024 16:48:02 +0100 Subject: [PATCH 3/3] Whatever lint --- cli/client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/client.go b/cli/client.go index e0ffbfd4..9210789f 100644 --- a/cli/client.go +++ b/cli/client.go @@ -23,7 +23,6 @@ import ( "crypto/x509" "errors" "fmt" - "log" "math" "math/rand" "net" @@ -265,7 +264,6 @@ func parseHosts(h string, resolveDNS bool) []string { patterns, perr := ellipses.FindEllipsesPatterns(host) if perr != nil { fatalIf(probe.NewError(perr), "Unable to parse host parameter") - log.Fatal(perr.Error()) } for _, lbls := range patterns.Expand() { dst = append(dst, strings.Join(lbls, "")) @@ -285,7 +283,6 @@ func parseHosts(h string, resolveDNS bool) []string { ips, err := net.LookupIP(host) if err != nil { fatalIf(probe.NewError(err), "Could not get IPs for "+hostport) - log.Fatal(err.Error()) } for _, ip := range ips { if port == "" {