Add --attempt-timeout

- Change --timeout to be a global interrupt
- Improve README

README.md

@@ -6,35 +6,37 @@ interrogates a service to determine if it is alive/ready.
 Whether the probe result corresponds to aliveness or readiness depends on how the service handles connections and/or requests on the specified address.
 
 
-### Probe Schemes
+### Schemes
 
-Probe supports three schemes of probing: tcp, http, & https.
+Probe supports http, http, and tcp. The scheme is specified by the fully qualified address.
 
 TCP probing simply opens a connection to the supplied address and port and then closes it.
 
 HTTP probing opens a connection to the supplied address, port, and path, makes an HTTP GET request, reads the response, and closes the connection.
 
 HTTPS probing acts like HTTP probing, except with TLS/SSL certification.
 
+HTTP and HTTPS attempts expect a successful response status code (>= 200 and < 400).
 
-### Example Usage
 
-TCP probing:
+#### Examples
+
+Open and close a TCP connection:
 
 ```
 probe tcp://example.com:80
 ```
 
-HTTP probing, with timeout:
+Make an HTTP GET request:
 
 ```
-probe --timeout 5s http://example.com/
+probe http://example.com/
 ```
 
-HTTPS probing, with timeout shortcut:
+Make an HTTPS GET request:
 
 ```
-probe -t 1s https://example.com/
+probe https://example.com/
 ```
 
 
@@ -47,6 +49,51 @@ probe -t 1s https://example.com/
 The error description will be printed to STDERR.
 
 
+### Retries
+
+Probe can optional retry attempts (e.g. `--max-attempts=2`).
+
+Related Options:
+
+- `--max-attempts` - Maximum number of attempts to make (unlimitted: -1) (default 1)
+- `--retry-delay` - Delay between attempts. Valid time units: ns, us (or µs), ms, s, m, h. (default 1s)
+
+#### Examples
+
+Retry 5 times with a half second delay between each attempt:
+
+```
+probe --max-attempts=5 --retry-delay=0.5s http://example.com/
+```
+
+
+### Timeouts
+
+Probe can optionally time out (e.g. `--timeout=5s`).
+
+Related Options:
+
+- `--timeout` - Time after which the attempt(s) will be interrupted. Valid time units: ns, us (or µs), ms, s, m, h. (default -1ns)
+- `--attempt-timeout` - Time after which each individual attempt will be interrupted. Valid time units: ns, us (or µs), ms, s, m, h. (default -1ns)
+
+#### Examples
+
+Retry for 30 seconds with a two second delay between each attempt and five second timeout for each attempt:
+
+```
+probe --timeout=30s --max-attempts=-1 --retry-delay=2s --attempt-timeout=5s http://example.com/
+```
+
+
+### DNS Resolution
+
+Probe uses Go's DNS resolver, which can be configured by environment variable to force Go-based or C-based resolution.
+
+See http://golang.org/pkg/net/#hdr-Name_Resolution for more details.
+
+If the Go-based resolver is used, there is no built-in DNS caching, which may or may not be desirable.
+
+
 ### Install
 
 #### From Source
@@ -100,23 +147,11 @@ make builder
 (Requires [Docker](https://docs.docker.com/installation/).)
 
 
-### DNS Resolution
-
-Probe uses Go's DNS resolver, which can be configured by environment variable to force Go-based or C-based resolution.
-
-See http://golang.org/pkg/net/#hdr-Name_Resolution for more details.
-
-If the Go-based resolver is used, there is no built-in DNS caching, which may or may not be desirable.
-
-
 ### TODO
 
 1. Add SSL certificate validation options (currently ignores cert validity).
-2. Detect timeouts better
-  - `request canceled while waiting for connection`
-  - `read tcp 93.184.216.34:443: use of closed network connection` (https://example.com/)
-3. Upload cross-platform pre-compiled binaries
-4. Add configurable DNS caching
+2. Upload cross-platform pre-compiled binaries
+3. Add configurable DNS caching
 
 ### License
 

config.go

@@ -20,20 +20,25 @@ type config struct {
 	timeout  *time.Duration
 	maxAttempts *int
 	retryDelay *time.Duration
+	attemptTimeout *time.Duration
 }
 
 func (c *config) addflags(s *flag.FlagSet) {
-	timeout := s.Duration("timeout", -1, "Timeout duration. "+validTimeUnits)
+	timeout := s.Duration("timeout", -1, "Time after which the attempt(s) will be interrupted. "+validTimeUnits)
 	s.DurationVar(timeout, "t", -1, "Shortcut for --timeout")
 	c.timeout = timeout
 
-	maxAttempts := s.Int("max-attempts", 1, "Maximum number of attempts to make (default=1, unlimitted=-1)")
+	maxAttempts := s.Int("max-attempts", 1, "Maximum number of attempts to make (unlimitted: -1)")
 	s.IntVar(maxAttempts, "a", 1, "Shortcut for --max-attempts")
 	c.maxAttempts = maxAttempts
 
 	delay := s.Duration("retry-delay", 1*time.Second, "Delay between attempts. "+validTimeUnits)
 	s.DurationVar(delay, "d", 1*time.Second, "Shortcut for --retry-delay")
 	c.retryDelay = delay
+
+	attemptTimeout := s.Duration("attempt-timeout", -1, "Time after which each individual attempt will be interrupted. "+validTimeUnits)
+	s.DurationVar(attemptTimeout, "at", -1, "Shortcut for --attempt-timeout")
+	c.attemptTimeout = attemptTimeout
 }
 
 func usage(s *flag.FlagSet) func() {

main.go

@@ -44,12 +44,22 @@ func main() {
 	}
 
 	if *c.maxAttempts == 0 {
-		fmt.Fprintf(os.Stderr, "Error: Invalid attempts %q - Expected an int > 0 or exactly -1 (unlimited)\n", addrArg)
+		fmt.Fprintf(os.Stderr, "Error: Invalid max-attempts %q - Expected an int > 0 or exactly -1 (unlimited)\n", addrArg)
 		os.Exit(2)
 	}
 
 	if *c.retryDelay < 0 {
-		fmt.Fprintf(os.Stderr, "Error: Invalid delay %q - Expected a duration >= 0\n", addrArg)
+		fmt.Fprintf(os.Stderr, "Error: Invalid retry-delay %q - Expected a duration >= 0\n", addrArg)
+		os.Exit(2)
+	}
+
+	if *c.timeout == 0 {
+		fmt.Fprintf(os.Stderr, "Error: Invalid timeout %q - Expected an duration > 0 or exactly -1 (unlimited)\n", addrArg)
+		os.Exit(2)
+	}
+
+	if *c.attemptTimeout == 0 {
+		fmt.Fprintf(os.Stderr, "Error: Invalid attempt-timeout %q - Expected an duration > 0 or exactly -1 (unlimited)\n", addrArg)
 		os.Exit(2)
 	}
 
@@ -60,8 +70,8 @@ func main() {
 	case SchemeTCP:
 		dialer := tcp.NewDialer()
 
-		if *c.timeout >= 0 {
-			dialer.Timeout = *c.timeout
+		if *c.attemptTimeout > 0 {
+			dialer.Timeout = *c.attemptTimeout
 		}
 
 		prober = tcp.NewProber(dialer)
@@ -71,8 +81,8 @@ func main() {
 	case SchemeHTTPS:
 		client := http.NewInsecureClient()
 
-		if *c.timeout >= 0 {
-			client.Timeout = *c.timeout
+		if *c.attemptTimeout > 0 {
+			client.Timeout = *c.attemptTimeout
 		}
 
 		prober = http.NewProber(client)
@@ -83,7 +93,26 @@ func main() {
 		os.Exit(2)
 	}
 
+	var exitTimer *time.Timer
+	if *c.timeout >= 0 {
+		exitTimer = time.NewTimer(*c.timeout)
+		go func() {
+			_, ok := <-exitTimer.C
+			if ok {
+				// channel still open means timeout occurred
+				fmt.Fprintf(os.Stderr, "Error: Timed out after %v\n", *c.timeout)
+				// main goroutine will be killed by timeout exit.
+				// http client & dialer don't seem to be interruptable, or we might do that instead.
+				os.Exit(1)
+			}
+			// otherwise timer was stopped
+		}()
+	}
+
 	err = makeAttempts(prober, address, *c.maxAttempts, *c.retryDelay)
+	if exitTimer != nil {
+		exitTimer.Stop()
+	}
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)