Skip to content

Commit

Permalink
Update to allow QuitWithoutEnvoyTimeout to be set and take precendenc…
Browse files Browse the repository at this point in the history
…e over WaitForEnvoyTimeout (#41)
  • Loading branch information
cbuto authored Nov 11, 2020
1 parent 895ae9c commit 0fc99fb
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 7 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ When the application exits, unless `NEVER_KILL_ISTIO_ON_FAILURE` has been set an
| `NEVER_KILL_ISTIO_ON_FAILURE` | If provided and set to `true`, `scuttle` will not instruct istio to exit if the main binary has exited with a non-zero exit code.
| `SCUTTLE_LOGGING` | If provided and set to `true`, `scuttle` will log various steps to the console which is helpful for debugging |
| `START_WITHOUT_ENVOY` | If provided and set to `true`, `scuttle` will not wait for envoy to be LIVE before starting the main application. However, it will still instruct envoy to exit.|
| `WAIT_FOR_ENVOY_TIMEOUT` | If provided and set to a valid `time.Duration` string greater than 0 seconds, `scuttle` will wait for that amount of time before starting the main application. By default, it will wait indefinitely.|
| `WAIT_FOR_ENVOY_TIMEOUT` | If provided and set to a valid `time.Duration` string greater than 0 seconds, `scuttle` will wait for that amount of time before starting the main application. By default, it will wait indefinitely. If `QUIT_WITHOUT_ENVOY_TIMEOUT` is set as well, it will take precedence over this variable |
| `ISTIO_QUIT_API` | If provided `scuttle` will send a POST to `/quitquitquit` at the given API. Should be in format `http://127.0.0.1:15020`. This is intended for Istio v1.3 and higher. When not given, Istio will be stopped using a `pkill` command.
| `GENERIC_QUIT_ENDPOINTS` | If provided `scuttle` will send a POST to the URL given. Multiple URLs are supported and must be provided as a CSV string. Should be in format `http://myendpoint.com` or `http://myendpoint.com,https://myotherendpoint.com`. The status code response is logged (if logging is enabled) but is not used. A 200 is treated the same as a 404 or 500. `GENERIC_QUIT_ENDPOINTS` is handled before Istio is stopped. |
| `QUIT_WITHOUT_ENVOY_TIMEOUT` | If provided and set to a valid duration, `scuttle` will exit if Envoy does not become available before the end of the timeout. If `START_WITHOUT_ENVOY` is also set, this variable will not be taken into account |
| `QUIT_WITHOUT_ENVOY_TIMEOUT` | If provided and set to a valid duration, `scuttle` will exit if Envoy does not become available before the end of the timeout and not continue with the passed in executable. If `START_WITHOUT_ENVOY` is also set, this variable will not be taken into account. Also, if `WAIT_FOR_ENVOY_TIMEOUT` is set, this variable will take precedence. |

## How Scuttle stops Istio

Expand Down
13 changes: 11 additions & 2 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,11 @@ func main() {
err := blockingCtx.Err()
if err == nil || errors.Is(err, context.Canceled) {
log("Blocking finished, Envoy has started")
} else if errors.Is(err, context.DeadlineExceeded) && config.QuitWithoutEnvoyTimeout > time.Duration(0) {
log("Blocking timeout reached and Envoy has not started, exiting scuttle")
os.Exit(1)
} else if errors.Is(err, context.DeadlineExceeded) {
log("Blocking timeout reached and Envoy has not started")
log("Blocking timeout reached and Envoy has not started, continuing with passed in executable")
} else {
panic(err.Error())
}
Expand Down Expand Up @@ -183,7 +186,9 @@ func waitForEnvoy() context.Context {
}
var blockingCtx context.Context
var cancel context.CancelFunc
if config.WaitForEnvoyTimeout > time.Duration(0) {
if config.QuitWithoutEnvoyTimeout > time.Duration(0) {
blockingCtx, cancel = context.WithTimeout(context.Background(), config.QuitWithoutEnvoyTimeout)
} else if config.WaitForEnvoyTimeout > time.Duration(0) {
blockingCtx, cancel = context.WithTimeout(context.Background(), config.WaitForEnvoyTimeout)
} else {
blockingCtx, cancel = context.WithCancel(context.Background())
Expand All @@ -201,6 +206,10 @@ func pollEnvoy(ctx context.Context, cancel context.CancelFunc) {
// We wait forever for envoy to start. In practice k8s will kill the pod if we take too long.
b.MaxElapsedTime = config.WaitForEnvoyTimeout

if config.QuitWithoutEnvoyTimeout > time.Duration(0) {
b.MaxElapsedTime = config.QuitWithoutEnvoyTimeout
}

_ = backoff.Retry(func() error {
pollCount++
rsp := typhon.NewRequest(ctx, "GET", url, nil).Send().Response()
Expand Down
7 changes: 4 additions & 3 deletions main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ func TestWaitTillTimeoutForEnvoy(t *testing.T) {
fmt.Println("Starting TestWaitTillTimeoutForEnvoy")
os.Setenv("QUIT_WITHOUT_ENVOY_TIMEOUT", "500ms")
os.Setenv("ENVOY_ADMIN_API", badServer.URL)
initTestingEnv()
dur, _ := time.ParseDuration("500ms")
config.QuitWithoutEnvoyTimeout = dur
blockingCtx := waitForEnvoy()
Expand All @@ -156,13 +157,13 @@ func TestWaitTillTimeoutForEnvoy(t *testing.T) {
case <-time.After(1 * time.Second):
t.Fatal("Context did not timeout")
case <-blockingCtx.Done():
if !errors.Is(blockingCtx.Err(), context.Canceled) {
if !errors.Is(blockingCtx.Err(), context.DeadlineExceeded) {
t.Fatalf("Context contains wrong error: %s", blockingCtx.Err())
}
}
}

// Tests scuttle will continue after WAIT_FOR_ENVOY_TIMEOUT expires and enovy is not ready
// Tests scuttle will continue after WAIT_FOR_ENVOY_TIMEOUT expires and envoy is not ready
func TestWaitForEnvoyTimeoutContinueWithoutEnvoy(t *testing.T) {
fmt.Println("Starting TestWaitForEnvoyTimeoutContinueWithoutEnvoy")
os.Setenv("WAIT_FOR_ENVOY_TIMEOUT", "5s")
Expand All @@ -173,7 +174,7 @@ func TestWaitForEnvoyTimeoutContinueWithoutEnvoy(t *testing.T) {
err := blockingCtx.Err()
if err == nil || !errors.Is(err, context.DeadlineExceeded) {
fmt.Println("TestWaitForEnvoyTimeoutContinueWithoutEnvoy err", err)
// Err is nil (enovy is up)
// Err is nil (envoy is up)
// or Err is set, but is not a cancellation err
// we expect a cancellation when the time is up
t.Fail()
Expand Down

0 comments on commit 0fc99fb

Please sign in to comment.