-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: graceful shutdown when leader lost
Signed-off-by: zwtop <[email protected]>
- Loading branch information
Showing
2 changed files
with
110 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package server | ||
|
||
import ( | ||
"context" | ||
|
||
"k8s.io/klog/v2" | ||
|
||
"github.com/everoute/runtime/pkg/options" | ||
) | ||
|
||
// GracefulShutdown graceful shutdown apiserver when leader lost | ||
// make sure api should available before the apiserver shutdown | ||
// stopped until context done, or another node becomes leader ready | ||
func GracefulShutdown(ctx context.Context, config *options.RecommendedConfig) { | ||
electionCh := make(chan string) | ||
|
||
go func() { | ||
defer close(electionCh) | ||
electionClient := config.LeaderElectionClient | ||
if electionClient == nil { | ||
return | ||
} | ||
// check if leader has been changed | ||
if ld := electionClient.GetLeader(); ld != "" && ld != electionClient.Identity() { | ||
electionCh <- ld | ||
return | ||
} | ||
// until leading state update | ||
for electionClient.UntilLeadingStateUpdate(ctx.Done()) { | ||
if ld := electionClient.GetLeader(); ld != "" && ld != electionClient.Identity() { | ||
electionCh <- ld | ||
} | ||
} | ||
}() | ||
|
||
select { | ||
case <-ctx.Done(): | ||
klog.Fatalf("stopped when context done: %s", ctx.Err()) | ||
case ld, _ := <-electionCh: | ||
klog.Fatalf("stopped when %s becomes leader", ld) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
package server_test | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"testing" | ||
"time" | ||
|
||
"github.com/agiledragon/gomonkey/v2" | ||
. "github.com/onsi/gomega" | ||
"k8s.io/apimachinery/pkg/util/rand" | ||
"k8s.io/klog/v2" | ||
|
||
"github.com/everoute/runtime/pkg/options" | ||
"github.com/everoute/runtime/pkg/server" | ||
. "github.com/everoute/runtime/pkg/util/testing" | ||
) | ||
|
||
func TestGracefulShutdown(t *testing.T) { | ||
RegisterTestingT(t) | ||
|
||
ctx := context.Background() | ||
messageCh := make(chan string, 10) | ||
patches := gomonkey.ApplyFunc(klog.Fatalf, func(format string, args ...interface{}) { | ||
messageCh <- fmt.Sprintf(format, args...) | ||
}) | ||
defer patches.Reset() | ||
|
||
t.Run("should shutdown when another node has been becomes leader", func(t *testing.T) { | ||
electionClient := NewFakeLeaderElectionClient(rand.String(20)) | ||
electionClient.SetLeader(rand.String(20)) | ||
go server.GracefulShutdown(ctx, &options.RecommendedConfig{LeaderElectionClient: electionClient}) | ||
|
||
select { | ||
case msg := <-messageCh: | ||
Expect(msg).Should(ContainSubstring("becomes leader")) | ||
case <-time.After(time.Second): | ||
t.Fatalf("unexpect timeout wait graceful shutdown") | ||
} | ||
}) | ||
|
||
t.Run("should shutdown when another node becomes leader", func(t *testing.T) { | ||
electionClient := NewFakeLeaderElectionClient(rand.String(20)) | ||
go func() { time.Sleep(200 * time.Millisecond); electionClient.SetLeader(rand.String(20)) }() | ||
go server.GracefulShutdown(ctx, &options.RecommendedConfig{LeaderElectionClient: electionClient}) | ||
|
||
select { | ||
case msg := <-messageCh: | ||
Expect(msg).Should(ContainSubstring("becomes leader")) | ||
case <-time.After(time.Second): | ||
t.Fatalf("unexpect timeout wait graceful shutdown") | ||
} | ||
}) | ||
|
||
t.Run("should shutdown when context done", func(t *testing.T) { | ||
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) | ||
defer cancel() | ||
electionClient := NewFakeLeaderElectionClient(rand.String(20)) | ||
go server.GracefulShutdown(ctx, &options.RecommendedConfig{LeaderElectionClient: electionClient}) | ||
|
||
select { | ||
case msg := <-messageCh: | ||
Expect(msg).Should(ContainSubstring("stopped when context done")) | ||
case <-time.After(time.Second): | ||
t.Fatalf("unexpect timeout wait graceful shutdown") | ||
} | ||
}) | ||
} |