test: check leader not ready in test (#8380)

ref #8350 Signed-off-by: okJiang <[email protected]> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com>
tikv · Jul 11, 2024 · 9eab28f · 9eab28f
1 parent c50993b
commit 9eab28f
Show file tree

Hide file tree

Showing 41 changed files with 98 additions and 69 deletions.
diff --git a/tests/autoscaling/autoscaling_test.go b/tests/autoscaling/autoscaling_test.go
@@ -40,7 +40,7 @@ func TestAPI(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	leaderServer := cluster.GetLeaderServer()
 	re.NoError(leaderServer.BootstrapCluster())

diff --git a/tests/compatibility/version_upgrade_test.go b/tests/compatibility/version_upgrade_test.go
@@ -37,7 +37,7 @@ func TestStoreRegister(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 	re.NoError(leaderServer.BootstrapCluster())
 
@@ -54,14 +54,14 @@ func TestStoreRegister(t *testing.T) {
 	_, err = svr.PutStore(context.Background(), putStoreRequest)
 	re.NoError(err)
 	// FIX ME: read v0.0.0 in sometime
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	version := leaderServer.GetClusterVersion()
 	// Restart all PDs.
 	err = cluster.StopAll()
 	re.NoError(err)
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	leaderServer = cluster.GetLeaderServer()
 	re.NotNil(leaderServer)
@@ -91,7 +91,7 @@ func TestRollingUpgrade(t *testing.T) {
 	defer cluster.Destroy()
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 	re.NoError(leaderServer.BootstrapCluster())
 

diff --git a/tests/dashboard/race_test.go b/tests/dashboard/race_test.go
@@ -35,7 +35,7 @@ func TestCancelDuringStarting(t *testing.T) {
 	re.NoError(err)
 	defer cluster.Destroy()
 	re.NoError(cluster.RunInitialServers())
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	time.Sleep(60 * time.Millisecond)
 	cancel()

diff --git a/tests/dashboard/service_test.go b/tests/dashboard/service_test.go
@@ -131,7 +131,7 @@ func (suite *dashboardTestSuite) testDashboard(re *require.Assertions, internalP
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	servers := cluster.GetServers()
 	leader := cluster.GetLeaderServer()
 	leaderAddr := leader.GetAddr()

diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go
@@ -656,7 +656,7 @@ func (suite *followerForwardAndHandleTestSuite) SetupSuite() {
 	re.NoError(err)
 	suite.cluster = cluster
 	suite.endpoints = runServer(re, cluster)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leader := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr())
 	suite.regionID = regionIDAllocator.alloc()

diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go
@@ -60,6 +60,7 @@ func (suite *serverRegisterTestSuite) SetupSuite() {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.clusterID = strconv.FormatUint(suite.pdLeader.GetClusterID(), 10)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()

diff --git a/tests/integrations/mcs/resourcemanager/resource_manager_test.go b/tests/integrations/mcs/resourcemanager/resource_manager_test.go
@@ -78,6 +78,7 @@ func (suite *resourceManagerClientTestSuite) SetupSuite() {
 	suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{})
 	re.NoError(err)
 	leader := suite.cluster.GetServer(suite.cluster.WaitLeader())
+	re.NotNil(leader)
 	waitLeader(re, suite.client, leader.GetAddr())
 
 	suite.initGroups = []*rmpb.ResourceGroup{
@@ -1046,7 +1047,7 @@ func (suite *resourceManagerClientTestSuite) TestBasicResourceGroupCURD() {
 		serverList = append(serverList, s)
 	}
 	re.NoError(tests.RunServers(serverList))
-	suite.cluster.WaitLeader()
+	re.NotEmpty(suite.cluster.WaitLeader())
 	// re-connect client as well
 	suite.client, err = pd.NewClientWithContext(suite.ctx, suite.cluster.GetConfig().GetClientURLs(), pd.SecurityOption{})
 	re.NoError(err)

diff --git a/tests/integrations/mcs/resourcemanager/server_test.go b/tests/integrations/mcs/resourcemanager/server_test.go
@@ -43,6 +43,7 @@ func TestResourceManagerServer(t *testing.T) {
 	re.NoError(err)
 
 	leaderName := cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	leader := cluster.GetServer(leaderName)
 
 	s, cleanup := tests.StartSingleResourceManagerTestServer(ctx, re, leader.GetAddr(), tempurl.Alloc())

diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go
@@ -64,6 +64,7 @@ func (suite *configTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	re.NoError(suite.pdLeaderServer.BootstrapCluster())
 	// Force the coordinator to be prepared to initialize the schedulers.

diff --git a/tests/integrations/mcs/scheduling/meta_test.go b/tests/integrations/mcs/scheduling/meta_test.go
@@ -55,6 +55,7 @@ func (suite *metaTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	re.NoError(suite.pdLeaderServer.BootstrapCluster())
 }

diff --git a/tests/integrations/mcs/scheduling/rule_test.go b/tests/integrations/mcs/scheduling/rule_test.go
@@ -53,6 +53,7 @@ func (suite *ruleTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoint = suite.pdLeaderServer.GetAddr()
 	re.NoError(suite.pdLeaderServer.BootstrapCluster())

diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go
@@ -70,6 +70,7 @@ func (suite *serverTestSuite) SetupSuite() {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	re.NoError(suite.pdLeader.BootstrapCluster())
@@ -116,6 +117,7 @@ func (suite *serverTestSuite) TestAllocIDAfterLeaderChange() {
 	re.NotEqual(uint64(0), id)
 	suite.cluster.ResignLeader()
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	time.Sleep(time.Second)
@@ -635,6 +637,7 @@ func (suite *multipleServerTestSuite) SetupSuite() {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	re.NoError(suite.pdLeader.BootstrapCluster())

diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go
@@ -65,6 +65,7 @@ func (suite *tsoAPITestSuite) SetupTest() {
 	err = suite.pdCluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.pdCluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	pdLeaderServer := suite.pdCluster.GetServer(leaderName)
 	re.NoError(pdLeaderServer.BootstrapCluster())
 	suite.backendEndpoints = pdLeaderServer.GetAddr()
@@ -155,6 +156,7 @@ func TestTSOServerStartFirst(t *testing.T) {
 	err = apiCluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := apiCluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	pdLeaderServer := apiCluster.GetServer(leaderName)
 	re.NoError(pdLeaderServer.BootstrapCluster())
 	re.NoError(err)

diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go
@@ -82,6 +82,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	re.NoError(suite.pdLeaderServer.BootstrapCluster())
 	suite.tsoCluster, err = tests.NewTestTSOCluster(suite.ctx, 2, suite.pdLeaderServer.GetAddr())

diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go
@@ -65,6 +65,7 @@ func (s *tsoProxyTestSuite) SetupSuite() {
 	err = s.apiCluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := s.apiCluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	s.apiLeader = s.apiCluster.GetServer(leaderName)
 	s.backendEndpoints = s.apiLeader.GetAddr()
 	re.NoError(s.apiLeader.BootstrapCluster())

diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go
@@ -77,6 +77,7 @@ func (suite *tsoServerTestSuite) SetupSuite() {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	re.NoError(suite.pdLeader.BootstrapCluster())
@@ -173,6 +174,7 @@ func checkTSOPath(re *require.Assertions, isAPIServiceMode bool) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	pdLeader := cluster.GetServer(leaderName)
 	re.NoError(pdLeader.BootstrapCluster())
 	backendEndpoints := pdLeader.GetAddr()
@@ -226,6 +228,7 @@ func TestWaitAPIServiceReady(t *testing.T) {
 		err = cluster.RunInitialServers()
 		re.NoError(err)
 		leaderName := cluster.WaitLeader()
+		re.NotEmpty(leaderName)
 		pdLeader := cluster.GetServer(leaderName)
 		return cluster, pdLeader.GetAddr()
 	}
@@ -283,6 +286,7 @@ func NewAPIServerForward(re *require.Assertions) APIServerForward {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	re.NoError(suite.pdLeader.BootstrapCluster())
@@ -557,6 +561,7 @@ func (suite *CommonTestSuite) SetupSuite() {
 	re.NoError(err)
 
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeader = suite.cluster.GetServer(leaderName)
 	suite.backendEndpoints = suite.pdLeader.GetAddr()
 	re.NoError(suite.pdLeader.BootstrapCluster())

diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go
@@ -97,6 +97,7 @@ func (suite *tsoClientTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	re.NoError(suite.pdLeaderServer.BootstrapCluster())
 	suite.backendEndpoints = suite.pdLeaderServer.GetAddr()
@@ -331,10 +332,12 @@ func (suite *tsoClientTestSuite) TestUpdateAfterResetTSO() {
 		// Resign leader to trigger the TSO resetting.
 		re.NoError(failpoint.Enable("github.com/tikv/pd/server/updateAfterResetTSO", "return(true)"))
 		oldLeaderName := suite.cluster.WaitLeader()
+		re.NotEmpty(oldLeaderName)
 		err := suite.cluster.GetServer(oldLeaderName).ResignLeader()
 		re.NoError(err)
 		re.NoError(failpoint.Disable("github.com/tikv/pd/server/updateAfterResetTSO"))
 		newLeaderName := suite.cluster.WaitLeader()
+		re.NotEmpty(newLeaderName)
 		re.NotEqual(oldLeaderName, newLeaderName)
 		// Request a new TSO.
 		testutil.Eventually(re, func() bool {
@@ -494,6 +497,7 @@ func TestMixedTSODeployment(t *testing.T) {
 	re.NoError(err)
 
 	leaderServer := cluster.GetServer(cluster.WaitLeader())
+	re.NotNil(leaderServer)
 	backendEndpoints := leaderServer.GetAddr()
 
 	apiSvr, err := cluster.JoinAPIServer(ctx)
@@ -516,6 +520,7 @@ func TestMixedTSODeployment(t *testing.T) {
 			time.Sleep(time.Duration(n) * time.Second)
 			leaderServer.ResignLeader()
 			leaderServer = cluster.GetServer(cluster.WaitLeader())
+			re.NotNil(leaderServer)
 		}
 		cancel1()
 	}()
@@ -534,6 +539,7 @@ func TestUpgradingAPIandTSOClusters(t *testing.T) {
 	err = apiCluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := apiCluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	pdLeader := apiCluster.GetServer(leaderName)
 	backendEndpoints := pdLeader.GetAddr()
 

diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go
@@ -79,6 +79,7 @@ func (suite *tsoConsistencyTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	backendEndpoints := suite.pdLeaderServer.GetAddr()
 	if suite.legacy {

diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go
@@ -77,6 +77,7 @@ func (suite *tsoServerTestSuite) SetupSuite() {
 	err = suite.cluster.RunInitialServers()
 	re.NoError(err)
 	leaderName := suite.cluster.WaitLeader()
+	re.NotEmpty(leaderName)
 	suite.pdLeaderServer = suite.cluster.GetServer(leaderName)
 	backendEndpoints := suite.pdLeaderServer.GetAddr()
 	if suite.legacy {

diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go
@@ -742,7 +742,7 @@ func TestRemovingProgress(t *testing.T) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leader := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr())
 	clusterID := leader.GetClusterID()
@@ -938,7 +938,7 @@ func TestPreparingProgress(t *testing.T) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leader := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr())
 	clusterID := leader.GetClusterID()

diff --git a/tests/server/cluster/cluster_work_test.go b/tests/server/cluster/cluster_work_test.go
@@ -40,7 +40,7 @@ func TestValidRequestRegion(t *testing.T) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leaderServer.GetAddr())
 	clusterID := leaderServer.GetClusterID()
@@ -84,7 +84,7 @@ func TestAskSplit(t *testing.T) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leaderServer.GetAddr())
 	clusterID := leaderServer.GetClusterID()
@@ -141,7 +141,7 @@ func TestSuspectRegions(t *testing.T) {
 	err = cluster.RunInitialServers()
 	re.NoError(err)
 
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 	grpcPDClient := testutil.MustNewGrpcClient(re, leaderServer.GetAddr())
 	clusterID := leaderServer.GetClusterID()

diff --git a/tests/server/id/id_test.go b/tests/server/id/id_test.go
@@ -43,7 +43,7 @@ func TestID(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	leaderServer := cluster.GetLeaderServer()
 	var last uint64
@@ -89,7 +89,7 @@ func TestCommand(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	leaderServer := cluster.GetLeaderServer()
 	req := &pdpb.AllocIDRequest{Header: testutil.NewRequestHeader(leaderServer.GetClusterID())}
@@ -115,7 +115,7 @@ func TestMonotonicID(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	leaderServer := cluster.GetLeaderServer()
 	var last1 uint64
@@ -127,7 +127,7 @@ func TestMonotonicID(t *testing.T) {
 	}
 	err = cluster.ResignLeader()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer = cluster.GetLeaderServer()
 	var last2 uint64
 	for i := uint64(0); i < 10; i++ {
@@ -138,7 +138,7 @@ func TestMonotonicID(t *testing.T) {
 	}
 	err = cluster.ResignLeader()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer = cluster.GetLeaderServer()
 	id, err := leaderServer.GetAllocator().Alloc()
 	re.NoError(err)
@@ -162,7 +162,7 @@ func TestPDRestart(t *testing.T) {
 
 	err = cluster.RunInitialServers()
 	re.NoError(err)
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 	leaderServer := cluster.GetLeaderServer()
 
 	var last uint64
@@ -175,7 +175,7 @@ func TestPDRestart(t *testing.T) {
 
 	re.NoError(leaderServer.Stop())
 	re.NoError(leaderServer.Run())
-	cluster.WaitLeader()
+	re.NotEmpty(cluster.WaitLeader())
 
 	for i := uint64(0); i < 10; i++ {
 		id, err := leaderServer.GetAllocator().Alloc()