Add stability to TestGateway and TestBasic tests (#1766)

* checking job status when timing out

* change token type in basic tests

* retry when generating tokens, sometimes getting the policy API times out

test/integration/acceptance/custom/basic/basic.go

@@ -76,9 +76,21 @@ func (r *BasicTestRunner) Setup(ctx context.Context, createOptsPublic types.Site
 
 	const secretFile = "/tmp/public_basic_1_secret.yaml"
 	if tokenType == "claim" {
-		err = pub1Cluster.VanClient.TokenClaimCreateFile(ctx, types.DefaultVanName, []byte(createOptsPublic.Password), 15*time.Minute, 1, secretFile)
+		err = utils.RetryError(3*time.Second, 5, func() error {
+			err := pub1Cluster.VanClient.TokenClaimCreateFile(ctx, types.DefaultVanName, []byte(createOptsPublic.Password), 15*time.Minute, 1, secretFile)
+			if err == nil {
+				return nil
+			}
+			return err
+		})
 	} else {
-		err = pub1Cluster.VanClient.ConnectorTokenCreateFile(ctx, types.DefaultVanName, secretFile)
+		err = utils.RetryError(3*time.Second, 5, func() error {
+			err := pub1Cluster.VanClient.ConnectorTokenCreateFile(ctx, types.DefaultVanName, secretFile)
+			if err == nil {
+				return nil
+			}
+			return err
+		})
 	}
 	assert.Assert(t, err)
 
@@ -232,6 +244,7 @@ func (r *BasicTestRunner) Run(ctx context.Context, t *testing.T) {
 			skip:       base.MultipleClusters(),
 			skipReason: SkipReasonIngressNone,
 			testSync:   true,
+			tokenType:  "claim",
 			createOptsPublic: types.SiteConfigSpec{
 				SkupperName:         "",
 				RouterMode:          string(types.TransportModeInterior),
@@ -297,9 +310,10 @@ func (r *BasicTestRunner) Run(ctx context.Context, t *testing.T) {
 			},
 		},
 		{
-			id:       "interiors-ingress-default",
-			doc:      "Connecting two interiors with ingress=default (route if available or loadbalancer)",
-			testSync: false,
+			id:        "interiors-ingress-default",
+			doc:       "Connecting two interiors with ingress=default (route if available or loadbalancer)",
+			testSync:  false,
+			tokenType: "claim",
 			createOptsPublic: types.SiteConfigSpec{
 				SkupperName:         "",
 				RouterMode:          string(types.TransportModeInterior),

test/utils/k8s/job.go

@@ -175,19 +175,34 @@ func WaitForJob(ns string, kubeClient kubernetes.Interface, jobName string, time
 		case <-timeoutCh:
 			return nil, fmt.Errorf("Timeout: Job is still active: %s", jobName)
 		case <-tick:
-			job, _ := jobsClient.Get(context.TODO(), jobName, metav1.GetOptions{})
+			job, err := jobsClient.Get(context.TODO(), jobName, metav1.GetOptions{})
+
+			if err != nil {
+				return nil, err // Handle the error
+			}
 
 			if job.Status.Active > 0 {
 				fmt.Println("Job is still active")
 			} else if len(job.Status.Conditions) > 0 {
-				if job.Status.Conditions[0].Type == batchv1.JobComplete {
+				jobComplete := false
+				jobFailed := false
+				for _, condition := range job.Status.Conditions {
+					if condition.Type == batchv1.JobComplete {
+						jobComplete = true
+					} else if condition.Type == batchv1.JobFailed {
+						jobFailed = true
+					}
+				}
+
+				if jobComplete {
 					fmt.Println("Job Successful!")
 					return job, nil
-				} else if job.Status.Conditions[0].Type == batchv1.JobFailed {
+				} else if jobFailed {
 					statusJson, _ := json.Marshal(job.Status)
 					fmt.Printf("Job failed?, status = %v\n", string(statusJson))
 					return job, fmt.Errorf("Job failed. Status: %s", string(statusJson))
 				}
+
 			} else {
 				fmt.Println("Waiting on job condition")
 			}