Cherry-pick #4565 #4573 (#4576)

* update: support to wait to update function (#4565) * update: support to wait to update function Signed-off-by: sivchari <[email protected]> * rearrange: updatefunctionfromsource Signed-off-by: sivchari <[email protected]> --------- Signed-off-by: sivchari <[email protected]> * Wait ECS taskset stable (#4573) * Wait ECS taskset stable Signed-off-by: khanhtc1202 <[email protected]> * Change number of retry time Signed-off-by: khanhtc1202 <[email protected]> * No sleep required Signed-off-by: khanhtc1202 <[email protected]> --------- Signed-off-by: khanhtc1202 <[email protected]> --------- Signed-off-by: sivchari <[email protected]> Signed-off-by: khanhtc1202 <[email protected]> Co-authored-by: sivchari <[email protected]>
pipe-cd · Aug 28, 2023 · 6859ce9 · 6859ce9
1 parent 6b0499c
commit 6859ce9
Show file tree

Hide file tree

Showing 2 changed files with 58 additions and 14 deletions.
diff --git a/pkg/app/piped/platformprovider/ecs/client.go b/pkg/app/piped/platformprovider/ecs/client.go
@@ -35,10 +35,13 @@ import (
 )
 
 const (
-	// Retry wait service stable check.
-	retryServiceStable = 40
-	// Interval wait service stable check.
+	// ServiceStable's constants.
+	retryServiceStable         = 40
 	retryServiceStableInterval = 15 * time.Second
+
+	// TaskSetStable's constants.
+	retryTaskSetStable         = 40
+	retryTaskSetStableInterval = 15 * time.Second
 )
 
 type client struct {
@@ -216,6 +219,34 @@ func (c *client) CreateTaskSet(ctx context.Context, service types.Service, taskD
 	if err != nil {
 		return nil, fmt.Errorf("failed to create ECS task set %s: %w", *taskDefinition.TaskDefinitionArn, err)
 	}
+
+	// Wait created TaskSet to be stable.
+	waitInput := &ecs.DescribeTaskSetsInput{
+		Cluster:  service.ClusterArn,
+		Service:  service.ServiceArn,
+		TaskSets: []string{*output.TaskSet.TaskSetArn},
+	}
+
+	retry := backoff.NewRetry(retryTaskSetStable, backoff.NewConstant(retryTaskSetStableInterval))
+	_, err = retry.Do(ctx, func() (interface{}, error) {
+		output, err := c.ecsClient.DescribeTaskSets(ctx, waitInput)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get ECS task set %s: %w", *taskDefinition.TaskDefinitionArn, err)
+		}
+		if len(output.TaskSets) == 0 {
+			return nil, fmt.Errorf("failed to get ECS task set %s: task sets empty", *taskDefinition.TaskDefinitionArn)
+		}
+		taskSet := output.TaskSets[0]
+		if taskSet.StabilityStatus == types.StabilityStatusSteadyState {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("task set %s is not stable", *taskDefinition.TaskDefinitionArn)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to wait ECS task set %s stable: %w", *taskDefinition.TaskDefinitionArn, err)
+	}
+
 	return output.TaskSet, nil
 }
 
@@ -252,11 +283,7 @@ func (c *client) WaitServiceStable(ctx context.Context, service types.Service) e
 		Cluster:  service.ClusterArn,
 		Services: []string{*service.ServiceArn},
 	}
-	// Wait before first checking the service state due to the logic checking service
-	// stable currently is based on `pendingCount`, which could always be `0` when
-	// the service deployment has started running.
-	// TODO: Wait until a new task is started instead of sleeping.
-	time.Sleep(30 * time.Second)
+
 	retry := backoff.NewRetry(retryServiceStable, backoff.NewConstant(retryServiceStableInterval))
 	_, err := retry.Do(ctx, func() (interface{}, error) {
 		output, err := c.ecsClient.DescribeServices(ctx, input)

diff --git a/pkg/app/piped/platformprovider/lambda/client.go b/pkg/app/piped/platformprovider/lambda/client.go
@@ -227,6 +227,11 @@ func (c *client) UpdateFunction(ctx context.Context, fm FunctionManifest) error
 }
 
 func (c *client) UpdateFunctionFromSource(ctx context.Context, fm FunctionManifest, zip io.Reader) error {
+	// Update function configuration.
+	if err := c.updateFunctionConfiguration(ctx, fm); err != nil {
+		return err
+	}
+
 	data, err := io.ReadAll(zip)
 	if err != nil {
 		return err
@@ -249,11 +254,6 @@ func (c *client) UpdateFunctionFromSource(ctx context.Context, fm FunctionManife
 		return fmt.Errorf("failed to update function code for Lambda function %s: %w", fm.Spec.Name, err)
 	}
 
-	// Update function configuration.
-	if err = c.updateFunctionConfiguration(ctx, fm); err != nil {
-		return err
-	}
-
 	// Tag/Untag function if necessary.
 	return c.updateTagsConfig(ctx, fm)
 }
@@ -295,7 +295,24 @@ func (c *client) updateFunctionConfiguration(ctx context.Context, fm FunctionMan
 	if !updateFunctionConfigurationSucceed {
 		return fmt.Errorf("failed to update configuration for Lambda function %s: %w", fm.Spec.Name, err)
 	}
-	return nil
+
+	// Wait until function updated successfully.
+	retry = backoff.NewRetry(RequestRetryTime, backoff.NewConstant(RetryIntervalDuration))
+	input := &lambda.GetFunctionInput{
+		FunctionName: aws.String(fm.Spec.Name),
+	}
+	_, err = retry.Do(ctx, func() (any, error) {
+		output, err := c.client.GetFunction(ctx, input)
+		if err != nil {
+			return nil, err
+		}
+		if output.Configuration.LastUpdateStatus != types.LastUpdateStatusSuccessful {
+			return nil, fmt.Errorf("failed to update Lambda function %s, status code %v, error reason %s",
+				fm.Spec.Name, output.Configuration.LastUpdateStatus, *output.Configuration.LastUpdateStatusReason)
+		}
+		return nil, nil
+	})
+	return err
 }
 
 func (c *client) PublishFunction(ctx context.Context, fm FunctionManifest) (string, error) {