DEVPROD-15543 Allow retries for 413 (#8797)

evergreen-ci · Mar 10, 2025 · 3b09379 · 3b09379
1 parent 72bff40
commit 3b09379
Show file tree

Hide file tree

Showing 6 changed files with 10 additions and 5 deletions.
diff --git a/agent/command/git.go b/agent/command/git.go
@@ -165,7 +165,6 @@ func getProjectMethodAndToken(ctx context.Context, comm client.Communicator, td
 
 	owner := conf.ProjectRef.Owner
 	repo := conf.ProjectRef.Repo
-	// chaya
 	appToken, err := comm.CreateInstallationTokenForClone(ctx, td, owner, repo)
 	if err != nil {
 		return "", "", errors.Wrap(err, "creating app token")

diff --git a/agent/internal/client/base_client.go b/agent/internal/client/base_client.go
@@ -713,6 +713,10 @@ func (c *baseCommunicator) GenerateTasks(ctx context.Context, td TaskData, jsonB
 	info := requestInfo{
 		method:   http.MethodPost,
 		taskData: &td,
+		// When generated tasks are large and evergreen is under load, we may not be able to ingest the
+		// data fast enough leading to a buffer overflow and a 413 status code. Therefore, a 413 status
+		// code in this case is transitive and we should retry.
+		retryOn413: true,
 	}
 	info.path = fmt.Sprintf("task/%s/generate", td.ID)
 	resp, err := c.retryRequest(ctx, info, jsonBytes)

diff --git a/agent/internal/client/request.go b/agent/internal/client/request.go
@@ -24,6 +24,7 @@ type requestInfo struct {
 	taskData *TaskData
 
 	retryOnInvalidBody bool
+	retryOn413         bool
 }
 
 func (c *baseCommunicator) newRequest(method, path, taskID, taskSecret string, data any) (*http.Request, error) {
@@ -146,6 +147,7 @@ func (c *baseCommunicator) retryRequest(ctx context.Context, info requestInfo, d
 	opts := utility.RetryRequestOptions{
 		RetryOptions:       c.retry,
 		RetryOnInvalidBody: info.retryOnInvalidBody,
+		RetryOn413:         info.retryOn413,
 	}
 	resp, err := utility.RetryRequest(ctx, r, opts)
 	if err != nil && resp != nil && resp.StatusCode == 400 {

diff --git a/config.go b/config.go
@@ -37,7 +37,7 @@ var (
 
 	// Agent version to control agent rollover. The format is the calendar date
 	// (YYYY-MM-DD).
-	AgentVersion = "2025-03-10"
+	AgentVersion = "2025-03-11"
 )
 
 const (

diff --git a/go.mod b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/evergreen-ci/poplar v0.0.0-20250226220147-608ee5d893b2
 	github.com/evergreen-ci/shrub v0.0.0-20250224222152-c8b72a51163b
 	github.com/evergreen-ci/timber v0.0.0-20250225175618-52d1e1841945
-	github.com/evergreen-ci/utility v0.0.0-20250224222128-c2a9c8dfbc87
+	github.com/evergreen-ci/utility v0.0.0-20250307192120-575c9dda4d76
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-github/v52 v52.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

diff --git a/go.sum b/go.sum
@@ -235,8 +235,8 @@ github.com/evergreen-ci/test-selection-client v0.0.0-20250211161638-10b5478f11ff
 github.com/evergreen-ci/timber v0.0.0-20250225175618-52d1e1841945 h1:brav0wdGU4QGOGlzLPjT5UdxzJBDwytmLC+FdF7EG7c=
 github.com/evergreen-ci/timber v0.0.0-20250225175618-52d1e1841945/go.mod h1:j76Uq4ABEq6Eulrq1otaNJGxWh/lo74IlfxWvAeOZb4=
 github.com/evergreen-ci/utility v0.0.0-20230104160902-3f0e05a638bd/go.mod h1:vkCEVgfCMIDajzbG/PHZURszI2Y1SuFqNWX9EUxmLLI=
-github.com/evergreen-ci/utility v0.0.0-20250224222128-c2a9c8dfbc87 h1:cBuYKFje/Y089ab5Rm6TaFYhzcFWZOjCIOyIAgfGihs=
-github.com/evergreen-ci/utility v0.0.0-20250224222128-c2a9c8dfbc87/go.mod h1:AGAekBl7zMjMvHB6yqkVkE81JuHBxBPaPozF49DsU70=
+github.com/evergreen-ci/utility v0.0.0-20250307192120-575c9dda4d76 h1:+iM3OKvVmGYH+xFnNt4mV4xI1m/9zBqy1zVmpYkrzz8=
+github.com/evergreen-ci/utility v0.0.0-20250307192120-575c9dda4d76/go.mod h1:AGAekBl7zMjMvHB6yqkVkE81JuHBxBPaPozF49DsU70=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=