diff --git a/providers/gcloud/gcloudProvider.go b/providers/gcloud/gcloudProvider.go index 27a7212653..9aeda07548 100644 --- a/providers/gcloud/gcloudProvider.go +++ b/providers/gcloud/gcloudProvider.go @@ -5,10 +5,8 @@ import ( "encoding/json" "errors" "fmt" - "log" "regexp" "strings" - "time" "github.com/StackExchange/dnscontrol/v4/models" "github.com/StackExchange/dnscontrol/v4/pkg/diff2" @@ -488,70 +486,3 @@ func (g *gcloudProvider) EnsureZoneExists(domain string) error { g.zones[domain+"."], err = g.client.ManagedZones.Create(g.project, mz).Do() return err } - -const ( - initialBackoff = time.Second * 10 // First delay duration - maxBackoff = time.Minute * 3 // Maximum backoff delay -) - -// backoff is the amount of time to sleep if a 429 or 504 is received. -// It is doubled after each use. -var ( - backoff = initialBackoff - backoff404 = false // Set if the last call requested a retry of a 404 -) - -func retryNeeded(resp *googleapi.ServerResponse, err error) bool { - if err != nil { - return false // Not an error. - } - serr, ok := err.(*googleapi.Error) - if !ok { - return false // Not a google error. - } - if serr.Code == 200 { - backoff = initialBackoff // Reset - return false // Success! No need to retry. - } - - if serr.Code == 404 { - // serr.Code == 404 happens occasionally when GCLOUD hasn't - // finished updating the database yet. We pause and retry - // exactly once. There should be a better way to do this, such as - // a callback that would tell us a transaction is complete. - if backoff404 { - backoff404 = false - return false // Give up. We've done this already. - } - log.Printf("Special 404 pause-and-retry for GCLOUD: Pausing %s\n", backoff) - time.Sleep(backoff) - backoff404 = true - return true // Request a retry. - } - backoff404 = false - - if serr.Code != 429 && serr.Code != 502 && serr.Code != 503 { - return false // Not an error that permits retrying. - } - - // TODO(tlim): In theory, resp.Header has a header that says how - // long to wait but I haven't been able to capture that header in - // the wild. If you get these "RUNCHANGE HEAD" messages, please - // file a bug with the contents! - - if resp != nil { - log.Printf("NOTE: If you see this message, please file a bug with the output below:\n") - log.Printf("RUNCHANGE CODE = %+v\n", resp.HTTPStatusCode) - log.Printf("RUNCHANGE HEAD = %+v\n", resp.Header) - } - - // a simple exponential back-off - log.Printf("Pausing due to ratelimit: %v seconds\n", backoff) - time.Sleep(backoff) - backoff = backoff + (backoff / 2) - if backoff > maxBackoff { - backoff = maxBackoff - } - - return true // Request the API call be re-tried. -} diff --git a/providers/gcloud/retry.go b/providers/gcloud/retry.go new file mode 100644 index 0000000000..0df9b88e1f --- /dev/null +++ b/providers/gcloud/retry.go @@ -0,0 +1,94 @@ +package gcloud + +import ( + "log" + "time" + + "google.golang.org/api/googleapi" +) + +const ( + initialBackoff = time.Second * 10 // First delay duration + maxBackoff = time.Minute * 3 // Maximum backoff delay +) + +// backoff is the amount of time to sleep if a 429 or 504 is received. +// It is doubled after each use. +var ( + backoff = initialBackoff + backoff404 = false // Set if the last call requested a retry of a 404 + backoff502 = false // Set if the last call requested a retry of a 502 +) + +func retryNeeded(resp *googleapi.ServerResponse, err error) bool { + if err == nil { + return false // Not an error. + } + + serr, ok := err.(*googleapi.Error) + if !ok { + return false // Not a google error. + } + + if serr.Code == 200 { + backoff = initialBackoff // Reset + return false // Success! No need to retry. + } + + if serr.Code == 404 { + // serr.Code == 404 happens occasionally when GCLOUD hasn't + // finished updating the database yet. We pause and retry + // exactly once. There should be a better way to do this, such as + // a callback that would tell us a transaction is complete. + if backoff404 { + backoff404 = false + return false // Give up. We've done this already. + } + log.Printf("Special 404 pause-and-retry for GCLOUD: Pausing %s\n", backoff) + time.Sleep(backoff) + backoff404 = true + return true // Request a retry. + } + backoff404 = false + + if serr.Code == 502 { + // serr.Code == 502 happens occasionally when "The server + // encountered a temporary error and could not complete your + // request. Please try again in 30 seconds. That’s all we know." + // We pause and retry exactly once. + if backoff502 { + backoff502 = false + return false // Give up. We've done this already. + } + log.Printf("Special 502 pause-and-retry for GCLOUD: Pausing %s\n", backoff) + time.Sleep(31 * time.Second) + backoff502 = true + return true // Request a retry. + } + backoff502 = false + + if serr.Code != 429 && serr.Code != 503 { + return false // Not an error that permits retrying. + } + + // TODO(tlim): In theory, resp.Header has a header that says how + // long to wait but I haven't been able to capture that header in + // the wild. If you get these "RUNCHANGE HEAD" messages, please + // file a bug with the contents! + + if resp != nil { + log.Printf("NOTE: If you see this message, please file a bug with the output below:\n") + log.Printf("RUNCHANGE CODE = %+v\n", resp.HTTPStatusCode) + log.Printf("RUNCHANGE HEAD = %+v\n", resp.Header) + } + + // a simple exponential back-off + log.Printf("Pausing due to ratelimit: %v seconds\n", backoff) + time.Sleep(backoff) + backoff = backoff + (backoff / 2) + if backoff > maxBackoff { + backoff = maxBackoff + } + + return true // Request the API call be re-tried. +}