From 266e586f1386f942363cefac19d630d205abedf2 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Tue, 30 Apr 2024 11:45:31 -0400 Subject: [PATCH] Adjust work queue rate limiter config Scale testing revealed excessive artificial delays from the bucket rate limiter used for the informer work queues. This was due to many failure requeues/retries occurring for many items in a short amount of time such that both the exponential and bucket rate limiters kicked in resulting in delays > 2 hours. We can make some adjustments to the rate limiter config to alleviate excessive delays. - Increase the exponential rate limiter base delay from 5 to 50 ms. This reduces the number of re-queues before hitting the max delay from 13 to 10. At 5 ms: 1: Delay: 5ms 2: Delay: 10ms 3: Delay: 20ms 4: Delay: 40ms 5: Delay: 80ms 6: Delay: 160ms 7: Delay: 320ms 8: Delay: 640ms 9: Delay: 1.28s 10: Delay: 2.56s 11: Delay: 5.12s 12: Delay: 10.24s 13: Delay: 20.48s 14: Delay: 30s At 50 ms: 1: Delay: 50ms 2: Delay: 100ms 3: Delay: 200ms 4: Delay: 400ms 5: Delay: 800ms 6: Delay: 1.6s 7: Delay: 3.2s 8: Delay: 6.4s 9: Delay: 12.8s 10: Delay: 25.6s 11: Delay: 30s - Increase the bucket rate limiter burst size from 100 to 500. This is the number of items at which the limiter will start adding delays if the rate is exceeded. - Cap the maximum delay at 5 min. Signed-off-by: Tom Pantelis --- pkg/workqueue/queue.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/workqueue/queue.go b/pkg/workqueue/queue.go index 15a698e4..ec5e3cea 100644 --- a/pkg/workqueue/queue.go +++ b/pkg/workqueue/queue.go @@ -52,12 +52,18 @@ var logger = log.Logger{Logger: logf.Log.WithName("WorkQueue")} func New(name string) Interface { return &queueType{ - RateLimitingInterface: workqueue.NewNamedRateLimitingQueue(workqueue.NewMaxOfRateLimiter( - // exponential per-item rate limiter - workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 30*time.Second), - // overall rate limiter (not per item) - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, - ), name), + RateLimitingInterface: workqueue.NewRateLimitingQueueWithConfig( + // caps the maximum wait + workqueue.NewWithMaxWaitRateLimiter( + workqueue.NewMaxOfRateLimiter( + // exponential per-item rate limiter + workqueue.NewItemExponentialFailureRateLimiter(50*time.Millisecond, 30*time.Second), + // overall rate limiter (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 500)}, + ), 5*time.Minute), + workqueue.RateLimitingQueueConfig{ + Name: name, + }), name: name, } }