From 6b3b97187d689707d4caa50c5133db8c55ecff96 Mon Sep 17 00:00:00 2001 From: Neelesh Thakur Date: Mon, 17 May 2021 16:05:19 -0600 Subject: [PATCH] retry the lock operation only when someone else is holding the lock We were retrying the lock operation for 5 minutes on all the errors. Create() ends up calling LeaseWithRetries() which has 30 retries with 2 sec sleep. If kvdb is down, the 5 min wait was causing unnecessary delay. Now, we use the 5 minutes timeout only when someone is holding the lock. Signed-off-by: Neelesh Thakur --- etcd/v3/kv_etcd.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etcd/v3/kv_etcd.go b/etcd/v3/kv_etcd.go index 17caafdb..41c91cda 100644 --- a/etcd/v3/kv_etcd.go +++ b/etcd/v3/kv_etcd.go @@ -696,7 +696,10 @@ func (et *etcdKV) LockWithTimeout( lockTag := ec.LockerIDInfo{LockerID: lockerID} kvPair, err := et.Create(key, lockTag, ttl) startTime := time.Now() - for count := 0; err != nil; count++ { + + // Create() will return ErrExist if the lock is being held already. In that case, + // we need to retry. + for count := 0; err == kvdb.ErrExist; count++ { time.Sleep(duration) kvPair, err = et.Create(key, lockTag, ttl) if count > 0 && count%15 == 0 && err != nil {