From 5bea0b7534e428e0ac916fcc30802c5294a1ee97 Mon Sep 17 00:00:00 2001
From: pk910 <philipp@pk910.de>
Date: Thu, 24 Aug 2023 13:51:35 +0200
Subject: [PATCH] skip epochs during sync if synchronization failed more than
 10 times

---
 indexer/synchronizer.go | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/indexer/synchronizer.go b/indexer/synchronizer.go
index 89bedb7a..a4961962 100644
--- a/indexer/synchronizer.go
+++ b/indexer/synchronizer.go
@@ -79,15 +79,27 @@ func (sync *synchronizerState) runSync() {
 	sync.cachedBlocks = make(map[uint64]*CacheBlock)
 	sync.cachedSlot = 0
 	isComplete := false
+	retryCount := 0
 	synclogger.Infof("synchronization started. Head epoch: %v", sync.currentEpoch)
 
 	for {
 		// synchronize next epoch
 		syncEpoch := sync.currentEpoch
 
-		synclogger.Infof("synchronizing epoch %v", syncEpoch)
-		done, err := sync.syncEpoch(syncEpoch)
-		if done {
+		lastRetry := retryCount >= 10
+		if lastRetry {
+			synclogger.Infof("synchronizing epoch %v (retry: %v, last retry!)", syncEpoch, retryCount)
+		} else if retryCount > 0 {
+			synclogger.Infof("synchronizing epoch %v (retry: %v)", syncEpoch, retryCount)
+		} else {
+			synclogger.Infof("synchronizing epoch %v", syncEpoch)
+		}
+		done, err := sync.syncEpoch(syncEpoch, lastRetry)
+		if done || lastRetry {
+			if err != nil {
+				synclogger.Warnf("synchronization of epoch %v failed: %v - skipping epoch", syncEpoch, err)
+			}
+			retryCount = 0
 			finalizedEpoch, _ := sync.indexer.indexerCache.getFinalizedHead()
 			sync.stateMutex.Lock()
 			syncEpoch++
@@ -99,6 +111,7 @@ func (sync *synchronizerState) runSync() {
 			}
 		} else if err != nil {
 			synclogger.Warnf("synchronization of epoch %v failed: %v - Retrying in 10 sec...", syncEpoch, err)
+			retryCount++
 			time.Sleep(10 * time.Second)
 		}
 
@@ -134,7 +147,7 @@ func (sync *synchronizerState) checkKillChan(timeout time.Duration) bool {
 	}
 }
 
-func (sync *synchronizerState) syncEpoch(syncEpoch uint64) (bool, error) {
+func (sync *synchronizerState) syncEpoch(syncEpoch uint64, lastTry bool) (bool, error) {
 	if db.IsEpochSynchronized(syncEpoch) {
 		return true, nil
 	}
@@ -144,8 +157,10 @@ func (sync *synchronizerState) syncEpoch(syncEpoch uint64) (bool, error) {
 	// load epoch assignments
 	epochAssignments, err := client.rpcClient.GetEpochAssignments(syncEpoch)
 	if err != nil || epochAssignments == nil {
-		synclogger.Errorf("error fetching epoch %v duties: %v", syncEpoch, err)
-		return false, err
+		return false, fmt.Errorf("error fetching epoch %v duties: %v", syncEpoch, err)
+	}
+	if epochAssignments.AttestorAssignments == nil && !lastTry {
+		return false, fmt.Errorf("error fetching epoch %v duties: attestor assignments empty", syncEpoch)
 	}
 
 	if sync.checkKillChan(0) {
@@ -196,7 +211,7 @@ func (sync *synchronizerState) syncEpoch(syncEpoch uint64) (bool, error) {
 	}
 	epochStats.loadValidatorStats(client, epochAssignments.DependendStateRef)
 
-	if epochStats.validatorStats == nil {
+	if epochStats.validatorStats == nil && !lastTry {
 		return false, fmt.Errorf("error fetching validator stats for epoch %v: %v", syncEpoch, err)
 	}
 	if sync.checkKillChan(0) {