Fix src overwrite (#1215)

This change includes 2 changes: - ensure that the rewritten of statedb_info.json has a correct block height - refactor ReadStateDbInfo such that only statedb path is required.
Fantom-foundation · Oct 31, 2024 · 7bddcf6 · 7bddcf6
1 parent 9b96504
commit 7bddcf6
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 25 deletions.
diff --git a/executor/extension/primer/primer.go b/executor/extension/primer/primer.go
@@ -58,7 +58,13 @@ func (p *stateDbPrimer[T]) PreRun(_ executor.State[T], ctx *executor.Context) (e
 	// is used to determine block from which the priming starts
 	var primingStartBlock uint64
 	if p.cfg.IsExistingStateDb {
-		stateDbInfo, err := utils.ReadStateDbInfo(filepath.Join(p.cfg.StateDbSrc, utils.PathToDbInfo))
+		var stateDbInfo utils.StateDbInfo
+		var err error
+		if p.cfg.ShadowDb {
+			stateDbInfo, err = utils.ReadStateDbInfo(filepath.Join(p.cfg.StateDbSrc, utils.PathToPrimaryStateDb))
+		} else {
+			stateDbInfo, err = utils.ReadStateDbInfo(p.cfg.StateDbSrc)
+		}
 		if err != nil {
 			return fmt.Errorf("cannot read state db info; %w", err)
 		}

diff --git a/executor/extension/statedb/archive_db_block_checker.go b/executor/extension/statedb/archive_db_block_checker.go
@@ -47,7 +47,7 @@ func (c *archiveBlockChecker[T]) PreRun(executor.State[T], *executor.Context) er
 	var archiveLastBlock uint64
 
 	if c.cfg.ShadowDb {
-		primeDbInfo, err := utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToPrimaryStateDb, utils.PathToDbInfo))
+		primeDbInfo, err := utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToPrimaryStateDb))
 		if err != nil {
 			return fmt.Errorf("cannot read state db info for primary db; %v", err)
 		}
@@ -56,7 +56,7 @@ func (c *archiveBlockChecker[T]) PreRun(executor.State[T], *executor.Context) er
 			return fmt.Errorf("prime state db %v does not contain archive", filepath.Join(c.cfg.StateDbSrc, utils.PathToPrimaryStateDb))
 		}
 
-		shadowDbInfo, err := utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToShadowStateDb, utils.PathToDbInfo))
+		shadowDbInfo, err := utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToShadowStateDb))
 		if err != nil {
 			return fmt.Errorf("cannot read state db info for shadow db; %v", err)
 		}
@@ -68,7 +68,7 @@ func (c *archiveBlockChecker[T]) PreRun(executor.State[T], *executor.Context) er
 		archiveLastBlock = umath.Min(shadowDbInfo.Block, primeDbInfo.Block)
 
 	} else {
-		stateDbInfo, err := utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToDbInfo))
+		stateDbInfo, err := utils.ReadStateDbInfo(c.cfg.StateDbSrc)
 		if err != nil {
 			return fmt.Errorf("cannot read state db info; %v", err)
 		}

diff --git a/executor/extension/statedb/live_db_block_checker.go b/executor/extension/statedb/live_db_block_checker.go
@@ -52,12 +52,12 @@ func (c *liveDbBlockChecker[T]) PreRun(executor.State[T], *executor.Context) err
 	)
 
 	if c.cfg.ShadowDb {
-		primeDbInfo, err = utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToPrimaryStateDb, utils.PathToDbInfo))
+		primeDbInfo, err = utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToPrimaryStateDb))
 		if err != nil {
 			return fmt.Errorf("cannot read state db info for primary db; %v", err)
 		}
 
-		shadowDbInfo, err = utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToShadowStateDb, utils.PathToDbInfo))
+		shadowDbInfo, err = utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToShadowStateDb))
 		if err != nil {
 			return fmt.Errorf("cannot read state db info for shadow db; %v", err)
 		}
@@ -70,7 +70,7 @@ func (c *liveDbBlockChecker[T]) PreRun(executor.State[T], *executor.Context) err
 		lastBlock = primeDbInfo.Block
 
 	} else {
-		primeDbInfo, err = utils.ReadStateDbInfo(filepath.Join(c.cfg.StateDbSrc, utils.PathToDbInfo))
+		primeDbInfo, err = utils.ReadStateDbInfo(c.cfg.StateDbSrc)
 		if err != nil {
 			return fmt.Errorf("cannot read state db info; %v", err)
 		}

diff --git a/executor/extension/statedb/state_db_manager.go b/executor/extension/statedb/state_db_manager.go
@@ -93,10 +93,23 @@ func (m *stateDbManager[T]) PreRun(_ executor.State[T], ctx *executor.Context) e
 		m.log.Warningf("--keep-db is not used. Directory %v with DB will be removed at the end of this run.", ctx.StateDbPath)
 	}
 
-	if err := utils.WriteStateDbInfo(ctx.StateDbPath, m.cfg, 0, gc.Hash{}, false); err != nil {
-		return fmt.Errorf("failed to create state-db info file; %v", err)
+	// Set state-db info to incomplete state at the beginning of the run.
+	// If state-db info exists, read block number and hash from it.
+	var blockNum uint64
+	var blockHash gc.Hash
+	if m.cfg.IsExistingStateDb {
+		dbinfo, err := utils.ReadStateDbInfo(ctx.StateDbPath)
+		if err != nil {
+			return fmt.Errorf("failed to read state-db info file; %v", err)
+		}
+		m.log.Infof("Resuming from block %v", blockNum)
+		blockNum = dbinfo.Block
 	}
 
+	// Mark state-db info with incomplete state.
+	if err := utils.WriteStateDbInfo(ctx.StateDbPath, m.cfg, blockNum, blockHash, false); err != nil {
+		return fmt.Errorf("failed to create state-db info file; %v", err)
+	}
 	return nil
 }
 

diff --git a/executor/extension/statedb/state_db_manager_test.go b/executor/extension/statedb/state_db_manager_test.go
@@ -189,7 +189,7 @@ func TestStateDbManager_StateDbInfoExistenceAndReadable(t *testing.T) {
 		t.Fatalf("failed to find %v of stateDbInfo; %v", utils.PathToDbInfo, err)
 	}
 
-	_, err = utils.ReadStateDbInfo(filename)
+	_, err = utils.ReadStateDbInfo(expectedPath)
 	if err != nil {
 		t.Fatal("failed to read statedb_info.json")
 	}
@@ -223,15 +223,15 @@ func TestStateDbManager_OverrideArchiveMode(t *testing.T) {
 	}
 
 	expectedName := fmt.Sprintf("state_db_%v_%v_%v", cfg.DbImpl, cfg.DbVariant, state.Block)
-	DbPath := filepath.Join(cfg.DbTmp, expectedName)
+	dbPath := filepath.Join(cfg.DbTmp, expectedName)
 
-	filename := filepath.Join(DbPath, utils.PathToDbInfo)
+	filename := filepath.Join(dbPath, utils.PathToDbInfo)
 
 	if _, err := os.Stat(filename); err != nil {
 		t.Fatalf("failed to find %v of stateDbInfo; %v", utils.PathToDbInfo, err)
 	}
 
-	stateDbInfo, err := utils.ReadStateDbInfo(filename)
+	stateDbInfo, err := utils.ReadStateDbInfo(dbPath)
 	if err != nil {
 		t.Fatal("failed to read statedb_info.json")
 	}
@@ -257,7 +257,7 @@ func TestStateDbManager_OverrideArchiveMode(t *testing.T) {
 			cfg.DbImpl = "carmen"
 			cfg.ChainID = utils.MainnetChainID
 			cfg.ArchiveMode = true
-			cfg.StateDbSrc = DbPath
+			cfg.StateDbSrc = dbPath
 			if test.readOnlyTool {
 				cfg.SetStateDbSrcReadOnly()
 			}
@@ -314,7 +314,7 @@ func TestStateDbManager_OverrideArchiveVariant(t *testing.T) {
 		t.Fatalf("failed to find %v of stateDbInfo; %v", utils.PathToDbInfo, err)
 	}
 
-	stateDbInfo, err := utils.ReadStateDbInfo(filename)
+	stateDbInfo, err := utils.ReadStateDbInfo(dbPath)
 	if err != nil {
 		t.Fatal("failed to read statedb_info.json")
 	}
@@ -710,7 +710,7 @@ func TestStateDbManager_PreRunCreatesZeroStateDbInfo(t *testing.T) {
 		t.Fatalf("failed to create stateDb; %v", err)
 	}
 
-	info, err := utils.ReadStateDbInfo(filepath.Join(ctx.StateDbPath, utils.PathToDbInfo))
+	info, err := utils.ReadStateDbInfo(ctx.StateDbPath)
 	if err != nil {
 		t.Fatalf("cannot read state-db info: %v", err)
 	}

diff --git a/utils/statedb.go b/utils/statedb.go
@@ -101,10 +101,9 @@ func useExistingStateDB(cfg *Config) (state.StateDB, string, error) {
 		cfg.PathToStateDb = filepath.Join(cfg.PathToStateDb, PathToPrimaryStateDb)
 	}
 
-	stateDbInfoFile := filepath.Join(cfg.PathToStateDb, PathToDbInfo)
-	stateDbInfo, err = ReadStateDbInfo(stateDbInfoFile)
+	stateDbInfo, err = ReadStateDbInfo(cfg.PathToStateDb)
 	if err != nil {
-		return nil, "", fmt.Errorf("cannot read StateDb cfg file '%v'; %v", stateDbInfoFile, err)
+		return nil, "", fmt.Errorf("cannot read StateDb cfg file in '%v'; %v", cfg.PathToStateDb, err)
 	}
 
 	// If the state db is in read-only mode, set archive config as in statedb_info
@@ -143,10 +142,9 @@ func useExistingStateDB(cfg *Config) (state.StateDB, string, error) {
 	)
 
 	shadowDbPath = filepath.Join(cfg.StateDbSrc, PathToShadowStateDb)
-	shadowDbInfoFile := filepath.Join(shadowDbPath, PathToDbInfo)
-	shadowDbInfo, err = ReadStateDbInfo(shadowDbInfoFile)
+	shadowDbInfo, err = ReadStateDbInfo(shadowDbPath)
 	if err != nil {
-		return nil, "", fmt.Errorf("cannot read ShadowDb cfg file '%v'; %v", shadowDbInfoFile, err)
+		return nil, "", fmt.Errorf("cannot read ShadowDb info file in '%v'; %v", shadowDbPath, err)
 	}
 
 	cfg.ShadowImpl = shadowDbInfo.Impl

diff --git a/utils/statedb_info.go b/utils/statedb_info.go
@@ -134,8 +134,9 @@ func WriteStateDbInfo(directory string, cfg *Config, block uint64, root common.H
 }
 
 // ReadStateDbInfo reads meta file of loaded stateDB
-func ReadStateDbInfo(filename string) (StateDbInfo, error) {
+func ReadStateDbInfo(dbpath string) (StateDbInfo, error) {
 	var dbinfo StateDbInfo
+	filename := filepath.Join(dbpath, PathToDbInfo)
 	file, err := os.ReadFile(filename)
 	if err != nil {
 		return dbinfo, fmt.Errorf("failed to read %v; %v", filename, err)

diff --git a/utils/statedb_info_test.go b/utils/statedb_info_test.go
@@ -42,8 +42,7 @@ func TestStatedbInfo_WriteReadStateDbInfo(t *testing.T) {
 			}
 
 			// Getting the DB info file path and call for reading from it
-			dbInfoFile := filepath.Join(cfg.StateDbSrc, PathToDbInfo)
-			dbInfo, err := ReadStateDbInfo(dbInfoFile)
+			dbInfo, err := ReadStateDbInfo(cfg.StateDbSrc)
 			if err != nil {
 				t.Fatalf("failed to read from DB info json file: %v", err)
 			}