Fix manual generation (#796)

* Fix manual generation

* Add option to skip state-hash scrapping

cmd/util-db/db/gen_deleted_accounts.go

@@ -134,16 +134,16 @@ func genDeletedAccountsAction(ctx *cli.Context) error {
 	}
 	defer ddb.Close()
 
-	return GenDeletedAccountsAction(cfg, ddb, cfg.First)
+	return GenDeletedAccountsAction(cfg, ddb, cfg.First, cfg.Last)
 }
 
 // GenDeletedAccountsAction replays transactions and record self-destructed accounts and resurrected accounts.
-func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountDB, firstBlock uint64) error {
+func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountDB, firstBlock uint64, lastBlock uint64) error {
 	var err error
 
 	log := logger.NewLogger(cfg.LogLevel, "Generate Deleted Accounts")
 
-	log.Infof("chain-id: %v", cfg.ChainID)
+	log.Noticef("Generate deleted accounts from block %v to block %v", firstBlock, lastBlock)
 
 	start := time.Now()
 	sec := time.Since(start).Seconds()
@@ -157,7 +157,7 @@ func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountD
 
 	for iter.Next() {
 		tx := iter.Value()
-		if tx.Block > cfg.Last {
+		if tx.Block > lastBlock {
 			break
 		}
 

cmd/util-db/db/generate.go

@@ -41,20 +41,20 @@ var GenerateCommand = cli.Command{
 	Flags: []cli.Flag{
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
-		&utils.GenesisFlag,
+		&utils.OperaDbFlag,
 		&utils.WorldStateFlag,
 		&utils.OperaBinaryFlag,
 		&utils.OutputFlag,
-		&utils.TargetEpochFlag,
 		&utils.UpdateBufferSizeFlag,
+		&utils.SkipStateHashScrappingFlag,
 		&substate.WorkersFlag,
 		&logger.LogLevelFlag,
 	},
 	Description: `
-The db generate command requires events as an argument:
-<events>
-
-<events> are fed into the opera database (either existing or genesis needs to be specified), processing them generates updated aida-db.`,
+The db generate command requires 4 arguments:
+<firstBlock> <lastBlock> <firstEpoch> <lastEpoch>
+This command is designed for manual generation of deletion, updateset and patch just from substates in aidadb.
+`,
 }
 
 type generator struct {
@@ -82,6 +82,11 @@ func generate(ctx *cli.Context) error {
 		return err
 	}
 
+	err = g.prepareManualGenerate(ctx, cfg)
+	if err != nil {
+		return fmt.Errorf("prepareManualGenerate: %v; make sure you have correct substate range already recorded in aidadb", err)
+	}
+
 	if err = g.Generate(); err != nil {
 		return err
 	}
@@ -91,6 +96,64 @@ func generate(ctx *cli.Context) error {
 	return printMetadata(g.cfg.AidaDb)
 }
 
+// prepareManualGenerate prepares generator for manual generation
+func (g *generator) prepareManualGenerate(ctx *cli.Context, cfg *utils.Config) (err error) {
+	if ctx.Args().Len() != 4 {
+		return fmt.Errorf("generate command requires exactly 4 arguments - first block, last block, first epoch, last epoch")
+	}
+
+	g.opera.firstBlock, g.opera.lastBlock, err = utils.SetBlockRange(ctx.Args().Get(0), ctx.Args().Get(1), cfg.ChainID)
+	if err != nil {
+		return err
+	}
+
+	g.opera.firstEpoch, g.opera.lastEpoch, err = utils.SetBlockRange(ctx.Args().Get(2), ctx.Args().Get(3), cfg.ChainID)
+	if err != nil {
+		return err
+	}
+
+	if g.opera.firstBlock > g.opera.lastBlock {
+		return fmt.Errorf("generation range first block %v cannot be greater than available last block %v", g.opera.firstBlock, g.opera.lastBlock)
+	}
+
+	if g.opera.firstEpoch > g.opera.lastEpoch {
+		return fmt.Errorf("generation range first epoch %v cannot be greater than available last epoch %v", g.opera.firstEpoch, g.opera.lastEpoch)
+	}
+
+	firstSubstate := substate.NewSubstateDB(g.aidaDb).GetFirstSubstate()
+	lastSubstate, err := substate.NewSubstateDB(g.aidaDb).GetLastSubstate()
+	if err != nil {
+		return fmt.Errorf("cannot get last substate; %v", err)
+	}
+
+	if firstSubstate.Env.Number > g.opera.firstBlock {
+		return fmt.Errorf("generation range first block %v cannot be greater than first substate block %v", g.opera.firstBlock, firstSubstate.Env.Number)
+	}
+
+	if lastSubstate.Env.Number < g.opera.lastBlock {
+		return fmt.Errorf("generation range last block %v cannot be greater than last substate block %v", g.opera.lastBlock, lastSubstate.Env.Number)
+	}
+
+	firstAvailableEpoch, err := utils.FindEpochNumber(firstSubstate.Env.Number, g.cfg.ChainID)
+	if err != nil {
+		return fmt.Errorf("cannot find first epoch number; %v", err)
+	}
+
+	lastAvailableEpoch, err := utils.FindEpochNumber(lastSubstate.Env.Number, g.cfg.ChainID)
+	if err != nil {
+		return fmt.Errorf("cannot find last epoch number; %v", err)
+	}
+
+	if g.opera.firstEpoch < firstAvailableEpoch {
+		return fmt.Errorf("generation range first epoch %v cannot be less than first available epoch %v", g.opera.firstEpoch, firstAvailableEpoch)
+	}
+
+	if g.opera.lastEpoch > lastAvailableEpoch {
+		return fmt.Errorf("generation range last epoch %v cannot be greater than last available epoch %v", g.opera.lastEpoch, lastAvailableEpoch)
+	}
+	return nil
+}
+
 // newGenerator returns new instance of generator
 func newGenerator(ctx *cli.Context, cfg *utils.Config) (*generator, error) {
 	if cfg.AidaDb == "" {
@@ -135,9 +198,13 @@ func (g *generator) Generate() error {
 		return err
 	}
 
-	err = g.runStateHashScraper(g.ctx)
-	if err != nil {
-		return fmt.Errorf("cannot scrape state hashes; %v", err)
+	if g.cfg.SkipStateHashScrapping {
+		g.log.Noticef("Skipping state hash scraping...")
+	} else {
+		err = g.runStateHashScraper(g.ctx)
+		if err != nil {
+			return fmt.Errorf("cannot scrape state hashes; %v", err)
+		}
 	}
 
 	err = g.runDbHashGeneration(err)
@@ -271,7 +338,7 @@ func (g *generator) processDeletedAccounts(ddb *substate.DestroyedAccountDB) err
 	start = time.Now()
 	g.log.Noticef("Generating DeletionDb...")
 
-	err = GenDeletedAccountsAction(g.cfg, ddb, 0)
+	err = GenDeletedAccountsAction(g.cfg, ddb, 0, g.opera.lastBlock)
 	if err != nil {
 		return fmt.Errorf("cannot doGenerations deleted accounts; %v", err)
 	}
@@ -333,7 +400,7 @@ func (g *generator) createPatch() (string, error) {
 
 	// creating patch name
 	// add leading zeroes to filename to make it sortable
-	patchName := fmt.Sprintf("aida-db-%09s", strconv.FormatUint(g.opera.lastEpoch, 10))
+	patchName := fmt.Sprintf("%s-%s", strconv.FormatUint(g.opera.firstEpoch, 10), strconv.FormatUint(g.opera.lastEpoch, 10))
 	patchPath := filepath.Join(g.cfg.Output, patchName)
 
 	g.cfg.TargetDb = patchPath
@@ -346,8 +413,9 @@ func (g *generator) createPatch() (string, error) {
 	}
 
 	err = CreatePatchClone(g.cfg, g.aidaDb, patchDb, g.opera.firstEpoch, g.opera.lastEpoch, g.opera.isNew)
-
-	g.log.Notice("Patch metadata")
+	if err != nil {
+		return "", fmt.Errorf("cannot create patch clone; %v", err)
+	}
 
 	// metadata
 	err = utils.ProcessPatchLikeMetadata(patchDb, g.cfg.LogLevel, g.cfg.First, g.cfg.Last, g.opera.firstEpoch,