diff --git a/cmd/util-db/db/autogen.go b/cmd/util-db/db/autogen.go index 1f3a6eace..4b27218a3 100644 --- a/cmd/util-db/db/autogen.go +++ b/cmd/util-db/db/autogen.go @@ -54,10 +54,12 @@ func autogen(ctx *cli.Context) error { return fmt.Errorf("GENERATION BLOCKED: autogen failed in last run; %v", locked) } - g, err := newGenerator(ctx, cfg) + var g *generator + g, err = prepareAutogen(ctx, cfg) if err != nil { - return err + return fmt.Errorf("cannot start autogen; %v", err) } + err = autogenRun(cfg, g) if err != nil { errLock := setLock(cfg, err.Error()) @@ -68,6 +70,47 @@ func autogen(ctx *cli.Context) error { return err } +// prepareAutogen prepares autogen generation +func prepareAutogen(ctx *cli.Context, cfg *utils.Config) (*generator, error) { + g, err := newGenerator(ctx, cfg) + if err != nil { + return nil, err + } + + err = g.opera.init() + if err != nil { + return nil, err + } + + // remove worldstate directory if it was created + defer func(log *logging.Logger) { + if cfg.WorldStateDb != "" { + err = os.RemoveAll(cfg.WorldStateDb) + if err != nil { + log.Criticalf("can't remove temporary folder: %v; %v", cfg.WorldStateDb, err) + } + } + }(g.log) + + // user specified targetEpoch + if cfg.TargetEpoch > 0 { + g.targetEpoch = cfg.TargetEpoch + } else { + err = g.calculatePatchEnd() + if err != nil { + return nil, err + } + } + + MustCloseDB(g.aidaDb) + + // start epoch is last epoch + 1 + if g.opera.firstEpoch > g.targetEpoch { + return nil, fmt.Errorf("supplied targetEpoch %d is already reached; latest generated epoch %d", g.targetEpoch, g.opera.firstEpoch-1) + } + return g, nil +} + // setLock creates lockfile in case of error while generating func setLock(cfg *utils.Config, message string) error { lockFile := cfg.AidaDb + ".autogen.lock" @@ -98,44 +141,18 @@ func getLock(cfg *utils.Config) (string, error) { // autogenRun is used to record/update aida-db func autogenRun(cfg *utils.Config, g *generator) error { - err := g.opera.init() - if err != nil { - return err - } - - // remove worldstate directory if it was created - defer func(log *logging.Logger) { - if cfg.WorldStateDb != "" { - err = os.RemoveAll(cfg.WorldStateDb) - if err != nil { - log.Criticalf("can't remove temporary folder: %v; %v", cfg.WorldStateDb, err) - } - } - }(g.log) - - err = g.calculatePatchEnd() - if err != nil { - return err - } - - if cfg.TargetEpoch > 0 { - g.stopAtEpoch = cfg.TargetEpoch - } - - g.log.Noticef("Starting substate generation %d - %d", g.opera.lastEpoch+1, g.stopAtEpoch) - - MustCloseDB(g.aidaDb) + g.log.Noticef("Starting substate generation %d - %d", g.opera.firstEpoch, g.targetEpoch) start := time.Now() // stop opera to be able to export events - errCh := startOperaRecording(g.cfg, g.stopAtEpoch) + errCh := startOperaRecording(g.cfg, g.targetEpoch) // wait for opera recording response err, ok := <-errCh if ok && err != nil { return err } - g.log.Noticef("Recording for epoch range %d - %d finished. It took: %v", g.cfg.Db, g.opera.lastEpoch+1, g.stopAtEpoch, time.Since(start).Round(1*time.Second)) + g.log.Noticef("Recording for epoch range %d - %d finished. It took: %v", g.cfg.Db, g.opera.firstBlock, g.targetEpoch, time.Since(start).Round(1*time.Second)) g.log.Noticef("Total elapsed time: %v", time.Since(g.start).Round(1*time.Second)) // reopen aida-db diff --git a/cmd/util-db/db/generate.go b/cmd/util-db/db/generate.go index ed4861a40..85046c949 100644 --- a/cmd/util-db/db/generate.go +++ b/cmd/util-db/db/generate.go @@ -35,7 +35,7 @@ const ( var GenerateCommand = cli.Command{ Action: generate, Name: "generate", - Usage: "generates aida-db from given events", + Usage: "generates full aida-db from substatedb - generates deletiondb and updatesetdb, merges them into aida-db and then creates a patch", Flags: []cli.Flag{ &utils.AidaDbFlag, &utils.ChainIDFlag, @@ -61,7 +61,7 @@ type generator struct { md *utils.AidaDbMetadata aidaDb ethdb.Database opera *aidaOpera - stopAtEpoch uint64 + targetEpoch uint64 dbHash []byte start time.Time } @@ -485,21 +485,16 @@ func (g *generator) createTarGz(filePath string, fileName string) interface{} { // calculatePatchEnd retrieve epoch at which will next patch generation end func (g *generator) calculatePatchEnd() error { - err := g.opera.getOperaBlockAndEpoch(false) - if err != nil { - return err - } - // load last finished epoch to calculate next target - g.stopAtEpoch = g.opera.lastEpoch + g.targetEpoch = g.opera.firstEpoch // next patch will be at least X epochs large if g.cfg.ChainID == 250 { // mainnet currently takes about 250 epochs per day - g.stopAtEpoch += 250 + g.targetEpoch += 250 } else { // generic value - about 3 days on testnet 4002 - g.stopAtEpoch += 50 + g.targetEpoch += 50 } headEpochNumber, err := utils.FindHeadEpochNumber(g.cfg.ChainID) @@ -508,8 +503,8 @@ func (g *generator) calculatePatchEnd() error { } // if current generator is too far in history, start generation to the current head - if headEpochNumber > g.stopAtEpoch { - g.stopAtEpoch = headEpochNumber + if headEpochNumber > g.targetEpoch { + g.targetEpoch = headEpochNumber } return nil diff --git a/cmd/util-db/db/opera.go b/cmd/util-db/db/opera.go index ac9916cab..549ac65db 100644 --- a/cmd/util-db/db/opera.go +++ b/cmd/util-db/db/opera.go @@ -76,11 +76,12 @@ func (opera *aidaOpera) init() error { opera.firstBlock++ } - opera.log.Noticef("Opera is starting at block: %v", opera.firstBlock) + opera.log.Noticef("Opera block from last run is: %v", opera.firstBlock) // starting generation one block later - opera.cfg.First = opera.firstBlock + 1 + opera.firstBlock += 1 opera.firstEpoch += 1 + return nil } diff --git a/cmd/util-db/db/utils.go b/cmd/util-db/db/utils.go index 579daeb5a..43f871fd7 100644 --- a/cmd/util-db/db/utils.go +++ b/cmd/util-db/db/utils.go @@ -212,7 +212,7 @@ func startOperaRecording(cfg *utils.Config, syncUntilEpoch uint64) chan error { cmd := exec.Command(getOperaBinary(cfg), "--datadir", cfg.Db, "--recording", "--substate-db", cfg.SubstateDb, "--exitwhensynced.epoch", strconv.FormatUint(syncUntilEpoch+1, 10)) err := runCommand(cmd, nil, log) if err != nil { - errChan <- fmt.Errorf("unable record opera substates %v; binary %v ; %v", cfg.Db, getOperaBinary(cfg), err) + errChan <- fmt.Errorf("unable to record opera substates %v; binary %v ; %v", cfg.Db, getOperaBinary(cfg), err) } }() return errChan diff --git a/utils/config.go b/utils/config.go index e8fadef63..bae52b6bb 100644 --- a/utils/config.go +++ b/utils/config.go @@ -591,7 +591,10 @@ func reportNewConfig(cfg *Config, log *logging.Logger) { log.Infof("Archive mode: disabled") } log.Infof("Used VM implementation: %v", cfg.VmImpl) + log.Infof("Aida DB directory: %v", cfg.AidaDb) + log.Infof("Substate DB directory: %v", cfg.SubstateDb) log.Infof("Update DB directory: %v", cfg.UpdateDb) + log.Infof("Delete DB directory: %v", cfg.DeletionDb) if cfg.SkipPriming { log.Infof("Priming: Skipped") } else {