Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix manual generation #796

Merged
merged 5 commits into from
Nov 9, 2023
Merged

Fix manual generation #796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2f7a0fa
Fix manual generation
Nov 6, 2023
ea2ce5a
Add option to skip state-hash scrapping
Nov 7, 2023
3488314
Fix deletion db generation
Nov 7, 2023
dffd167
Change patch name to epoch range
Nov 7, 2023
94acb12
Update cmd/util-db/db/generate.go
matejmode Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/util-db/db/gen_deleted_accounts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,16 +134,16 @@ func genDeletedAccountsAction(ctx *cli.Context) error {
}
defer ddb.Close()

return GenDeletedAccountsAction(cfg, ddb, cfg.First)
return GenDeletedAccountsAction(cfg, ddb, cfg.First, cfg.Last)
}

// GenDeletedAccountsAction replays transactions and record self-destructed accounts and resurrected accounts.
func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountDB, firstBlock uint64) error {
func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountDB, firstBlock uint64, lastBlock uint64) error {
var err error

log := logger.NewLogger(cfg.LogLevel, "Generate Deleted Accounts")

log.Infof("chain-id: %v", cfg.ChainID)
log.Noticef("Generate deleted accounts from block %v to block %v", firstBlock, lastBlock)

start := time.Now()
sec := time.Since(start).Seconds()
Expand All @@ -157,7 +157,7 @@ func GenDeletedAccountsAction(cfg *utils.Config, ddb *substate.DestroyedAccountD

for iter.Next() {
tx := iter.Value()
if tx.Block > cfg.Last {
if tx.Block > lastBlock {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

most of the time the last block is excluded, is this what we need here?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cfg.Last is not set because the command doesn't intake range from arguments. Therefore lastBlock is supplied explicitly from g.opera.lastBLock

break
}

Expand Down
94 changes: 81 additions & 13 deletions cmd/util-db/db/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,20 @@ var GenerateCommand = cli.Command{
Flags: []cli.Flag{
&utils.AidaDbFlag,
&utils.ChainIDFlag,
&utils.GenesisFlag,
&utils.OperaDbFlag,
&utils.WorldStateFlag,
&utils.OperaBinaryFlag,
&utils.OutputFlag,
&utils.TargetEpochFlag,
&utils.UpdateBufferSizeFlag,
&utils.SkipStateHashScrappingFlag,
&substate.WorkersFlag,
&logger.LogLevelFlag,
},
Description: `
The db generate command requires events as an argument:
<events>

<events> are fed into the opera database (either existing or genesis needs to be specified), processing them generates updated aida-db.`,
The db generate command requires 4 arguments:
<firstBlock> <lastBlock> <firstEpoch> <lastEpoch>
This command is designed for manual generation of deletion, updateset and patch just from substates in aidadb.
`,
}

type generator struct {
Expand Down Expand Up @@ -82,6 +82,11 @@ func generate(ctx *cli.Context) error {
return err
}

err = g.prepareManualGenerate(ctx, cfg)
if err != nil {
return fmt.Errorf("prepareManualGenerate: %v; make sure you have correct substate range already recorded in aidadb", err)
}

if err = g.Generate(); err != nil {
return err
}
Expand All @@ -91,6 +96,64 @@ func generate(ctx *cli.Context) error {
return printMetadata(g.cfg.AidaDb)
}

// prepareManualGenerate prepares generator for manual generation
func (g *generator) prepareManualGenerate(ctx *cli.Context, cfg *utils.Config) (err error) {
if ctx.Args().Len() != 4 {
return fmt.Errorf("generate command requires exactly 4 arguments - first block, last block, first epoch, last epoch")
}

g.opera.firstBlock, g.opera.lastBlock, err = utils.SetBlockRange(ctx.Args().Get(0), ctx.Args().Get(1), cfg.ChainID)
if err != nil {
return err
}

g.opera.firstEpoch, g.opera.lastEpoch, err = utils.SetBlockRange(ctx.Args().Get(2), ctx.Args().Get(3), cfg.ChainID)
if err != nil {
return err
}

if g.opera.firstBlock > g.opera.lastBlock {
return fmt.Errorf("generation range first block %v cannot be greater than available last block %v", g.opera.firstBlock, g.opera.lastBlock)
}

if g.opera.firstEpoch > g.opera.lastEpoch {
return fmt.Errorf("generation range first epoch %v cannot be greater than available last epoch %v", g.opera.firstEpoch, g.opera.lastEpoch)
}

firstSubstate := substate.NewSubstateDB(g.aidaDb).GetFirstSubstate()
lastSubstate, err := substate.NewSubstateDB(g.aidaDb).GetLastSubstate()
if err != nil {
return fmt.Errorf("cannot get last substate; %v", err)
}

if firstSubstate.Env.Number > g.opera.firstBlock {
return fmt.Errorf("generation range first block %v cannot be greater than first substate block %v", g.opera.firstBlock, firstSubstate.Env.Number)
}

if lastSubstate.Env.Number < g.opera.lastBlock {
return fmt.Errorf("generation range last block %v cannot be greater than last substate block %v", g.opera.lastBlock, lastSubstate.Env.Number)
}

firstAvailableEpoch, err := utils.FindEpochNumber(firstSubstate.Env.Number, g.cfg.ChainID)
if err != nil {
return fmt.Errorf("cannot find first epoch number; %v", err)
}

lastAvailableEpoch, err := utils.FindEpochNumber(lastSubstate.Env.Number, g.cfg.ChainID)
if err != nil {
return fmt.Errorf("cannot find last epoch number; %v", err)
}

if g.opera.firstEpoch < firstAvailableEpoch {
return fmt.Errorf("generation range first epoch %v cannot be less than first available epoch %v", g.opera.firstEpoch, firstAvailableEpoch)
}

if g.opera.lastEpoch > lastAvailableEpoch {
return fmt.Errorf("generation range last epoch %v cannot be greater than last available epoch %v", g.opera.lastEpoch, lastAvailableEpoch)
}
return nil
}

// newGenerator returns new instance of generator
func newGenerator(ctx *cli.Context, cfg *utils.Config) (*generator, error) {
if cfg.AidaDb == "" {
Expand Down Expand Up @@ -135,9 +198,13 @@ func (g *generator) Generate() error {
return err
}

err = g.runStateHashScraper(g.ctx)
if err != nil {
return fmt.Errorf("cannot scrape state hashes; %v", err)
if g.cfg.SkipStateHashScrapping {
g.log.Noticef("Skipping state hash scraping...")
} else {
err = g.runStateHashScraper(g.ctx)
if err != nil {
return fmt.Errorf("cannot scrape state hashes; %v", err)
}
}

err = g.runDbHashGeneration(err)
Expand Down Expand Up @@ -271,7 +338,7 @@ func (g *generator) processDeletedAccounts(ddb *substate.DestroyedAccountDB) err
start = time.Now()
g.log.Noticef("Generating DeletionDb...")

err = GenDeletedAccountsAction(g.cfg, ddb, 0)
err = GenDeletedAccountsAction(g.cfg, ddb, 0, g.opera.lastBlock)
if err != nil {
return fmt.Errorf("cannot doGenerations deleted accounts; %v", err)
}
Expand Down Expand Up @@ -333,7 +400,7 @@ func (g *generator) createPatch() (string, error) {

// creating patch name
// add leading zeroes to filename to make it sortable
patchName := fmt.Sprintf("aida-db-%09s", strconv.FormatUint(g.opera.lastEpoch, 10))
patchName := fmt.Sprintf("%s-%s", strconv.FormatUint(g.opera.firstEpoch, 10), strconv.FormatUint(g.opera.lastEpoch, 10))
patchPath := filepath.Join(g.cfg.Output, patchName)

g.cfg.TargetDb = patchPath
Expand All @@ -346,8 +413,9 @@ func (g *generator) createPatch() (string, error) {
}

err = CreatePatchClone(g.cfg, g.aidaDb, patchDb, g.opera.firstEpoch, g.opera.lastEpoch, g.opera.isNew)

g.log.Notice("Patch metadata")
if err != nil {
return "", fmt.Errorf("cannot create patch clone; %v", err)
}

// metadata
err = utils.ProcessPatchLikeMetadata(patchDb, g.cfg.LogLevel, g.cfg.First, g.cfg.Last, g.opera.firstEpoch,
Expand Down
Loading