Merge pull request #23 from CortexFoundation/dev

Chain db recover when lost some blocks in fatal

block.go

@@ -7,6 +7,7 @@ import (
 
 type BlockContent struct {
 	x string
+	n uint64
 }
 
 func (t BlockContent) CalculateHash() ([]byte, error) {

chaindb.go

@@ -142,7 +142,7 @@ func (fs *ChainDB) initMerkleTree() error {
 	}
 	fs.tree = tr
 	for _, block := range fs.blocks {
-		if err := fs.addLeaf(block, true); err != nil {
+		if err := fs.addLeaf(block, false); err != nil {
 			panic("Storage merkletree construct failed")
 		}
 	}
@@ -157,17 +157,25 @@ func (fs *ChainDB) Metrics() time.Duration {
 }
 
 //Make sure the block group is increasing by number
-func (fs *ChainDB) addLeaf(block *types.Block, init bool) error {
+func (fs *ChainDB) addLeaf(block *types.Block, mess bool) error {
 	number := block.Number
-	leaf := BlockContent{x: block.Hash.String()}
+	leaf := BlockContent{x: block.Hash.String(), n: number}
 
 	if len(fs.leaves) >= params.LEAFS {
+		if mess {
+			//todo
+		}
 		fs.leaves = nil
 		fs.leaves = append(fs.leaves, BlockContent{x: hexutil.Encode(fs.tree.MerkleRoot())})
 		log.Debug("Next tree level", "leaf", len(fs.leaves), "root", hexutil.Encode(fs.tree.MerkleRoot()))
 	}
 
 	fs.leaves = append(fs.leaves, leaf)
+	if mess {
+		sort.Slice(fs.leaves, func(i, j int) bool {
+			return fs.leaves[i].(BlockContent).n < fs.leaves[j].(BlockContent).n
+		})
+	}
 
 	if err := fs.tree.RebuildTreeWith(fs.leaves); err == nil {
 		if err := fs.writeRoot(number, fs.tree.MerkleRoot()); err != nil {
@@ -339,60 +347,81 @@ func (fs *ChainDB) progress(f *types.FileInfo, init bool) (bool, error) {
 	return update, err
 }
 
+func (fs *ChainDB) find(b *types.Block) (bool, error) {
+	i := sort.Search(len(fs.blocks), func(i int) bool { return fs.blocks[i].Number >= b.Number })
+	if i < len(fs.blocks) && fs.blocks[i].Number == b.Number {
+		return true, nil
+	} else {
+		return false, nil
+	}
+}
+
 //func (fs *ChainDB) addBlock(b *Block, record bool) error {
 func (fs *ChainDB) AddBlock(b *types.Block) error {
-	if b.Number < fs.LastListenBlockNumber {
-		return nil
-	}
+	//if b.Number < fs.LastListenBlockNumber {
+	//	return nil
+	//}
 
 	if fs.metrics {
 		defer func(start time.Time) { fs.treeUpdates += time.Since(start) }(time.Now())
 	}
-	if b.Number > fs.CheckPoint {
+	//u := false
+	//if b.Number > fs.CheckPoint {
+	//	u = true
+	//} else {
+	//	if exist, _ := fs.find(b); !exist {
+	//		log.Warn("Find a missing ancient block", "number", b.Number)
+	//		u = true
+	//	} else {
+	//		return nil
+	//	}
+	//}
+
+	//if u {
+	if fs.GetBlockByNumber(b.Number) != nil {
+		return nil
+	}
 
-		if err := fs.db.Update(func(tx *bolt.Tx) error {
-			buk, err := tx.CreateBucketIfNotExists([]byte("blocks_" + fs.version))
-			if err != nil {
-				return err
-			}
-			v, err := json.Marshal(b)
-			if err != nil {
-				return err
-			}
-			k, err := json.Marshal(b.Number)
-			if err != nil {
-				return err
-			}
+	if err := fs.db.Update(func(tx *bolt.Tx) error {
+		buk, err := tx.CreateBucketIfNotExists([]byte("blocks_" + fs.version))
+		if err != nil {
+			return err
+		}
+		v, err := json.Marshal(b)
+		if err != nil {
+			return err
+		}
+		k, err := json.Marshal(b.Number)
+		if err != nil {
+			return err
+		}
 
-			return buk.Put(k, v)
-		}); err == nil {
-			if err := fs.appendBlock(b); err == nil {
-				fs.txs += uint64(len(b.Txs))
-				if err := fs.addLeaf(b, false); err == nil {
-					if err := fs.writeCheckPoint(); err == nil {
-						fs.CheckPoint = b.Number
-					}
+		return buk.Put(k, v)
+	}); err == nil {
+		//if err := fs.appendBlock(b); err == nil {
+		fs.blocks = append(fs.blocks, b)
+		fs.txs += uint64(len(b.Txs))
+		mes := false
+		if b.Number <= fs.CheckPoint {
+			mes = true
+		}
+		if err := fs.addLeaf(b, mes); err == nil {
+			if err := fs.writeCheckPoint(); err == nil {
+				if !mes {
+					fs.CheckPoint = b.Number
 				}
 			}
-		} else {
-			return err
 		}
+		//}
+	} else {
+		return err
 	}
+	//}
 
 	fs.LastListenBlockNumber = b.Number
 	return fs.Flush()
 }
 
-func (fs *ChainDB) appendBlock(b *types.Block) error {
-	if len(fs.blocks) == 0 || fs.blocks[len(fs.blocks)-1].Number < b.Number {
-		log.Debug("Append block", "number", b.Number)
-		fs.blocks = append(fs.blocks, b)
-	} else {
-		return errors.New("err block duplicated")
-	}
-	return nil
-}
-
 func (fs *ChainDB) Version() string {
 	return fs.version
 }

sync.go

@@ -125,13 +125,15 @@ func (m *Monitor) IndexCheck() error {
 
 		version := m.fs.GetRootByNumber(checkpoint.TfsCheckPoint)
 		if common.BytesToHash(version) != checkpoint.TfsRoot {
-			log.Warn("Fs storage is reloading ...", "name", m.ckp.Name, "number", checkpoint.TfsCheckPoint, "version", common.BytesToHash(version), "checkpoint", checkpoint.TfsRoot, "blocks", len(m.fs.Blocks()), "files", len(m.fs.Files()), "txs", m.fs.Txs())
+			m.lastNumber = 0
 			if m.lastNumber > checkpoint.TfsCheckPoint {
-				m.lastNumber = 0
-				if err := m.fs.Reset(); err != nil {
-					return err
-				}
+				m.fs.LastListenBlockNumber = 0
+				//m.lastNumber = 0
+				//if err := m.fs.Reset(); err != nil {
+				//	return err
+				//}
 			}
+			log.Warn("Fs storage is reloading ...", "name", m.ckp.Name, "number", checkpoint.TfsCheckPoint, "version", common.BytesToHash(version), "checkpoint", checkpoint.TfsRoot, "blocks", len(m.fs.Blocks()), "files", len(m.fs.Files()), "txs", m.fs.Txs(), "lastNumber", m.lastNumber, "last in db", m.fs.LastListenBlockNumber)
 		} else {
 			log.Info("Fs storage version check passed", "name", m.ckp.Name, "number", checkpoint.TfsCheckPoint, "version", common.BytesToHash(version), "blocks", len(m.fs.Blocks()), "files", len(m.fs.Files()), "txs", m.fs.Txs())
 		}
@@ -338,7 +340,6 @@ func (m *Monitor) parseBlockTorrentInfo(b *types.Block) (bool, error) {
 				record = true
 			} else if tx.IsFlowControl() {
 				if tx.Recipient == nil {
-					log.Trace("Recipient is nil", "num", b.Number)
 					continue
 				}
 				addr := *tx.Recipient
@@ -351,7 +352,7 @@ func (m *Monitor) parseBlockTorrentInfo(b *types.Block) (bool, error) {
 				if err != nil {
 					return false, err
 				}
-
+				//todo
 				if receipt.Status != 1 {
 					continue
 				}

types/common.go

@@ -130,7 +130,7 @@ type Block struct {
 	Number uint64      `json:"number"           gencodec:"required"`
 	Hash   common.Hash `json:"Hash"             gencodec:"required"`
 	//ParentHash common.Hash   `json:"parentHash"       gencodec:"required"`
-	Txs []Transaction `json:"Transactions"     gencodec:"required"`
+	Txs []Transaction `json:"transactions"     gencodec:"required"`
 }
 
 type blockMarshaling struct {
@@ -140,9 +140,9 @@ type blockMarshaling struct {
 //go:generate gencodec -type Receipt -field-override receiptMarshaling -out gen_receipt_json.go
 type Receipt struct {
 	// Contract Address
-	ContractAddr *common.Address `json:"ContractAddress"`
+	ContractAddr *common.Address `json:"contractAddress"`
 	// Transaction Hash
-	TxHash *common.Hash `json:"TransactionHash"  gencodec:"required"`
+	TxHash *common.Hash `json:"transactionHash"  gencodec:"required"`
 	//Receipt   *TxReceipt      `json:"receipt"  rlp:"nil"`
 	GasUsed uint64 `json:"gasUsed" gencodec:"required"`
 	Status  uint64 `json:"status"`
@@ -155,10 +155,10 @@ type receiptMarshaling struct {
 
 //go:generate gencodec -type FileMeta -out gen_filemeta_json.go
 type FileMeta struct {
-	InfoHash metainfo.Hash `json:"InfoHash"         gencodec:"required"`
+	InfoHash metainfo.Hash `json:"infoHash"         gencodec:"required"`
 	//	Name     string        `json:"Name"             gencodec:"required"`
 	// The raw size of the file counted in bytes
-	RawSize uint64 `json:"RawSize"          gencodec:"required"`
+	RawSize uint64 `json:"rawSize"          gencodec:"required"`
 	//BlockNum uint64 `json:"BlockNum"         gencodec:"required"`
 }