Merge pull request #816 from gzliudan/reduce_chaindb_size

all: reduce the chain db size

core/blockchain.go

@@ -100,7 +100,23 @@ const (
 	triesInMemory       = 128
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
-	BlockChainVersion = 3
+	//
+	// Changelog:
+	//
+	// - Version 4
+	//   The following incompatible database changes were added:
+	//   * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted
+	//   * the `Bloom` field of receipt is deleted
+	//   * the `BlockIndex` and `TxIndex` fields of txlookup are deleted
+	// - Version 5
+	//  The following incompatible database changes were added:
+	//    * the `TxHash`, `GasCost`, and `ContractAddress` fields are no longer stored for a receipt
+	//    * the `TxHash`, `GasCost`, and `ContractAddress` fields are computed by looking up the
+	//      receipts' corresponding block
+	// - Version 6
+	//  The following incompatible database changes were added:
+	//    * Transaction lookup information stores the corresponding block number instead of block hash
+	BlockChainVersion uint64 = 6
 
 	// Maximum length of chain to cache by block's number
 	blocksHashCacheLimit = 900
@@ -1220,7 +1236,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		}
 		// Compute all the non-consensus fields of the receipts
 		if err := receipts.DeriveFields(bc.chainConfig, blockHash, blockNumber, block.BaseFee(), block.Transactions()); err != nil {
-			return i, fmt.Errorf("failed to set receipts data: %v", err)
+			return i, fmt.Errorf("failed to derive receipts data: %v", err)
 		}
 		// Write all the data out into the database
 		rawdb.WriteBody(batch, blockHash, blockNumber, block.Body())

core/rawdb/accessors_chain.go

@@ -476,18 +476,6 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 	}
 }
 
-// storedReceiptRLP is the storage encoding of a receipt.
-// Re-definition in core/types/receipt.go.
-type storedReceiptRLP struct {
-	PostStateOrStatus []byte
-	CumulativeGasUsed uint64
-	Bloom             types.Bloom
-	TxHash            common.Hash
-	ContractAddress   common.Address
-	Logs              []*types.LogForStorage
-	GasUsed           uint64
-}
-
 // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps
 // the list of logs. When decoding a stored receipt into this object we
 // avoid creating the bloom filter.
@@ -497,7 +485,7 @@ type receiptLogs struct {
 
 // DecodeRLP implements rlp.Decoder.
 func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error {
-	var stored storedReceiptRLP
+	var stored types.ReceiptForStorage
 	if err := s.Decode(&stored); err != nil {
 		return err
 	}

core/rawdb/accessors_chain_test.go

@@ -19,6 +19,7 @@ package rawdb
 import (
 	"bytes"
 	"encoding/hex"
+	"fmt"
 	"math/big"
 	"os"
 	"testing"
@@ -339,22 +340,48 @@ func TestBlockReceiptStorage(t *testing.T) {
 	if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); len(rs) == 0 {
 		t.Fatalf("no receipts returned")
 	} else {
-		for i := 0; i < len(receipts); i++ {
-			rlpHave, _ := rlp.EncodeToBytes(rs[i])
-			rlpWant, _ := rlp.EncodeToBytes(receipts[i])
-
-			if !bytes.Equal(rlpHave, rlpWant) {
-				t.Fatalf("receipt #%d: receipt mismatch: have %v, want %v", i, rs[i], receipts[i])
-			}
+		if err := checkReceiptsRLP(rs, receipts); err != nil {
+			t.Fatalf("fail to checkReceiptsRLP %v", err)
 		}
 	}
-	// Delete the receipt slice and check purge
+	// Delete the body and ensure that the receipts are no longer returned (metadata can't be recomputed)
+	DeleteBody(db, hash, 0)
+	if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); rs != nil {
+		t.Fatalf("receipts returned when body was deleted: %v", rs)
+	}
+	// Ensure that receipts without metadata can be returned without the block body too
+	if err := checkReceiptsRLP(ReadRawReceipts(db, hash, 0), receipts); err != nil {
+		t.Fatalf("fail to checkReceiptsRLP %v", err)
+	}
+	// Sanity check that body alone without the receipt is a full purge
+	WriteBody(db, hash, 0, body)
+
 	DeleteReceipts(db, hash, 0)
 	if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 {
 		t.Fatalf("deleted receipts returned: %v", rs)
 	}
 }
 
+func checkReceiptsRLP(have, want types.Receipts) error {
+	if len(have) != len(want) {
+		return fmt.Errorf("receipts sizes mismatch: have %d, want %d", len(have), len(want))
+	}
+	for i := 0; i < len(want); i++ {
+		rlpHave, err := rlp.EncodeToBytes(have[i])
+		if err != nil {
+			return err
+		}
+		rlpWant, err := rlp.EncodeToBytes(want[i])
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(rlpHave, rlpWant) {
+			return fmt.Errorf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant))
+		}
+	}
+	return nil
+}
+
 // Tests that logs associated with a single block can be retrieved.
 func TestReadLogs(t *testing.T) {
 	db := NewMemoryDatabase()

core/rawdb/accessors_indexes.go

@@ -17,6 +17,8 @@
 package rawdb
 
 import (
+	"math/big"
+
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/core/types"
 	"github.com/XinFinOrg/XDPoSChain/ethdb"
@@ -25,45 +27,37 @@ import (
 	"github.com/XinFinOrg/XDPoSChain/rlp"
 )
 
-type TxLookupEntry struct {
-	BlockHash  common.Hash
-	BlockIndex uint64
-	Index      uint64
-}
-
 // ReadTxLookupEntry retrieves the positional metadata associated with a transaction
 // hash to allow retrieving the transaction or receipt by hash.
-func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (common.Hash, uint64, uint64) {
-	// Load the positional metadata from disk and bail if it fails
+func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
 	data, _ := db.Get(txLookupKey(hash))
 	if len(data) == 0 {
-		return common.Hash{}, 0, 0
+		return nil
+	}
+	// Database v6 tx lookup just stores the block number
+	if len(data) < common.HashLength {
+		number := new(big.Int).SetBytes(data).Uint64()
+		return &number
 	}
-	// Parse and return the contents of the lookup entry
-	var entry TxLookupEntry
+	// Database v4-v5 tx lookup format just stores the hash
+	if len(data) == common.HashLength {
+		return ReadHeaderNumber(db, common.BytesToHash(data))
+	}
+	// Finally try database v3 tx lookup format
+	var entry LegacyTxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
-		log.Error("Invalid lookup entry RLP", "hash", hash, "err", err)
-		return common.Hash{}, 0, 0
+		log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
+		return nil
 	}
-	return entry.BlockHash, entry.BlockIndex, entry.Index
+	return &entry.BlockIndex
 }
 
 // WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from
 // a block, enabling hash based transaction and receipt lookups.
 func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) {
-	// Iterate over each transaction and encode its metadata
-	for i, tx := range block.Transactions() {
-		entry := TxLookupEntry{
-			BlockHash:  block.Hash(),
-			BlockIndex: block.NumberU64(),
-			Index:      uint64(i),
-		}
-		data, err := rlp.EncodeToBytes(entry)
-		if err != nil {
-			log.Crit("Failed to RLP encode TxLookupEntry", "err", err)
-		}
-		if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
-			log.Crit("Failed to store tx lookup entry", "err", err)
+	for _, tx := range block.Transactions() {
+		if err := db.Put(txLookupKey(tx.Hash()), block.Number().Bytes()); err != nil {
+			log.Crit("Failed to store transaction lookup entry", "err", err)
 		}
 	}
 }
@@ -76,63 +70,82 @@ func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) {
 // ReadTransaction retrieves a specific transaction from the database, along with
 // its added positional metadata.
 func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
-	// Retrieve the lookup metadata and resolve the transaction from the body
-	blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
-
-	if blockHash != (common.Hash{}) {
-		body := ReadBody(db, blockHash, blockNumber)
-		if body == nil || len(body.Transactions) <= int(txIndex) {
-			log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex)
+	blockNumber := ReadTxLookupEntry(db, hash)
+	if blockNumber == nil {
+		// return nil, common.Hash{}, 0, 0
+		// TODO(daniel): delete the following old codes
+		// Old transaction representation, load the transaction and it's metadata separately
+		data, _ := db.Get(hash.Bytes())
+		if len(data) == 0 {
 			return nil, common.Hash{}, 0, 0
 		}
-		return body.Transactions[txIndex], blockHash, blockNumber, txIndex
-	}
-	// Old transaction representation, load the transaction and it's metadata separately
-	data, _ := db.Get(hash.Bytes())
-	if len(data) == 0 {
-		return nil, common.Hash{}, 0, 0
+		var tx types.Transaction
+		if err := rlp.DecodeBytes(data, &tx); err != nil {
+			return nil, common.Hash{}, 0, 0
+		}
+		// Retrieve the blockchain positional metadata
+		data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...))
+		if len(data) == 0 {
+			return nil, common.Hash{}, 0, 0
+		}
+		var entry LegacyTxLookupEntry
+		if err := rlp.DecodeBytes(data, &entry); err != nil {
+			return nil, common.Hash{}, 0, 0
+		}
+		return &tx, entry.BlockHash, entry.BlockIndex, entry.Index
 	}
-	var tx types.Transaction
-	if err := rlp.DecodeBytes(data, &tx); err != nil {
+	blockHash := ReadCanonicalHash(db, *blockNumber)
+	if blockHash == (common.Hash{}) {
 		return nil, common.Hash{}, 0, 0
 	}
-	// Retrieve the blockchain positional metadata
-	data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...))
-	if len(data) == 0 {
+	body := ReadBody(db, blockHash, *blockNumber)
+	if body == nil {
+		log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
 		return nil, common.Hash{}, 0, 0
 	}
-	var entry TxLookupEntry
-	if err := rlp.DecodeBytes(data, &entry); err != nil {
-		return nil, common.Hash{}, 0, 0
+	for txIndex, tx := range body.Transactions {
+		if tx.Hash() == hash {
+			return tx, blockHash, *blockNumber, uint64(txIndex)
+		}
 	}
-	return &tx, entry.BlockHash, entry.BlockIndex, entry.Index
+	log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
+	return nil, common.Hash{}, 0, 0
 }
 
 // ReadReceipt retrieves a specific transaction receipt from the database, along with
 // its added positional metadata.
 func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) {
-	// Retrieve the lookup metadata and resolve the receipt from the receipts
-	blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash)
-
-	if blockHash != (common.Hash{}) {
-		receipts := ReadReceipts(db, blockHash, blockNumber, config)
-		if len(receipts) <= int(receiptIndex) {
-			log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex)
+	// Retrieve the context of the receipt based on the transaction hash
+	blockNumber := ReadTxLookupEntry(db, hash)
+	if blockNumber == nil {
+		// return nil, common.Hash{}, 0, 0
+		// TODO(daniel): delete the following old codes
+		// Old receipt representation, load the receipt and set an unknown metadata
+		data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...))
+		if len(data) == 0 {
 			return nil, common.Hash{}, 0, 0
 		}
-		return receipts[receiptIndex], blockHash, blockNumber, receiptIndex
+		var receipt types.ReceiptForStorage
+		err := rlp.DecodeBytes(data, &receipt)
+		if err != nil {
+			log.Error("Invalid receipt RLP", "hash", hash, "err", err)
+			return nil, common.Hash{}, 0, 0
+		}
+		return (*types.Receipt)(&receipt), common.Hash{}, 0, 0
 	}
-	// Old receipt representation, load the receipt and set an unknown metadata
-	data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...))
-	if len(data) == 0 {
+	blockHash := ReadCanonicalHash(db, *blockNumber)
+	if blockHash == (common.Hash{}) {
 		return nil, common.Hash{}, 0, 0
 	}
-	var receipt types.ReceiptForStorage
-	err := rlp.DecodeBytes(data, &receipt)
-	if err != nil {
-		log.Error("Invalid receipt RLP", "hash", hash, "err", err)
+	// Read all the receipts from the block and return the one with the matching hash
+	receipts := ReadReceipts(db, blockHash, *blockNumber, config)
+	for receiptIndex, receipt := range receipts {
+		if receipt.TxHash == hash {
+			return receipt, blockHash, *blockNumber, uint64(receiptIndex)
+		}
 	}
-	return (*types.Receipt)(&receipt), common.Hash{}, 0, 0
+	log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
+	return nil, common.Hash{}, 0, 0
 }
 
 // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given