Merge pull request #1293 from silasdavis/metadata-state

Split out metadata cache and interfaces

CHANGELOG.md

@@ -1,4 +1,9 @@
 # [Hyperledger Burrow](https://github.com/hyperledger/burrow) Changelog
+## [0.29.1] - 2019-10-10
+### Changed
+- [State] Split metadata and account state to be kinder to downstream EVM integrators
+
+
 ## [0.29.0] - 2019-10-08
 ### Changed
 - [Config] Reverted rename of ValidatorAddress to Address in config (each Burrow node has a specific validator key it uses for signing whether or not it is running as a validator right now)
@@ -578,6 +583,7 @@ This release marks the start of Eris-DB as the full permissioned blockchain node
   - [Blockchain] Fix getBlocks to respect block height cap.
 
 
+[0.29.1]: https://github.com/hyperledger/burrow/compare/v0.29.0...v0.29.1
 [0.29.0]: https://github.com/hyperledger/burrow/compare/v0.28.2...v0.29.0
 [0.28.2]: https://github.com/hyperledger/burrow/compare/v0.28.1...v0.28.2
 [0.28.1]: https://github.com/hyperledger/burrow/compare/v0.28.0...v0.28.1

NOTES.md

@@ -1,24 +1,3 @@
 ### Changed
-- [Config] Reverted rename of ValidatorAddress to Address in config (each Burrow node has a specific validator key it uses for signing whether or not it is running as a validator right now)
-
-### Fixed 
-- [EVM] Return integer overflow error code (not stack overflow) for integer overflow errors
-- [Docs] Fix broken examples
-- [Deploy] Set input on QueryContract jobs correctly
-- [EVM] Fix byte-printing for DebugOpcodes run mode
-- [Crypto] Use Tendermint-compatible secp256k1 addressing
-- [Natives] Make natives first class contracts and establish Dispatcher and Callable as a common calling convention for natives, EVM, and WASM (pending for WASM).
-- [Natives] Fix Ethereum precompile addresses (addresses were padded on right instead of the left)
-
-
-### Added
-- [Web3] Implemented Ethereum web3 JSON RPC including sendRawTransaction!
-- [Docs] Much docs (see also: https://www.hyperledger.org/blog/2019/10/08/burrow-the-boring-blockchain)
-- [Docs] Generate github pages docs index with docsify: https://hyperledger.github.io/burrow/
-- [JS] Publish burrow.js to @hyperledger/burrow
-- [State] Store EVM ABI and contract metadata on-chain see [GetMetadata](https://github.com/hyperledger/burrow/blob/e80aad5d8fac1f67dbfec61ea75670f9a38c61a1/protobuf/rpcquery.proto#L25)
-- [Tendermint] Upgrade to v0.32.3
-- [Execution] Added IdentifyTx for introducing nodes (binding their NodeID to ValidatorAddress)
-- [Natives] Implement Ethereum precompile number 5 - modular exponentiation
-
+- [State] Split metadata and account state to be kinder to downstream EVM integrators
 

acm/acmstate/metadata_cache.go

@@ -0,0 +1,82 @@
+package acmstate
+
+import (
+	"sync"
+)
+
+type metadataInfo struct {
+	metadata string
+	updated  bool
+}
+
+type MetadataCache struct {
+	backend MetadataReader
+	m       sync.Map
+}
+
+func NewMetadataCache(backend MetadataReader) *MetadataCache {
+	return &MetadataCache{
+		backend: backend,
+	}
+}
+
+func (cache *MetadataCache) SetMetadata(metahash MetadataHash, metadata string) error {
+	cache.m.Store(metahash, &metadataInfo{updated: true, metadata: metadata})
+	return nil
+}
+
+func (cache *MetadataCache) GetMetadata(metahash MetadataHash) (string, error) {
+	metaInfo, err := cache.getMetadata(metahash)
+	if err != nil {
+		return "", err
+	}
+
+	return metaInfo.metadata, nil
+}
+
+// Syncs changes to the backend in deterministic order. Sends storage updates before updating
+// the account they belong so that storage values can be taken account of in the update.
+func (cache *MetadataCache) Sync(st MetadataWriter) error {
+	var err error
+	cache.m.Range(func(key, value interface{}) bool {
+		hash := key.(MetadataHash)
+		info := value.(*metadataInfo)
+		if info.updated {
+			err = st.SetMetadata(hash, info.metadata)
+			if err != nil {
+				return false
+			}
+		}
+		return true
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cache *MetadataCache) Flush(output MetadataWriter, backend MetadataReader) error {
+	err := cache.Sync(output)
+	if err != nil {
+		return err
+	}
+	cache.m = sync.Map{}
+	return nil
+}
+
+// Get the cache accountInfo item creating it if necessary
+func (cache *MetadataCache) getMetadata(metahash MetadataHash) (*metadataInfo, error) {
+	value, ok := cache.m.Load(metahash)
+	if !ok {
+		metadata, err := cache.backend.GetMetadata(metahash)
+		if err != nil {
+			return nil, err
+		}
+		metaInfo := &metadataInfo{
+			metadata: metadata,
+		}
+		cache.m.Store(metahash, metaInfo)
+		return metaInfo, nil
+	}
+	return value.(*metadataInfo), nil
+}

acm/acmstate/state.go

@@ -110,12 +110,12 @@ type StorageIterable interface {
 	IterateStorage(address crypto.Address, consumer func(key binary.Word256, value []byte) error) (err error)
 }
 
-type MetadataGetter interface {
+type MetadataReader interface {
 	// Get an Metadata by its hash. This is content-addressed
 	GetMetadata(metahash MetadataHash) (string, error)
 }
 
-type MetadataSetter interface {
+type MetadataWriter interface {
 	// Set an Metadata according to it keccak-256 hash.
 	SetMetadata(metahash MetadataHash, metadata string) error
 }
@@ -135,7 +135,6 @@ type AccountStatsGetter interface {
 type Reader interface {
 	AccountGetter
 	StorageGetter
-	MetadataGetter
 }
 
 type Iterable interface {
@@ -158,7 +157,6 @@ type IterableStatsReader interface {
 type Writer interface {
 	AccountUpdater
 	StorageSetter
-	MetadataSetter
 }
 
 // Read and write account and storage state
@@ -167,6 +165,11 @@ type ReaderWriter interface {
 	Writer
 }
 
+type MetadataReaderWriter interface {
+	MetadataReader
+	MetadataWriter
+}
+
 type IterableReaderWriter interface {
 	Iterable
 	Reader

acm/acmstate/state_cache.go

@@ -19,7 +19,6 @@ type Cache struct {
 	name     string
 	backend  Reader
 	accounts map[crypto.Address]*accountInfo
-	metadata map[MetadataHash]*metadataInfo
 	readonly bool
 }
 
@@ -31,11 +30,6 @@ type accountInfo struct {
 	updated bool
 }
 
-type metadataInfo struct {
-	metadata string
-	updated  bool
-}
-
 type CacheOption func(*Cache) *Cache
 
 // Returns a Cache that wraps an underlying Reader to use on a cache miss, can write to an output Writer
@@ -44,7 +38,6 @@ func NewCache(backend Reader, options ...CacheOption) *Cache {
 	cache := &Cache{
 		backend:  backend,
 		accounts: make(map[crypto.Address]*accountInfo),
-		metadata: make(map[MetadataHash]*metadataInfo),
 	}
 	for _, option := range options {
 		option(cache)
@@ -99,28 +92,6 @@ func (cache *Cache) UpdateAccount(account *acm.Account) error {
 	return nil
 }
 
-func (cache *Cache) GetMetadata(metahash MetadataHash) (string, error) {
-	metaInfo, err := cache.getMetadata(metahash)
-	if err != nil {
-		return "", err
-	}
-
-	return metaInfo.metadata, nil
-}
-
-func (cache *Cache) SetMetadata(metahash MetadataHash, metadata string) error {
-	if cache.readonly {
-		return errors.Errorf(errors.Codes.IllegalWrite, "SetMetadata called in read-only context on metadata hash: %v", metahash)
-	}
-
-	cache.Lock()
-	defer cache.Unlock()
-
-	cache.metadata[metahash] = &metadataInfo{updated: true, metadata: metadata}
-
-	return nil
-}
-
 func (cache *Cache) RemoveAccount(address crypto.Address) error {
 	if cache.readonly {
 		return errors.Errorf(errors.Codes.IllegalWrite, "RemoveAccount called on read-only account %v", address)
@@ -268,14 +239,6 @@ func (cache *Cache) Sync(st Writer) error {
 		accInfo.RUnlock()
 	}
 
-	for metahash, metadataInfo := range cache.metadata {
-		if metadataInfo.updated {
-			err := st.SetMetadata(metahash, metadataInfo.metadata)
-			if err != nil {
-				return err
-			}
-		}
-	}
 	return nil
 }
 
@@ -327,26 +290,3 @@ func (cache *Cache) get(address crypto.Address) (*accountInfo, error) {
 	}
 	return accInfo, nil
 }
-
-// Get the cache accountInfo item creating it if necessary
-func (cache *Cache) getMetadata(metahash MetadataHash) (*metadataInfo, error) {
-	cache.RLock()
-	metaInfo := cache.metadata[metahash]
-	cache.RUnlock()
-	if metaInfo == nil {
-		cache.Lock()
-		defer cache.Unlock()
-		metaInfo = cache.metadata[metahash]
-		if metaInfo == nil {
-			metadata, err := cache.backend.GetMetadata(metahash)
-			if err != nil {
-				return nil, err
-			}
-			metaInfo = &metadataInfo{
-				metadata: metadata,
-			}
-			cache.metadata[metahash] = metaInfo
-		}
-	}
-	return metaInfo, nil
-}

core/processes.go

@@ -309,12 +309,8 @@ func GRPCLauncher(kern *Kernel, conf *rpc.ServerConfig, keyConfig *keys.KeysConf
 				}
 				keys.RegisterKeysServer(grpcServer, ks)
 			}
-
-			nameRegState := kern.State
-			nodeRegState := kern.State
-			proposalRegState := kern.State
-			rpcquery.RegisterQueryServer(grpcServer, rpcquery.NewQueryServer(kern.State, nameRegState, nodeRegState, proposalRegState,
-				kern.Blockchain, kern.State, nodeView, kern.Logger))
+			rpcquery.RegisterQueryServer(grpcServer, rpcquery.NewQueryServer(kern.State, kern.Blockchain, nodeView,
+				kern.Logger))
 
 			txCodec := txs.NewProtobufCodec()
 			rpctransact.RegisterTransactServer(grpcServer,

execution/contexts/call_context.go

@@ -21,13 +21,14 @@ import (
 const GasLimit = uint64(1000000)
 
 type CallContext struct {
-	EVM        *evm.EVM
-	State      acmstate.ReaderWriter
-	Blockchain engine.Blockchain
-	RunCall    bool
-	Logger     *logging.Logger
-	tx         *payload.CallTx
-	txe        *exec.TxExecution
+	EVM           *evm.EVM
+	State         acmstate.ReaderWriter
+	MetadataState acmstate.MetadataReaderWriter
+	Blockchain    engine.Blockchain
+	RunCall       bool
+	Logger        *logging.Logger
+	tx            *payload.CallTx
+	txe           *exec.TxExecution
 }
 
 func (ctx *CallContext) Execute(txe *exec.TxExecution, p payload.Payload) error {
@@ -121,6 +122,7 @@ func (ctx *CallContext) Deliver(inAcc, outAcc *acm.Account, value uint64) error
 	createContract := ctx.tx.Address == nil
 	caller := inAcc.Address
 	txCache := acmstate.NewCache(ctx.State, acmstate.Named("TxCache"))
+	metaCache := acmstate.NewMetadataCache(ctx.MetadataState)
 
 	var callee crypto.Address
 	var code []byte
@@ -141,7 +143,7 @@ func (ctx *CallContext) Deliver(inAcc, outAcc *acm.Account, value uint64) error
 			"init_code", code)
 
 		// store abis
-		err = native.UpdateContractMeta(txCache, callee, ctx.tx.ContractMeta)
+		err = native.UpdateContractMeta(txCache, metaCache, callee, ctx.tx.ContractMeta)
 		if err != nil {
 			return err
 		}
@@ -198,7 +200,7 @@ func (ctx *CallContext) Deliver(inAcc, outAcc *acm.Account, value uint64) error
 			ctx.txe.PushError(errors.Wrap(err, "call error"))
 		} else {
 			ctx.Logger.TraceMsg("Successful execution")
-			err := txCache.Sync(ctx.State)
+			err = ctx.Sync(txCache, metaCache)
 			if err != nil {
 				return err
 			}
@@ -234,7 +236,7 @@ func (ctx *CallContext) Deliver(inAcc, outAcc *acm.Account, value uint64) error
 					return err
 				}
 			}
-			err := txCache.Sync(ctx.State)
+			err = ctx.Sync(txCache, metaCache)
 			if err != nil {
 				return err
 			}
@@ -259,3 +261,11 @@ func (ctx *CallContext) CallEvents(err error) {
 		ctx.txe.Input(*ctx.tx.Address, errors.AsException(err))
 	}
 }
+
+func (ctx *CallContext) Sync(cache *acmstate.Cache, metaCache *acmstate.MetadataCache) error {
+	err := cache.Sync(ctx.State)
+	if err != nil {
+		return err
+	}
+	return metaCache.Sync(ctx.MetadataState)
+}