Merge pull request #1223 from hyperledger/develop

v0.28.2

CHANGELOG.md

@@ -1,4 +1,9 @@
 # [Hyperledger Burrow](https://github.com/hyperledger/burrow) Changelog
+## [0.28.2] - 2019-08-21
+### Fixed
+- [Vent] The new decode event ABI _before_ filter provides more keys but means vent must have access to all possible LogEvent ABIs when it is started. This is not practical in general so we now will will only err if an event matches but we have no ABI. This means we might not notice we have forgot to include an ABI since an event that _would_ have matched on an ABI spec field (prefixed 'Event') will not just not match, and so fail silently. 
+
+
 ## [0.28.1] - 2019-08-21
 ### Fixed
 - [Vent] Log for _vent_log insert now faithfully captures what is being inserted
@@ -547,6 +552,7 @@ This release marks the start of Eris-DB as the full permissioned blockchain node
   - [Blockchain] Fix getBlocks to respect block height cap.
 
 
+[0.28.2]: https://github.com/hyperledger/burrow/compare/v0.28.1...v0.28.2
 [0.28.1]: https://github.com/hyperledger/burrow/compare/v0.28.0...v0.28.1
 [0.28.0]: https://github.com/hyperledger/burrow/compare/v0.27.0...v0.28.0
 [0.27.0]: https://github.com/hyperledger/burrow/compare/v0.26.2...v0.27.0

NOTES.md

@@ -1,7 +1,3 @@
 ### Fixed
-- [Vent] Log for _vent_log insert now faithfully captures what is being inserted
-- [Vent] Remove arbitrary 100 character limits on system table text fields
-
-### Added
-- [JS] Burrow.js now included in Burrow repo and tested with Burrow CI! Future burrow.js releases will now match version of Burrow.
+- [Vent] The new decode event ABI _before_ filter provides more keys but means vent must have access to all possible LogEvent ABIs when it is started. This is not practical in general so we now will will only err if an event matches but we have no ABI. This means we might not notice we have forgot to include an ABI since an event that _would_ have matched on an ABI spec field (prefixed 'Event') will not just not match, and so fail silently. 
 

event/query/builder.go

@@ -47,16 +47,16 @@ func (pq parsedQuery) Query() (Query, error) {
 	return pq.query, nil
 }
 
-// A yet-to-parsed query
-type String string
-
 func Must(qry Query, err error) Query {
 	if err != nil {
 		panic(fmt.Errorf("could not compile: %v", qry))
 	}
 	return qry
 }
 
+// A yet-to-be-parsed query
+type String string
+
 func (qs String) Query() (Query, error) {
 	if isEmpty(string(qs)) {
 		return Empty{}, nil

event/query/query.go

@@ -51,9 +51,9 @@ func New(s string) (*PegQuery, error) {
 	p := &QueryParser{
 		Buffer: s,
 	}
-	//p.Expression.explainer = func(format string, args ...interface{}) {
-	//	fmt.Printf(format, args...)
-	//}
+	p.Expression.explainer = func(format string, args ...interface{}) {
+		fmt.Printf(format, args...)
+	}
 	err := p.Init()
 	if err != nil {
 		return nil, err

project/history.go

@@ -48,6 +48,10 @@ func FullVersion() string {
 // release tagging script: ./scripts/tag_release.sh
 var History relic.ImmutableHistory = relic.NewHistory("Hyperledger Burrow", "https://github.com/hyperledger/burrow").
 	MustDeclareReleases(
+		"0.28.2 - 2019-08-21",
+		`### Fixed
+- [Vent] The new decode event ABI _before_ filter provides more keys but means vent must have access to all possible LogEvent ABIs when it is started. This is not practical in general so we now will will only err if an event matches but we have no ABI. This means we might not notice we have forgot to include an ABI since an event that _would_ have matched on an ABI spec field (prefixed 'Event') will not just not match, and so fail silently. 
+`,
 		"0.28.1 - 2019-08-21",
 		`### Fixed
 - [Vent] Log for _vent_log insert now faithfully captures what is being inserted

vent/service/block_consumer.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hyperledger/burrow/execution/evm/abi"
 	"github.com/hyperledger/burrow/execution/exec"
 	"github.com/hyperledger/burrow/logging"
+	"github.com/hyperledger/burrow/logging/structure"
 	"github.com/hyperledger/burrow/vent/sqlsol"
 	"github.com/hyperledger/burrow/vent/types"
 	"github.com/pkg/errors"
@@ -77,13 +78,18 @@ func NewBlockConsumer(projection *sqlsol.Projection, opt sqlsol.SpecOpt, getEven
 						continue
 					}
 
+					var tagged query.Tagged = event
 					eventID := event.Log.SolidityEventID()
-					eventSpec, err := getEventSpec(eventID, event.Log.Address)
-					if err != nil {
-						return errors.Wrapf(err, "could not get ABI for solidity event with id %v at address %v",
-							eventID, event.Log.Address)
+					eventSpec, eventSpecErr := getEventSpec(eventID, event.Log.Address)
+					if eventSpecErr != nil {
+						logger.InfoMsg("could not get ABI for solidity event",
+							structure.ErrorKey, eventSpecErr,
+							"event_id", eventID,
+							"address", event.Log.Address)
+					} else {
+						// Since we have the event ABI we will allow matching on ABI fields
+						tagged = query.TagsFor(event, query.TaggedPrefix("Event", eventSpec))
 					}
-					tagged := query.TagsFor(event, query.TaggedPrefix("Event", eventSpec))
 
 					// see which spec filter matches with the one in event data
 					for _, eventClass := range projection.Spec {
@@ -95,6 +101,11 @@ func NewBlockConsumer(projection *sqlsol.Projection, opt sqlsol.SpecOpt, getEven
 
 						// there's a matching filter, add data to the rows
 						if qry.Matches(tagged) {
+							if eventSpecErr != nil {
+								return errors.Wrapf(eventSpecErr, "could not get ABI for solidity event matching "+
+									"projection filter \"%s\" with id %v at address %v",
+									eventClass.Filter, eventID, event.Log.Address)
+							}
 
 							logger.InfoMsg("Matched event", "header", event.Header,
 								"filter", eventClass.Filter)

vent/service/block_consumer_test.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"fmt"
 	"math/big"
 	"testing"
 	"time"
@@ -20,29 +21,10 @@ import (
 func TestBlockConsumer(t *testing.T) {
 	doneCh := make(chan struct{})
 	eventCh := make(chan types.EventData, 100)
-	longFilter := "(Log1Text = 'a' OR Log1Text = 'b' OR Log1Text = 'frogs') AND EventName = 'ManyTypes'"
-	tableName := "Events"
-	projection, err := sqlsol.NewProjection(types.ProjectionSpec{
-		{
-			TableName: tableName,
-			Filter:    longFilter,
-			FieldMappings: []*types.EventFieldMapping{
-				{
-					Field:         "direction",
-					Type:          types.EventFieldTypeString,
-					ColumnName:    "direction",
-					BytesToString: true,
-				},
-			},
-		},
-	})
-	require.NoError(t, err)
 
 	spec, err := abi.ReadSpec(solidity.Abi_EventEmitter)
 	require.NoError(t, err)
 
-	blockConsumer := NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logging.NewNoopLogger())
-
 	type args struct {
 		Direction []byte
 		Trueism   bool
@@ -51,7 +33,7 @@ func TestBlockConsumer(t *testing.T) {
 		Bignum    int8
 		Hash      string
 	}
-	eventSpec := spec.EventsByName["ManyTypes"]
+	manyTypesEventSpec := spec.EventsByName["ManyTypes"]
 
 	bignum := big.NewInt(1000)
 	in := args{
@@ -64,31 +46,151 @@ func TestBlockConsumer(t *testing.T) {
 	}
 	direction := "frogs"
 	copy(in.Direction, direction)
-	topics, data, err := abi.PackEvent(&eventSpec, in)
+	topics, data, err := abi.PackEvent(&manyTypesEventSpec, in)
 	require.NoError(t, err)
-
-	txe := &exec.TxExecution{
-		TxHeader: &exec.TxHeader{},
-	}
-	err = txe.Log(&exec.LogEvent{
+	log := &exec.LogEvent{
 		Address: crypto.Address{},
 		Data:    data,
 		Topics:  topics,
+	}
+
+	logger := logging.NewNoopLogger()
+	fieldMappings := []*types.EventFieldMapping{
+		{
+			Field:         "direction",
+			Type:          types.EventFieldTypeString,
+			ColumnName:    "direction",
+			BytesToString: true,
+		},
+	}
+	t.Run("Consume matching event", func(t *testing.T) {
+		spec, err := abi.ReadSpec(solidity.Abi_EventEmitter)
+		require.NoError(t, err)
+
+		longFilter := "(Log1Text = 'a' OR Log1Text = 'b' OR Log1Text = 'c' OR Log1Text = 'frogs') AND EventName = 'ManyTypes'"
+		require.True(t, len(longFilter) > 100)
+		tableName := "Events"
+		projection, err := sqlsol.NewProjection(types.ProjectionSpec{
+			{
+				TableName:     tableName,
+				Filter:        longFilter,
+				FieldMappings: fieldMappings,
+			},
+		})
+		require.NoError(t, err)
+		blockConsumer := NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logger)
+		tables, err := consumeBlock(blockConsumer, eventCh, log)
+		require.NoError(t, err)
+		rows := tables[tableName]
+		assert.Len(t, rows, 1)
+		assert.Equal(t, direction, rows[0].RowData["direction"])
 	})
-	require.NoError(t, err)
+
+	t.Run("Consume matching event without ABI", func(t *testing.T) {
+		spec, err := abi.ReadSpec(solidity.Abi_EventEmitter)
+		require.NoError(t, err)
+
+		// Remove the ABI
+		delete(spec.EventsByID, manyTypesEventSpec.ID)
+
+		tableName := "Events"
+		// Here we are using a filter that matches - but we no longer have ABI
+		projection, err := sqlsol.NewProjection(types.ProjectionSpec{
+			{
+				TableName:     tableName,
+				Filter:        "Log1Text = 'a' OR Log1Text = 'b' OR Log1Text = 'frogs'",
+				FieldMappings: fieldMappings,
+			},
+		})
+		require.NoError(t, err)
+		blockConsumer := NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logger)
+		_, err = consumeBlock(blockConsumer, eventCh, log)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "could not find ABI")
+	})
+
+	t.Run("Consume non-matching event without ABI", func(t *testing.T) {
+		spec, err := abi.ReadSpec(solidity.Abi_EventEmitter)
+		require.NoError(t, err)
+
+		// Remove the ABI
+		delete(spec.EventsByID, manyTypesEventSpec.ID)
+
+		tableName := "Events"
+		// Here we are using a filter that matches - but we no longer have ABI
+		projection, err := sqlsol.NewProjection(types.ProjectionSpec{
+			{
+				TableName:     tableName,
+				Filter:        "ThisIsNotAKey = 'bar'",
+				FieldMappings: fieldMappings,
+			},
+		})
+		require.NoError(t, err)
+		blockConsumer := NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logger)
+		_, err = consumeBlock(blockConsumer, eventCh, log)
+		require.Equal(t, errTimeout, err)
+	})
+
+	// This is possibly 'bad' behaviour - since you may be missing an ABI - but for now it is expected. On-chain ABIs
+	// ought to solve this
+	t.Run("Consume event that doesn't match without ABI tags", func(t *testing.T) {
+		// This is the case where we silently fail - it would match if we had the ABI - but since we don't it doesn't
+		// and we just carry on
+		tableName := "Events"
+		// Here we are using a filter that matches - but we no longer have ABI
+		projection, err := sqlsol.NewProjection(types.ProjectionSpec{
+			{
+				TableName:     tableName,
+				Filter:        "EventName = 'ManyTypes'",
+				FieldMappings: fieldMappings,
+			},
+		})
+		require.NoError(t, err)
+
+		spec, err := abi.ReadSpec(solidity.Abi_EventEmitter)
+		require.NoError(t, err)
+
+		blockConsumer := NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logger)
+		_, err = consumeBlock(blockConsumer, eventCh, log)
+		// Check matches
+		require.NoError(t, err)
+
+		// Now Remove the ABI - should timeout indicating we did not match the event, but it wasn't an error
+		delete(spec.EventsByID, manyTypesEventSpec.ID)
+		blockConsumer = NewBlockConsumer(projection, sqlsol.None, spec.GetEventAbi, eventCh, doneCh, logger)
+		_, err = consumeBlock(blockConsumer, eventCh, log)
+		require.Equal(t, errTimeout, err)
+	})
+}
+
+const timeout = time.Second
+
+var errTimeout = fmt.Errorf("timed out after %s waiting for consumer to emit block event", timeout)
+
+func consumeBlock(blockConsumer func(*exec.BlockExecution) error, eventCh <-chan types.EventData,
+	logEvents ...*exec.LogEvent) (map[string]types.EventDataTable, error) {
 
 	block := &exec.BlockExecution{
 		Header: &tmTypes.Header{},
 	}
-	block.AppendTxs(txe)
-	err = blockConsumer(block)
-	require.NoError(t, err)
+	for _, logEvent := range logEvents {
+		txe := &exec.TxExecution{
+			TxHeader: &exec.TxHeader{},
+		}
+		err := txe.Log(logEvent)
+		if err != nil {
+			return nil, err
+		}
+		block.AppendTxs(txe)
+	}
+	err := blockConsumer(block)
+	if err != nil {
+		return nil, err
+	}
 	select {
-	case <-time.After(time.Second * 5):
-		t.Fatalf("timed out waiting for consumer to emit block event")
+	case <-time.After(timeout):
+		return nil, errTimeout
 	case ed := <-eventCh:
-		rows := ed.Tables[tableName]
-		assert.Len(t, rows, 1)
-		assert.Equal(t, direction, rows[0].RowData["direction"])
+		return ed.Tables, nil
 	}
 }

vent/sqldb/sqldb.go

@@ -383,13 +383,13 @@ loop:
 				"block_height", eventData.BlockHeight,
 				"tx_hash", txHash,
 				"row_action", row.Action,
-				"row_data", rowData,
+				"row_data", string(rowData),
 				"sql_query", sqlQuery,
-				"sql_values", sqlValues,
+				"sql_values", string(sqlValues),
 			)
 
-			if _, err = logStmt.Exec(chainID, tableName, eventName, row.EventClass.GetFilter(), eventData.BlockHeight, txHash,
-				row.Action, rowData, sqlQuery, sqlValues); err != nil {
+			if _, err = logStmt.Exec(chainID, tableName, eventName, row.EventClass.GetFilter(), eventData.BlockHeight,
+				txHash, row.Action, rowData, sqlQuery, sqlValues); err != nil {
 				db.Log.InfoMsg("Error inserting into log", "err", err)
 				break loop // exits from all loops -> continue in close log stmt
 			}