From 1f11e93a286c7cb324a136f2ac2ca8ea00a09c94 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:13:26 -0400 Subject: [PATCH 01/21] base script on GITHUB_ACTIONS --- tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 13e4dde..9b3c2cc 100644 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -3,7 +3,7 @@ set -euxo pipefail listen_address= if [[ $TEST_LOCAL_CODEGEN == "true" ]]; then - if [[ -n "${CI:-}" ]]; then + if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then # Codegen address when running test listen_address="http://172.17.0.1:9000" else From 42b6c8f5b8a16eb7b07f393bed091a12163d16ce Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:16:34 -0400 Subject: [PATCH 02/21] add ENV GITHUB_ACTIONS to true --- .github/workflows/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e75f357..afb4cad 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,6 +18,7 @@ jobs: env: TEST_LOCAL_CODEGEN: "true" RUN_INTEGRATION_TESTS: "true" + GITHUB_ACTIONS: "true" CODEGEN_MAINNET_API_KEY: ${{ secrets.CODEGEN_MAINNET_API_KEY }} REGISTRY: ghcr.io steps: From 16a738ac1a64105796f8cd0ab465d91dbe9fe4dd Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:21:52 -0400 Subject: [PATCH 03/21] fix script --- tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 tests/entrypoint.sh diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh old mode 100644 new mode 100755 index 9b3c2cc..e46e363 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -3,7 +3,7 @@ set -euxo pipefail listen_address= if [[ $TEST_LOCAL_CODEGEN == "true" ]]; then - if [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then + if [[ $GITHUB_ACTIONS == "true" ]]; then # Codegen address when running test listen_address="http://172.17.0.1:9000" else From 2f6a30a313e3f5c269dc16cb87895d762db2d5e7 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:29:50 -0400 Subject: [PATCH 04/21] fix not local path --- tests/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index e46e363..6e192b4 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -2,15 +2,15 @@ set -euxo pipefail listen_address= -if [[ $TEST_LOCAL_CODEGEN == "true" ]]; then - if [[ $GITHUB_ACTIONS == "true" ]]; then +if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then + if [[ ${GITHUB_ACTIONS:-false} == "true" ]]; then # Codegen address when running test listen_address="http://172.17.0.1:9000" else listen_address="http://host.docker.internal:9000" fi else - listen_address= https://codegen-staging.substreams.dev + listen_address="https://codegen-staging.substreams.dev" fi substreams init --state-file /app/generator.json --force-download-cwd --codegen-endpoint $listen_address From 120bfc1d3284d102ae697105352fd49bff695e33 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:32:00 -0400 Subject: [PATCH 05/21] replace GITHUB_ACTIONS by CI --- tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 6e192b4..76923d6 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -3,7 +3,7 @@ set -euxo pipefail listen_address= if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then - if [[ ${GITHUB_ACTIONS:-false} == "true" ]]; then + if [[ "${CI:-""}" != "" ]]; then # Codegen address when running test listen_address="http://172.17.0.1:9000" else From 897c334e3cc586072d31b261e20a38ef81797e78 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:41:09 -0400 Subject: [PATCH 06/21] test ahah --- tests/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 76923d6..0baa73e 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euxo pipefail +echo "ahah $CI" + listen_address= if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then if [[ "${CI:-""}" != "" ]]; then From 246c71fb5571964c5ee7a9f730031318c408bc55 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:43:00 -0400 Subject: [PATCH 07/21] add CI=true --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index afb4cad..7b7a096 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,7 +18,7 @@ jobs: env: TEST_LOCAL_CODEGEN: "true" RUN_INTEGRATION_TESTS: "true" - GITHUB_ACTIONS: "true" + CI: "true" CODEGEN_MAINNET_API_KEY: ${{ secrets.CODEGEN_MAINNET_API_KEY }} REGISTRY: ghcr.io steps: From 546ec9591237a9f73d38db34458fecacb74c1f3c Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:53:38 -0400 Subject: [PATCH 08/21] update script --- tests/entrypoint.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 0baa73e..eefd4b8 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash set -euxo pipefail -echo "ahah $CI" - listen_address= if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then - if [[ "${CI:-""}" != "" ]]; then + if [[ ${CI:-""} != "" ]]; then # Codegen address when running test listen_address="http://172.17.0.1:9000" else From 97c195e6573de7d0a462e22ed15a1d3533e0d653 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:57:57 -0400 Subject: [PATCH 09/21] update script again --- tests/entrypoint.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index eefd4b8..49f88f4 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -3,11 +3,11 @@ set -euxo pipefail listen_address= if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then - if [[ ${CI:-""} != "" ]]; then + if [[ -z $CI ]]; then + listen_address="http://host.docker.internal:9000" + else # Codegen address when running test listen_address="http://172.17.0.1:9000" - else - listen_address="http://host.docker.internal:9000" fi else listen_address="https://codegen-staging.substreams.dev" From ba319922465fdf7a9497c91c224d49571dc7a7a5 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 15:59:17 -0400 Subject: [PATCH 10/21] update script --- tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoint.sh b/tests/entrypoint.sh index 49f88f4..c2d1ec0 100755 --- a/tests/entrypoint.sh +++ b/tests/entrypoint.sh @@ -3,7 +3,7 @@ set -euxo pipefail listen_address= if [[ ${TEST_LOCAL_CODEGEN:-false} == "true" ]]; then - if [[ -z $CI ]]; then + if [[ -z ${CI} ]]; then listen_address="http://host.docker.internal:9000" else # Codegen address when running test From 5894954cce81f8efc2157edf5fff61ffe52802f6 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 16:06:11 -0400 Subject: [PATCH 11/21] fix for good --- tests/integration_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration_test.go b/tests/integration_test.go index 4eb6ce0..9b79bb0 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -136,6 +136,8 @@ func TestIntegration(t *testing.T) { fmt.Sprintf("%s:/app/generator.json", c.stateFile), "-e", fmt.Sprintf("TEST_LOCAL_CODEGEN=%s", os.Getenv("TEST_LOCAL_CODEGEN")), + "-e", + fmt.Sprintf("CI=%s", os.Getenv("CI")), "test-image", } From a473e76264157bc98eef36545af114c68cbca57a Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Fri, 18 Oct 2024 16:11:58 -0400 Subject: [PATCH 12/21] fix starknet-events integration test --- tests/starknet-events/generator.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/starknet-events/generator.json b/tests/starknet-events/generator.json index 1bf695e..8d1ed2c 100644 --- a/tests/starknet-events/generator.json +++ b/tests/starknet-events/generator.json @@ -1,5 +1,5 @@ { - "generator": "starknet-events", + "generator": "starknet-events-beta", "state": { "name": "my_project", "chainName": "starknet-mainnet", From 89df0a449741937dcc8b1e2a761e5b29a5f9e5b3 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Mon, 21 Oct 2024 11:55:02 -0400 Subject: [PATCH 13/21] fix codegen starknet --- starknet-events/abi.go | 66 ++++++++++++++++++--- starknet-events/contract.go | 58 ++++++++++++------ starknet-events/convo.go | 2 +- starknet-events/templates/src/lib.rs.gotmpl | 2 +- starknet-events/utils.go | 50 ++++++++++++++++ 5 files changed, 151 insertions(+), 27 deletions(-) create mode 100644 starknet-events/utils.go diff --git a/starknet-events/abi.go b/starknet-events/abi.go index 7edc782..b3887a2 100644 --- a/starknet-events/abi.go +++ b/starknet-events/abi.go @@ -1,26 +1,78 @@ package starknet_events import ( - starknetABI "github.com/dipdup-io/starknet-go-api/pkg/abi" + "encoding/json" + "github.com/streamingfast/substreams-codegen/loop" ) type ABI struct { - decodedAbi *starknetABI.Abi - raw string + decodedEvents StarknetEvents + raw string +} + +type StarknetEvents []*StarknetEvent + +type StarknetEvent struct { + CommonAttribute + + Variants []CommonAttribute `json:"variants"` } -type StarknetABI struct { +type OtherItem struct { + CommonAttribute +} +type CommonAttribute struct { + Type string `json:"type"` + Name string `json:"name"` + Kind string `json:"kind"` +} + +const ( + EventType = "event" +) + +func (s *StarknetEvents) ExtractEvents(data []byte) error { + var Attributes []CommonAttribute + if err := json.Unmarshal(data, &Attributes); err != nil { + return err + } + + items := make([]interface{}, 0) + + for _, attribute := range Attributes { + switch attribute.Type { + case EventType: + items = append(items, &StarknetEvent{}) + default: + items = append(items, &OtherItem{}) + } + } + + if err := json.Unmarshal(data, &items); err != nil { + return err + } + + for _, item := range items { + switch i := item.(type) { + case *StarknetEvent: + *s = append(*s, i) + default: + continue + } + } + + return nil } func CmdDecodeABI(contract *Contract) loop.Cmd { return func() loop.Msg { - contractABI := starknetABI.Abi{} - err := contractABI.UnmarshalJSON(contract.RawABI) + events := StarknetEvents{} + err := events.ExtractEvents(contract.RawABI) if err != nil { panic("decoding contract abi") } - return ReturnRunDecodeContractABI{Abi: &ABI{&contractABI, string(contract.RawABI)}, Err: err} + return ReturnRunDecodeContractABI{Abi: &ABI{events, string(contract.RawABI)}, Err: err} } } diff --git a/starknet-events/contract.go b/starknet-events/contract.go index 4dbb38c..61d7c1b 100644 --- a/starknet-events/contract.go +++ b/starknet-events/contract.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/NethermindEth/juno/core/felt" - starknetRPC "github.com/NethermindEth/starknet.go/rpc" ) @@ -17,12 +16,19 @@ type Alias struct { NewName string } +func NewAlias(oldName, newName string) *Alias { + return &Alias{ + OldName: oldName, + NewName: newName, + } +} + type Contract struct { Name string `json:"name,omitempty"` Address string `json:"address"` InitialBlock *uint64 `json:"initialBlock"` - Aliases []Alias `json:"aliases"` + Aliases []*Alias `json:"aliases"` RawABI json.RawMessage `json:"rawAbi,omitempty"` Abi *ABI @@ -43,34 +49,50 @@ func (c *Contract) IdentifierCapitalize() string { return strings.ToUpper(string(c.Name[0])) + c.Name[1:] } func (c *Contract) SetAliases() { - events := c.Abi.decodedAbi.EventsBySelector + events := c.Abi.decodedEvents - aliases := make([]Alias, 0) + aliases := make([]*Alias, 0) seen := make(map[string]struct{}) - for _, eventItem := range events { - eventName := eventItem.Name - - splitEventName := strings.Split(eventName, "::") - - lastPart := splitEventName[len(splitEventName)-1] - - if _, found := seen[lastPart]; found { - if len(splitEventName) < 2 { - panic("parsed event name does not contain enough parts to have an alias") + // Based on Starknet documentation, we assume that in each contract, it exists a Event which is an enum containing all other events... (https://docs.starknet.io/architecture-and-concepts/smart-contracts/contract-abi/) + // Finding this "golden" event is not an easy path, as multiple enum with the same name can exist in the ABI... + // We need to detect the Golden Event to avoid applying Alias on it... + potentialsGoldenEvent := make(map[string]*StarknetEvent) + for _, event := range events { + eventName := event.Name + lastPart, newName := eventNameInfo(eventName) + + if lastPart == "Event" { + // Event which are not enum, we can safely apply alias + if event.Kind != "enum" { + alias := NewAlias(eventName, newName) + aliases = append(aliases, alias) + continue } - alias := Alias{ - OldName: eventName, - NewName: splitEventName[len(splitEventName)-2] + lastPart, - } + potentialsGoldenEvent[event.Name] = event + continue + } + if _, found := seen[lastPart]; found { + alias := NewAlias(eventName, newName) aliases = append(aliases, alias) } seen[lastPart] = struct{}{} } + if len(potentialsGoldenEvent) == 1 { + c.Aliases = aliases + return + } + + goldenName := detectGoldenEvent(potentialsGoldenEvent) + if goldenName == "" { + panic("no golden event found") + } + + aliases = setNonGoldenAliases(potentialsGoldenEvent, goldenName, aliases) c.Aliases = aliases } diff --git a/starknet-events/convo.go b/starknet-events/convo.go index 141feda..a1bbc7d 100644 --- a/starknet-events/convo.go +++ b/starknet-events/convo.go @@ -72,7 +72,7 @@ func (c *Convo) NextStep() loop.Cmd { return cmd(AskContractAddress{}) } - if contract.Abi == nil || contract.Abi.decodedAbi == nil { + if contract.Abi == nil || contract.Abi.decodedEvents == nil { // if the user pasted an empty ABI, we would restart the process or choosing a contract address if contract.emptyABI { contract.Address = "" // reset the address diff --git a/starknet-events/templates/src/lib.rs.gotmpl b/starknet-events/templates/src/lib.rs.gotmpl index ecb06e0..ca4aabf 100644 --- a/starknet-events/templates/src/lib.rs.gotmpl +++ b/starknet-events/templates/src/lib.rs.gotmpl @@ -51,7 +51,7 @@ fn map_{{ $contract.Identifier }}_events(transactions: Transactions) -> Result Date: Tue, 6 Aug 2024 15:50:39 -0400 Subject: [PATCH 14/21] add index module on codegens --- .../templates/entities/substreams.yaml.gotmpl | 96 +++++++++++++++++++ ethfull/templates/substreams.yaml.gotmpl | 0 evm-events-calls/convo.go | 8 +- evm-events-calls/state.go | 59 ++++++++++-- .../templates/sql/substreams.yaml.gotmpl | 11 +++ .../templates/substreams.yaml.gotmpl | 13 +++ 6 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 ethfull/templates/entities/substreams.yaml.gotmpl create mode 100644 ethfull/templates/substreams.yaml.gotmpl diff --git a/ethfull/templates/entities/substreams.yaml.gotmpl b/ethfull/templates/entities/substreams.yaml.gotmpl new file mode 100644 index 0000000..b7cda24 --- /dev/null +++ b/ethfull/templates/entities/substreams.yaml.gotmpl @@ -0,0 +1,96 @@ +specVersion: v0.1.0 +package: + name: {{ .ModuleName }} + version: v0.1.0 + +imports: + graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .GraphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .GraphImportVersion }}.spkg + entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .EntityChangeImportVersion }}/substreams-entity-change-v{{ .EntityChangeImportVersion }}.spkg + ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg + +protobuf: + files: + - contract.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: +{{ range $ddsContract := .DynamicContracts }} + - name: store_{{ $ddsContract.Identifier }}_created + kind: store + initialBlock: {{ with $ddsContract.FactoryInitialBlock }}{{ . }}{{ else }}0{{ end }} + updatePolicy: set + valueType: proto:dynamic_datasource + inputs: + - source: sf.ethereum.type.v2.Block +{{- end}} +{{- if .TrackAnyEvents }} + - name: map_events + kind: map + blockFilter: + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} + initialBlock: {{ .MustLowestStartBlock }} + inputs: + - source: sf.ethereum.type.v2.Block +{{- range $ddsContract := .DynamicContracts }} + {{- if $ddsContract.TrackEvents }} + - store: store_{{ $ddsContract.Identifier }}_created + {{- end }} +{{- end }} + output: + type: proto:contract.v1.Events +{{- end }} + +{{- if and $.TrackAnyEvents $.TrackAnyCalls }} + - name: map_events_calls + kind: map + initialBlock: {{ .MustLowestStartBlock }} + inputs: + - map: map_events + - map: map_calls + output: + type: proto:contract.v1.EventsCalls +{{- end }} + +{{- if .TrackAnyCalls }} + - name: map_calls + kind: map + initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_calls + {{ $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} + inputs: + - source: sf.ethereum.type.v2.Block +{{- range $ddsContract := .DynamicContracts }} + {{- if $ddsContract.TrackCalls }} + - store: store_{{ $ddsContract.Identifier }}_created + {{- end }} +{{- end }} + output: + type: proto:contract.v1.Calls +{{- end }}{{/* if TrackAnyCalls */}} + + - name: graph_out + kind: map + initialBlock: {{ .MustLowestStartBlock }} + inputs: +{{- if .TrackAnyEvents }} + - map: map_events +{{- end }} +{{- if .TrackAnyCalls }} + - map: map_calls +{{- end }} + output: + type: proto:sf.substreams.entity.v1.EntityChanges + +network: {{ .ChainName }} diff --git a/ethfull/templates/substreams.yaml.gotmpl b/ethfull/templates/substreams.yaml.gotmpl new file mode 100644 index 0000000..e69de29 diff --git a/evm-events-calls/convo.go b/evm-events-calls/convo.go index 5847896..178e3c3 100644 --- a/evm-events-calls/convo.go +++ b/evm-events-calls/convo.go @@ -136,12 +136,12 @@ func (c *Convo) NextStep() (out loop.Cmd) { if dynContract.Abi == nil { // if the user pasted an empty ABI, we would restart the process or choosing a contract address if dynContract.emptyABI { - dynContract.referenceContractAddress = "" // reset the reference address + dynContract.ReferenceContractAddress = "" // reset the reference address dynContract.emptyABI = false // reset the flag return notifyContext(cmd(AskContractAddress{})) } if dynContract.RawABI == nil { - if dynContract.referenceContractAddress == "" { + if dynContract.ReferenceContractAddress == "" { if p.ChainConfig().ApiEndpoint == "" { return notifyContext(cmd(AskDynamicContractABI{})) } @@ -275,7 +275,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { } contract := c.State.dynamicContractOf(factory.Name) - contract.referenceContractAddress = inputAddress + contract.ReferenceContractAddress = inputAddress return c.NextStep() @@ -422,7 +422,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { contract := c.State.dynamicContractOf(factory.Name) if msg.err != nil { return loop.Seq( - c.Msg().Messagef("Cannot fetch the ABI for dynamic contract %q (%s)", contract.referenceContractAddress, msg.err).Cmd(), + c.Msg().Messagef("Cannot fetch the ABI for dynamic contract %q (%s)", contract.ReferenceContractAddress, msg.err).Cmd(), cmd(AskDynamicContractABI{}), ) } diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index f0af20b..60275a5 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -75,6 +75,53 @@ func isValidChainName(input string) bool { return ChainConfigByID[input] != nil } +func (p *Project) GenerateEventsBlockFilterQuery() string { + var query string + for i, contract := range p.Contracts { + if i == 0 { + query = fmt.Sprintf("evt_addr:%s", contract.Address) + continue + } + + query += fmt.Sprintf(" || evt_addr:%s", contract.Address) + } + + for i, dynamicContract := range p.DynamicContracts { + if i == 0 { + query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) + fmt.Println("query", query) + continue + } + + query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) + fmt.Println("query", query) + } + + return query +} + +func (p *Project) GenerateCallsBlockFilterQuery() string { + var query string + for i, contract := range p.Contracts { + if i == 0 { + query = fmt.Sprintf("call_to:%s", contract.Address) + continue + } + + query += fmt.Sprintf(" || call_to:%s", contract.Address) + } + + for i, dynamicContract := range p.DynamicContracts { + if i == 0 { + query += fmt.Sprintf(" || call_to:%s", dynamicContract.ReferenceContractAddress) + continue + } + + query += fmt.Sprintf(" || call_to:%s", dynamicContract.ReferenceContractAddress) + } + return query +} + func (p *Project) TrackAnyCalls() bool { for _, contract := range p.Contracts { if contract.TrackCalls { @@ -268,7 +315,7 @@ type DynamicContract struct { ParentContractName string `json:"parentContractName"` parentContract *Contract - referenceContractAddress string + ReferenceContractAddress string `json:"referenceContractAddress"` } func (d DynamicContract) FactoryInitialBlock() uint64 { @@ -279,7 +326,7 @@ func (d DynamicContract) ParentContract() *Contract { return d.parentContract func (d DynamicContract) Identifier() string { return d.Name } func (d DynamicContract) IdentifierSnakeCase() string { return kace.Snake(d.Name) } func (d DynamicContract) FetchABI(chainConfig *ChainConfig) (abi string, err error) { - a, err := getContractABIFollowingProxy(context.Background(), d.referenceContractAddress, chainConfig) + a, err := getContractABIFollowingProxy(context.Background(), d.ReferenceContractAddress, chainConfig) if err != nil { return "", err } @@ -317,7 +364,7 @@ func validateContractAddress(p *Project, address string) error { } for _, dynamicContract := range p.DynamicContracts { - if dynamicContract.referenceContractAddress == address { + if dynamicContract.ReferenceContractAddress == address { return fmt.Errorf("contract address %s already exists in the project", address) } } @@ -347,12 +394,12 @@ func validateIncomingState(p *Project) error { return fmt.Errorf("contract with name %s already exists in the project", dynamicContract.Name) } - if _, found := uniqueContractAddresses[dynamicContract.referenceContractAddress]; found { - return fmt.Errorf("contract address %s already exists in the project", dynamicContract.referenceContractAddress) + if _, found := uniqueContractAddresses[dynamicContract.ReferenceContractAddress]; found { + return fmt.Errorf("contract address %s already exists in the project", dynamicContract.ReferenceContractAddress) } uniqueContractNames[dynamicContract.Name] = struct{}{} - uniqueContractAddresses[dynamicContract.referenceContractAddress] = struct{}{} + uniqueContractAddresses[dynamicContract.ReferenceContractAddress] = struct{}{} } return nil diff --git a/evm-events-calls/templates/sql/substreams.yaml.gotmpl b/evm-events-calls/templates/sql/substreams.yaml.gotmpl index 547fedd..0d3ec95 100644 --- a/evm-events-calls/templates/sql/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/sql/substreams.yaml.gotmpl @@ -6,6 +6,7 @@ package: imports: sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .SQLImportVersion }}/substreams-sink-sql-protodefs-v{{ .SQLImportVersion }}.spkg database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .DatabaseChangeImportVersion }}/substreams-database-change-v{{ .DatabaseChangeImportVersion }}.spkg + ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg protobuf: files: @@ -33,6 +34,11 @@ modules: - name: map_events kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} @@ -60,6 +66,11 @@ modules: - name: map_calls kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_calls + {{ $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $ddsContract := .DynamicContracts }} diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index c6c65ed..aa04a9b 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -3,6 +3,9 @@ package: name: {{ .ModuleName }} version: v0.1.0 +imports: + ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg + protobuf: files: - contract.proto @@ -32,6 +35,11 @@ modules: - name: map_events kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} @@ -46,6 +54,11 @@ modules: - name: map_calls kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_calls + {{ $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} From 4157cdd161b43bb1a84607780f465375f4469857 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Tue, 22 Oct 2024 12:06:19 -0400 Subject: [PATCH 15/21] remove prints --- evm-events-calls/state.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index 60275a5..4370160 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -89,12 +89,10 @@ func (p *Project) GenerateEventsBlockFilterQuery() string { for i, dynamicContract := range p.DynamicContracts { if i == 0 { query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) - fmt.Println("query", query) continue } query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) - fmt.Println("query", query) } return query From add77bd89bd98e4844390b761315b0ec142d0c71 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Tue, 22 Oct 2024 12:07:11 -0400 Subject: [PATCH 16/21] remove ethfull entities --- .../templates/entities/substreams.yaml.gotmpl | 96 ------------------- ethfull/templates/substreams.yaml.gotmpl | 0 2 files changed, 96 deletions(-) delete mode 100644 ethfull/templates/entities/substreams.yaml.gotmpl delete mode 100644 ethfull/templates/substreams.yaml.gotmpl diff --git a/ethfull/templates/entities/substreams.yaml.gotmpl b/ethfull/templates/entities/substreams.yaml.gotmpl deleted file mode 100644 index b7cda24..0000000 --- a/ethfull/templates/entities/substreams.yaml.gotmpl +++ /dev/null @@ -1,96 +0,0 @@ -specVersion: v0.1.0 -package: - name: {{ .ModuleName }} - version: v0.1.0 - -imports: - graph: https://github.com/streamingfast/substreams-sink-subgraph/releases/download/v{{ .GraphImportVersion }}/substreams-sink-subgraph-protodefs-v{{ .GraphImportVersion }}.spkg - entity: https://github.com/streamingfast/substreams-entity-change/releases/download/v{{ .EntityChangeImportVersion }}/substreams-entity-change-v{{ .EntityChangeImportVersion }}.spkg - ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg - -protobuf: - files: - - contract.proto - importPaths: - - ./proto - -binaries: - default: - type: wasm/rust-v1 - file: ./target/wasm32-unknown-unknown/release/substreams.wasm - -modules: -{{ range $ddsContract := .DynamicContracts }} - - name: store_{{ $ddsContract.Identifier }}_created - kind: store - initialBlock: {{ with $ddsContract.FactoryInitialBlock }}{{ . }}{{ else }}0{{ end }} - updatePolicy: set - valueType: proto:dynamic_datasource - inputs: - - source: sf.ethereum.type.v2.Block -{{- end}} -{{- if .TrackAnyEvents }} - - name: map_events - kind: map - blockFilter: - module: ethcommon:index_events - {{- $eventQuery := .GenerateEventsBlockFilterQuery }} - query: - string: {{ $eventQuery }} - initialBlock: {{ .MustLowestStartBlock }} - inputs: - - source: sf.ethereum.type.v2.Block -{{- range $ddsContract := .DynamicContracts }} - {{- if $ddsContract.TrackEvents }} - - store: store_{{ $ddsContract.Identifier }}_created - {{- end }} -{{- end }} - output: - type: proto:contract.v1.Events -{{- end }} - -{{- if and $.TrackAnyEvents $.TrackAnyCalls }} - - name: map_events_calls - kind: map - initialBlock: {{ .MustLowestStartBlock }} - inputs: - - map: map_events - - map: map_calls - output: - type: proto:contract.v1.EventsCalls -{{- end }} - -{{- if .TrackAnyCalls }} - - name: map_calls - kind: map - initialBlock: {{ .MustLowestStartBlock }} - blockFilter: - module: ethcommon:index_calls - {{ $callQuery := .GenerateCallsBlockFilterQuery }} - query: - string: {{ $callQuery }} - inputs: - - source: sf.ethereum.type.v2.Block -{{- range $ddsContract := .DynamicContracts }} - {{- if $ddsContract.TrackCalls }} - - store: store_{{ $ddsContract.Identifier }}_created - {{- end }} -{{- end }} - output: - type: proto:contract.v1.Calls -{{- end }}{{/* if TrackAnyCalls */}} - - - name: graph_out - kind: map - initialBlock: {{ .MustLowestStartBlock }} - inputs: -{{- if .TrackAnyEvents }} - - map: map_events -{{- end }} -{{- if .TrackAnyCalls }} - - map: map_calls -{{- end }} - output: - type: proto:sf.substreams.entity.v1.EntityChanges - -network: {{ .ChainName }} diff --git a/ethfull/templates/substreams.yaml.gotmpl b/ethfull/templates/substreams.yaml.gotmpl deleted file mode 100644 index e69de29..0000000 From 16f01f3fa33742095dc6087a43f3ed111c61da0c Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Tue, 22 Oct 2024 12:08:05 -0400 Subject: [PATCH 17/21] clean yaml --- evm-events-calls/templates/substreams.yaml.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index aa04a9b..b420598 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -56,7 +56,7 @@ modules: initialBlock: {{ .MustLowestStartBlock }} blockFilter: module: ethcommon:index_calls - {{ $callQuery := .GenerateCallsBlockFilterQuery }} + {{- $callQuery := .GenerateCallsBlockFilterQuery }} query: string: {{ $callQuery }} inputs: From 5a724682eb7c81eb4ca912113e42c8339395a031 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Wed, 30 Oct 2024 13:25:06 -0400 Subject: [PATCH 18/21] fix event_calls block filters --- evm-events-calls/state.go | 63 +++++++++++++------ .../templates/substreams.yaml.gotmpl | 8 +++ 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index 4370160..dfb2afa 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -75,24 +75,51 @@ func isValidChainName(input string) bool { return ChainConfigByID[input] != nil } +func (p *Project) ApplyEventsBlockFilter() bool { + for _, dcontract := range p.DynamicContracts { + if dcontract.TrackEvents { + return false + } + } + + for _, contract := range p.Contracts { + if contract.TrackEvents { + return true + } + } + + return false +} + +func (p *Project) ApplyCallsBlockFilter() bool { + for _, dcontract := range p.DynamicContracts { + if dcontract.TrackCalls { + return false + } + } + + for _, contract := range p.Contracts { + if contract.TrackCalls { + return true + } + } + + return false +} + func (p *Project) GenerateEventsBlockFilterQuery() string { var query string - for i, contract := range p.Contracts { - if i == 0 { - query = fmt.Sprintf("evt_addr:%s", contract.Address) + for _, contract := range p.Contracts { + if !contract.TrackEvents { continue } - query += fmt.Sprintf(" || evt_addr:%s", contract.Address) - } - - for i, dynamicContract := range p.DynamicContracts { - if i == 0 { - query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) + if query == "" { + query = fmt.Sprintf("evt_addr:%s", contract.Address) continue } - query += fmt.Sprintf(" || evt_addr:%s", dynamicContract.ReferenceContractAddress) + query += fmt.Sprintf(" || evt_addr:%s", contract.Address) } return query @@ -100,23 +127,19 @@ func (p *Project) GenerateEventsBlockFilterQuery() string { func (p *Project) GenerateCallsBlockFilterQuery() string { var query string - for i, contract := range p.Contracts { - if i == 0 { - query = fmt.Sprintf("call_to:%s", contract.Address) + for _, contract := range p.Contracts { + if !contract.TrackCalls { continue } - query += fmt.Sprintf(" || call_to:%s", contract.Address) - } - - for i, dynamicContract := range p.DynamicContracts { - if i == 0 { - query += fmt.Sprintf(" || call_to:%s", dynamicContract.ReferenceContractAddress) + if query == "" { + query = fmt.Sprintf("call_to:%s", contract.Address) continue } - query += fmt.Sprintf(" || call_to:%s", dynamicContract.ReferenceContractAddress) + query += fmt.Sprintf(" || call_to:%s", contract.Address) } + return query } diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index b420598..6419e36 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -27,6 +27,10 @@ modules: initialBlock: {{ with $ddsContract.ParentContract.InitialBlock }}{{ . }}{{ else }}0{{ end }} updatePolicy: set valueType: proto:dynamic_datasource + blockFilter: + module: ethcommon:index_events + query: + string: evt_addr:{{ $ddsContract.ParentContract.Address }} inputs: - source: sf.ethereum.type.v2.Block {{- end}} @@ -35,11 +39,13 @@ modules: - name: map_events kind: map initialBlock: {{ .MustLowestStartBlock }} + {{- if .ApplyEventsBlockFilter }} blockFilter: module: ethcommon:index_events {{- $eventQuery := .GenerateEventsBlockFilterQuery }} query: string: {{ $eventQuery }} + {{- end }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} @@ -54,11 +60,13 @@ modules: - name: map_calls kind: map initialBlock: {{ .MustLowestStartBlock }} + {{- if .ApplyCallsBlockFilter }} blockFilter: module: ethcommon:index_calls {{- $callQuery := .GenerateCallsBlockFilterQuery }} query: string: {{ $callQuery }} + {{- end }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} From 3a3855c88d2ad6d199d37e10be40f3c987008ddc Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Wed, 30 Oct 2024 14:59:35 -0400 Subject: [PATCH 19/21] fix spaces --- .../templates/substreams.yaml.gotmpl | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index 6419e36..596ac30 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -28,9 +28,9 @@ modules: updatePolicy: set valueType: proto:dynamic_datasource blockFilter: - module: ethcommon:index_events - query: - string: evt_addr:{{ $ddsContract.ParentContract.Address }} + module: ethcommon:index_events + query: + string: evt_addr:{{ $ddsContract.ParentContract.Address }} inputs: - source: sf.ethereum.type.v2.Block {{- end}} @@ -41,10 +41,10 @@ modules: initialBlock: {{ .MustLowestStartBlock }} {{- if .ApplyEventsBlockFilter }} blockFilter: - module: ethcommon:index_events - {{- $eventQuery := .GenerateEventsBlockFilterQuery }} - query: - string: {{ $eventQuery }} + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} {{- end }} inputs: - source: sf.ethereum.type.v2.Block @@ -62,10 +62,10 @@ modules: initialBlock: {{ .MustLowestStartBlock }} {{- if .ApplyCallsBlockFilter }} blockFilter: - module: ethcommon:index_calls - {{- $callQuery := .GenerateCallsBlockFilterQuery }} - query: - string: {{ $callQuery }} + module: ethcommon:index_calls + {{- $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} {{- end }} inputs: - source: sf.ethereum.type.v2.Block From 73c24c3b1c43d64c2534a2e04bac00b8db0c1334 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Wed, 30 Oct 2024 15:40:46 -0400 Subject: [PATCH 20/21] add evt_sig to filter --- evm-events-calls/state.go | 4 +++- evm-events-calls/templates/substreams.yaml.gotmpl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index dfb2afa..4d69dfe 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -342,7 +342,9 @@ type DynamicContract struct { func (d DynamicContract) FactoryInitialBlock() uint64 { return *d.parentContract.InitialBlock } - +func (d DynamicContract) GenerateStoreQuery() string { + return fmt.Sprintf("evt_addr:%s || evt_sig:%s", d.parentContract.Address, "0x"+d.parentContract.FactoryCreationEvent) +} func (d DynamicContract) ParentContract() *Contract { return d.parentContract } func (d DynamicContract) Identifier() string { return d.Name } func (d DynamicContract) IdentifierSnakeCase() string { return kace.Snake(d.Name) } diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index 596ac30..0af7026 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -30,7 +30,7 @@ modules: blockFilter: module: ethcommon:index_events query: - string: evt_addr:{{ $ddsContract.ParentContract.Address }} + string: {{ $ddsContract.GenerateStoreQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- end}} From 0ec39822397a848c862730def002f6177f7d6462 Mon Sep 17 00:00:00 2001 From: arnaudberger Date: Wed, 30 Oct 2024 15:54:47 -0400 Subject: [PATCH 21/21] change or to && --- evm-events-calls/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index 4d69dfe..51983a2 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -343,7 +343,7 @@ func (d DynamicContract) FactoryInitialBlock() uint64 { return *d.parentContract.InitialBlock } func (d DynamicContract) GenerateStoreQuery() string { - return fmt.Sprintf("evt_addr:%s || evt_sig:%s", d.parentContract.Address, "0x"+d.parentContract.FactoryCreationEvent) + return fmt.Sprintf("evt_addr:%s && evt_sig:%s", d.parentContract.Address, "0x"+d.parentContract.FactoryCreationEvent) } func (d DynamicContract) ParentContract() *Contract { return d.parentContract } func (d DynamicContract) Identifier() string { return d.Name }