From 14df7fbb07802b886c5783a08ab2c3ae3295ccd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Thu, 1 Aug 2024 16:36:43 -0400 Subject: [PATCH 1/3] rework api keys to fetch ABIs --- ethfull/apicalls.go | 28 ++++++++++++++++++---------- ethfull/chain_configs.go | 5 +++++ ethfull/state.go | 7 ++++--- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/ethfull/apicalls.go b/ethfull/apicalls.go index a601c3f..6eb89d7 100644 --- a/ethfull/apicalls.go +++ b/ethfull/apicalls.go @@ -17,8 +17,6 @@ import ( "github.com/tidwall/gjson" ) -var etherscanAPIKey = os.Getenv("SUBDEV_ETHERSCAN_API_KEY") - var httpClient = http.Client{ Transport: dhttp.NewLoggingRoundTripper(zlog, tracer, http.DefaultTransport), Timeout: 30 * time.Second, @@ -37,20 +35,20 @@ func getContractABIFollowingProxy(ctx context.Context, contractAddress string, c } return &ABI{abi, abiContent}, nil } - abi, abiContent, wait, err := getContractABI(ctx, contractAddress, chain.ApiEndpoint) + abi, abiContent, wait, err := getContractABI(ctx, contractAddress, chain.ApiEndpoint, os.Getenv(chain.APIKeyEnvVar)) if err != nil { return nil, err } <-wait.C - implementationAddress, wait, err := getProxyContractImplementation(ctx, contractAddress, chain.ApiEndpoint) + implementationAddress, wait, err := getProxyContractImplementation(ctx, contractAddress, chain.ApiEndpoint, os.Getenv(chain.APIKeyEnvVar)) if err != nil { return nil, err } <-wait.C if implementationAddress != "" { - implementationABI, implementationABIContent, wait, err := getContractABI(ctx, implementationAddress, chain.ApiEndpoint) + implementationABI, implementationABIContent, wait, err := getContractABI(ctx, implementationAddress, chain.ApiEndpoint, os.Getenv(chain.APIKeyEnvVar)) if err != nil { return nil, err } @@ -121,8 +119,11 @@ func getContractABIDirect(ctx context.Context, address string, endpoint string) } -func getContractABI(ctx context.Context, address string, endpoint string) (*eth.ABI, string, *time.Timer, error) { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=contract&action=getabi&address=%s&apiKey=%s", endpoint, address, etherscanAPIKey), nil) +func getContractABI(ctx context.Context, address string, endpoint string, apiKey string) (*eth.ABI, string, *time.Timer, error) { + if apiKey != "" { + apiKey = fmt.Sprintf("&apiKey=%s", apiKey) + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=contract&action=getabi&address=%s%s", endpoint, address, apiKey), nil) if err != nil { return nil, "", nil, fmt.Errorf("new request: %w", err) } @@ -158,9 +159,12 @@ func getContractABI(ctx context.Context, address string, endpoint string) (*eth. } // getProxyContractImplementation returns the implementation address and a timer to wait before next call -func getProxyContractImplementation(ctx context.Context, address string, endpoint string) (string, *time.Timer, error) { +func getProxyContractImplementation(ctx context.Context, address string, endpoint string, apiKey string) (string, *time.Timer, error) { + if apiKey != "" { + apiKey = fmt.Sprintf("&apiKey=%s", apiKey) + } // check for proxy contract's implementation - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=contract&action=getsourcecode&address=%s&apiKey=%s", endpoint, address, etherscanAPIKey), nil) + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=contract&action=getsourcecode&address=%s%s", endpoint, address, apiKey), nil) if err != nil { return "", nil, fmt.Errorf("new request: %w", err) @@ -236,7 +240,11 @@ func getContractInitialBlock(ctx context.Context, chain *ChainConfig, contractAd return initBlock, nil } - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=account&action=txlist&address=%s&page=1&offset=1&sort=asc&apikey=%s", chain.ApiEndpoint, contractAddress, etherscanAPIKey), nil) + apiKey := "" + if key := os.Getenv(chain.APIKeyEnvVar); key != "" { + apiKey = fmt.Sprintf("&apiKey=%s", key) + } + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/api?module=account&action=txlist&address=%s&page=1&offset=1&sort=asc%s", chain.ApiEndpoint, contractAddress, apiKey), nil) if err != nil { return chain.FirstStreamableBlock, fmt.Errorf("new request: %w", err) } diff --git a/ethfull/chain_configs.go b/ethfull/chain_configs.go index a1c9b36..c47b651 100644 --- a/ethfull/chain_configs.go +++ b/ethfull/chain_configs.go @@ -18,6 +18,7 @@ type ChainConfig struct { FirstStreamableBlock uint64 Network string SupportsCalls bool + APIKeyEnvVar string abiCache map[string]*ABI initialBlockCache map[string]uint64 @@ -36,6 +37,7 @@ var ChainConfigByID = map[string]*ChainConfig{ abiCache: make(map[string]*ABI), initialBlockCache: make(map[string]uint64), SupportsCalls: true, + APIKeyEnvVar: "CODEGEN_MAINNET_API_KEY", }, "bnb": { DisplayName: "BNB", @@ -47,6 +49,7 @@ var ChainConfigByID = map[string]*ChainConfig{ abiCache: make(map[string]*ABI), initialBlockCache: make(map[string]uint64), SupportsCalls: true, + APIKeyEnvVar: "CODEGEN_BNB_API_KEY", }, "polygon": { DisplayName: "Polygon", @@ -58,6 +61,7 @@ var ChainConfigByID = map[string]*ChainConfig{ abiCache: make(map[string]*ABI), initialBlockCache: make(map[string]uint64), SupportsCalls: true, + APIKeyEnvVar: "CODEGEN_POLYGON_API_KEY", }, "amoy": { DisplayName: "Polygon Amoy Testnet", @@ -113,6 +117,7 @@ var ChainConfigByID = map[string]*ChainConfig{ abiCache: make(map[string]*ABI), initialBlockCache: make(map[string]uint64), SupportsCalls: false, + APIKeyEnvVar: "CODEGEN_OPTIMISM_API_KEY", }, "avalanche": { DisplayName: "Avalanche C-chain", diff --git a/ethfull/state.go b/ethfull/state.go index 65e11d7..8f9cb48 100644 --- a/ethfull/state.go +++ b/ethfull/state.go @@ -5,14 +5,15 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/codemodus/kace" - "github.com/golang-cz/textcase" - "github.com/huandu/xstrings" "math" "regexp" "strings" "time" + "github.com/codemodus/kace" + "github.com/golang-cz/textcase" + "github.com/huandu/xstrings" + "github.com/streamingfast/eth-go" ) From a1f5ef0b60f1dddd447d024c4e26b79734cb9aa7 Mon Sep 17 00:00:00 2001 From: Giuliano Francescangeli Date: Fri, 2 Aug 2024 10:04:42 -0400 Subject: [PATCH 2/3] Adding Base to codegen --- ethfull/chain_configs.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ethfull/chain_configs.go b/ethfull/chain_configs.go index c47b651..eafb04d 100644 --- a/ethfull/chain_configs.go +++ b/ethfull/chain_configs.go @@ -153,6 +153,18 @@ var ChainConfigByID = map[string]*ChainConfig{ initialBlockCache: make(map[string]uint64), SupportsCalls: true, }, + "base-mainnet": { + DisplayName: "Base Mainnet", + ExplorerLink: "https://basescan.org", + ApiEndpoint: "https://api.basescan.org", + FirehoseEndpoint: "base-mainnet.streamingfast.io", + FirstStreamableBlock: 0, + Network: "base-mainnet", + abiCache: make(map[string]*ABI), + initialBlockCache: make(map[string]uint64), + SupportsCalls: true, + APIKeyEnvVar: "CODEGEN_BASE_API_KEY", + }, } func init() { From 00c0e3aaa4b889b401fca89f94f1e5061ca732ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Fri, 2 Aug 2024 15:24:04 -0400 Subject: [PATCH 3/3] add substreams::skip_empty_output to all generated code, bump substreams-rs to 0.5.21 --- ethfull/templates/Cargo.toml.gotmpl | 2 +- ethfull/templates/entities/README.md | 10 ---------- ethfull/templates/src/lib.rs.gotmpl | 2 ++ ethfull/testoutput/bayc_triggers/README.md | 10 ---------- ethfull/testoutput/bayc_triggers/substreams/Cargo.toml | 2 +- ethfull/testoutput/bayc_triggers/substreams/src/lib.rs | 2 ++ ethfull/testoutput/complex_abi/Cargo.toml | 2 +- ethfull/testoutput/complex_abi/src/lib.rs | 2 ++ ethfull/testoutput/uniswap_factory_v3/README.md | 10 ---------- .../uniswap_factory_v3/substreams/Cargo.toml | 2 +- .../uniswap_factory_v3/substreams/src/lib.rs | 1 + .../substreams/Cargo.toml | 2 +- .../substreams/src/lib.rs | 2 ++ injective-events/templates/Cargo.toml.gotmpl | 2 +- injective-events/templates/lib.rs.gotmpl | 1 + starknet/templates/Cargo.toml.gotmpl | 2 +- starknet/templates/txfilter/lib.rs.gotmpl | 2 ++ 17 files changed, 19 insertions(+), 37 deletions(-) diff --git a/ethfull/templates/Cargo.toml.gotmpl b/ethfull/templates/Cargo.toml.gotmpl index ba18bf0..02931ed 100644 --- a/ethfull/templates/Cargo.toml.gotmpl +++ b/ethfull/templates/Cargo.toml.gotmpl @@ -14,7 +14,7 @@ num-bigint = "0.4" num-traits = "0.2.15" prost = "0.11" prost-types = "0.11" -substreams = "0.5" +substreams = "0.5.21" substreams-ethereum = "0.9" substreams-database-change = "1" substreams-entity-change = "1" diff --git a/ethfull/templates/entities/README.md b/ethfull/templates/entities/README.md index 39500e8..55f1b40 100644 --- a/ethfull/templates/entities/README.md +++ b/ethfull/templates/entities/README.md @@ -30,16 +30,6 @@ To run a local `graph-node` instance, you will need to install Docker. You can d To run the proto assembly script bindings, you will need to install the `buf` [cli](https://buf.build/docs/installation). -## Run the entire stack with the `run-local.sh` script - -You can run the entire stack (`docker`, `npm` installations and `graph` creation with deployment) by running the below script - -```bash -./run-local.sh -``` - -However, if you want to run each commen individually, follow the instructions below: - ## Install npm and nodeJS packages Run the following command in the `root` of the repository: diff --git a/ethfull/templates/src/lib.rs.gotmpl b/ethfull/templates/src/lib.rs.gotmpl index ceb06d5..c75d800 100644 --- a/ethfull/templates/src/lib.rs.gotmpl +++ b/ethfull/templates/src/lib.rs.gotmpl @@ -428,6 +428,7 @@ fn map_events(blk: eth::Block) -> Result Result { let mut events = contract::Events::default(); map_bayc_events(&blk, &mut events); + substreams::skip_empty_output(); Ok(events) } #[substreams::handlers::map] fn map_calls(blk: eth::Block) -> Result { let mut calls = contract::Calls::default(); map_bayc_calls(&blk, &mut calls); + substreams::skip_empty_output(); Ok(calls) } diff --git a/ethfull/testoutput/complex_abi/Cargo.toml b/ethfull/testoutput/complex_abi/Cargo.toml index e63a30e..f06375d 100644 --- a/ethfull/testoutput/complex_abi/Cargo.toml +++ b/ethfull/testoutput/complex_abi/Cargo.toml @@ -14,7 +14,7 @@ num-bigint = "0.4" num-traits = "0.2.15" prost = "0.11" prost-types = "0.11" -substreams = "0.5" +substreams = "0.5.21" substreams-ethereum = "0.9" substreams-database-change = "1" substreams-entity-change = "1" diff --git a/ethfull/testoutput/complex_abi/src/lib.rs b/ethfull/testoutput/complex_abi/src/lib.rs index d5ca4a8..7a54c35 100644 --- a/ethfull/testoutput/complex_abi/src/lib.rs +++ b/ethfull/testoutput/complex_abi/src/lib.rs @@ -1048,6 +1048,7 @@ fn map_events( let mut events = contract::Events::default(); map_ewqocontraadd123_events(&blk, &mut events); map_test_events(&blk, &store_test, &mut events); + substreams::skip_empty_output(); Ok(events) } #[substreams::handlers::map] @@ -1059,6 +1060,7 @@ fn map_calls( let mut calls = contract::Calls::default(); map_ewqocontraadd123_calls(&blk, &mut calls); map_test_calls(&blk, &store_test, &mut calls); + substreams::skip_empty_output(); Ok(calls) } #[substreams::handlers::map] diff --git a/ethfull/testoutput/uniswap_factory_v3/README.md b/ethfull/testoutput/uniswap_factory_v3/README.md index 39500e8..55f1b40 100644 --- a/ethfull/testoutput/uniswap_factory_v3/README.md +++ b/ethfull/testoutput/uniswap_factory_v3/README.md @@ -30,16 +30,6 @@ To run a local `graph-node` instance, you will need to install Docker. You can d To run the proto assembly script bindings, you will need to install the `buf` [cli](https://buf.build/docs/installation). -## Run the entire stack with the `run-local.sh` script - -You can run the entire stack (`docker`, `npm` installations and `graph` creation with deployment) by running the below script - -```bash -./run-local.sh -``` - -However, if you want to run each commen individually, follow the instructions below: - ## Install npm and nodeJS packages Run the following command in the `root` of the repository: diff --git a/ethfull/testoutput/uniswap_factory_v3/substreams/Cargo.toml b/ethfull/testoutput/uniswap_factory_v3/substreams/Cargo.toml index 11fe5fb..78cc69e 100644 --- a/ethfull/testoutput/uniswap_factory_v3/substreams/Cargo.toml +++ b/ethfull/testoutput/uniswap_factory_v3/substreams/Cargo.toml @@ -14,7 +14,7 @@ num-bigint = "0.4" num-traits = "0.2.15" prost = "0.11" prost-types = "0.11" -substreams = "0.5" +substreams = "0.5.21" substreams-ethereum = "0.9" substreams-database-change = "1" substreams-entity-change = "1" diff --git a/ethfull/testoutput/uniswap_factory_v3/substreams/src/lib.rs b/ethfull/testoutput/uniswap_factory_v3/substreams/src/lib.rs index 183b4cf..47b9058 100644 --- a/ethfull/testoutput/uniswap_factory_v3/substreams/src/lib.rs +++ b/ethfull/testoutput/uniswap_factory_v3/substreams/src/lib.rs @@ -87,6 +87,7 @@ fn map_unifactory_events(blk: ð::Block, events: &mut contract::Events) { fn map_events(blk: eth::Block) -> Result { let mut events = contract::Events::default(); map_unifactory_events(&blk, &mut events); + substreams::skip_empty_output(); Ok(events) } diff --git a/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/Cargo.toml b/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/Cargo.toml index 785528d..28224e2 100644 --- a/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/Cargo.toml +++ b/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/Cargo.toml @@ -14,7 +14,7 @@ num-bigint = "0.4" num-traits = "0.2.15" prost = "0.11" prost-types = "0.11" -substreams = "0.5" +substreams = "0.5.21" substreams-ethereum = "0.9" substreams-database-change = "1" substreams-entity-change = "1" diff --git a/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/src/lib.rs b/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/src/lib.rs index 77419e7..40ec4da 100644 --- a/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/src/lib.rs +++ b/ethfull/testoutput/uniswap_v3_triggers_dynamic_datasources/substreams/src/lib.rs @@ -712,6 +712,7 @@ fn map_events( let mut events = contract::Events::default(); map_factory_events(&blk, &mut events); map_pools_events(&blk, &store_pools, &mut events); + substreams::skip_empty_output(); Ok(events) } #[substreams::handlers::map] @@ -723,6 +724,7 @@ fn map_calls( let mut calls = contract::Calls::default(); map_factory_calls(&blk, &mut calls); map_pools_calls(&blk, &store_pools, &mut calls); + substreams::skip_empty_output(); Ok(calls) } diff --git a/injective-events/templates/Cargo.toml.gotmpl b/injective-events/templates/Cargo.toml.gotmpl index 89785ce..f71cfcc 100644 --- a/injective-events/templates/Cargo.toml.gotmpl +++ b/injective-events/templates/Cargo.toml.gotmpl @@ -14,7 +14,7 @@ strip = "debuginfo" crate-type = ["cdylib"] [dependencies] -substreams = "^0.5.19" +substreams = "0.5.21" substreams-database-change = "1" hex = "0.4.3" prost = "0.11" diff --git a/injective-events/templates/lib.rs.gotmpl b/injective-events/templates/lib.rs.gotmpl index 576a77a..ba914d8 100644 --- a/injective-events/templates/lib.rs.gotmpl +++ b/injective-events/templates/lib.rs.gotmpl @@ -10,6 +10,7 @@ fn db_out(events: EventList) -> Result Result