From f7022be264bb33f3b8912432a149313baa3d679d Mon Sep 17 00:00:00 2001 From: Daniel Beal Date: Wed, 25 Sep 2024 15:00:45 +0900 Subject: [PATCH 1/4] batch feed_ids to pyth api another optimization that was forgotten about in the first pass --- src/synthetix/utils/multicall.py | 151 ++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 51 deletions(-) diff --git a/src/synthetix/utils/multicall.py b/src/synthetix/utils/multicall.py index 7019395..b977428 100644 --- a/src/synthetix/utils/multicall.py +++ b/src/synthetix/utils/multicall.py @@ -1,3 +1,4 @@ +from eth_typing import HexStr import requests import base64 from web3.exceptions import ContractCustomError @@ -79,13 +80,13 @@ def decode_erc7412_oracle_data_required_error(snx, error): raise Exception("Error data can not be decoded") -def make_fulfillment_request(snx, address, price_update_data, fee, args): +def make_pyth_fulfillment_request(snx, address, update_type, feed_ids, price_update_data, publish_time_or_staleness, fee): erc_contract = snx.web3.eth.contract( address=address, abi=snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["abi"], ) - update_type, publish_time_or_staleness, feed_ids = args + #update_type, publish_time_or_staleness, feed_ids = args encoded_args = encode( ["uint8", "uint64", "bytes32[]", "bytes[]"], [update_type, publish_time_or_staleness, feed_ids, price_update_data], @@ -99,64 +100,62 @@ def make_fulfillment_request(snx, address, price_update_data, fee, args): ).build_transaction({"value": value, "gas": None}) return update_tx["to"], update_tx["data"], update_tx["value"] - -def handle_erc7412_error(snx, error, calls): - "When receiving a ERC7412 error, will return an updated list of calls with the required price updates" +class PythVaaRequest: + feed_ids: list[HexStr] = [] + publish_time = 0 + fee = 0 + +class ERC7412Requests: + pyth_address = '' + pyth_latest: list[HexStr] = [] + pyth_latest_fee = 0 + pyth_vaa: list[PythVaaRequest] = [] + +def aggregate_erc7412_price_requests(snx, error, requests = None) + "Figures out all the prices that have been requested by an ERC7412 error and puts them all in aggregated requests" + if not requests: + requests = ERC7412Requests() if type(error) is ContractCustomError and error.data.startswith(SELECTOR_ERRORS): errors = decode_erc7412_errors_error(error.data) # TODO: execute in parallel for sub_error in errors: - sub_calls = handle_erc7412_error(snx, sub_error, []) - calls = sub_calls + calls + aggregate_erc7412_price_requests(snx, sub_error, requests) - return calls if type(error) is ContractCustomError and ( error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED) or error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED_WITH_FEE) ): # decode error data - address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error( - snx, error.data - ) - update_type = args[0] - - if update_type == 1: - # fetch the data from pyth for those feed ids - if not snx.is_fork: - pyth_data = snx.pyth.get_price_from_ids(feed_ids) - price_update_data = pyth_data["price_update_data"] - else: - # if it's a fork, get the price for the latest block - # this avoids providing "future" prices to the contract on a fork - block = snx.web3.eth.get_block("latest") - - # set a manual 60 second staleness - publish_time = block.timestamp - 60 - pyth_data = snx.pyth.get_price_from_ids( - feed_ids, publish_time=publish_time - ) - price_update_data = pyth_data["price_update_data"] - - # create a new request - to, data, value = make_fulfillment_request( - snx, address, price_update_data, fee, args - ) - elif update_type == 2: - # fetch the data from pyth for those feed ids - pyth_data = snx.pyth.get_price_from_ids(feed_ids, publish_time=args[1]) - price_update_data = pyth_data["price_update_data"] - - # create a new request - to, data, value = make_fulfillment_request( - snx, address, price_update_data, fee, args + update_type = None + address = '' + feed_ids = [] + fee = 0 + args = [] + try: + address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error( + snx, error.data ) - else: - snx.logger.error(f"Unknown update type: {update_type}") - raise error - - calls = [(to, True, value, data)] + calls - return calls + update_type = args[0] + except: + pass + + if update_type: + requests.pyth_address = address + if update_type == 1: + # fetch the data from pyth for those feed ids + requests.pyth_latest = requests.pyth_latest + feed_ids + requests.pyth_latest_fee = requests.pyth_latest_fee + fee + elif update_type == 2: + # fetch the data from pyth for those feed ids + vaa_request = PythVaaRequest() + vaa_request.feed_ids = feed_ids + vaa_request.publish_time = args[1] + vaa_request.fee = fee + requests.pyth_vaa.append(vaa_request) + else: + snx.logger.error(f"Unknown update type: {update_type}") + raise error else: try: is_nonce_error = ( @@ -169,11 +168,61 @@ def handle_erc7412_error(snx, error, calls): if is_nonce_error: snx.logger.debug(f"Error is related to nonce, resetting nonce") snx.nonce = snx.web3.eth.get_transaction_count(snx.address) - return calls + return requests else: snx.logger.debug(f"Error is not related to oracle data") raise error + return requests + +def handle_erc7412_error(snx, error): + "When receiving a ERC7412 error, will return an updated list of calls with the required price updates" + requests = aggregate_erc7412_price_requests(snx, error) + + calls = [] + + if requests.pyth_latest: + # fetch the data from pyth for those feed ids + if not snx.is_fork: + pyth_data = snx.pyth.get_price_from_ids(requests.pyth_latest) + price_update_data = pyth_data["price_update_data"] + else: + # if it's a fork, get the price for the latest block + # this avoids providing "future" prices to the contract on a fork + block = snx.web3.eth.get_block("latest") + + # set a manual 60 second staleness + publish_time = block.timestamp - 60 + pyth_data = snx.pyth.get_price_from_ids( + requests.pyth_latest, publish_time=publish_time + ) + price_update_data = pyth_data["price_update_data"] + + # create a new request + # TODO: the actual number should go here for staleness + to, data, value = make_pyth_fulfillment_request( + snx, requests.pyth_address, 1, requests.pyth_latest, price_update_data, 3600, requests.pyth_latest_fee + ) + + calls.append((to, True, value, data)) + + if requests.pyth_vaa: + for r in requests.pyth_vaa: + # fetch the data from pyth for those feed ids + pyth_data = snx.pyth.get_price_from_ids(r.feed_ids, publish_time=r.publish_time) + price_update_data = pyth_data["price_update_data"] + + # create a new request + to, data, value = make_pyth_fulfillment_request( + snx, requests.pyth_address, 2, r.feed_ids, price_update_data, r.publish_time, r.fee + ) + + calls.append((to, True, value, data)) + + # note: more calls (ex. new oracle providers) can be added here in the future + + return calls + def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]): # prepare the initial call @@ -209,7 +258,7 @@ def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]): snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"): @@ -244,7 +293,7 @@ def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"): snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls def multicall_erc7412( From 2169de217ef441ae7470b611e6734a1da8cef628 Mon Sep 17 00:00:00 2001 From: Troy Date: Wed, 25 Sep 2024 16:52:25 -0600 Subject: [PATCH 2/4] update batching --- src/synthetix/perps/perps.py | 11 +++---- src/synthetix/utils/multicall.py | 53 ++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/synthetix/perps/perps.py b/src/synthetix/perps/perps.py index 8d08f29..a9aa934 100644 --- a/src/synthetix/perps/perps.py +++ b/src/synthetix/perps/perps.py @@ -7,7 +7,7 @@ call_erc7412, multicall_erc7412, write_erc7412, - make_fulfillment_request, + make_pyth_fulfillment_request, ) from .constants import DISABLED_MARKETS from .perps_utils import unpack_bfp_configuration, unpack_bfp_configuration_by_id @@ -230,15 +230,14 @@ def _prepare_oracle_call(self, market_names: [str] = []): price_metadata = pyth_data["meta"] # prepare the oracle call - raw_feed_ids = [decode_hex(feed_id) for feed_id in feed_ids] - args = (1, 30, raw_feed_ids) - - to, data, _ = make_fulfillment_request( + to, data, _ = make_pyth_fulfillment_request( self.snx, self.snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["address"], + 1, # update type + feed_ids, price_update_data, + 30, # staleness tolerance 0, - args, ) value = len(market_names) diff --git a/src/synthetix/utils/multicall.py b/src/synthetix/utils/multicall.py index b977428..4a3860a 100644 --- a/src/synthetix/utils/multicall.py +++ b/src/synthetix/utils/multicall.py @@ -80,38 +80,51 @@ def decode_erc7412_oracle_data_required_error(snx, error): raise Exception("Error data can not be decoded") -def make_pyth_fulfillment_request(snx, address, update_type, feed_ids, price_update_data, publish_time_or_staleness, fee): +def make_pyth_fulfillment_request( + snx, + address, + update_type, + feed_ids, + price_update_data, + publish_time_or_staleness, + fee, +): + # log all of the inputs erc_contract = snx.web3.eth.contract( address=address, abi=snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["abi"], ) - #update_type, publish_time_or_staleness, feed_ids = args + # update_type, publish_time_or_staleness, feed_ids = args + feed_ids = [decode_hex(f) for f in feed_ids] encoded_args = encode( ["uint8", "uint64", "bytes32[]", "bytes[]"], [update_type, publish_time_or_staleness, feed_ids, price_update_data], ) # assume 1 wei per price update - value = fee if fee > 0 else len(price_update_data) * 1 + value = fee if fee > 0 else len(feed_ids) * 1 update_tx = erc_contract.functions.fulfillOracleQuery( encoded_args ).build_transaction({"value": value, "gas": None}) return update_tx["to"], update_tx["data"], update_tx["value"] + class PythVaaRequest: feed_ids: list[HexStr] = [] publish_time = 0 fee = 0 + class ERC7412Requests: - pyth_address = '' + pyth_address = "" pyth_latest: list[HexStr] = [] pyth_latest_fee = 0 pyth_vaa: list[PythVaaRequest] = [] -def aggregate_erc7412_price_requests(snx, error, requests = None) + +def aggregate_erc7412_price_requests(snx, error, requests=None): "Figures out all the prices that have been requested by an ERC7412 error and puts them all in aggregated requests" if not requests: requests = ERC7412Requests() @@ -120,7 +133,9 @@ def aggregate_erc7412_price_requests(snx, error, requests = None) # TODO: execute in parallel for sub_error in errors: - aggregate_erc7412_price_requests(snx, sub_error, requests) + requests = aggregate_erc7412_price_requests(snx, sub_error, requests) + + return requests if type(error) is ContractCustomError and ( error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED) @@ -128,7 +143,7 @@ def aggregate_erc7412_price_requests(snx, error, requests = None) ): # decode error data update_type = None - address = '' + address = "" feed_ids = [] fee = 0 args = [] @@ -175,10 +190,10 @@ def aggregate_erc7412_price_requests(snx, error, requests = None) return requests + def handle_erc7412_error(snx, error): "When receiving a ERC7412 error, will return an updated list of calls with the required price updates" requests = aggregate_erc7412_price_requests(snx, error) - calls = [] if requests.pyth_latest: @@ -201,7 +216,13 @@ def handle_erc7412_error(snx, error): # create a new request # TODO: the actual number should go here for staleness to, data, value = make_pyth_fulfillment_request( - snx, requests.pyth_address, 1, requests.pyth_latest, price_update_data, 3600, requests.pyth_latest_fee + snx, + requests.pyth_address, + 1, + requests.pyth_latest, + price_update_data, + 3600, + requests.pyth_latest_fee, ) calls.append((to, True, value, data)) @@ -209,12 +230,20 @@ def handle_erc7412_error(snx, error): if requests.pyth_vaa: for r in requests.pyth_vaa: # fetch the data from pyth for those feed ids - pyth_data = snx.pyth.get_price_from_ids(r.feed_ids, publish_time=r.publish_time) + pyth_data = snx.pyth.get_price_from_ids( + r.feed_ids, publish_time=r.publish_time + ) price_update_data = pyth_data["price_update_data"] # create a new request to, data, value = make_pyth_fulfillment_request( - snx, requests.pyth_address, 2, r.feed_ids, price_update_data, r.publish_time, r.fee + snx, + requests.pyth_address, + 2, + r.feed_ids, + price_update_data, + r.publish_time, + r.fee, ) calls.append((to, True, value, data)) @@ -346,4 +375,4 @@ def multicall_erc7412( snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls From 22086b88528a205ef93dfc245b2cabe8f3fda734 Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 26 Sep 2024 14:47:56 -0600 Subject: [PATCH 3/4] improve batching --- src/synthetix/pyth/pyth.py | 16 ++++++++++++---- src/synthetix/utils/multicall.py | 7 +++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/synthetix/pyth/pyth.py b/src/synthetix/pyth/pyth.py index 1b12988..afe3931 100644 --- a/src/synthetix/pyth/pyth.py +++ b/src/synthetix/pyth/pyth.py @@ -96,7 +96,17 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None): :return: List of price update data :rtype: [bytes] | None """ - self.logger.info(f"Fetching Pyth data for {len(feed_ids)} markets") + market_names = ",".join( + [ + self.symbol_lookup[feed_id] + for feed_id in feed_ids + if feed_id in self.symbol_lookup + ] + ) + self.logger.info( + f"Fetching Pyth data for {len(feed_ids)} markets ({market_names}) @ {publish_time if publish_time else 'latest'}" + ) + self.logger.debug(f"Fetching data for feed ids: {feed_ids}") params = {"ids[]": feed_ids, "encoding": "hex"} @@ -113,9 +123,7 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None): if response.text and "Price ids not found" in response.text: self.logger.info(f"Removing missing price feeds: {response.text}") feed_ids = [ - feed_id - for feed_id in feed_ids - if feed_id not in response.text + feed_id for feed_id in feed_ids if feed_id not in response.text ] return self._fetch_prices(feed_ids, publish_time=publish_time) diff --git a/src/synthetix/utils/multicall.py b/src/synthetix/utils/multicall.py index 4a3860a..9bfbdd2 100644 --- a/src/synthetix/utils/multicall.py +++ b/src/synthetix/utils/multicall.py @@ -136,7 +136,6 @@ def aggregate_erc7412_price_requests(snx, error, requests=None): requests = aggregate_erc7412_price_requests(snx, sub_error, requests) return requests - if type(error) is ContractCustomError and ( error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED) or error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED_WITH_FEE) @@ -167,7 +166,7 @@ def aggregate_erc7412_price_requests(snx, error, requests=None): vaa_request.feed_ids = feed_ids vaa_request.publish_time = args[1] vaa_request.fee = fee - requests.pyth_vaa.append(vaa_request) + requests.pyth_vaa = requests.pyth_vaa + [vaa_request] else: snx.logger.error(f"Unknown update type: {update_type}") raise error @@ -196,7 +195,7 @@ def handle_erc7412_error(snx, error): requests = aggregate_erc7412_price_requests(snx, error) calls = [] - if requests.pyth_latest: + if len(requests.pyth_latest) > 0: # fetch the data from pyth for those feed ids if not snx.is_fork: pyth_data = snx.pyth.get_price_from_ids(requests.pyth_latest) @@ -227,7 +226,7 @@ def handle_erc7412_error(snx, error): calls.append((to, True, value, data)) - if requests.pyth_vaa: + if len(requests.pyth_vaa) > 0: for r in requests.pyth_vaa: # fetch the data from pyth for those feed ids pyth_data = snx.pyth.get_price_from_ids( From a57d227b31de98afc51b2a54d934d0ccd5ac3e8a Mon Sep 17 00:00:00 2001 From: Troy Date: Thu, 26 Sep 2024 15:58:21 -0600 Subject: [PATCH 4/4] version bump --- src/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.py b/src/setup.py index 6c57068..f86265d 100644 --- a/src/setup.py +++ b/src/setup.py @@ -2,7 +2,7 @@ setup( name="synthetix", - version="0.1.19", + version="0.1.20", description="Synthetix protocol SDK", long_description=open("README.md").read(), long_description_content_type="text/markdown",