From 6ca283dd9bcd9fe6810dd4b8edd95b98d08a300a Mon Sep 17 00:00:00 2001 From: Mikko Ohtamaa Date: Wed, 9 Aug 2023 13:04:42 +0200 Subject: [PATCH] Aave reserves data (#145) - Add: Aave v3 reserve data queries --- CHANGELOG.md | 4 + eth_defi/aave_v3/deployer.py | 1 + eth_defi/aave_v3/events.py | 3 +- eth_defi/aave_v3/reserve.py | 351 +++++++++++++++++++++ eth_defi/chain.py | 11 +- eth_defi/trade.py | 2 +- eth_defi/uniswap_v2/analysis.py | 2 +- eth_defi/uniswap_v3/analysis.py | 2 +- tests/aave_v3/test_aave_v3_reserve_data.py | 64 ++++ 9 files changed, 435 insertions(+), 5 deletions(-) create mode 100644 eth_defi/aave_v3/reserve.py create mode 100644 tests/aave_v3/test_aave_v3_reserve_data.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c8b9cb9d..01330c26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Current + +- Add: Aave v3 reserve data queries + # 0.22.7 - Fix: Decimal place adjustment when calculating Uniswap v3 fees diff --git a/eth_defi/aave_v3/deployer.py b/eth_defi/aave_v3/deployer.py index bce1e779..3b829d50 100644 --- a/eth_defi/aave_v3/deployer.py +++ b/eth_defi/aave_v3/deployer.py @@ -355,6 +355,7 @@ # this is the same as mainnet deployment "PoolProxy": "0x763e69d24a03c0c8B256e470D9fE9e0753504D07", "PoolDataProvider": "0x09635F643e140090A9A8Dcd712eD6285858ceBef", + "PoolAddressProvider": "0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f", # https://github.com/aave/aave-v3-periphery/blob/1fdd23b38cc5b6c095687b3c635c4d761ff75c4c/contracts/mocks/testnet-helpers/Faucet.sol "Faucet": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", # TestnetERC20 https://github.com/aave/aave-v3-periphery/blob/1fdd23b38cc5b6c095687b3c635c4d761ff75c4c/contracts/mocks/testnet-helpers/TestnetERC20.sol#L12 diff --git a/eth_defi/aave_v3/events.py b/eth_defi/aave_v3/events.py index c2fc049d..6a40892d 100644 --- a/eth_defi/aave_v3/events.py +++ b/eth_defi/aave_v3/events.py @@ -23,7 +23,8 @@ from eth_defi.event_reader.conversion import ( convert_int256_bytes_to_int, convert_uint256_string_to_address, - decode_data, convert_jsonrpc_value_to_int, + decode_data, + convert_jsonrpc_value_to_int, ) from eth_defi.event_reader.logresult import LogContext from eth_defi.event_reader.reader import LogResult, read_events_concurrent diff --git a/eth_defi/aave_v3/reserve.py b/eth_defi/aave_v3/reserve.py new file mode 100644 index 00000000..c14ca49f --- /dev/null +++ b/eth_defi/aave_v3/reserve.py @@ -0,0 +1,351 @@ +"""Aave v3 pool statistics. + +- Reads Aave pool metrics on-chain + +- Relies on a lot of undocumented Aave v3 source code to pull out the data + +- Based on + + - https://github.com/aave/aave-utilities + - https://github.com/aave/aave-utilities/tree/master/packages/contract-helpers/src/v3-UiPoolDataProvider-contract + - https://github.com/aave/aave-v3-periphery/blob/master/contracts/misc/UiPoolDataProviderV3.sol + - https://github.com/aave/aave-ui/blob/f34f1cfc4fa6c1128b31eaa70b37b5b2109d1dc5/src/libs/pool-data-provider/hooks/use-v2-protocol-data-with-rpc.tsx#L62 + - https://github.com/aave/aave-utilities/blob/664e92b5c7710e8060d4dcac5d6c0ebb48bb069f/packages/math-utils/src/formatters/user/index.ts#L95 + - https://github.com/aave/aave-utilities/blob/664e92b5c7710e8060d4dcac5d6c0ebb48bb069f/packages/math-utils/src/formatters/reserve/index.ts#L310 + +- Aave contracts deployment registry https://docs.aave.com/developers/deployed-contracts/v3-mainnet + +""" +from dataclasses import dataclass +from typing import List, TypeAlias, Tuple, TypedDict, Dict + +from web3 import Web3 +from web3._utils.abi import named_tree +from web3.contract import Contract + +from eth_defi.aave_v3.deployer import AaveDeployer +from eth_defi.event_reader.conversion import convert_jsonrpc_value_to_int + +#: +#: Chain id -> labelled address mapping from Aave documentation +#: +_addresses = { + # Polygon + 137: { + "PoolAddressProvider": "0xa97684ead0e402dC232d5A977953DF7ECBaB3CDb", + "UiPoolDataProviderV3": "0xC69728f11E9E6127733751c8410432913123acf1", + } +} + + +class AaveContractsNotConfigured(Exception): + """We lack hardcoded data of Aave contract addresses.""" + + +@dataclass +class HelperContracts: + """Contracts needed to resolve reserve info on Aave v3.""" + + #: Which EVM chian + chain_id: int + + #: See + #: - https://github.com/aave/aave-v3-periphery/blob/master/contracts/misc/interfaces/IUiPoolDataProviderV3.sol + #: - https://github.com/aave/aave-v3-periphery/blob/master/contracts/misc/UiPoolDataProviderV3.sol + ui_pool_data_provider: Contract + + #: See https://github.com/aave/aave-v3-core/blob/27a6d5c83560694210849d4abf09a09dec8da388/contracts/interfaces/IPoolAddressesProvider.sol#L5 + pool_addresses_provider: Contract + + +#: Quick and dirty "any" Solidity value hack +StructVal: TypeAlias = str | bool | int + + +class AggreatedReserveData(TypedDict): + """Rough mapping of AggreatedReserveData in Aave v3 Solidity source code.""" + + underlyingAsset: StructVal + name: StructVal + symbol: StructVal + decimals: StructVal + baseLTVasCollateral: StructVal + reserveLiquidationThreshold: StructVal + reserveLiquidationBonus: StructVal + reserveFactor: StructVal + usageAsCollateralEnabled: StructVal + borrowingEnabled: StructVal + stableBorrowRateEnabled: StructVal + isActive: StructVal + isFrozen: StructVal + liquidityIndex: StructVal + variableBorrowIndex: StructVal + liquidityRate: StructVal + variableBorrowRate: StructVal + stableBorrowRate: StructVal + lastUpdateTimestamp: StructVal + aTokenAddress: StructVal + stableDebtTokenAddress: StructVal + variableDebtTokenAddress: StructVal + interestRateStrategyAddress: StructVal + availableLiquidity: StructVal + totalPrincipalStableDebt: StructVal + averageStableRate: StructVal + stableDebtLastUpdateTimestamp: StructVal + totalScaledVariableDebt: StructVal + priceInMarketReferenceCurrency: StructVal + priceOracle: StructVal + variableRateSlope1: StructVal + variableRateSlope2: StructVal + stableRateSlope1: StructVal + stableRateSlope2: StructVal + baseStableBorrowRate: StructVal + baseVariableBorrowRate: StructVal + optimalUsageRatio: StructVal + isPaused: StructVal + isSiloedBorrowing: StructVal + accruedToTreasury: StructVal + unbacked: StructVal + isolationModeTotalDebt: StructVal + flashLoanEnabled: StructVal + debtCeiling: StructVal + debtCeilingDecimals: StructVal + eModeCategoryId: StructVal + borrowCap: StructVal + supplyCap: StructVal + eModeLtv: StructVal + eModeLiquidationThreshold: StructVal + eModeLiquidationBonus: StructVal + eModePriceSource: StructVal + eModeLabel: StructVal + borrowableInIsolation: StructVal + + +class BaseCurrencyInfo(TypedDict): + """Rough mapping of BaseCurrencyInfo in Aave v3 Solidity source code.""" + + marketReferenceCurrencyUnit: StructVal + marketReferenceCurrencyPriceInUsd: StructVal + networkBaseTokenPriceInUsd: StructVal + networkBaseTokenPriceDecimals: StructVal + + +class JSONSerialisableReserveData(TypedDict): + """JSON friendly way to store Aave v3 protocol reserve status. + + All ints are converted to JavaScript to avoid BigInt issues. + + .. note :: + + This data is not useful until JavaScript based formatters from + aave-utilities are applied. As writing of this, these formatters are only + available as undocumented JavaScript code in this repository. + `See the repository for more information `__. + + """ + + #: Which chain this was one + chain_id: int + + #: When this fetch was performed + block_number: int + + #: When this fetch was performed + block_hash: str + + #: Unix timestamp when this fetch was performed + timestamp: int + + #: ERC-20 address -> reserve info mapping + reserves: Dict[str, AggreatedReserveData] + + #: Chainlink currency conversion multipliers + base_currency_info: BaseCurrencyInfo + + +def get_helper_contracts(web3: Web3) -> HelperContracts: + """Get helper contracts need to read Aave reserve data. + + :raise AaveContractsNotConfigured: + If we do not have labelled addresses for this chain + """ + chain_id = web3.eth.chain_id + + if chain_id not in _addresses: + raise AaveContractsNotConfigured(f"Chain {chain_id} does not have Aave v3 addresses configured") + + deployer = AaveDeployer() + Contract = deployer.get_contract(web3, "UiPoolDataProviderV3.json") + Contract.decode_tuples = False + ui_pool_data_provider = Contract(Web3.to_checksum_address(_addresses[chain_id]["UiPoolDataProviderV3"])) + + Contract = deployer.get_contract(web3, "PoolAddressesProvider.json") + Contract.decode_tuples = False + pool_addresses_provider = Contract(Web3.to_checksum_address(_addresses[chain_id]["PoolAddressProvider"])) + return HelperContracts( + chain_id, + ui_pool_data_provider, + pool_addresses_provider, + ) + + +def fetch_reserves(contracts: HelperContracts) -> List[str]: + """Enumerate available reserves. + + https://github.com/aave/aave-v3-core/blob/27a6d5c83560694210849d4abf09a09dec8da388/contracts/interfaces/IPool.sol#L603 + + :return: + Returns the list of the underlying assets of all the initialized reserves. + + List of ERC-20 addresses. + """ + reserve_list = contracts.ui_pool_data_provider.functions.getReservesList(contracts.pool_addresses_provider.address).call() + return reserve_list + + +def fetch_reserve_data( + contracts: HelperContracts, + block_identifier=None, +) -> Tuple[List[AggreatedReserveData], BaseCurrencyInfo]: + """Fetch data for all reserves. + + :param contracts: + Helper contracts needed to pull the data + + :return: + List of data of all reserves, currency data from ChainLink used to convert this info for display + + """ + func = contracts.ui_pool_data_provider.functions.getReservesData(contracts.pool_addresses_provider.address) + aggregated_reserve_data, base_currency_info = func.call(block_identifier=block_identifier) + + # Manually decode anonymous tuples to named struct fields + outputs = func.abi["outputs"] + AggregatedReserveData = outputs[0]["components"] + BaseCurrencyInfo = outputs[1]["components"] + + aggregated_reserve_data_decoded = [named_tree(AggregatedReserveData, a) for a in aggregated_reserve_data] + base_currency_info_decoded = named_tree(BaseCurrencyInfo, base_currency_info) + return aggregated_reserve_data_decoded, base_currency_info_decoded + + +def fetch_aave_reserves_snapshop(web3: Web3, block_identifier=None) -> JSONSerialisableReserveData: + """Get a snapshot of all Aave reserves at a certain point of time. + + See :py:class:`JSONSerialisableReserveData` for notes on how to read the output. + + Example: + + .. code-block:: python + + # Read Polygon Aave v3 reserves data at current block + snapshot = fetch_aave_reserves_snapshop(web3) + + Example output: + + .. code-block:: text + + {'block_number': 46092890, + 'block_hash': '0x66b91e13e66978632d7687fa37d61994a092194dd83ab800c4b3fbbfbbc4b882', + 'timestamp': 1691574096, + 'reserves': {'0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063': {'underlyingAsset': '0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063', + 'name': '(PoS) Dai Stablecoin', + 'symbol': 'DAI', + 'decimals': '18', + 'baseLTVasCollateral': '7600', + 'reserveLiquidationThreshold': '8100', + 'reserveLiquidationBonus': '10500', + 'reserveFactor': '1000', + 'usageAsCollateralEnabled': True, + 'borrowingEnabled': True, + 'stableBorrowRateEnabled': True, + 'isActive': True, + 'isFrozen': False, + 'liquidityIndex': '1022026858597482843618393800', + 'variableBorrowIndex': '1039320957656647363864994430', + 'liquidityRate': '28850861922310792585422606', + 'variableBorrowRate': '39579583454495318816309720', + 'stableBorrowRate': '54947447931811914852038715', + 'lastUpdateTimestamp': '1691574072', + 'aTokenAddress': '0x82E64f49Ed5EC1bC6e43DAD4FC8Af9bb3A2312EE', + 'stableDebtTokenAddress': '0xd94112B5B62d53C9402e7A60289c6810dEF1dC9B', + 'variableDebtTokenAddress': '0x8619d80FB0141ba7F184CbF22fd724116D9f7ffC', + 'interestRateStrategyAddress': '0xA9F3C3caE095527061e6d270DBE163693e6fda9D', + 'availableLiquidity': '1889483036044495898670614', + 'totalPrincipalStableDebt': '411830124610128093102375', + 'averageStableRate': '55554322387136738659305167', + 'stableDebtLastUpdateTimestamp': '1691573968', + 'totalScaledVariableDebt': '6509001421349391268535081', + 'priceInMarketReferenceCurrency': '99970000', + 'priceOracle': '0x4746DeC9e833A82EC7C2C1356372CcF2cfcD2F3D', + 'variableRateSlope1': '40000000000000000000000000', + 'variableRateSlope2': '750000000000000000000000000', + 'stableRateSlope1': '5000000000000000000000000', + 'stableRateSlope2': '750000000000000000000000000', + 'baseStableBorrowRate': '50000000000000000000000000', + 'baseVariableBorrowRate': '0', + 'optimalUsageRatio': '800000000000000000000000000', + 'isPaused': False, + 'isSiloedBorrowing': False, + 'accruedToTreasury': '142442743829638527556', + 'unbacked': '0', + 'isolationModeTotalDebt': '0', + 'flashLoanEnabled': True, + 'debtCeiling': '0', + 'debtCeilingDecimals': '2', + 'eModeCategoryId': '1', + 'borrowCap': '30000000', + 'supplyCap': '45000000', + 'eModeLtv': '9300', + 'eModeLiquidationThreshold': '9500', + 'eModeLiquidationBonus': '10100', + 'eModePriceSource': '0x0000000000000000000000000000000000000000', + 'eModeLabel': 'Stablecoins', + 'borrowableInIsolation': True}, + '0x53E0bca35eC356BD5ddDFebbD1Fc0fD03FaBad39': {'underlyingAsset': '0x53E0bca35eC356BD5ddDFebbD1Fc0fD03FaBad39', + 'name': 'ChainLink Token', + 'symbol': 'LINK', + 'decimals': '18', + + + :param web3: + Web3 connection for some of the chain for which we have Aave v3 contract data available. + + :param block_identifier: + Block when to take the snapshot. + + If not given, use the latest block. + + :return: + JSON friendly dict where all ints are converted to string + """ + + helpers = get_helper_contracts(web3) + + if block_identifier is None: + block_identifier = web3.eth.block_number + + block = web3.eth.get_block(block_identifier) + + aggregated_reserve_data, base_currency_info = fetch_reserve_data(helpers) + + reserve_map = {a["underlyingAsset"]: _to_json_friendly(a) for a in aggregated_reserve_data} + + return JSONSerialisableReserveData( + chain_id=helpers.chain_id, + block_number=convert_jsonrpc_value_to_int(block["number"]), + block_hash=block["hash"].hex(), + timestamp=convert_jsonrpc_value_to_int(block["timestamp"]), + reserves=reserve_map, + base_currency_info=base_currency_info, + ) + + +def _to_json_friendly(d: dict) -> dict: + """Deal with JavaScript lacking of good number types""" + result = {} + for k, v in d.items(): + if type(v) == int: + v = str(v) + result[k] = v + return result diff --git a/eth_defi/chain.py b/eth_defi/chain.py index 04beeed3..66e3d328 100644 --- a/eth_defi/chain.py +++ b/eth_defi/chain.py @@ -12,7 +12,7 @@ import requests from web3 import Web3, HTTPProvider -from web3.middleware import geth_poa_middleware +from web3.middleware import geth_poa_middleware, construct_time_based_cache_middleware from web3.providers import JSONBaseProvider, BaseProvider from web3.types import RPCEndpoint, RPCResponse from web3.datastructures import NamedElementOnion @@ -226,3 +226,12 @@ def fetch_block_timestamp(web3: Web3, block_number: int) -> datetime.datetime: timestamp = convert_jsonrpc_value_to_int(block["timestamp"]) time = datetime.datetime.utcfromtimestamp(timestamp) return time + + +def install_retry_middleware(web3: Web3): + """Install gracefully HTTP request retry middleware. + + In the case your Internet connection or JSON-RPC node has issues, + gracefully do exponential backoff retries. + """ + web3.middleware_onion.inject(http_retry_request_with_sleep_middleware, layer=0) diff --git a/eth_defi/trade.py b/eth_defi/trade.py index 57c4a97c..78e9db69 100644 --- a/eth_defi/trade.py +++ b/eth_defi/trade.py @@ -20,7 +20,7 @@ class TradeResult: def get_effective_gas_price_gwei(self) -> Decimal: return Decimal(self.effective_gas_price) / Decimal(10**9) - + def get_cost_of_gas(self) -> Decimal: """This will return the gas cost of the transaction in blockchain's native currency e.g. in ETH on Ethereum.""" return Decimal(self.gas_used) * Decimal(self.effective_gas_price) / Decimal(10**18) diff --git a/eth_defi/uniswap_v2/analysis.py b/eth_defi/uniswap_v2/analysis.py index aff1c053..81bfdcbf 100644 --- a/eth_defi/uniswap_v2/analysis.py +++ b/eth_defi/uniswap_v2/analysis.py @@ -190,7 +190,7 @@ def analyse_trade_by_receipt(web3: Web3, uniswap: UniswapV2Deployment, tx: dict, price = amount_out_cleaned / amount_in_cleaned - lp_fee_paid = float(amount_in * pair_fee / 10 ** in_token_details.decimals) if pair_fee else None + lp_fee_paid = float(amount_in * pair_fee / 10**in_token_details.decimals) if pair_fee else None return TradeSuccess( gas_used, diff --git a/eth_defi/uniswap_v3/analysis.py b/eth_defi/uniswap_v3/analysis.py index 61f693e5..d16eddb6 100644 --- a/eth_defi/uniswap_v3/analysis.py +++ b/eth_defi/uniswap_v3/analysis.py @@ -137,7 +137,7 @@ def analyse_trade_by_receipt( price = pool.convert_price_to_human(tick) # Return price of token0/token1 amount_in = amount0 if amount0 > 0 else amount1 - lp_fee_paid = float(amount_in * pool.fee / 10 ** in_token_details.decimals) + lp_fee_paid = float(amount_in * pool.fee / 10**in_token_details.decimals) return TradeSuccess( gas_used, diff --git a/tests/aave_v3/test_aave_v3_reserve_data.py b/tests/aave_v3/test_aave_v3_reserve_data.py new file mode 100644 index 00000000..341f2435 --- /dev/null +++ b/tests/aave_v3/test_aave_v3_reserve_data.py @@ -0,0 +1,64 @@ +"""Tests for reading reserve data.""" +import os + +import pytest +import requests +from web3 import Web3, HTTPProvider + +from eth_defi.aave_v3.reserve import HelperContracts, get_helper_contracts, fetch_reserves, fetch_reserve_data +from eth_defi.aave_v3.reserve import fetch_aave_reserves_snapshop +from eth_defi.chain import install_chain_middleware, install_retry_middleware + + +JSON_RPC_POLYGON = os.environ.get("JSON_RPC_POLYGON", "https://polygon-rpc.com") +pytestmark = pytest.mark.skipif(not JSON_RPC_POLYGON, reason="This test needs Polygon node via JSON_RPC_POLYGON") + + +@pytest.fixture(scope="module") +def web3(): + """Live Polygon web3 instance.""" + web3 = Web3(HTTPProvider(JSON_RPC_POLYGON, session=requests.Session())) + web3.middleware_onion.clear() + install_chain_middleware(web3) + install_retry_middleware(web3) + return web3 + + +@pytest.fixture(scope="module") +def helpers(web3) -> HelperContracts: + return get_helper_contracts(web3) + + +def test_aave_v3_fetch_reserve_list( + web3: Web3, + helpers: HelperContracts, +): + """Get the list of reserve assets.""" + reserves = fetch_reserves(helpers) + assert "0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174" in reserves + + +def test_aave_v3_fetch_reserve_data( + web3: Web3, + helpers: HelperContracts, +): + """Get the reserve data.""" + + aggregated_reserve_data, base_currency_info = fetch_reserve_data(helpers) + assert aggregated_reserve_data[0]["underlyingAsset"] == "0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063" + assert aggregated_reserve_data[0]["symbol"] == "DAI" + + one_usd = base_currency_info["marketReferenceCurrencyUnit"] + assert one_usd == 100000000 # ChainLink units are used, so one USD multiplier has 8 decimal places + + +def test_aave_v3_fetch_reserve_snapshot( + web3: Web3, +): + """Get the reserve data snapshot.""" + + snapshot = fetch_aave_reserves_snapshop(web3) + assert snapshot["chain_id"] == 137 + assert snapshot["timestamp"] > 0 + assert snapshot["block_number"] > 0 + assert snapshot["reserves"]["0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063"]["symbol"] == "DAI"