Add TRD support for Dexter liquidity pool payouts (tzstats) (tezos-re…

…ward-distributor-organization#277)

* Add storage parsing for dexter implementation
* Add a full working pipeline for payouts for dexter-based delegations
* Update rpc_reward_api.py
* update dexter utils file and change function names
* Update requirements.txt
* Calculate frozen rewards with tzkt
* Contributor: amzid, Effort=41h
* Reviewer: dansan566, Effort=1h

.travis.yml

@@ -20,6 +20,7 @@ install:
     # Requirements to run tests
     - pip install pycodestyle
     - pip install parameterized
+    - pip install parse
 
 # Addons necessary for documentation.
 addons:

examples/tz1boot1pK9h2BVGXdyvfQSv8kd1LQM6H889.yaml

@@ -22,6 +22,7 @@ rules_map:
   KT1Ao8UXNJ9Dz71Wx3m8yzYNdnNQp2peqtM0: TOE
   KT1VyxJWhe9oz3v4qwTp2U6Rb17ocHGpJmW0: TOB
   KT19cJWfbDNXT4azVbgTBvtLMeqweuHH8W20: TOF
+  KT1DextebDNXT4azVbgTBvtLMeqweuHH8W20: Dexter
   mindelegation: TOB
 plugins:
   enabled:

requirements.txt

@@ -1,4 +1,5 @@
 requests==2.20.0
 base58==1.0.2
 pyyaml>=4.2b1
-fysom==2.1.5
+fysom==2.1.5
+parse

src/Dexter/dexter_utils.py

@@ -0,0 +1,102 @@
+from time import sleep
+import requests
+from parse import parse
+
+from log_config import main_logger
+logger = main_logger
+
+
+def get_dexter_balance_map(contract_id, snapshot_block, api_provider):
+    big_map_id = api_provider.get_big_map_id(contract_id)
+    listLPs = api_provider.get_liquidity_providers_list(big_map_id, snapshot_block)
+    balanceMap = {}
+    totalLiquidity = 0
+    for LP in listLPs:
+        balanceMap[LP] = {'liquidity_share': listLPs[LP]}
+        totalLiquidity += listLPs[LP]
+    api_provider.update_current_balances_dexter(balanceMap)
+    return balanceMap, totalLiquidity
+
+
+def process_original_delegators_map(delegator_map, contract_id, snapshot_block, api_provider):
+
+    contract_balance = delegator_map[contract_id]['staking_balance']
+    dexter_liquidity_provider_map, totalLiquidity = get_dexter_balance_map(contract_id, snapshot_block, api_provider)
+
+    del delegator_map[contract_id]
+
+    for delegator in dexter_liquidity_provider_map:
+        balance = int(dexter_liquidity_provider_map[delegator]['liquidity_share'] * contract_balance / totalLiquidity)
+        if delegator in delegator_map:
+            delegator_map[delegator]['staking_balance'] += balance
+        else:
+            delegator_map[delegator] = {}
+            delegator_map[delegator]['staking_balance'] = balance
+            delegator_map[delegator]['current_balance'] = dexter_liquidity_provider_map[delegator]['current_balance']
+
+    # url = 'https://api.tzstats.com/tables/op?hash=onydXMUCP5JFQp19VfFk6WJ54LftNxo3a34sw1uE3CbbxhmMNw3&limit=1000&columns=receiver,volume'
+    # resp = requests.get(url, timeout=5)
+    # payouts = resp.json()
+    # for payout in payouts:
+    #    addr = payout[0]
+    #    if (addr in dexter_liquidity_provider_map) and (dexter_liquidity_provider_map[addr]['liquidity_share'] != 0):
+    #        print(addr, payout[1], payout[1] / (dexter_liquidity_provider_map[addr]['liquidity_share'] / totalLiquidity))
+    #    else:
+    #        print(addr, payout[1])
+    # print(sum_delegated_liquidity, contract_balance)
+
+
+def parse_dexter_storage(storage_input):
+    '''
+    Dexter Exchange Contract represents a liquidity pool contract and has the following form
+    storage (pair (big_map %accounts (address :owner)
+                                 (pair (nat :balance)
+                                       (map (address :spender)
+                                            (nat :allowance))))
+              (pair (pair (bool :selfIsUpdatingTokenPool)
+                          (pair (bool :freezeBaker)
+                                (nat :lqtTotal)))
+                    (pair (pair (address :manager)
+                                (address :tokenAddress))
+                          (pair (nat :tokenPool)
+                                (mutez :xtzPool)))));
+    '''
+    storage = {}
+    try:  # Json map format
+        storage['big_map_id'] = storage_input['args'][0]['int']
+
+        storage['selfIsUpdatingTokenPool'] = storage_input['args'][1]['args'][0]['args'][0]['prim']
+
+        storage['freezeBaker'] = storage_input['args'][1]['args'][0]['args'][1]['args'][0]['prim']
+        storage['lqtTotal'] = storage_input['args'][1]['args'][0]['args'][1]['args'][1]['int']
+
+        storage['manager'] = storage_input['args'][1]['args'][1]['args'][0]['args'][0]['string']
+        storage['tokenAddress'] = storage_input['args'][1]['args'][1]['args'][0]['args'][1]['string']
+        storage['tokenPool'] = storage_input['args'][1]['args'][1]['args'][1]['args'][0]['int']
+        storage['xtzPool'] = storage_input['args'][1]['args'][1]['args'][1]['args'][0]['int']
+
+        return storage
+
+    except Exception:
+        try:  # storage through rpc query
+            data = parse('Pair {} (Pair (Pair {} (Pair {} {})) (Pair (Pair "{}" "{}") (Pair {} {})))', storage_input)
+            storage_fields = ['big_map_id', 'selfIsUpdatingTokenPool', 'freezeBaker', 'lqtTotal', 'manager', 'tokenAddress',
+                              'tokenPool', 'xtzPool']
+            for i in range(len(storage_fields)):
+                storage[storage_fields[i]] = data[i]
+            return storage
+        except Exception:
+            logger.warn('Parsing dexter storage not successful')
+            return storage
+
+
+# def test_dexter_implementation(contract_id = 'KT1Puc9St8wdNoGtLiD2WXaHbWU7styaxYhD', snapshot_block = 'BMQn5rnV1U5snTAmocdqzBgtGWd9kpUYnGHTh9zBhVWm5Mh5e5v'):
+#     storage = getContractStorage_rpc(contract_id, snapshot_block)
+#     listLPs = getLiquidityProvidersList_tzstats(storage['big_map_id'])
+#     balanceMap = {}
+#     lqdt_ttl = 0
+#     for i, LP in enumerate(listLPs):
+#         print("{}/{}".format(i, len(listLPs)))
+#         balanceMap[LP] = getBalanceFromBigMap_rpc(storage['big_map_id'], listLPs[LP], snapshot_block)
+#         lqdt_ttl += balanceMap[LP]
+#     assert(lqdt_ttl == int(storage['lqtTotal']))

src/api/reward_api.py

@@ -4,7 +4,11 @@
 class RewardApi(ABC):
     def __init__(self):
         super().__init__()
+        self.dexter_contracts_set = []
 
     @abstractmethod
     def get_rewards_for_cycle_map(self, cycle):
         pass
+
+    def set_dexter_contracts_set(self, dexter_contracts_set):
+        self.dexter_contracts_set = dexter_contracts_set

src/calc/test_calculatePhase0.py

@@ -20,6 +20,7 @@ def test_calculate(self):
         }
 
         api = ProviderFactory(provider='prpc').newRewardApi(nw, BAKING_ADDRESS, '')
+
         model = api.get_rewards_for_cycle_map(410)
 
         phase0 = CalculatePhase0(model)

src/config/yaml_baking_conf_parser.py

@@ -5,7 +5,8 @@
     FULL_SUPPORTERS_SET, MIN_DELEGATION_AMT, PAYMENT_ADDRESS, SPECIALS_MAP, \
     DELEGATOR_PAYS_XFER_FEE, REACTIVATE_ZEROED, DELEGATOR_PAYS_RA_FEE, \
     RULES_MAP, MIN_DELEGATION_KEY, TOF, TOB, TOE, EXCLUDED_DELEGATORS_SET_TOB, \
-    EXCLUDED_DELEGATORS_SET_TOE, EXCLUDED_DELEGATORS_SET_TOF, DEST_MAP, PLUGINS_CONF
+    EXCLUDED_DELEGATORS_SET_TOE, EXCLUDED_DELEGATORS_SET_TOF, DEST_MAP, PLUGINS_CONF, DEXTER, \
+    CONTRACTS_SET
 from util.address_validator import AddressValidator
 from util.fee_validator import FeeValidator
 
@@ -58,6 +59,8 @@ def process(self):
         addr_validator = AddressValidator("dest_map")
         conf_obj[DEST_MAP] = {k: v for k, v in conf_obj[RULES_MAP].items() if addr_validator.isaddress(v)}
 
+        conf_obj[CONTRACTS_SET] = set([k for k, v in conf_obj[RULES_MAP].items() if v.lower() == DEXTER])
+
         # default destination for min_delegation filtered account rewards
         if MIN_DELEGATION_KEY not in conf_obj[RULES_MAP]:
             conf_obj[EXCLUDED_DELEGATORS_SET_TOB].add(MIN_DELEGATION_KEY)

src/model/baking_conf.py

@@ -21,12 +21,14 @@
 EXCLUDED_DELEGATORS_SET_TOE = "__excluded_delegators_set_toe"
 EXCLUDED_DELEGATORS_SET_TOF = "__excluded_delegators_set_tof"
 DEST_MAP = "__destination_map"
+CONTRACTS_SET = '__contracts_set'
 
 # destination map
 TOF = "TOF"
 TOB = "TOB"
 TOE = "TOE"
 MIN_DELEGATION_KEY = 'mindelegation'
+DEXTER = 'dexter'
 
 
 class BakingConf:
@@ -83,6 +85,9 @@ def get_delegator_pays_ra_fee(self):
     def get_rule_map(self):
         return self.get_attribute(RULES_MAP)
 
+    def get_contracts_set(self):
+        return self.get_attribute(CONTRACTS_SET)
+
     def get_dest_map(self):
         return self.get_attribute(DEST_MAP)
 

src/pay/payment_producer.py

@@ -46,6 +46,12 @@ def __init__(self, name, initial_payment_cycle, network_config, payments_dir, ca
         self.reward_api = provider_factory.newRewardApi(
             network_config, self.baking_address, self.node_url, node_url_public, api_base_url)
         self.block_api = provider_factory.newBlockApi(network_config, self.node_url, api_base_url)
+        dexter_contracts_set = baking_cfg.get_contracts_set()
+        if len(dexter_contracts_set) > 0 and not (self.reward_api.name == 'tzstats'):
+            logger.warn("The Dexter funktionality is currently supported only using tzstats."
+                        "The contract address will be treated as a normal delegator.")
+        else:
+            self.reward_api.set_dexter_contracts_set(dexter_contracts_set)
 
         self.fee_calc = service_fee_calc
         self.initial_payment_cycle = initial_payment_cycle
@@ -159,20 +165,20 @@ def run(self):
 
                         # Paying upcoming cycles (-R in [-6, -11] )
                         if pymnt_cycle >= current_cycle:
-                            if self.reward_api.name == 'tzstats':
+                            if self.reward_api.name == 'tzstats' or self.reward_api.name == 'tzkt':
                                 logger.warn("Please note that you are doing payouts for future rewards!!! These rewards are not earned yet, they are an estimation given by tzstats.")
                                 result = self.try_to_pay(pymnt_cycle, expected_reward=True)
                             else:
-                                logger.error("This feature is only possible using tzstats. Please consider changing the provider using the -P flag.")
+                                logger.error("This feature is currently not possible using the rpc provider. Please consider changing the provider using the -P flag.")
                                 self.exit()
                                 break
                         # Paying cycles with frozen rewards (-R in [-1, -5] )
                         elif pymnt_cycle >= current_cycle - self.nw_config['NB_FREEZE_CYCLE']:
-                            if self.reward_api.name == 'tzstats':
+                            if self.reward_api.name == 'tzstats' or self.reward_api.name == 'tzkt':
                                 logger.warn("Please note that you are doing payouts for frozen rewards!!!")
                                 result = self.try_to_pay(pymnt_cycle)
                             else:
-                                logger.error("This feature is only possible using tzstats. Please consider changing the provider using the -P flag or wait until the rewards are unfrozen.")
+                                logger.error("This feature is currently not possible using the rpc provider. Please consider changing the provider using the -P flag.")
                                 self.exit()
                                 break
                         # If user wants to offset payments within a cycle, check here

src/rpc/rpc_reward_api.py

@@ -5,6 +5,7 @@
 from api.reward_api import RewardApi
 from log_config import main_logger
 from model.reward_provider_model import RewardProviderModel
+from Dexter import dexter_utils as dxtz
 
 logger = main_logger
 
@@ -13,6 +14,8 @@
 COMM_BLOCK = "{}/chains/main/blocks/{}"
 COMM_SNAPSHOT = COMM_BLOCK + "/context/raw/json/cycle/{}/roll_snapshot"
 COMM_DELEGATE_BALANCE = "{}/chains/main/blocks/{}/context/contracts/{}/balance"
+COMM_CONTRACT_STORAGE = "{}/chains/main/blocks/{}/context/contracts/{}/storage"
+COMM_BIGMAP_QUERY = "{}/chains/main/blocks/{}/context/big_maps/{}/{}"
 
 
 class RpcRewardApiImpl(RewardApi):
@@ -61,6 +64,10 @@ def get_rewards_for_cycle_map(self, cycle):
             logger.warning("Please wait until the rewards and fees for cycle {} are unfrozen".format(cycle))
             reward_data["total_rewards"] = 0
 
+        _, snapshot_level = self.__get_roll_snapshot_block_level(cycle, current_level)
+        for delegator in self.dexter_contracts_set:
+            dxtz.process_original_delegators_map(reward_data["delegators"], delegator, snapshot_level)
+
         reward_model = RewardProviderModel(reward_data["delegate_staking_balance"], reward_data["total_rewards"],
                                            reward_data["delegators"])
 
@@ -135,6 +142,61 @@ def update_current_balances(self, reward_logs):
                 logger.warning("update_current_balances - unexpected error: {}".format(e), exc_info=True)
                 raise e from e
 
+    def get_contract_storage(self, contract_id, block):
+        get_contract_storage_request = COMM_CONTRACT_STORAGE.format(self.node_url, block, contract_id)
+
+        contract_storage_response = None
+
+        while not contract_storage_response:
+            sleep(0.4)  # Be nice to public RPC
+            try:
+                contract_storage_response = self.do_rpc_request(get_contract_storage_request, time_out=5)
+            except requests.exceptions.RequestException as e:
+                # Catch HTTP-related errors and retry
+                logger.debug("Fetching contract storage {} failed, will retry: {}", contract_id, e)
+                sleep(2.0)
+            except Exception as e:
+                # Anything else, raise up
+                raise e from e
+
+        return contract_storage_response
+
+    def get_big_map_id(self, contract_id):
+        storage = self.get_contract_storage(contract_id, 'head')
+        parsed_storage = dxtz.parse_dexter_storage(storage)
+        return parsed_storage['big_map_id']
+
+    def get_address_value_from_big_map(self, big_map_id, address_script_expr, snapshot_block):
+        get_address_value_request = COMM_BIGMAP_QUERY.format(self.node_url, snapshot_block, big_map_id, address_script_expr)
+
+        address_value_response = None
+
+        while not address_value_response:
+            sleep(0.4)  # Be nice to public RPC
+            try:
+                address_value_response = self.do_rpc_request(get_address_value_request, time_out=5)
+            except requests.exceptions.RequestException as e:
+                # Catch HTTP-related errors and retry
+                logger.debug("Fetching address value {} failed, will retry: {}", address_script_expr, e)
+                sleep(2.0)
+            except Exception as e:
+                # Anything else, raise up
+                raise e from e
+
+        return address_value_response
+
+    def get_liquidity_provider_balance(self, big_map_id, address_script_expr, snapshot_block):
+        big_map_value = self.get_address_value_from_big_map(big_map_id, address_script_expr, snapshot_block)
+        int(big_map_value.json()['args'][0]['int'])
+
+    def get_liquidity_providers_list(self, big_map_id, snapshot_block, verbose=False):
+        pass
+
+    def update_current_balances_dexter(self, balanceMap):
+        for address in balanceMap:
+            curr_balance = self.__get_current_balance_of_delegator(address)
+            balanceMap[address].update({"current_balance": curr_balance})
+
     def __get_current_level(self):
         head = self.do_rpc_request(COMM_HEAD.format(self.node_url))
         current_level = int(head["metadata"]["level"]["level"])
@@ -179,7 +241,6 @@ def __get_delegators_and_delgators_balances(self, cycle, current_level):
 
             # Loop over delegators; get snapshot balance, and current balance
             for idx, delegator in enumerate(delegators_addresses):
-
                 # create new dictionary for each delegator
                 d_info = {"staking_balance": 0, "current_balance": 0}
 

src/tzkt/tzkt_api.py

@@ -292,3 +292,6 @@ def get_protocol_by_cycle(self, cycle: int) -> dict:
         }
         """
         return self._request(f'protocols/cycles/{cycle}')
+
+    def get_snapshot_level(self, cycle):
+        return self._request(f'cycles/{cycle}')['snapshotLevel']

src/tzkt/tzkt_reward_api.py

@@ -100,6 +100,10 @@ def get_rewards_for_cycle_map(self, cycle, expected_reward=False) -> RewardProvi
             if item['balance'] > 0
         }
 
+        # snapshot_level = self.api.get_snapshot_level(cycle)
+        # for delegator in self.dexter_contracts_set:
+        #    dxtz.process_original_delegators_map(delegators_balances, delegator, snapshot_level)
+
         return RewardProviderModel(delegate_staking_balance, total_reward_amount, delegators_balances)
 
     def update_current_balances(self, reward_logs: List[RewardLog]):

src/tzstats/tzstats_reward_api.py

@@ -3,6 +3,7 @@
 from log_config import main_logger
 from model.reward_provider_model import RewardProviderModel
 from tzstats.tzstats_reward_provider_helper import TzStatsRewardProviderHelper
+from Dexter import dexter_utils as dxtz
 
 logger = main_logger
 
@@ -25,6 +26,10 @@ def get_rewards_for_cycle_map(self, cycle, expected_reward=False):
         total_reward_amount = root["total_reward_amount"]
         delegators_balances_dict = root["delegators_balances"]
 
+        snapshot_level = self.helper.get_snapshot_level(cycle)
+        for delegator in self.dexter_contracts_set:
+            dxtz.process_original_delegators_map(delegators_balances_dict, delegator, snapshot_level, self.helper)
+
         return RewardProviderModel(delegate_staking_balance, total_reward_amount, delegators_balances_dict)
 
     def update_current_balances(self, reward_logs):