Merge pull request #166 from atomicals/develop

Fix CBOR encode error on deep copy and various fixes

.github/workflows/tests.yml

@@ -21,6 +21,11 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
+      - name: Setup Python caches
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py','requirements.txt','requirements-test.txt') }}
       - name: Install dependencies
         run: |
           sudo apt-get update

electrumx/lib/util_atomicals.py

@@ -39,7 +39,7 @@
 import pickle
 import math
 from electrumx.lib.hash import sha256, double_sha256
-from cbor2 import dumps, loads, CBORDecodeError
+from cbor2 import dumps, loads, CBORDecodeError, CBORTag
 from collections.abc import Mapping
 from functools import reduce
 from merkletools import MerkleTools
@@ -1282,27 +1282,32 @@ def encode_tx_hash_hex(state):
         cloned_state[encode_tx_hash_hex(key)] = encode_tx_hash_hex(value)
     return cloned_state 
 
-# Auto detect any bytes data and encoded it
+
+# Auto encodes data into structured bytes data.
 def auto_encode_bytes_elements(state):
     if isinstance(state, bytes):
         return {
             '$b': state.hex(),
             '$len': sys.getsizeof(state),
             '$auto': True
         }
-    if not isinstance(state, dict) and not isinstance(state, list):
-        return state 
-
+
+    if isinstance(state, CBORTag):
+        dumped_bytes = dumps(state)
+        return auto_encode_bytes_elements(dumped_bytes)
+
     if isinstance(state, list):
         reformatted_list = []
         for item in state:
             reformatted_list.append(auto_encode_bytes_elements(item))
-        return reformatted_list 
+        return reformatted_list
+
+    if isinstance(state, dict):
+        for key, value in state.items():
+            state[key] = auto_encode_bytes_elements(value)
+
+    return state
 
-    for key, value in state.items():
-        state[key] = auto_encode_bytes_elements(value)
-    return state 
-
 
 # Base atomical commit to reveal delay allowed
 def is_within_acceptable_blocks_for_general_reveal(commit_height, reveal_location_height):

electrumx/server/block_processor.py

@@ -1617,9 +1617,10 @@ def put_or_delete_init_state_updates(self, mint_info, data_payload, Delete):
         height = mint_info['reveal_location_height']
 
         # Make a deep copy of the data payload and remove the reserved sections
-        copied_data_state = copy.deepcopy(data_payload)
-        # Remove any of the reserved sections
-        copied_data_state.pop('args', None)
+        copied_data_state = {}
+        for k, v in data_payload.items():
+            if k != 'args':
+                copied_data_state[k] = v
         init_payload_bytes = dumps(copied_data_state)
         op_struct = {
             'op': 'mod',
@@ -2566,7 +2567,11 @@ def populate_dmitem_subtype_specific_fields(self, atomical):
                 f'parent container not found atomical_id={atomical_id}, '
                 f'parent_container={parent_container}',
             )
-        atomical['$parent_container_name'] = parent_container['$container']
+        # The parent container name may not be populated if it's still in the mempool,
+        # or it's not settled realm request yet. Therefore, check to make sure it exists
+        # before we can populate this dmitem's container name.
+        if parent_container.get('$container'):
+            atomical['$parent_container_name'] = parent_container['$container']
         if status == 'verified' and candidate_id == atomical['atomical_id']:
             atomical['subtype'] = 'dmitem'
             atomical['$dmitem'] = request_dmitem

electrumx/server/controller.py

@@ -97,7 +97,7 @@ async def serve(self, shutdown_event):
         Daemon = env.coin.DAEMON
         BlockProcessor = env.coin.BLOCK_PROCESSOR
 
-        async with Daemon(env.coin, env.daemon_url) as daemon:
+        async with Daemon(env.coin, env.daemon_url, proxy_url=env.daemon_proxy_url) as daemon:
             db = DB(env)
             bp = BlockProcessor(env, db, daemon, notifications)
 

electrumx/server/daemon.py

@@ -54,20 +54,24 @@ def __init__(
             max_workqueue=10,
             init_retry=0.25,
             max_retry=4.0,
+            proxy_url=None
     ):
         self.coin = coin
         self.logger = class_logger(__name__, self.__class__.__name__)
         self.url_index = None
         self.urls = []
         self.set_url(url)
+        self.proxy_url: str | None = proxy_url
+        if proxy_url:
+            self.logger.info(f'Using proxy {proxy_url} for daemon.')
         # Limit concurrent RPC calls to this number.
         # See DEFAULT_HTTP_WORKQUEUE in bitcoind, which is typically 16
         self.workqueue_semaphore = asyncio.Semaphore(value=max_workqueue)
         self.init_retry = init_retry
         self.max_retry = max_retry
         self._height = None
         self.available_rpcs = {}
-        self.session = None
+        self.session: aiohttp.ClientSession | None = None
 
         self._networkinfo_cache = (None, 0)
         self._networkinfo_lock = asyncio.Lock()
@@ -117,7 +121,7 @@ def failover(self):
     async def _send_data(self, data):
         async with self.workqueue_semaphore:
             if self.session:
-                async with self.session.post(self.current_url(), data=data) as resp:
+                async with self.session.post(self.current_url(), data=data, proxy=self.proxy_url) as resp:
                     kind = resp.headers.get('Content-Type', None)
                     if kind == 'application/json':
                         return await resp.json(loads=json_deserialize)

electrumx/server/env.py

@@ -45,6 +45,7 @@ def __init__(self, coin=None):
 
         self.db_dir = self.required('DB_DIRECTORY')
         self.daemon_url = self.required('DAEMON_URL')
+        self.daemon_proxy_url = self.default('DAEMON_PROXY_URL', None)
         if coin is not None:
             assert issubclass(coin, Coin)
             self.coin = coin