feat: add erc1155 support

add erc1155 support, include token_transfer and receipt

fix blockchain-etl#351

docs/commands.md

@@ -128,11 +128,11 @@ First extract token addresses from `contracts.json`
 (Exported with [export_contracts](#export_contracts)):
 
 ```bash
-> ethereumetl filter_items -i contracts.json -p "item['is_erc20'] or item['is_erc721']" | \
+> ethereumetl filter_items -i contracts.json -p "item
+['is_erc20'] or item['is_erc721'] or item['is_erc1155']" | \
 ethereumetl extract_field -f address -o token_addresses.txt
 ```
-
-Then export ERC20 / ERC721 tokens:
+Then export ERC20 / ERC721 / ERC1155 tokens:
 
 ```bash
 > ethereumetl export_tokens --token-addresses token_addresses.txt \

docs/schema.md

@@ -96,13 +96,14 @@ topics                   | string      |
 
 ## contracts.csv
 
-Column                       |    Type     |
+Column                       | Type        |
 -----------------------------|-------------|
 address                      | address     |
 bytecode                     | hex_string  |
 function_sighashes           | string      |
 is_erc20                     | boolean     |
 is_erc721                    | boolean     |
+is_erc1155                   | boolean     |          
 block_number                 | bigint      |
 
 ---

ethereumetl/cli/export_token_transfers.py

@@ -44,7 +44,7 @@
               help='The URI of the web3 provider e.g. file://$HOME/Library/Ethereum/geth.ipc or http://localhost:8545/')
 @click.option('-t', '--tokens', default=None, show_default=True, type=str, multiple=True, help='The list of token addresses to filter by.')
 def export_token_transfers(start_block, end_block, batch_size, output, max_workers, provider_uri, tokens):
-    """Exports ERC20/ERC721 transfers."""
+    """Exports ERC20/ERC721/ERC1155 transfers."""
     job = ExportTokenTransfersJob(
         start_block=start_block,
         end_block=end_block,

ethereumetl/cli/extract_token_transfers.py

@@ -41,7 +41,7 @@
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
 @click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
 def extract_token_transfers(logs, batch_size, output, max_workers, values_as_strings=False):
-    """Extracts ERC20/ERC721 transfers from logs file."""
+    """Extracts ERC20/ERC721/ERC1155 transfers from logs file."""
     with smart_open(logs, 'r') as logs_file:
         if logs.endswith('.json'):
             logs_reader = (json.loads(line) for line in logs_file)

ethereumetl/domain/contract.py

@@ -28,4 +28,5 @@ def __init__(self):
         self.function_sighashes = []
         self.is_erc20 = False
         self.is_erc721 = False
+        self.is_erc1155 = False
         self.block_number = None

ethereumetl/domain/token_transfer.py

@@ -30,3 +30,9 @@ def __init__(self):
         self.transaction_hash = None
         self.log_index = None
         self.block_number = None
+        # support ERC1155
+        self.operator_address = None
+        self.id = None
+        self.ids = []
+        self.values = []
+

ethereumetl/jobs/export_contracts_job.py

@@ -81,6 +81,7 @@ def _get_contract(self, contract_address, rpc_result):
         contract.function_sighashes = function_sighashes
         contract.is_erc20 = self.contract_service.is_erc20_contract(function_sighashes)
         contract.is_erc721 = self.contract_service.is_erc721_contract(function_sighashes)
+        contract.is_erc1155 = self.contract_service.is_erc1155_contract(function_sighashes)
 
         return contract
 

ethereumetl/jobs/exporters/contracts_item_exporter.py

@@ -29,6 +29,7 @@
     'function_sighashes',
     'is_erc20',
     'is_erc721',
+    'is_erc1155',
     'block_number',
 ]
 

ethereumetl/jobs/exporters/receipts_and_logs_item_exporter.py

@@ -44,7 +44,12 @@
     'block_number',
     'address',
     'data',
-    'topics'
+    'topics',
+    'topic1',
+    'topic2',
+    'topic3',
+    'topic4',
+
 ]
 
 

ethereumetl/jobs/exporters/token_transfers_item_exporter.py

@@ -30,7 +30,11 @@
     'value',
     'transaction_hash',
     'log_index',
-    'block_number'
+    'block_number',
+    'operator_address',
+    'id',
+    'ids',
+    'values',
 ]
 
 

ethereumetl/jobs/extract_contracts_job.py

@@ -74,6 +74,7 @@ def _extract_contracts(self, traces):
             contract.function_sighashes = function_sighashes
             contract.is_erc20 = self.contract_service.is_erc20_contract(function_sighashes)
             contract.is_erc721 = self.contract_service.is_erc721_contract(function_sighashes)
+            contract.is_erc1155 = self.contract_service.is_erc1155_contract(function_sighashes)
 
             contracts.append(contract)
 

ethereumetl/jobs/extract_tokens_job.py

@@ -33,7 +33,7 @@ def _export(self):
         self.batch_work_executor.execute(self.contracts_iterable, self._export_tokens_from_contracts)
 
     def _export_tokens_from_contracts(self, contracts):
-        tokens = [contract for contract in contracts if contract.get('is_erc20') or contract.get('is_erc721')]
+        tokens = [contract for contract in contracts if contract.get('is_erc20') or contract.get('is_erc721') or contract.get('is_erc1155')]
 
         for token in tokens:
             self._export_token(token_address=token['address'], block_number=token['block_number'])

ethereumetl/mappers/contract_mapper.py

@@ -41,5 +41,6 @@ def contract_to_dict(self, contract):
             'function_sighashes': contract.function_sighashes,
             'is_erc20': contract.is_erc20,
             'is_erc721': contract.is_erc721,
+            'is_erc1155': contract.is_erc1155,
             'block_number': contract.block_number
         }

ethereumetl/mappers/receipt_log_mapper.py

@@ -76,7 +76,11 @@ def receipt_log_to_dict(self, receipt_log):
             'block_number': receipt_log.block_number,
             'address': receipt_log.address,
             'data': receipt_log.data,
-            'topics': receipt_log.topics
+            'topics': receipt_log.topics,
+            'topic1': receipt_log.topics[0] if len(receipt_log.topics) > 0 else '',
+            'topic2': receipt_log.topics[1] if len(receipt_log.topics) > 1 else '',
+            'topic3': receipt_log.topics[2] if len(receipt_log.topics) > 2 else '',
+            'topic4': receipt_log.topics[3] if len(receipt_log.topics) > 3 else '',
         }
 
     def dict_to_receipt_log(self, dict):
@@ -91,6 +95,7 @@ def dict_to_receipt_log(self, dict):
         receipt_log.data = dict.get('data')
 
         topics = dict.get('topics')
+
         if isinstance(topics, str):
             if len(topics.strip()) == 0:
                 receipt_log.topics = []

ethereumetl/mappers/token_transfer_mapper.py

@@ -32,4 +32,8 @@ def token_transfer_to_dict(self, token_transfer):
             'transaction_hash': token_transfer.transaction_hash,
             'log_index': token_transfer.log_index,
             'block_number': token_transfer.block_number,
+            'operator_address': token_transfer.operator_address,
+            'id': token_transfer.id,
+            'ids': token_transfer.ids,
+            'values': token_transfer.values,
         }

ethereumetl/service/eth_contract_service.py

@@ -33,8 +33,9 @@ def get_function_sighashes(self, bytecode):
             evm_code.disassemble(bytecode)
             basic_blocks = evm_code.basicblocks
             if basic_blocks and len(basic_blocks) > 0:
-                init_block = basic_blocks[0]
-                instructions = init_block.instructions
+                # init_block = basic_blocks[0]
+                # instructions = init_block.instructions
+                instructions = [inst for block in basic_blocks for inst in block.instructions]
                 push4_instructions = [inst for inst in instructions if inst.name == 'PUSH4']
                 return sorted(list(set('0x' + inst.operand for inst in push4_instructions)))
             else:
@@ -69,6 +70,16 @@ def is_erc721_contract(self, function_sighashes):
                c.implements_any_of('transfer(address,uint256)', 'transferFrom(address,address,uint256)') and \
                c.implements('approve(address,uint256)')
 
+    # https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1155.md
+    # https://github.com/OpenZeppelin/openzeppelin-contracts/blob/master/contracts/token/ERC1155/ERC1155.sol
+    def is_erc1155_contract(self, function_sighashes):
+        c = ContractWrapper(function_sighashes)
+        return c.implements('balanceOf(address,uint256)') and \
+               c.implements('safeTransferFrom(address,address,uint256,uint256,bytes)') and \
+               c.implements('safeBatchTransferFrom(address,address,uint256[],uint256[],bytes)')
+               # c.implements('balanceOfBatch(address[],uint256[])') and \
+               # c.implements('setApprovalForAll(address,bool)') and \
+               # c.implements('isApprovedForAll(address,address)')
 
 def clean_bytecode(bytecode):
     if bytecode is None or bytecode == '0x':

ethereumetl/service/eth_token_service.py

@@ -21,7 +21,7 @@
 # SOFTWARE.
 import logging
 
-from web3.exceptions import BadFunctionCallOutput
+from web3.exceptions import BadFunctionCallOutput, ContractLogicError
 
 from ethereumetl.domain.token import EthToken
 from ethereumetl.erc20_abi import ERC20_ABI, ERC20_ABI_ALTERNATIVE_1
@@ -82,7 +82,7 @@ def _call_contract_function(self, func):
         # OverflowError exception happens if the return type of the function doesn't match the expected type
         result = call_contract_function(
             func=func,
-            ignore_errors=(BadFunctionCallOutput, OverflowError, ValueError),
+            ignore_errors=(ContractLogicError, BadFunctionCallOutput, OverflowError, ValueError),
             default_value=None)
 
         if self._function_call_result_transformer is not None:

ethereumetl/service/token_transfer_extractor.py

@@ -29,17 +29,20 @@
 
 # https://ethereum.stackexchange.com/questions/12553/understanding-logs-and-log-blooms
 TRANSFER_EVENT_TOPIC = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
-logger = logging.getLogger(__name__)
+TRANSFER_SINGLE_EVENT_TOPIC = '0xc3d58168c5ae7397731d063d5bbf3d657854427343f4c083240f7aacaa2d0f62'
+TRANSFER_BATCH_EVENT_TOPIC = '0x4a39dc06d4c0dbc64b70af90fd698a233a518aa5d07e595d983b8c0526c8f7fb'
 
+logger = logging.getLogger(__name__)
 
 class EthTokenTransferExtractor(object):
+
     def extract_transfer_from_log(self, receipt_log):
 
         topics = receipt_log.topics
         if topics is None or len(topics) < 1:
             # This is normal, topics can be empty for anonymous events
             return None
-
+        # support ERC20/ERC721
         if (topics[0]).casefold() == TRANSFER_EVENT_TOPIC:
             # Handle unindexed event fields
             topics_with_data = topics + split_to_words(receipt_log.data)
@@ -53,12 +56,57 @@ def extract_transfer_from_log(self, receipt_log):
             token_transfer.token_address = to_normalized_address(receipt_log.address)
             token_transfer.from_address = word_to_address(topics_with_data[1])
             token_transfer.to_address = word_to_address(topics_with_data[2])
-            token_transfer.value = hex_to_dec(topics_with_data[3])
+            # token_transfer.value = hex_to_dec(topics_with_data[3])
+            if len(topics) != 4:
+                token_transfer.value = hex_to_dec(topics_with_data[3])
+            else:
+                token_transfer.id = hex_to_dec(topics_with_data[3])
             token_transfer.transaction_hash = receipt_log.transaction_hash
             token_transfer.log_index = receipt_log.log_index
             token_transfer.block_number = receipt_log.block_number
             return token_transfer
+        elif (topics[0]).casefold() == TRANSFER_SINGLE_EVENT_TOPIC:
+            # Handle unindexed event fields
+            topics_with_data = topics + split_to_words(receipt_log.data)
+            # if the number of topics and fields in data part != 4, then it's a weird event
+            if len(topics_with_data) != 6:
+                logger.warning("The number of topics and data parts is not equal to 6 in log {} of transaction {}"
+                               .format(receipt_log.log_index, receipt_log.transaction_hash))
+                return None
 
+            token_transfer = EthTokenTransfer()
+            token_transfer.token_address = to_normalized_address(receipt_log.address)
+            token_transfer.from_address = word_to_address(topics_with_data[2])
+            token_transfer.to_address = word_to_address(topics_with_data[3])
+            token_transfer.operator_address = word_to_address(topics_with_data[1])
+            token_transfer.value = hex_to_dec(topics_with_data[5])
+            token_transfer.id = hex_to_dec(topics_with_data[4])
+            token_transfer.transaction_hash = receipt_log.transaction_hash
+            token_transfer.log_index = receipt_log.log_index
+            token_transfer.block_number = receipt_log.block_number
+            return token_transfer
+        elif (topics[0]).casefold() == TRANSFER_BATCH_EVENT_TOPIC:
+            # Handle unindexed event fields
+            topics_with_data = topics + split_to_words(receipt_log.data)
+            # if the number of topics and fields in data part != 4, then it's a weird event
+            # if len(topics_with_data) != 10:
+            logger.info("The number of topics and data parts is equal to {} in log {} of transaction {}"
+                        .format(len(topics_with_data), receipt_log.log_index, receipt_log.transaction_hash))
+            # return None
+            nums = hex_to_dec(topics_with_data[6])
+            token_transfer = EthTokenTransfer()
+            token_transfer.token_address = to_normalized_address(receipt_log.address)
+            token_transfer.from_address = word_to_address(topics_with_data[2])
+            token_transfer.to_address = word_to_address(topics_with_data[3])
+            token_transfer.operator_address = word_to_address(topics_with_data[1])
+            # nums = hex_to_dec(topics_with_data[6])
+            for num in range(1, nums + 1):
+                token_transfer.ids.append(hex_to_dec(topics_with_data[num + 6]))
+                token_transfer.values.append(hex_to_dec(topics_with_data[num + 8]))
+            token_transfer.transaction_hash = receipt_log.transaction_hash
+            token_transfer.log_index = receipt_log.log_index
+            token_transfer.block_number = receipt_log.block_number
+            return token_transfer
         return None
 
 

ethereumetl/streaming/enrich.py

@@ -187,6 +187,7 @@ def enrich_contracts(blocks, contracts):
             'function_sighashes',
             'is_erc20',
             'is_erc721',
+            'is_erc1155',
             'block_number'
         ],
         [

ethereumetl/streaming/postgres_tables.py

@@ -148,6 +148,7 @@
     Column('function_sighashes', ARRAY(String)),
     Column('is_erc20', Boolean),
     Column('is_erc721', Boolean),
+    Column('is_erc1155', Boolean),
     Column('block_number', BigInteger),
     PrimaryKeyConstraint('address', 'block_number', name='contracts_pk'),
 )

schemas/aws/contracts.sql

@@ -3,7 +3,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS contracts (
     bytecode STRING,
     function_sighashes STRING,
     is_erc20 BOOLEAN,
-    is_erc721 BOOLEAN
+    is_erc721 BOOLEAN,
+    is_erc1155 BOOLEAN
 )
 PARTITIONED BY (date STRING)
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'