multichain automated dex trades

.github/workflows/dbt_run.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   dbt-test:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
-    timeout-minutes: 90
+    timeout-minutes: 300
 
     steps:
       - name: Check out repository code

dbt_subprojects/dex/macros/models/_decoding/evm_event_decoding_base.sql

@@ -12,13 +12,26 @@ FROM TABLE (
     decode_evm_event (
       abi => '{{abi}}',
       input => TABLE (
-        SELECT l.* 
+        SELECT  block_number,
+                block_time,
+                block_date,
+                date_trunc('month', block_date) as block_month,
+                block_hash,
+                contract_address, 
+                topic0,
+                topic1,
+                topic2,
+                topic3,
+                data,
+                tx_hash, 
+                index as evt_index,
+                tx_index, 
+                tx_from,
+                tx_to
         FROM {{logs}} l
         WHERE topic0 = {{topic0}}
             {% if is_incremental() %}
             AND {{ incremental_predicate('block_time') }}
-            {% else %}
-            AND block_date >= (SELECT MIN(block_date) FROM {{logs}} WHERE topic0 = {{topic0}})
             {% endif %}
       )
     )

dbt_subprojects/dex/macros/models/_decoding/uniswap_exposed_blockchains_list.sql

@@ -0,0 +1,37 @@
+{% macro uniswap_old_blockchains_list() %}
+   {{ return([
+       "arbitrum"
+       , "avalanche_c"
+       , "base"
+       , "bnb"
+       , "ethereum"
+       , "gnosis"
+       , "polygon"
+    ]) }}
+{% endmacro %} 
+
+
+{% macro uniswap_new_blockchains_list() %}
+   {{ return([
+        "blast"
+       , "bob"
+       , "celo"
+       , "degen"
+       , "fantom"
+       , "kaia"
+       , "linea"
+       , "mantle"
+       , "mode"
+       , "nova"
+       , "ronin"
+       , "scroll"
+       , "sei"
+       , "worldchain"
+       , "zkevm"
+       , "zksync"
+       , "zora"
+    ]) }}
+{% endmacro %} 
+
+
+-- "optimism" has some problems with traces, need to fix later

dbt_subprojects/dex/macros/models/_decoding/uniswap_v2/uniswap_v2_factory_event_decoding.sql

@@ -35,5 +35,8 @@
 {% set topic0 = '0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9' %}
 
 {{ evm_event_decoding_base(logs, abi, topic0) }}
+{% if is_incremental() %}
+WHERE {{ incremental_predicate('block_time') }}
+{% endif %}
 
 {% endmacro %}

dbt_subprojects/dex/macros/models/_decoding/uniswap_v2/uniswap_v2_pool_event_decoding.sql

@@ -47,5 +47,7 @@
 {% set topic0 = '0xd78ad95fa46c994b6551d0da85fc275fe613ce37657fb8d5e3d130840159d822' %}
 
 {{ evm_event_decoding_base(logs, abi, topic0) }}
-
+{% if is_incremental() %}
+WHERE {{ incremental_predicate('block_time') }}
+{% endif %}
 {% endmacro %}

dbt_subprojects/dex/macros/models/_decoding/uniswap_v2/uniswap_v2_trades.sql

@@ -1,26 +1,27 @@
 {% macro uniswap_v2_forks_trades(
-    blockchain = null
-    , dex_type = 'uni-v2'
-    , project = null
+    dex_type = 'uni-v2'
     , version = null
     , Pair_evt_Swap = null
     , Factory_evt_PairCreated = null
-    , pair_column_name = 'pair'
     )
 %}
 
 WITH evt_swap AS (
     SELECT
-        block_number
+        blockchain
+        , block_number
         , block_time
         , to
         , contract_address
         , tx_hash
-        , index
+        , evt_index
         , amount0In
         , amount0Out
         , amount1In
         , amount1Out
+        , tx_from
+        , tx_to
+        , tx_index
     FROM {{ Pair_evt_Swap }}
     {% if is_incremental() %}
     WHERE {{ incremental_predicate('block_time') }}
@@ -30,7 +31,8 @@ WITH evt_swap AS (
 , dexs AS
 (
     SELECT
-        t.block_number
+        t.blockchain
+        , t.block_number
         , t.block_time
         , t.to AS taker
         , t.contract_address AS maker
@@ -40,85 +42,51 @@ WITH evt_swap AS (
         , CASE WHEN amount0In = UINT256 '0' OR amount1Out = UINT256 '0' THEN f.token1 ELSE f.token0 END AS token_sold_address
         , t.contract_address AS project_contract_address
         , t.tx_hash
-        , t.index AS evt_index
+        , t.evt_index
         , f.contract_address as factory_address
+        , t.tx_from
+        , t.tx_to
+        , t.tx_index
     FROM
         evt_swap t
     INNER JOIN
         {{ Factory_evt_PairCreated }} f
-        ON f.{{ pair_column_name }} = t.contract_address
-    INNER JOIN {{ source(blockchain, 'creation_traces') }} ct
-        ON f.{{ pair_column_name }} = ct.address
+        ON f.pair = t.contract_address 
+        AND f.blockchain = t.blockchain
+    INNER JOIN (
+        SELECT address, "from", blockchain, MAX(block_number) as latest_block
+        FROM {{ source('evms', 'creation_traces') }}
+        GROUP BY address, "from", blockchain
+    ) ct
+        ON f.pair = ct.address
         AND f.contract_address = ct."from"
+        AND ct.blockchain = t.blockchain
 )
 
-, base_trades AS (
-    SELECT
-        '{{ blockchain }}' AS blockchain
-        , '{{project}}' AS project
-        , '{{ version }}' AS version
-        , '{{dex_type}}' AS dex_type
-        , CAST(date_trunc('month', dexs.block_time) AS date) AS block_month
-        , CAST(date_trunc('day', dexs.block_time) AS date) AS block_date
-        , dexs.block_time
-        , dexs.block_number
-        , dexs.token_bought_amount_raw
-        , dexs.token_sold_amount_raw
-        , dexs.token_bought_address
-        , dexs.token_sold_address
-        , dexs.taker
-        , dexs.maker
-        , dexs.project_contract_address
-        , dexs.tx_hash
-        , dexs.evt_index
-        , dexs.factory_address
-    FROM
-        dexs
-)
-
-SELECT  base_trades.blockchain
-        , COALESCE(contracts.namespace, base_trades.project) AS project
-        , base_trades.version
-        , base_trades.dex_type
-        , base_trades.factory_address
-        , base_trades.block_month
-        , base_trades.block_date
-        , base_trades.block_time
-        , base_trades.block_number
-        , base_trades.token_bought_amount_raw
-        , base_trades.token_sold_amount_raw
-        , base_trades.token_bought_address
-        , base_trades.token_sold_address
-        , base_trades.taker
-        , base_trades.maker
-        , base_trades.project_contract_address
-        , base_trades.tx_hash
-        , base_trades.evt_index
-FROM base_trades
-INNER JOIN (
-    SELECT
-        tx_hash,
-        array_agg(DISTINCT contract_address) as contract_addresses
-    FROM {{ source('tokens', 'transfers') }}
-    WHERE blockchain = '{{ blockchain }}'
-        {% if is_incremental() %}
-        AND {{ incremental_predicate('block_time') }}
-        {% endif %}
-    GROUP BY tx_hash
-) AS transfers
-ON transfers.tx_hash = base_trades.tx_hash
-    AND contains(transfers.contract_addresses, base_trades.token_bought_address)
-    AND contains(transfers.contract_addresses, base_trades.token_sold_address)
-LEFT JOIN (
-    SELECT
-        address,
-        blockchain,
-        namespace
-    FROM {{ source('evms', 'contracts') }}
-    WHERE blockchain = '{{ blockchain }}'
-        AND namespace IS NOT NULL
-) AS contracts
-ON base_trades.project_contract_address = contracts.address
-  AND base_trades.blockchain = contracts.blockchain
+SELECT
+    blockchain
+    , '{{ version }}' AS version
+    , '{{dex_type}}' AS dex_type
+    , CAST(date_trunc('month', block_time) AS date) AS block_month
+    , CAST(date_trunc('day', block_time) AS date) AS block_date
+    , block_time
+    , block_number
+    , token_bought_amount_raw
+    , token_sold_amount_raw
+    , token_bought_address
+    , token_sold_address
+    , taker
+    , maker
+    , project_contract_address
+    , tx_hash
+    , evt_index
+    , factory_address
+    , tx_from
+    , tx_to
+    , tx_index
+FROM dexs
+{% if is_incremental() %}
+WHERE {{ incremental_predicate('block_time') }}
+{% endif %}
 
 {% endmacro %}

dbt_subprojects/dex/macros/models/_decoding/uniswap_v3/uniswap_v3_factory_event_decoding.sql

@@ -7,6 +7,8 @@
 {% set topic0 = '0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118' %}
 
 {{ evm_event_decoding_base(logs, abi, topic0) }}
-
+{% if is_incremental() %}
+WHERE {{ incremental_predicate('block_time') }}
+{% endif %}
 {% endmacro %}
 

dbt_subprojects/dex/macros/models/_decoding/uniswap_v3/uniswap_v3_pool_event_decoding.sql

@@ -7,6 +7,8 @@
 {% set topic0 = '0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67' %}
 
 {{ evm_event_decoding_base(logs, abi, topic0) }}
-
+{% if is_incremental() %}
+WHERE {{ incremental_predicate('block_time') }}
+{% endif %}
 {% endmacro %}