Token filter module

CHANGELOG.md

@@ -1,11 +1,14 @@
-# Current
+# 0.23
 
 - Add: `fix_dex_price_data` for wrangling DEX price feeds and separate reusable `wrangle` module
 - Add: `resample_dataframe` utility function. Useful for resampling dataframes with multiple columns that are not candlestick data 
 - Add: `fix_bad_wicks(bad_open_close_threshold)` to massage the data - There are ~60 broken open price data points for U/ni v3 price feeds, cause unknown
 - Add: `filter_for_blacklisted_tokens`
 - Add: `DEXPair.from_series` helper
-- 
+- Add: `filter_pairs_default()` that will remove known bad tokens from the trading pair universe with multiple heurestics rules
+- Add: `build_liquidity_summary()` for creating trading universes based on the available liquidity
+- Internal change: Move trading pair filtering logic to its own module
+
 # 0.22.14
 
 - Fix: Fix for inconcistencies in lending data downloads

tests/test_liquidity_universe.py

@@ -6,6 +6,9 @@
 from tradingstrategy.liquidity import GroupedLiquidityUniverse, LiquidityDataUnavailable
 from tradingstrategy.pair import DEXPair, LegacyPairUniverse, PandasPairUniverse
 from tradingstrategy.timebucket import TimeBucket
+from tradingstrategy.utils.forward_fill import forward_fill
+from tradingstrategy.utils.liquidity_filter import build_liquidity_summary
+from tradingstrategy.utils.token_filter import filter_pairs_default
 
 
 def test_grouped_liquidity(persistent_test_client: Client):
@@ -171,3 +174,33 @@ def test_merge_liquidity_samples(persistent_test_client: Client):
 def test_empty_liquididty_universe():
     universe = GroupedLiquidityUniverse.create_empty()
     assert universe.get_sample_count() == 0
+
+
+def test_build_liquidity_summary(persistent_test_client: Client):
+    """See we can put together historical liquidity for backtest filtering.
+
+    - Get liquidity summary for all Uniswap v3 pairs on Etheruem
+    """
+
+    client = persistent_test_client
+
+    exchange_universe = client.fetch_exchange_universe()
+    exchange = exchange_universe.get_by_chain_and_slug(ChainId.ethereum, "uniswap-v3")
+    pairs_df = client.fetch_pair_universe().to_pandas()
+
+    pairs_df = filter_pairs_default(
+        pairs_df,
+        chain_id=ChainId.ethereum,
+        exchanges={exchange},
+        verbose_print=lambda x, y: x,  # Mute
+    )
+
+    liquidity_df = client.fetch_all_liquidity_samples(TimeBucket.d7).to_pandas()
+    liquidity_df = liquidity_df.loc[liquidity_df["pair_id"].isin(pairs_df["pair_id"])]  # Filter to our pair set before forward fill
+    liquidity_df = liquidity_df.set_index("timestamp").groupby("pair_id")
+    liquidity_df = forward_fill(liquidity_df, TimeBucket.d7.to_frequency(), columns=("close",))
+    historical_max, today = build_liquidity_summary(liquidity_df, pairs_df["pair_id"])
+    assert len(historical_max) > 100
+    print(historical_max.most_common(10))
+    for pair_id, liquidity_usd in historical_max.most_common(10):
+        assert liquidity_usd > 0, f"Got zero liquidity for pair {pair_id}"

tests/test_pair_universe.py

@@ -7,6 +7,7 @@
 from tradingstrategy.client import Client
 from tradingstrategy.exchange import ExchangeType, ExchangeNotFoundError
 from tradingstrategy.pair import PandasPairUniverse, resolve_pairs_based_on_ticker, PairNotFoundError, DEXPair, generate_address_columns, TokenNotFound, MultipleTokensWithSymbol
+from tradingstrategy.utils.token_filter import filter_pairs_default
 
 
 @pytest.fixture
@@ -441,3 +442,11 @@ def test_create_two_pair_universe_with_fee(persistent_test_client: Client):
     assert pair_universe.get_count() == 2
 
 
+def test_pair_universe_default_filter(persistent_test_client: Client):
+    """Test default filtering"""
+    client = persistent_test_client
+    pairs_df = client.fetch_pair_universe().to_pandas()
+    filtered_pairs_df = filter_pairs_default(pairs_df)
+    print("All pairs ({len(pairs)}, filtered pairs {len(filtered_pairs_}")
+    assert len(filtered_pairs_df) > 100
+