diff --git a/docs/data.rst b/docs/data.rst index 2224484..57cc467 100644 --- a/docs/data.rst +++ b/docs/data.rst @@ -1,390 +1,519 @@ Data ==== -Please see https://github.com/nkaz001/collect-binancefutures or :doc:`Data Preparation ` regarding collecting and converting the feed data. +Please see https://github.com/nkaz001/hftbacktest/tree/master/collector or +:doc:`Data Preparation ` regarding collecting and converting the feed data. Format ------ -`hftbacktest` can digest a `numpy` file such as `npz` or `npy` and pickled `pandas.DataFrame`. The data has 6 columns as follows in the following order. +`hftbacktest` can digest a `numpy` structured array. The data has 8 fields in the following order. +You can also find details in `Event `_. -* event: A type of event - -.. code-block:: python - - DEPTH_EVENT = 1 - TRADE_EVENT = 2 - DEPTH_CLEAR_EVENT = 3 - DEPTH_SNAPSHOT_EVENT = 4 - - -Event code above 100 is used for user-defined events. - -* exch_timestamp: exchange timestamp -* local_timestamp: local timestamp that your system receives -* side: side - 1: Buy(Bid) - -1: Sell(Ask) -* price: price -* qty: quantity - -Example -~~~~~~~ +* ev (u64): You can find the possible flag combinations in `Constants `_. +* exch_ts (i64): Exchange timestamp, which is the time at which the event occurs on the exchange. +* local_ts (i64): Local timestamp, which is the time at which the event is received by the local. +* px (f64): Price +* qty (f64): Quantity +* order_id (u64): Order ID is only for the L3 Market-By-Order feed. +* ival (i64): Reserved for an additional i64 value +* faval (f64): Reserved for an additional f64 value **Raw data** .. code-block:: - 1676419207212527 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} - 1676419207212584 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} + 1676419207212527000 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} + 1676419207212584000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} **Normalized data** .. list-table:: - :widths: 5 10 10 5 5 5 + :widths: 5 10 10 5 5 5 5 5 :header-rows: 1 - * - event - - exch_timestamp - - local_timestamp - - side - - price + * - ev + - exch_ts + - local_ts + - px - qty - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - order_id + - ival + - fval + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 2218.8 - 0.603 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 5000.00 - 2.641 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22160.60 - 0.008 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22172.30 - 0.551 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22173.40 - 0.073 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22174.50 - 0.006 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22176.80 - 0.157 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22177.90 - 0.425 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22181.20 - 0.260 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22182.30 - 3.918 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22182.90 - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22183.40 - 0.014 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22203.00 - 0.000 - - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22171.70 - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22187.30 - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22194.30 - 0.270 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22194.70 - 0.423 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22195.20 - 2.075 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | DEPTH_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22209.60 - 4.506 - * - 2 - - 1676419205116000 - - 1676419207212584 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | TRADE_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205116000000 + - 1676419207212584000 - 22177.90 - 0.001 + - 0 + - 0 + - 0.0 Validation ---------- -Before you start backtesting, you should check if the data is valid. The data that is received from crypto exchanges needs data cleaning and validation. +1. All timestamps must be in the correct order, chronological order. -1. All `timestamp` should be in the correct order. +There can be cases where an event happens before another at the exchange, resulting in an earlier exchange timestamp, +but it is received locally after the other event. +This reverses the chronological order of exchange and local timestamps. To handle this situation, hftbacktest uses the +:const:`EXCH_EVENT ` and :const:`LOCAL_EVENT ` flags. +Events flagged with :const:`EXCH_EVENT ` should be in chronological order according to the +exchange timestamp, while events flagged with :const:`LOCAL_EVENT ` should be in +chronological order according to the local timestamp. -2. You might find `local_timestamp` is advanced to `exch_timestamp` due to time-sync. As `local_timestamp - exch_timestamp` is used as latency, the value must be positive. +2. The exchange timestamp must be earlier than the local timestamp; the feed latency must be positive. -3. Even though `local_timestamp` is in the correct order, `exch_timestamp` can be in the incorrect order. +Due to potential errors in time synchronization between two sites, the local timestamp may be earlier than the exchange +timestamp, resulting in negative latency. The best way to address this is to improve time synchronization using PTP +(Precision Time Protocol), which minimizes the possibility of negative latency. +However, by adding a base latency or offsetting the size of the negative latency, you can ensure that the data remains +valid with only positive latencies, where the local timestamp is always later than the exchange timestamp of the event. -See the following example. `exch_timestamp` of `depth` feed is advanced to the prior `trade` feed even though `depth` feed -is received after `trade` feed. +See the following example. The exchange timestamp of the depth feed is advanced to the prior trade feed even though +the depth feed is received after the trade feed. .. code-block:: - 1676419207212385 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803051, 'p': '22177.90', 'q': '0.300', 'X': 'MARKET', 'm': True}} - 1676419207212480 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803052, 'p': '22177.90', 'q': '0.119', 'X': 'MARKET', 'm': True}} - 1676419207212527 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} - 1676419207212584 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} - 1676419207212621 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803054, 'p': '22177.90', 'q': '0.005', 'X': 'MARKET', 'm': True}} + 1676419207212385000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803051, 'p': '22177.90', 'q': '0.300', 'X': 'MARKET', 'm': True}} + 1676419207212480000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803052, 'p': '22177.90', 'q': '0.119', 'X': 'MARKET', 'm': True}} + 1676419207212527000 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} + 1676419207212584000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} + 1676419207212621000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803054, 'p': '22177.90', 'q': '0.005', 'X': 'MARKET', 'm': True}} -This should be converted into the following form. `hftbacktest` provides `correct` method to automatically correct this type of mess. +This should be converted into the following form. HftBacktest provides :meth:`correct_event_order ` +method to automatically correct this issue. :meth:`validate_event_order ` +helps to check if this issue exists. .. code-block:: - ... - 1676419207212385 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': -1, 's': 'BTCUSDT', 't': 3288803051, 'p': '22177.90', 'q': '0.300', 'X': 'MARKET', 'm': True}} - 1676419207212480 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': -1, 's': 'BTCUSDT', 't': 3288803052, 'p': '22177.90', 'q': '0.119', 'X': 'MARKET', 'm': True}} - 1676419207212527 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} - ... - -1 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803051, 'p': '22177.90', 'q': '0.300', 'X': 'MARKET', 'm': True}} - -1 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803052, 'p': '22177.90', 'q': '0.119', 'X': 'MARKET', 'm': True}} - 1676419207212584 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} - 1676419207212621 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803054, 'p': '22177.90', 'q': '0.005', 'X': 'MARKET', 'm': True}} + EXCH_EVENT 1676419207212527000 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} + EXCH_EVENT | LOCAL_EVENT 1676419207212385000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803051, 'p': '22177.90', 'q': '0.300', 'X': 'MARKET', 'm': True}} + EXCH_EVENT | LOCAL_EVENT 1676419207212480000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206968, 'T': 1676419205111, 's': 'BTCUSDT', 't': 3288803052, 'p': '22177.90', 'q': '0.119', 'X': 'MARKET', 'm': True}} + LOCAL_EVENT 1676419207212527000 {'stream': 'btcusdt@depth@0ms', 'data': {'e': 'depthUpdate', 'E': 1676419206974, 'T': 1676419205108, 's': 'BTCUSDT', 'U': 2505118837831, 'u': 2505118838224, 'pu': 2505118837821, 'b': [['2218.80', '0.603'], ['5000.00', '2.641'], ['22160.60', '0.008'], ['22172.30', '0.551'], ['22173.40', '0.073'], ['22174.50', '0.006'], ['22176.80', '0.157'], ['22177.90', '0.425'], ['22181.20', '0.260'], ['22182.30', '3.918'], ['22182.90', '0.000'], ['22183.40', '0.014'], ['22203.00', '0.000']], 'a': [['22171.70', '0.000'], ['22187.30', '0.000'], ['22194.30', '0.270'], ['22194.70', '0.423'], ['22195.20', '2.075'], ['22209.60', '4.506']]}} + EXCH_EVENT | LOCAL_EVENT 1676419207212584000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803053, 'p': '22177.90', 'q': '0.001', 'X': 'MARKET', 'm': True}} + EXCH_EVENT | LOCAL_EVENT 1676419207212621000 {'stream': 'btcusdt@trade', 'data': {'e': 'trade', 'E': 1676419206976, 'T': 1676419205116, 's': 'BTCUSDT', 't': 3288803054, 'p': '22177.90', 'q': '0.005', 'X': 'MARKET', 'm': True}} **Normalized data** .. list-table:: - :widths: 5 10 10 5 5 5 + :widths: 5 10 10 5 5 5 5 5 :header-rows: 1 - * - event - - exch_timestamp - - local_timestamp - - side - - price + * - ev + - exch_ts + - local_ts + - px - qty - + - order_id + - ival + - fval + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 2218.8 + - 0.603 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 5000.00 + - 2.641 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22160.60 + - 0.008 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22172.30 + - 0.551 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22173.40 + - 0.073 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22174.50 + - 0.006 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22176.80 + - 0.157 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22177.90 + - 0.425 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22181.20 + - 0.260 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22182.30 + - 3.918 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22182.90 + - 0.000 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22183.40 + - 0.014 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | EXCH_EVENT + - 1676419205108000000 + - 1676419207212527000 + - 22203.00 + - 0.000 + - 0 + - 0 + - 0.0 * - ... - - - - - - * - 2 - - -1 - - 1676419207212385 - - -1 + - + - + * - SELL_EVENT | TRADE_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205111000000 + - 1676419207212385000 - 22177.90 - 0.300 - * - 2 - - -1 - - 1676419207212480 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | TRADE_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419205111000000 + - 1676419207212480000 - 22177.90 - 0.119 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 2218.8 - 0.603 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 5000.00 - 2.641 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22160.60 - 0.008 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22172.30 - 0.551 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22173.40 - 0.073 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22174.50 - 0.006 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22176.80 - 0.157 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22177.90 - 0.425 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22181.20 - 0.260 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22182.30 - 3.918 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22182.90 - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22183.40 - 0.014 - * - 1 - - 1676419205108000 - - 1676419207212527 - - 1 + - 0 + - 0 + - 0.0 + * - BUY_EVENT | DEPTH_EVENT | LOCAL_EVENT + - 1676419205108000000 + - 1676419207212527000 - 22203.00 - 0.000 - - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22171.70 - - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22187.30 - - 0.000 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22194.30 - - 0.270 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22194.70 - - 0.423 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22195.20 - - 2.075 - * - 1 - - 1676419205108000 - - 1676419207212527 - - -1 - - 22209.60 - - 4.506 + - 0 + - 0 + - 0.0 * - ... - - - - - - * - 2 - - 1676419205111000 - - -1 - - -1 - - 22177.90 - - 0.300 - * - 2 - - 1676419205111000 - - -1 - - -1 - - 22177.90 - - 0.119 - * - 2 - - 1676419206976000 - - 1676419207212584 - - -1 + - + - + * - SELL_EVENT | TRADE_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419206976000000 + - 1676419207212584000 - 22177.90 - 0.001 - * - 2 - - 1676419206976000 - - 1676419207212621 - - -1 + - 0 + - 0 + - 0.0 + * - SELL_EVENT | TRADE_EVENT | EXCH_EVENT | LOCAL_EVENT + - 1676419206976000000 + - 1676419207212621000 - 22177.90 - 0.005 - -`-1` in `exch_timestamp` means that the event is not processed on exchange-side logic such as order fill. `-1` in `local_timestamp` -means that the event is not recognized by the local. + - 0 + - 0 + - 0.0 diff --git a/docs/index.rst b/docs/index.rst index a35ca3d..7c3ff1c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -266,6 +266,7 @@ Please see the `roadmap Backtester + Constants Statistics Data Validation Data Utilities diff --git a/docs/reference/constants.rst b/docs/reference/constants.rst new file mode 100644 index 0000000..3be843e --- /dev/null +++ b/docs/reference/constants.rst @@ -0,0 +1,52 @@ +Constants +========= + +.. autodata:: hftbacktest.types.EXCH_EVENT + +.. autodata:: hftbacktest.types.LOCAL_EVENT + +.. autodata:: hftbacktest.types.BUY_EVENT + +.. autodata:: hftbacktest.types.SELL_EVENT + +.. autodata:: hftbacktest.order.MARKET + +.. autodata:: hftbacktest.order.LIMIT + +.. autodata:: hftbacktest.order.BUY + +.. autodata:: hftbacktest.order.SELL + +.. autodata:: hftbacktest.order.NONE + +.. autodata:: hftbacktest.order.NEW + +.. autodata:: hftbacktest.order.EXPIRED + +.. autodata:: hftbacktest.order.FILLED + +.. autodata:: hftbacktest.order.PARTIALLY_FILLED + +.. autodata:: hftbacktest.order.CANCELED + +.. autodata:: hftbacktest.order.REJECTED + +.. autodata:: hftbacktest.order.GTC + +.. autodata:: hftbacktest.order.GTX + +.. autodata:: hftbacktest.order.FOK + +.. autodata:: hftbacktest.order.IOC + +.. autodata:: hftbacktest.types.ALL_ASSETS + +.. autodata:: hftbacktest.types.DEPTH_EVENT + +.. autodata:: hftbacktest.types.TRADE_EVENT + +.. autodata:: hftbacktest.types.DEPTH_CLEAR_EVENT + +.. autodata:: hftbacktest.types.DEPTH_SNAPSHOT_EVENT + +.. autodata:: hftbacktest.types.UNTIL_END_OF_DATA diff --git a/docs/reference/data_validation.rst b/docs/reference/data_validation.rst index 9b3813a..d1dd392 100644 --- a/docs/reference/data_validation.rst +++ b/docs/reference/data_validation.rst @@ -1,5 +1,5 @@ Data Validation =============== -.. automodule:: hftbacktest.data.validation +.. automodule:: hftbacktest.data :members: diff --git a/docs/reference/initialization.rst b/docs/reference/initialization.rst index dc7c916..0203249 100644 --- a/docs/reference/initialization.rst +++ b/docs/reference/initialization.rst @@ -6,4 +6,6 @@ Initialization :inherited-members: :member-order: bysource -.. autofunction:: hftbacktest.MultiAssetMultiExchangeBacktest +.. autofunction:: hftbacktest.HashMapMarketDepthBacktest + +.. autofunction:: hftbacktest.ROIVectorMarketDepthBacktest diff --git a/docs/tutorials/Getting Started.ipynb b/docs/tutorials/Getting Started.ipynb index 0074591..c2b896c 100644 --- a/docs/tutorials/Getting Started.ipynb +++ b/docs/tutorials/Getting Started.ipynb @@ -114,10 +114,8 @@ ")\n", "\n", "# HftBacktest provides several types of built-in market depth implementations.\n", - "# HashMapMarketDepthMultiAssetMultiExchangeBacktest constructs a MultiAssetMultiExchangeBacktest\n", - "# using a HashMap-based market depth implementation.\n", - "# Another useful implementation is ROIVectorMarketDepth, which is utilized in\n", - "# ROIVectorMarketDepthMultiAssetMultiExchangeBacktest.\n", + "# HashMapMarketDepthBacktest constructs a Backtest using a HashMap-based market depth implementation.\n", + "# Another useful implementation is ROIVectorMarketDepth, which is utilized in ROIVectorMarketDepthBacktest.\n", "# Please find the details in the document below.\n", "hbt = HashMapMarketDepthBacktest([asset])" ] diff --git a/examples/Getting Started.ipynb b/examples/Getting Started.ipynb index 0074591..c2b896c 100644 --- a/examples/Getting Started.ipynb +++ b/examples/Getting Started.ipynb @@ -114,10 +114,8 @@ ")\n", "\n", "# HftBacktest provides several types of built-in market depth implementations.\n", - "# HashMapMarketDepthMultiAssetMultiExchangeBacktest constructs a MultiAssetMultiExchangeBacktest\n", - "# using a HashMap-based market depth implementation.\n", - "# Another useful implementation is ROIVectorMarketDepth, which is utilized in\n", - "# ROIVectorMarketDepthMultiAssetMultiExchangeBacktest.\n", + "# HashMapMarketDepthBacktest constructs a Backtest using a HashMap-based market depth implementation.\n", + "# Another useful implementation is ROIVectorMarketDepth, which is utilized in ROIVectorMarketDepthBacktest.\n", "# Please find the details in the document below.\n", "hbt = HashMapMarketDepthBacktest([asset])" ] diff --git a/py-hftbacktest/hftbacktest/binding.py b/py-hftbacktest/hftbacktest/binding.py index 3d430c0..fd4fefa 100644 --- a/py-hftbacktest/hftbacktest/binding.py +++ b/py-hftbacktest/hftbacktest/binding.py @@ -421,6 +421,12 @@ def values(self) -> Values: break # Do what you need with the order. + Alternatively, ``has_next`` returns ``True`` if there is a next element and ``False`` otherwise, while also + moving the iterator to the next element internally. ``get`` method then returns the element moved to by the + ``has_next`` method. Since ``has_next`` internally moves the iterator, it should not be used solely to check if + there is a next element. + + **Example** .. code-block:: python @@ -640,11 +646,12 @@ def last_trades(self, asset_no: uint64) -> EVENT_ARRAY: def clear_last_trades(self, asset_no: uint64) -> None: """ - Clears the last trades occurring in the market from the buffer for :func:`trade`. + Clears the last trades occurring in the market from the buffer for :func:`last_trades`. Args: - asset_no: Asset number at which this command will be executed. If :const:`ALL_ASSETS`, all last trades in - any assets will be cleared. + asset_no: Asset number at which this command will be executed. + If :const:`ALL_ASSETS `, + all last trades in any assets will be cleared. """ hashmapbt_clear_last_trades(self.ptr, asset_no) @@ -654,7 +661,8 @@ def orders(self, asset_no: uint64) -> OrderDict: asset_no: Asset number from which orders will be retrieved. Returns: - An order dictionary where the keys are order IDs and the corresponding values are :class:`Order`s. + An order dictionary where the keys are order IDs and the corresponding values are + :class:`Order `. """ return OrderDict_(hashmapbt_orders(self.ptr, asset_no)) @@ -677,17 +685,18 @@ def submit_buy_order( exchange sides. price: Order price. qty: Quantity to buy. - time_in_force: Available options vary depending on the exchange model. See to the exchange model for - details. + time_in_force: Available options vary depending on the exchange model. See to the exchange model for details. + + * :const:`GTC ` + * :const:`GTX ` + * :const:`FOK ` + * :const:`IOC ` - * :const:`GTC` - * :const:`GTX` - * :const:`FOK` - * :const:`IOC` order_type: Available options vary depending on the exchange model. See to the exchange model for details. - * :const:`LIMIT` - * :const:`MARKET` + * :const:`LIMIT ` + * :const:`MARKET ` + wait: If `True`, wait until the order placement response is received. Returns: @@ -715,18 +724,19 @@ def submit_sell_order( order_id: The unique order ID; there should not be any existing order with the same ID on both local and exchange sides. price: Order price. - qty: Quantity to buy. - time_in_force: Available options vary depending on the exchange model. See to the exchange model for - details. + qty: Quantity to sell. + time_in_force: Available options vary depending on the exchange model. See to the exchange model for details. + + * :const:`GTC ` + * :const:`GTX ` + * :const:`FOK ` + * :const:`IOC ` - * :const:`GTC` - * :const:`GTX` - * :const:`FOK` - * :const:`IOC` order_type: Available options vary depending on the exchange model. See to the exchange model for details. - * :const:`LIMIT` - * :const:`MARKET` + * :const:`LIMIT ` + * :const:`MARKET ` + wait: If `True`, wait until the order placement response is received. Returns: @@ -754,12 +764,13 @@ def cancel(self, asset_no: uint64, order_id: uint64, wait: bool) -> int64: def clear_inactive_orders(self, asset_no: uint64) -> None: """ - Clears inactive orders from the local order dictionary whose status is neither :const:`NEW` nor - :const:`PARTIALLY_FILLED`. + Clears inactive orders from the local order dictionary whose status is neither + :const:`NEW ` nor :const:`PARTIALLY_FILLED `. Args: - asset_no: Asset number at which this command will be executed. If :const:`ALL_ASSETS`, all inactive orders - in any assets will be cleared. + asset_no: Asset number at which this command will be executed. + If :const:`ALL_ASSETS `, + all inactive orders in any assets will be cleared. """ hashmapbt_clear_inactive_orders(self.ptr, asset_no) @@ -1054,11 +1065,12 @@ def last_trades(self, asset_no: uint64) -> EVENT_ARRAY: def clear_last_trades(self, asset_no: uint64) -> None: """ - Clears the last trades occurring in the market from the buffer for :func:`trade`. + Clears the last trades occurring in the market from the buffer for :func:`last_trades`. Args: - asset_no: Asset number at which this command will be executed. If :const:`ALL_ASSETS`, all last trades in - any assets will be cleared. + asset_no: Asset number at which this command will be executed. + If :const:`ALL_ASSETS `, + all last trades in any assets will be cleared. """ roivecbt_clear_last_trades(self.ptr, asset_no) @@ -1068,7 +1080,8 @@ def orders(self, asset_no: uint64) -> OrderDict: asset_no: Asset number from which orders will be retrieved. Returns: - An order dictionary where the keys are order IDs and the corresponding values are :class:`Order`s. + An order dictionary where the keys are order IDs and the corresponding values are + :class:`Order `. """ return OrderDict_(roivecbt_orders(self.ptr, asset_no)) @@ -1091,17 +1104,18 @@ def submit_buy_order( exchange sides. price: Order price. qty: Quantity to buy. - time_in_force: Available options vary depending on the exchange model. See to the exchange model for - details. + time_in_force: Available options vary depending on the exchange model. See to the exchange model for details. + + * :const:`GTC ` + * :const:`GTX ` + * :const:`FOK ` + * :const:`IOC ` - * :const:`GTC` - * :const:`GTX` - * :const:`FOK` - * :const:`IOC` order_type: Available options vary depending on the exchange model. See to the exchange model for details. - * :const:`LIMIT` - * :const:`MARKET` + * :const:`LIMIT ` + * :const:`MARKET ` + wait: If `True`, wait until the order placement response is received. Returns: @@ -1129,18 +1143,19 @@ def submit_sell_order( order_id: The unique order ID; there should not be any existing order with the same ID on both local and exchange sides. price: Order price. - qty: Quantity to buy. - time_in_force: Available options vary depending on the exchange model. See to the exchange model for - details. + qty: Quantity to sell. + time_in_force: Available options vary depending on the exchange model. See to the exchange model for details. + + * :const:`GTC ` + * :const:`GTX ` + * :const:`FOK ` + * :const:`IOC ` - * :const:`GTC` - * :const:`GTX` - * :const:`FOK` - * :const:`IOC` order_type: Available options vary depending on the exchange model. See to the exchange model for details. - * :const:`LIMIT` - * :const:`MARKET` + * :const:`LIMIT ` + * :const:`MARKET ` + wait: If `True`, wait until the order placement response is received. Returns: @@ -1168,12 +1183,13 @@ def cancel(self, asset_no: uint64, order_id: uint64, wait: bool) -> int64: def clear_inactive_orders(self, asset_no: uint64) -> None: """ - Clears inactive orders from the local order dictionary whose status is neither :const:`NEW` nor - :const:`PARTIALLY_FILLED`. + Clears inactive orders from the local order dictionary whose status is neither + :const:`NEW ` nor :const:`PARTIALLY_FILLED `. Args: - asset_no: Asset number at which this command will be executed. If :const:`ALL_ASSETS`, all inactive orders - in any assets will be cleared. + asset_no: Asset number at which this command will be executed. + If :const:`ALL_ASSETS `, + all inactive orders in any assets will be cleared. """ roivecbt_clear_inactive_orders(self.ptr, asset_no) diff --git a/py-hftbacktest/hftbacktest/order.py b/py-hftbacktest/hftbacktest/order.py index 398bf02..416bb6e 100644 --- a/py-hftbacktest/hftbacktest/order.py +++ b/py-hftbacktest/hftbacktest/order.py @@ -9,22 +9,54 @@ UNSUPPORTED = 255 BUY = 1 +""" +In the market depth event, this indicates the bid side; in the market trade event, +it indicates that the trade initiator is a buyer. +""" + SELL = -1 +""" +In the market depth event, this indicates the ask side; in the market trade event, +it indicates that the trade initiator is a seller. +""" +#: NONE NONE = 0 + +#: NEW NEW = 1 + +#: EXPIRED EXPIRED = 2 + +#: FILLED FILLED = 3 + +#: CANCELED CANCELED = 4 + +#: PARTIALLY_FILLED PARTIALLY_FILLED = 5 + +#: REJECTED REJECTED = 6 -GTC = 0 # Good 'till cancel -GTX = 1 # Post only -FOK = 2 # Fill or kill -IOC = 3 # Immediate or cancel +#: Good 'till cancel +GTC = 0 + +#: Post only +GTX = 1 + +#: Fill or kill +FOK = 2 +#: Immediate or cancel +IOC = 3 + +#: LIMIT LIMIT = 0 + +#: MARKET MARKET = 1 diff --git a/py-hftbacktest/hftbacktest/stats/__init__.py b/py-hftbacktest/hftbacktest/stats/__init__.py index 3dadf5c..5b6e279 100644 --- a/py-hftbacktest/hftbacktest/stats/__init__.py +++ b/py-hftbacktest/hftbacktest/stats/__init__.py @@ -4,6 +4,7 @@ LinearAssetRecord ) from .metrics import ( + Metric, Ret, AnnualRet, SR, @@ -28,6 +29,7 @@ 'InverseAssetRecord', 'LinearAssetRecord', + 'Metric', 'Ret', 'AnnualRet', 'SR', diff --git a/py-hftbacktest/hftbacktest/types.py b/py-hftbacktest/hftbacktest/types.py index aeaaff7..59a20b0 100644 --- a/py-hftbacktest/hftbacktest/types.py +++ b/py-hftbacktest/hftbacktest/types.py @@ -1,26 +1,48 @@ -import sys from typing import Any import numpy as np -ALL_ASSETS = -1 # the maximum value of uint64_t +#: Indicates all assets. +ALL_ASSETS = -1 # the maximum value of uint64 +#: Indicates that the market depth is changed. DEPTH_EVENT = 1 + +#: Indicates that a trade occurs in the market. TRADE_EVENT = 2 + +#: Indicates that the market depth is cleared. DEPTH_CLEAR_EVENT = 3 + +#: Indicates that the market depth snapshot is received. DEPTH_SNAPSHOT_EVENT = 4 # todo: fix WAIT_ORDER_RESPONSE flags. WAIT_ORDER_RESPONSE_NONE = -1 WAIT_ORDER_RESPONSE_ANY = -2 -UNTIL_END_OF_DATA = sys.maxsize +#: Indicates that one should continue until the end of the data. +UNTIL_END_OF_DATA = 9223372036854775807 # the maximum value of int64 +#: Indicates that it is a valid event to be handled by the exchange processor at the exchange timestamp. EXCH_EVENT = 1 << 31 + +#: Indicates that it is a valid event to be handled by the local processor at the local timestamp. LOCAL_EVENT = 1 << 30 BUY_EVENT = 1 << 29 +""" +Indicates a buy, with specific meaning that can vary depending on the situation. +For example, when combined with a depth event, it means a bid-side event, while when combined with a trade event, +it means that the trade initiator is a buyer. +""" + SELL_EVENT = 1 << 28 +""" +Indicates a sell, with specific meaning that can vary depending on the situation. +For example, when combined with a depth event, it means an ask-side event, while when combined with a trade event, +it means that the trade initiator is a seller. +""" state_values_dtype = np.dtype( [