diff --git a/hftbacktest/__init__.py b/hftbacktest/__init__.py index c586af0..dd63cd9 100644 --- a/hftbacktest/__init__.py +++ b/hftbacktest/__init__.py @@ -5,10 +5,11 @@ from .reader import COL_EVENT, COL_EXCH_TIMESTAMP, COL_LOCAL_TIMESTAMP, COL_SIDE, COL_PRICE, COL_QTY, \ DEPTH_EVENT, DEPTH_CLEAR_EVENT, DEPTH_SNAPSHOT_EVENT, TRADE_EVENT, DataReader, Cache from .order import BUY, SELL, NONE, NEW, EXPIRED, FILLED, CANCELED, GTC, GTX, Order, OrderBus -from .backtest import SingleInstHftBacktest +from .backtest import SingleAssetHftBacktest from .data import validate_data, correct_local_timestamp, correct_exch_timestamp, correct from .proc.local import Local -from .proc.exchange import NoPartialFillExch +from .proc.nopartialfillexchange import NoPartialFillExchange +from .proc.partialfillexchange import PartialFillExchange from .marketdepth import MarketDepth from .state import State from .models.latencies import FeedLatency, ConstantLatency, ForwardFeedLatency, BackwardFeedLatency, IntpOrderLatency @@ -21,13 +22,14 @@ 'NONE', 'NEW', 'EXPIRED', 'FILLED', 'CANCELED', 'GTC', 'GTX', 'Order', 'HftBacktest', + 'NoPartialFillExchange', 'PartialFillExchange', 'ConstantLatency', 'FeedLatency', 'ForwardFeedLatency', 'BackwardFeedLatency', 'IntpOrderLatency', 'Linear', 'Inverse', 'RiskAverseQueueModel', 'LogProbQueueModel', 'IdentityProbQueueModel', 'SquareProbQueueModel', 'Stat', 'validate_data', 'correct_local_timestamp', 'correct_exch_timestamp', 'correct',) -__version__ = '1.3.1' +__version__ = '1.4.0' def HftBacktest( @@ -43,7 +45,8 @@ def HftBacktest( start_position=0, start_balance=0, start_fee=0, - trade_list_size=0 + trade_list_size=0, + exchange_model=None ): cache = Cache() @@ -160,7 +163,11 @@ def HftBacktest( order_latency, trade_list_size ) - exch = NoPartialFillExch( + + if exchange_model is None: + exchange_model = NoPartialFillExchange + + exch = exchange_model( exch_reader, exch_to_local_orders, local_to_exch_orders, @@ -170,4 +177,4 @@ def HftBacktest( queue_model ) - return SingleInstHftBacktest(local, exch) + return SingleAssetHftBacktest(local, exch) diff --git a/hftbacktest/assettype.py b/hftbacktest/assettype.py index d531b01..4a736bf 100644 --- a/hftbacktest/assettype.py +++ b/hftbacktest/assettype.py @@ -1,29 +1,34 @@ +from numba import int64 from numba.experimental import jitclass @jitclass -class _Linear: - def __init__(self): - pass +class LinearAsset: + contract_size: int64 + + def __init__(self, contract_size=1): + self.contract_size = contract_size def amount(self, exec_price, qty): - return exec_price * qty + return self.contract_size * exec_price * qty def equity(self, price, balance, position, fee): - return balance + position * price - fee + return balance + self.contract_size * position * price - fee @jitclass -class _Inverse: - def __init__(self): - pass +class InverseAsset: + contract_size: int64 + + def __init__(self, contract_size=1): + self.contract_size = contract_size def amount(self, exec_price, qty): - return qty / exec_price + return self.contract_size * qty / exec_price def equity(self, price, balance, position, fee): - return -balance - position / price - fee + return -balance - self.contract_size * position / price - fee -Linear = _Linear() -Inverse = _Inverse() +Linear = LinearAsset() +Inverse = InverseAsset() diff --git a/hftbacktest/backtest.py b/hftbacktest/backtest.py index 7631c9a..a31352e 100644 --- a/hftbacktest/backtest.py +++ b/hftbacktest/backtest.py @@ -1,10 +1,12 @@ from numba import int64, boolean, typeof from numba.experimental import jitclass +from . import BUY +from .order import LIMIT, SELL from .reader import WAIT_ORDER_RESPONSE_NONE, COL_LOCAL_TIMESTAMP, UNTIL_END_OF_DATA -class SingleInstHftBacktest_: +class SingleAssetHftBacktest_: def __init__(self, local, exch): self.local = local self.exch = exch @@ -109,15 +111,15 @@ def last_trades(self): def local_timestamp(self): return self.current_timestamp - def submit_buy_order(self, order_id, price, qty, time_in_force, wait=False): - self.local.submit_buy_order(order_id, price, qty, time_in_force, self.current_timestamp) + def submit_buy_order(self, order_id, price, qty, time_in_force, order_type=LIMIT, wait=False): + self.local.submit_order(order_id, BUY, price, qty, order_type, time_in_force, self.current_timestamp) if wait: return self.goto(UNTIL_END_OF_DATA, wait_order_response=order_id) return True - def submit_sell_order(self, order_id, price, qty, time_in_force, wait=False): - self.local.submit_sell_order(order_id, price, qty, time_in_force, self.current_timestamp) + def submit_sell_order(self, order_id, price, qty, time_in_force, order_type=LIMIT, wait=False): + self.local.submit_order(order_id, SELL, price, qty, order_type, time_in_force, self.current_timestamp) if wait: return self.goto(UNTIL_END_OF_DATA, wait_order_response=order_id) @@ -202,11 +204,11 @@ def goto(self, timestamp, wait_order_response=WAIT_ORDER_RESPONSE_NONE): return True -def SingleInstHftBacktest(local, exch): +def SingleAssetHftBacktest(local, exch): jitted = jitclass(spec=[ ('run', boolean), ('current_timestamp', int64), ('local', typeof(local)), ('exch', typeof(exch)), - ])(SingleInstHftBacktest_) + ])(SingleAssetHftBacktest_) return jitted(local, exch) diff --git a/hftbacktest/order.py b/hftbacktest/order.py index 8d016ab..cee5a21 100644 --- a/hftbacktest/order.py +++ b/hftbacktest/order.py @@ -13,14 +13,21 @@ EXPIRED = 2 FILLED = 3 CANCELED = 4 +PARTIALLY_FILLED = 5 GTC = 0 # Good 'till cancel GTX = 1 # Post only +FOK = 2 # Fill or kill +IOC = 3 # Immediate or cancel + +LIMIT = 0 +MARKET = 1 @jitclass class Order: qty: float64 + leaves_qty: float64 price_tick: int64 tick_size: float64 side: int8 @@ -30,9 +37,11 @@ class Order: local_timestamp: int64 req: int8 exec_price_tick: int64 + exec_qty: float64 order_id: int64 q: float64[:] - limit: boolean + maker: boolean + order_type: int8 def __init__( self, @@ -41,9 +50,11 @@ def __init__( tick_size, qty, side, - time_in_force + time_in_force, + order_type ): self.qty = qty + self.leaves_qty = qty self.price_tick = price_tick self.tick_size = tick_size self.side = side @@ -53,9 +64,16 @@ def __init__( self.local_timestamp = 0 self.req = NONE self.exec_price_tick = 0 + self.exec_qty = 0.0 self.order_id = order_id self.q = np.zeros(2, float64) - self.limit = False + self.maker = False + self.order_type = order_type + + @property + def limit(self): + # compatibility <= 1.3 + return self.maker @property def price(self): @@ -76,16 +94,19 @@ def copy(self): self.tick_size, self.qty, self.side, - self.time_in_force + self.time_in_force, + self.order_type ) + order.leaves_qty = self.leaves_qty order.exch_timestamp = self.exch_timestamp order.status = self.status order.local_timestamp = self.local_timestamp order.req = self.req order.exec_price_tick = self.exec_price_tick + order.exec_qty = self.exec_qty order.order_id = self.order_id - order.q = self.q - order.limit = self.limit + order.q[:] = self.q[:] + order.maker = self.maker return order diff --git a/hftbacktest/proc/local.py b/hftbacktest/proc/local.py index f5d6c93..a522ec0 100644 --- a/hftbacktest/proc/local.py +++ b/hftbacktest/proc/local.py @@ -3,7 +3,7 @@ from numba.experimental import jitclass from .proc import Proc, proc_spec -from ..order import BUY, SELL, NEW, CANCELED, FILLED, EXPIRED, NONE, Order +from ..order import BUY, SELL, NEW, CANCELED, FILLED, EXPIRED, NONE, Order, LIMIT from ..reader import COL_EVENT, COL_LOCAL_TIMESTAMP, COL_SIDE, COL_PRICE, COL_QTY, DEPTH_CLEAR_EVENT, DEPTH_EVENT, \ DEPTH_SNAPSHOT_EVENT, TRADE_EVENT, USER_DEFINED_EVENT @@ -78,18 +78,9 @@ def _process_data(self, row): self.user_data[i] = row[:] return 0 - def submit_buy_order(self, order_id, price, qty, time_in_force, current_timestamp): + def submit_order(self, order_id, side, price, qty, order_type, time_in_force, current_timestamp): price_tick = round(price / self.depth.tick_size) - order = Order(order_id, price_tick, self.depth.tick_size, qty, BUY, time_in_force) - order.req = NEW - exch_recv_timestamp = current_timestamp + self.order_latency.entry(current_timestamp, order, self) - - self.orders[order.order_id] = order - self.orders_to.append(order.copy(), exch_recv_timestamp) - - def submit_sell_order(self, order_id, price, qty, time_in_force, current_timestamp): - price_tick = round(price / self.depth.tick_size) - order = Order(order_id, price_tick, self.depth.tick_size, qty, SELL, time_in_force) + order = Order(order_id, price_tick, self.depth.tick_size, qty, side, time_in_force, order_type) order.req = NEW exch_recv_timestamp = current_timestamp + self.order_latency.entry(current_timestamp, order, self) diff --git a/hftbacktest/proc/exchange.py b/hftbacktest/proc/nopartialfillexchange.py similarity index 84% rename from hftbacktest/proc/exchange.py rename to hftbacktest/proc/nopartialfillexchange.py index 7b7888a..9fcdb9f 100644 --- a/hftbacktest/proc/exchange.py +++ b/hftbacktest/proc/nopartialfillexchange.py @@ -10,7 +10,7 @@ DEPTH_SNAPSHOT_EVENT, TRADE_EVENT -class NoPartialFillExch_(Proc): +class NoPartialFillExchange_(Proc): def __init__( self, reader, @@ -94,23 +94,43 @@ def _process_data(self, row): or (len(self.orders) < price_tick - self.depth.best_bid_tick): for order in list(self.orders.values()): if order.side == SELL: - self.__check_if_sell_filled(order, price_tick, qty, row[COL_EXCH_TIMESTAMP]) + self.__check_if_sell_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) else: for t in range(self.depth.best_bid_tick + 1, price_tick + 1): if t in self.sell_orders: for order in list(self.sell_orders[t].values()): - self.__check_if_sell_filled(order, price_tick, qty, row[COL_EXCH_TIMESTAMP]) + self.__check_if_sell_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) else: if (self.depth.best_ask_tick == INVALID_MAX) \ or (len(self.orders) < self.depth.best_ask_tick - price_tick): for order in list(self.orders.values()): if order.side == BUY: - self.__check_if_buy_filled(order, price_tick, qty, row[COL_EXCH_TIMESTAMP]) + self.__check_if_buy_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) else: for t in range(self.depth.best_ask_tick - 1, price_tick - 1, -1): if t in self.buy_orders: for order in list(self.buy_orders[t].values()): - self.__check_if_buy_filled(order, price_tick, qty, row[COL_EXCH_TIMESTAMP]) + self.__check_if_buy_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) return 0 def __check_if_sell_filled(self, order, price_tick, qty, timestamp): @@ -202,7 +222,10 @@ def __ack_new(self, order, timestamp): else: # The exchange accepts this order. self.orders[order.order_id] = order - o = self.buy_orders.setdefault(order.price_tick, Dict.empty(int64, order_ladder_ty)) + o = self.buy_orders.setdefault( + order.price_tick, + Dict.empty(int64, order_ladder_ty) + ) o[order.order_id] = order # Initialize the order's queue position. self.queue_model.new(order, self) @@ -224,7 +247,10 @@ def __ack_new(self, order, timestamp): else: # The exchange accepts this order. self.orders[order.order_id] = order - o = self.sell_orders.setdefault(order.price_tick, Dict.empty(int64, order_ladder_ty)) + o = self.sell_orders.setdefault( + order.price_tick, + Dict.empty(int64, order_ladder_ty) + ) o[order.order_id] = order # Initialize the order's queue position. self.queue_model.new(order, self) @@ -259,9 +285,18 @@ def __ack_cancel(self, order, timestamp): self.orders_to.append(exch_order.copy(), local_recv_timestamp) return local_recv_timestamp - def __fill(self, order, timestamp, limit, exec_price_tick=0, delete_order=True): - order.limit = limit - order.exec_price_tick = order.price_tick if limit else exec_price_tick + def __fill( + self, + order, + timestamp, + maker, + exec_price_tick=0, + delete_order=True + ): + order.maker = maker + order.exec_price_tick = order.price_tick if maker else exec_price_tick + order.exec_qty = order.leaves_qty + order.leaves_qty = 0 order.status = FILLED order.exch_timestamp = timestamp local_recv_timestamp = order.exch_timestamp + self.order_latency.response(timestamp, order, self) @@ -269,7 +304,6 @@ def __fill(self, order, timestamp, limit, exec_price_tick=0, delete_order=True): if delete_order: del self.orders[order.order_id] - if limit and delete_order: if order.side == BUY: del self.buy_orders[order.price_tick][order.order_id] else: @@ -280,7 +314,7 @@ def __fill(self, order, timestamp, limit, exec_price_tick=0, delete_order=True): return local_recv_timestamp -def NoPartialFillExch( +def NoPartialFillExchange( reader, orders_to_local, orders_from_local, @@ -295,7 +329,7 @@ def NoPartialFillExch( ('buy_orders', DictType(int64, order_ladder_ty)), ('queue_model', typeof(queue_model)) ] - )(NoPartialFillExch_) + )(NoPartialFillExchange_) return jitted( reader, orders_to_local, diff --git a/hftbacktest/proc/partialfillexchange.py b/hftbacktest/proc/partialfillexchange.py new file mode 100644 index 0000000..5e04e7b --- /dev/null +++ b/hftbacktest/proc/partialfillexchange.py @@ -0,0 +1,504 @@ +from numba import typeof, int64 +from numba.experimental import jitclass +from numba.typed.typeddict import Dict +from numba.types import DictType + +import numpy as np + +from .proc import Proc, proc_spec +from ..marketdepth import INVALID_MAX, INVALID_MIN +from ..order import BUY, SELL, NEW, CANCELED, FILLED, EXPIRED, PARTIALLY_FILLED, GTX, FOK, IOC, NONE, order_ladder_ty +from ..reader import COL_EVENT, COL_EXCH_TIMESTAMP, COL_SIDE, COL_PRICE, COL_QTY, DEPTH_CLEAR_EVENT, DEPTH_EVENT, \ + DEPTH_SNAPSHOT_EVENT, TRADE_EVENT + + +class PartialFillExchange_(Proc): + def __init__( + self, + reader, + orders_to_local, + orders_from_local, + depth, + state, + order_latency, + queue_model + ): + self._proc_init( + reader, + orders_to_local, + orders_from_local, + depth, + state, + order_latency + ) + self.sell_orders = Dict.empty(int64, order_ladder_ty) + self.buy_orders = Dict.empty(int64, order_ladder_ty) + self.queue_model = queue_model + + def _next_data_timestamp(self): + return self._next_data_timestamp_column(COL_EXCH_TIMESTAMP) + + def _process_recv_order(self, order, recv_timestamp, wait_resp, next_timestamp): + # Process a new order. + if order.req == NEW: + order.req = NONE + resp_timestamp = self.__ack_new(order, recv_timestamp) + + # Process a cancel order. + elif order.req == CANCELED: + order.req = NONE + resp_timestamp = self.__ack_cancel(order, recv_timestamp) + + else: + raise ValueError('req') + + # Check if the local waits for the order's response. + if wait_resp == order.order_id: + # If next_timestamp is valid, choose the earlier timestamp. + if next_timestamp > 0: + return min(resp_timestamp, next_timestamp) + else: + return resp_timestamp + + # Bypass next_timestamp + return next_timestamp + + def _process_data(self, row): + # Process a depth event + if row[COL_EVENT] == DEPTH_CLEAR_EVENT: + self.depth.clear_depth(row[COL_SIDE], row[COL_PRICE]) + elif row[COL_EVENT] == DEPTH_EVENT or row[COL_EVENT] == DEPTH_SNAPSHOT_EVENT: + if row[COL_SIDE] == BUY: + self.depth.update_bid_depth( + row[COL_PRICE], + row[COL_QTY], + row[COL_EXCH_TIMESTAMP], + self + ) + else: + self.depth.update_ask_depth( + row[COL_PRICE], + row[COL_QTY], + row[COL_EXCH_TIMESTAMP], + self + ) + + # Process a trade event + elif row[COL_EVENT] == TRADE_EVENT: + # Check if a user order is filled. + # To simplify the backtest and avoid a complex market-impact model, all user orders are + # considered to be small enough not to make any market impact. + price_tick = round(row[COL_PRICE] / self.depth.tick_size) + qty = row[COL_QTY] + # This side is a trade initiator's side. + if row[COL_SIDE] == BUY: + # Choose the faster computing path. + if (self.depth.best_bid_tick == INVALID_MIN) \ + or (len(self.orders) < price_tick - self.depth.best_bid_tick): + for order in list(self.orders.values()): + if order.side == SELL: + self.__check_if_sell_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) + else: + for t in range(self.depth.best_bid_tick + 1, price_tick + 1): + if t in self.sell_orders: + for order in list(self.sell_orders[t].values()): + self.__check_if_sell_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) + else: + # Choose the faster computing path. + if (self.depth.best_ask_tick == INVALID_MAX) \ + or (len(self.orders) < self.depth.best_ask_tick - price_tick): + for order in list(self.orders.values()): + if order.side == BUY: + self.__check_if_buy_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) + else: + for t in range(self.depth.best_ask_tick - 1, price_tick - 1, -1): + if t in self.buy_orders: + for order in list(self.buy_orders[t].values()): + self.__check_if_buy_filled( + order, + price_tick, + qty, + row[COL_EXCH_TIMESTAMP] + ) + return 0 + + def __check_if_sell_filled(self, order, price_tick, qty, timestamp): + if order.price_tick < price_tick: + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + elif order.price_tick == price_tick: + # Update the order's queue position. + self.queue_model.trade(order, qty, self) + if self.queue_model.is_filled(order, self): + q_qty = np.ceil(-order.q[0] / self.depth.lot_size) * self.depth.lot_size + exec_qty = min(q_qty, qty, order.leaves_qty) + self.__fill( + order, + exec_qty, + timestamp, + True + ) + + def __check_if_buy_filled(self, order, price_tick, qty, timestamp): + if order.price_tick > price_tick: + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + elif order.price_tick == price_tick: + # Update the order's queue position. + self.queue_model.trade(order, qty, self) + if self.queue_model.is_filled(order, self): + q_qty = np.ceil(-order.q[0] / self.depth.lot_size) * self.depth.lot_size + exec_qty = min(q_qty, qty, order.leaves_qty) + self.__fill( + order, + exec_qty, + timestamp, + True + ) + + def on_new(self, order): + self.queue_model.new(order, self) + + def on_bid_qty_chg( + self, + price_tick, + prev_qty, + new_qty, + timestamp + ): + if price_tick in self.buy_orders: + for order in self.buy_orders[price_tick].values(): + self.queue_model.depth(order, prev_qty, new_qty, self) + + def on_ask_qty_chg( + self, + price_tick, + prev_qty, + new_qty, + timestamp + ): + if price_tick in self.sell_orders: + for order in self.sell_orders[price_tick].values(): + self.queue_model.depth(order, prev_qty, new_qty, self) + + def on_best_bid_update(self, prev_best, new_best, timestamp): + # If the best has been significantly updated compared to the previous best, it would be better to iterate + # orders dict instead of order price ladder. + if (prev_best == INVALID_MIN) \ + or (len(self.orders) < new_best - prev_best): + for order in list(self.orders.values()): + if order.side == SELL and order.price_tick <= new_best: + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + else: + for t in range(prev_best + 1, new_best + 1): + if t in self.sell_orders: + for order in list(self.sell_orders[t].values()): + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + + def on_best_ask_update(self, prev_best, new_best, timestamp): + # If the best has been significantly updated compared to the previous best, it would be better to iterate + # orders dict instead of order price ladder. + if (prev_best == INVALID_MAX) \ + or (len(self.orders) < prev_best - new_best): + for order in list(self.orders.values()): + if order.side == BUY and new_best <= order.price_tick: + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + else: + for t in range(new_best, prev_best): + if t in self.buy_orders: + for order in list(self.buy_orders[t].values()): + self.__fill( + order, + order.leaves_qty, + timestamp, + True + ) + + def __ack_new(self, order, timestamp): + if order.side == BUY: + # Check if the buy order price is greater than or equal to the current best ask. + if order.price_tick >= self.depth.best_ask_tick: + if order.time_in_force == GTX: + order.status = EXPIRED + elif order.time_in_force == FOK: + # The order must be executed immediately in its entirety; otherwise, the entire order will be + # cancelled. + execute = False + cum_qty = 0 + for t in range(self.depth.best_ask_tick, order.price_tick + 1): + cum_qty += self.depth.ask_depth[t] + if round(cum_qty / self.depth.lot_size) >= round(order.qty / self.depth.lot_size): + execute = True + break + if execute: + for t in range(self.depth.best_ask_tick, order.price_tick + 1): + exec_qty = min(self.depth.ask_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + else: + order.status = EXPIRED + elif order.time_in_force == IOC: + # The order must be executed immediately + for t in range(self.depth.best_ask_tick, order.price_tick + 1): + exec_qty = min(self.depth.ask_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + order.status = EXPIRED + else: + # time_in_force == GTC + # Take the market. + for t in range(self.depth.best_ask_tick, order.price_tick): + exec_qty = min(self.depth.ask_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + # The buy order cannot remain in the ask book, as it cannot affect the market depth during + # backtesting based on market-data replay. So, even though it simulates partial fill, if the order + # size is not small enough, it introduces unreality. + return self.__fill( + order, + order.leaves_qty, + timestamp, + False, + exec_price_tick=order.price_tick, + delete_order=False + ) + else: + # The exchange accepts this order. + self.orders[order.order_id] = order + o = self.buy_orders.setdefault( + order.price_tick, + Dict.empty(int64, order_ladder_ty) + ) + o[order.order_id] = order + # Initialize the order's queue position. + self.queue_model.new(order, self) + order.status = NEW + else: + # Check if the sell order price is less than or equal to the current best bid. + if order.price_tick <= self.depth.best_bid_tick: + if order.time_in_force == GTX: + order.status = EXPIRED + elif order.time_in_force == FOK: + # The order must be executed immediately in its entirety; otherwise, the entire order will be + # cancelled. + execute = False + cum_qty = 0 + for t in range(self.depth.best_bid_tick, order.price_tick - 1, -1): + cum_qty += self.depth.bid_depth[t] + if round(cum_qty / self.depth.lot_size) >= round(order.qty / self.depth.lot_size): + execute = True + break + if execute: + for t in range(self.depth.best_bid_tick, order.price_tick - 1, -1): + exec_qty = min(self.depth.bid_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + else: + order.status = EXPIRED + elif order.time_in_force == IOC: + # The order must be executed immediately + for t in range(self.depth.best_bid_tick, order.price_tick - 1, -1): + exec_qty = min(self.depth.bid_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + order.status = EXPIRED + else: + # time_in_force == GTC + # Take the market. + for t in range(self.depth.best_bid_tick, order.price_tick, -1): + exec_qty = min(self.depth.bid_depth[t], order.qty) + local_recv_timestamp = self.__fill( + order, + exec_qty, + timestamp, + False, + exec_price_tick=t, + delete_order=False + ) + if order.status == FILLED: + return local_recv_timestamp + # The sell order cannot remain in the bid book, as it cannot affect the market depth during + # backtesting based on market-data replay. So, even though it simulates partial fill, if the order + # size is not small enough, it introduces unreality. + return self.__fill( + order, + order.leaves_qty, + timestamp, + False, + exec_price_tick=order.price_tick, + delete_order=False + ) + else: + # The exchange accepts this order. + self.orders[order.order_id] = order + o = self.sell_orders.setdefault( + order.price_tick, + Dict.empty(int64, order_ladder_ty) + ) + o[order.order_id] = order + # Initialize the order's queue position. + self.queue_model.new(order, self) + order.status = NEW + order.exch_timestamp = timestamp + local_recv_timestamp = timestamp + self.order_latency.response(timestamp, order, self) + self.orders_to.append(order.copy(), local_recv_timestamp) + return local_recv_timestamp + + def __ack_cancel(self, order, timestamp): + exch_order = self.orders.get(order.order_id) + + # The order can be already deleted due to fill or expiration. + if exch_order is None: + order.status = EXPIRED + order.exch_timestamp = timestamp + local_recv_timestamp = timestamp + self.order_latency.response(timestamp, order, self) + self.orders_to.append(order.copy(), local_recv_timestamp) + return local_recv_timestamp + + # Delete the order. + del self.orders[exch_order.order_id] + if exch_order.side == BUY: + del self.buy_orders[exch_order.price_tick][exch_order.order_id] + else: + del self.sell_orders[exch_order.price_tick][exch_order.order_id] + + # Make the response. + exch_order.status = CANCELED + exch_order.exch_timestamp = timestamp + local_recv_timestamp = timestamp + self.order_latency.response(timestamp, exch_order, self) + self.orders_to.append(exch_order.copy(), local_recv_timestamp) + return local_recv_timestamp + + def __fill( + self, + order, + exec_qty, + timestamp, + maker, + exec_price_tick=0, + delete_order=True + ): + order.maker = maker + order.exec_price_tick = order.price_tick if maker else exec_price_tick + order.exec_qty = exec_qty + order.leaves_qty -= exec_qty + order.status = PARTIALLY_FILLED if round(order.leaves_qty / self.depth.lot_size) > 0 else FILLED + order.exch_timestamp = timestamp + local_recv_timestamp = order.exch_timestamp + self.order_latency.response(timestamp, order, self) + + if delete_order and order.status == FILLED: + del self.orders[order.order_id] + + if order.side == BUY: + del self.buy_orders[order.price_tick][order.order_id] + else: + del self.sell_orders[order.price_tick][order.order_id] + + self.state.apply_fill(order) + self.orders_to.append(order.copy(), local_recv_timestamp) + return local_recv_timestamp + + +def PartialFillExchange( + reader, + orders_to_local, + orders_from_local, + depth, + state, + order_latency, + queue_model +): + jitted = jitclass( + spec=proc_spec(reader, state, order_latency) + [ + ('sell_orders', DictType(int64, order_ladder_ty)), + ('buy_orders', DictType(int64, order_ladder_ty)), + ('queue_model', typeof(queue_model)) + ] + )(PartialFillExchange_) + return jitted( + reader, + orders_to_local, + orders_from_local, + depth, + state, + order_latency, + queue_model + ) diff --git a/hftbacktest/stat.py b/hftbacktest/stat.py index b5c0767..a3abb3f 100644 --- a/hftbacktest/stat.py +++ b/hftbacktest/stat.py @@ -98,13 +98,14 @@ def equity(self, resample=None, include_fee=True): def sharpe(self, resample, include_fee=True, trading_days=365): pnl = self.equity(resample, include_fee=include_fee).diff() c = (24 * 60 * 60 * 1e9) / (pnl.index[1] - pnl.index[0]).value - return pnl.mean() / pnl.std() * np.sqrt(c * trading_days) + std = pnl.std() + return np.divide(pnl.mean(), std) * np.sqrt(c * trading_days) def sortino(self, resample, include_fee=True, trading_days=365): pnl = self.equity(resample, include_fee=include_fee).diff() std = pnl[pnl < 0].std() c = (24 * 60 * 60 * 1e9) / (pnl.index[1] - pnl.index[0]).value - return pnl.mean() / std * np.sqrt(c * trading_days) + return np.divide(pnl.mean(), std) * np.sqrt(c * trading_days) def riskreturnratio(self, include_fee=True): return self.annualised_return(include_fee=include_fee) / self.maxdrawdown(include_fee=include_fee) @@ -159,10 +160,10 @@ def summary(self, capital=None, resample='5min', trading_days=365): rs_pnl = rs_equity.diff() c = (24 * 60 * 60 * 1e9) / (rs_pnl.index[1] - rs_pnl.index[0]).value - sr = rs_pnl.mean() / rs_pnl.std() * np.sqrt(c * trading_days) + sr = np.divide(rs_pnl.mean(), rs_pnl.std()) * np.sqrt(c * trading_days) std = rs_pnl[rs_pnl < 0].std() - sortino = rs_pnl.mean() / std * np.sqrt(c * trading_days) + sortino = np.divide(rs_pnl.mean(), std) * np.sqrt(c * trading_days) max_equity = rs_equity.cummax() drawdown = rs_equity - max_equity diff --git a/hftbacktest/state.py b/hftbacktest/state.py index 0e09a8e..9727c86 100644 --- a/hftbacktest/state.py +++ b/hftbacktest/state.py @@ -23,16 +23,13 @@ def __init__( self.asset_type = asset_type def apply_fill(self, order): - fee = self.maker_fee if order.limit else self.taker_fee - fill_qty = order.qty * order.side - amount = self.asset_type.amount(order.exec_price, order.qty) - fill_amount = amount * order.side - fee_amount = amount * fee - self.position += fill_qty - self.balance -= fill_amount - self.fee += fee_amount + fee = self.maker_fee if order.maker else self.taker_fee + amount = self.asset_type.amount(order.exec_price, order.exec_qty) + self.position += (order.exec_qty * order.side) + self.balance -= (amount * order.side) + self.fee += (amount * fee) self.trade_num += 1 - self.trade_qty += order.qty + self.trade_qty += order.exec_qty self.trade_amount += amount def equity(self, mid):