Skip to content

Commit

Permalink
Merge pull request #17 from jhonabreul/feature-null-history-for-unsup…
Browse files Browse the repository at this point in the history 
…ported-securities

Return null on unsupported history requests
  • Loading branch information
@jhonabreul
jhonabreul authored Feb 27, 2024
2 parents da283bf + de6e19a commit 1bba7c5
Show file tree
Hide file tree
Showing 4 changed files with 134 additions and 115 deletions.
152 changes: 70 additions & 82 deletions QuantConnect.BybitBrokerage.Tests/BybitBrokerageHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
using QuantConnect.Securities;
using QuantConnect.Data.Market;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.HistoricalData;

namespace QuantConnect.BybitBrokerage.Tests
{
Expand All @@ -51,6 +50,8 @@ private static TestCaseData[] ValidHistory
{
get
{
TestGlobals.Initialize();

return new[]
{
// valid
Expand All @@ -62,40 +63,39 @@ private static TestCaseData[] ValidHistory
}
}

private static TestCaseData[] NoHistory
{
get
{
return new[]
{
new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Bybit),
Resolution.Second, Time.OneMinute, TickType.Trade),
new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Bybit),
Resolution.Minute, Time.OneHour, TickType.Quote),
};
}
}

private static TestCaseData[] InvalidHistory
{
get
{
return new[]
{
// invalid period, no error, empty result
new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade),
// invalid period
new TestCaseData(ETHUSDT, Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade),

// invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ"
// invalid symbol
new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Bybit), Resolution.Daily,
TimeSpan.FromDays(15), TickType.Trade),

//invalid security type
new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade),

// invalid resolution
new TestCaseData(ETHUSDT, Resolution.Second, Time.OneMinute, TickType.Trade),

// invalid tick type
new TestCaseData(ETHUSDT, Resolution.Minute, Time.OneHour, TickType.Quote),

// invalid market
new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute,
Time.OneMinute, TickType.Trade),

// invalid resolution for tick type
new TestCaseData(ETHUSDT, Resolution.Tick, TimeSpan.FromDays(15), TickType.OpenInterest),
new TestCaseData(ETHUSDT, Resolution.Minute, TimeSpan.FromDays(15), TickType.OpenInterest),
};
}
}


[Test]
[TestCaseSource(nameof(ValidHistory))]
public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType)
Expand All @@ -104,104 +104,92 @@ public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan p
}

[Test]
[TestCaseSource(nameof(NoHistory))]
[TestCaseSource(nameof(InvalidHistory))]
public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType)
public virtual void ReturnsNullOnInvalidHistoryRequest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType)
{
BaseHistoryTest(_brokerage, symbol, resolution, period, tickType, true);
}

protected static void BaseHistoryTest(Brokerage brokerage, Symbol symbol, Resolution resolution,
TimeSpan period, TickType tickType, bool emptyData)
TimeSpan period, TickType tickType, bool invalidRequest)
{
var historyProvider = new BrokerageHistoryProvider();
historyProvider.SetBrokerage(brokerage);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null,
null, null, null, null,
false, new DataPermissionManager(), null));

var marketHoursDatabase = MarketHoursDatabase.FromDataFolder();

var now = DateTime.UtcNow.AddDays(-1);
var requests = new[]
var request = new HistoryRequest(now.Add(-period),
now,
resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar),
symbol,
resolution,
marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType),
marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType),
resolution,
false,
false,
DataNormalizationMode.Adjusted,
tickType);

var history = brokerage.GetHistory(request)?.ToList();

if (invalidRequest)
{
new HistoryRequest(now.Add(-period),
now,
resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar),
symbol,
resolution,
marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType),
marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType),
resolution,
false,
false,
DataNormalizationMode.Adjusted,
tickType)
};

var historyArray = historyProvider.GetHistory(requests, TimeZones.Utc).ToArray();
foreach (var slice in historyArray)
Assert.IsNull(history);
return;
}

Assert.IsNotNull(history);

foreach (var data in history)
{
if (resolution == Resolution.Tick)
if (data is Tick tick)
{
foreach (var tick in slice.Ticks[symbol])
{
Log.Trace("{0}: {1} - {2} / {3}", tick.Time.ToStringInvariant("yyyy-MM-dd HH:mm:ss.fff"),
tick.Symbol, tick.BidPrice, tick.AskPrice);
}
Log.Trace("{0}: {1} - {2} / {3}", tick.Time.ToStringInvariant("yyyy-MM-dd HH:mm:ss.fff"),
tick.Symbol, tick.BidPrice, tick.AskPrice);
}
else if (slice.QuoteBars.TryGetValue(symbol, out var quoteBar))
else if (data is QuoteBar quoteBar)
{
Log.Trace($"QuoteBar: {quoteBar}");
}
else if (slice.Bars.TryGetValue(symbol, out var bar))
else if (data is TradeBar bar)
{
Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High,
bar.Low, bar.Close);
}
}

if (emptyData)
{
Assert.Zero(historyProvider.DataPointCount);
}
else
{
Assert.Greater(historyProvider.DataPointCount, 0);
}
Assert.Greater(history.Count, 0);

if (historyProvider.DataPointCount > 0)
{
// Ordered by time
Assert.That(historyArray, Is.Ordered.By("Time"));
// Ordered by time
Assert.That(history, Is.Ordered.By("Time"));

var timesArray = history.Select(x => x.Time).ToArray();
if (resolution != Resolution.Tick)
{
// No repeating bars
var timesArray = historyArray.Select(x => x.Time).ToArray();
Assert.AreEqual(timesArray.Length, timesArray.Distinct().Count());
}

foreach (var slice in historyArray)
{
var data = slice.AllData[0];
Assert.AreEqual(symbol, data.Symbol);
foreach (var data in history)
{
Assert.AreEqual(symbol, data.Symbol);

if (data.DataType != MarketDataType.Tick)
{
Assert.AreEqual(resolution.ToTimeSpan(), data.EndTime - data.Time);
}
if (data.DataType != MarketDataType.Tick)
{
Assert.AreEqual(resolution.ToTimeSpan(), data.EndTime - data.Time);
}
}

// No missing bars
if (resolution != Resolution.Tick && historyProvider.DataPointCount >= 2)
// No missing bars
if (resolution != Resolution.Tick && history.Count >= 2)
{
var diff = resolution.ToTimeSpan();
for (var i = 1; i < timesArray.Length; i++)
{
var diff = resolution.ToTimeSpan();
for (var i = 1; i < timesArray.Length; i++)
{
Assert.AreEqual(diff, timesArray[i] - timesArray[i - 1]);
}
Assert.AreEqual(diff, timesArray[i] - timesArray[i - 1]);
}
}

Log.Trace("Data points retrieved: " + historyProvider.DataPointCount);
Log.Trace("Data points retrieved: " + history.Count);
}

private Brokerage CreateBrokerage()
Expand Down
4 changes: 2 additions & 2 deletions QuantConnect.BybitBrokerage.Tests/BybitFuturesBrokerageHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ public override void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan
}

[Ignore("The brokerage is shared between different product categories, therefore this test is only required in the base class")]
public override void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType)
public override void ReturnsNullOnInvalidHistoryRequest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType)
{
base.GetEmptyHistory(symbol, resolution, period, tickType);
base.ReturnsNullOnInvalidHistoryRequest(symbol, resolution, period, tickType);
}
}
}
18 changes: 5 additions & 13 deletions QuantConnect.BybitBrokerage.ToolBox/BybitBrokerageDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,17 +65,6 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

if (tickType is not (TickType.Trade or TickType.OpenInterest))
{
return Enumerable.Empty<BaseData>();
}

if (!_symbolMapper.IsKnownLeanSymbol(symbol))
throw new ArgumentException($"The ticker {symbol.Value} is not available.");

if (endUtc < startUtc)
throw new ArgumentException("The end date must be greater or equal than the start date.");

var historyRequest = new HistoryRequest(
startUtc,
endUtc,
Expand All @@ -91,8 +80,7 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
tickType);

var brokerage = CreateBrokerage();
var data = brokerage.GetHistory(historyRequest);
return data;
return brokerage.GetHistory(historyRequest);
}

/// <summary>
Expand Down Expand Up @@ -145,6 +133,10 @@ public static void DownloadHistory(List<string> tickers, string resolution, stri
// Download the data
var symbol = downloader.GetSymbol(ticker, securityTypeEnum);
var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, fromDate, toDate, tickType: tickTypeEnum));
if (data == null)
{
continue;
}

// todo how to write open interest data
// Save the data (single resolution)
Expand Down
75 changes: 57 additions & 18 deletions QuantConnect.BybitBrokerage/BybitBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ public partial class BybitBrokerage : BaseWebsocketsBrokerage, IDataQueueHandler
private Lazy<BybitApi> _apiClientLazy;
private BrokerageConcurrentMessageHandler<WebSocketMessage> _messageHandler;

private bool _unsupportedAssetHistoryLogged;
private bool _unsupportedResolutionHistoryLogged;
private bool _unsupportedTickTypeHistoryLogged;
private bool _unsupportedResolutionOpenInterestHistoryLogged;
private bool _invalidTimeRangeHistoryLogged;

/// <summary>
/// Order provider
/// </summary>
Expand Down Expand Up @@ -143,32 +149,52 @@ public BybitBrokerage(string apiKey, string apiSecret, string restApiUrl, string
/// <returns>An enumerable of bars or ticks covering the span specified in the request</returns>
public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
{
if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol))
if (!CanSubscribe(request.Symbol))
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol",
$"Unknown symbol: {request.Symbol.Value}, no history returned"));
return Array.Empty<BaseData>();
if (!_unsupportedAssetHistoryLogged)
{
_unsupportedAssetHistoryLogged = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol",
$"Invalid symbol: {request.Symbol.Value}, no history returned"));
}

return null;
}

if (request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution is not supported, no history returned"));
return Array.Empty<BaseData>();
if (!_unsupportedResolutionHistoryLogged)
{
_unsupportedResolutionHistoryLogged = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution is not supported, no history returned"));
}

return null;
}

if (request.TickType is not (TickType.OpenInterest or TickType.Trade))
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType",
$"{request.TickType} tick type not supported, no history returned"));
return Array.Empty<BaseData>();
if (!_unsupportedTickTypeHistoryLogged)
{
_unsupportedTickTypeHistoryLogged = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType",
$"{request.TickType} tick type not supported, no history returned"));
}

return null;
}

if (!IsSupported(request.Symbol))
if (request.StartTimeUtc >= request.EndTimeUtc)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType",
$"{request.Symbol.SecurityType} security type not supported, no history returned"));
return Array.Empty<BaseData>();
if (!_invalidTimeRangeHistoryLogged)
{
_invalidTimeRangeHistoryLogged = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange",
"The history request start date must precede the end date, no history returned"));
}

return null;
}

var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
Expand Down Expand Up @@ -303,7 +329,10 @@ private Dictionary<Symbol, int> FetchSymbolWeights(BybitApi client, BybitProduct
/// <returns>returns true if brokerage supports the specified symbol; otherwise false</returns>
protected virtual bool CanSubscribe(Symbol symbol)
{
var baseCanSubscribe = !symbol.Value.Contains("UNIVERSE") && IsSupported(symbol) && symbol.ID.Market == MarketName;
var baseCanSubscribe = !symbol.Value.Contains("UNIVERSE") &&
IsSupported(symbol) &&
symbol.ID.Market == MarketName &&
_symbolMapper.IsKnownLeanSymbol(symbol);

if (baseCanSubscribe && symbol.SecurityType == SecurityType.CryptoFuture)
{
Expand Down Expand Up @@ -387,11 +416,21 @@ private IEnumerable<OpenInterest> GetOpenInterestHistory(string brokerageSymbol,
{
if (request.Resolution is not (Resolution.Hour or Resolution.Daily))
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"Only hourly and daily resolutions are supported for open interest history. No history returned"));
yield break;
if (!_unsupportedResolutionOpenInterestHistoryLogged)
{
_unsupportedResolutionOpenInterestHistoryLogged = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"Only hourly and daily resolutions are supported for open interest history. No history returned"));
}

return null;
}

return GetOpenInterestHistoryImpl(brokerageSymbol, request);
}

private IEnumerable<OpenInterest> GetOpenInterestHistoryImpl(string brokerageSymbol, HistoryRequest request)
{
var usingTempClient = MakeTempApiClient(
() => GetApiClient(_symbolMapper, null, Config.Get("bybit-api-url", "https://api.bybit.com"), null, null, BybitVIPLevel.VIP0),
out var client);
Expand Down

0 comments on commit 1bba7c5

Please sign in to comment.