diff --git a/QuantConnect.BybitBrokerage.Tests/BybitBrokerageHistoryProviderTests.cs b/QuantConnect.BybitBrokerage.Tests/BybitBrokerageHistoryProviderTests.cs index 4a2bb95..3725720 100644 --- a/QuantConnect.BybitBrokerage.Tests/BybitBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BybitBrokerage.Tests/BybitBrokerageHistoryProviderTests.cs @@ -24,7 +24,6 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.BybitBrokerage.Tests { @@ -51,6 +50,8 @@ private static TestCaseData[] ValidHistory { get { + TestGlobals.Initialize(); + return new[] { // valid @@ -62,40 +63,39 @@ private static TestCaseData[] ValidHistory } } - private static TestCaseData[] NoHistory - { - get - { - return new[] - { - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Bybit), - Resolution.Second, Time.OneMinute, TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Bybit), - Resolution.Minute, Time.OneHour, TickType.Quote), - }; - } - } - private static TestCaseData[] InvalidHistory { get { return new[] { - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), + // invalid period + new TestCaseData(ETHUSDT, Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" + // invalid symbol new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Bybit), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade), //invalid security type new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade), + + // invalid resolution + new TestCaseData(ETHUSDT, Resolution.Second, Time.OneMinute, TickType.Trade), + + // invalid tick type + new TestCaseData(ETHUSDT, Resolution.Minute, Time.OneHour, TickType.Quote), + + // invalid market + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, + Time.OneMinute, TickType.Trade), + + // invalid resolution for tick type + new TestCaseData(ETHUSDT, Resolution.Tick, TimeSpan.FromDays(15), TickType.OpenInterest), + new TestCaseData(ETHUSDT, Resolution.Minute, TimeSpan.FromDays(15), TickType.OpenInterest), }; } } - [Test] [TestCaseSource(nameof(ValidHistory))] public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) @@ -104,104 +104,92 @@ public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan p } [Test] - [TestCaseSource(nameof(NoHistory))] [TestCaseSource(nameof(InvalidHistory))] - public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) + public virtual void ReturnsNullOnInvalidHistoryRequest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) { BaseHistoryTest(_brokerage, symbol, resolution, period, tickType, true); } protected static void BaseHistoryTest(Brokerage brokerage, Symbol symbol, Resolution resolution, - TimeSpan period, TickType tickType, bool emptyData) + TimeSpan period, TickType tickType, bool invalidRequest) { - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, - null, null, null, null, - false, new DataPermissionManager(), null)); - var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); var now = DateTime.UtcNow.AddDays(-1); - var requests = new[] + var request = new HistoryRequest(now.Add(-period), + now, + resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar), + symbol, + resolution, + marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType), + marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType), + resolution, + false, + false, + DataNormalizationMode.Adjusted, + tickType); + + var history = brokerage.GetHistory(request)?.ToList(); + + if (invalidRequest) { - new HistoryRequest(now.Add(-period), - now, - resolution == Resolution.Tick ? typeof(Tick) : typeof(TradeBar), - symbol, - resolution, - marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType), - marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType), - resolution, - false, - false, - DataNormalizationMode.Adjusted, - tickType) - }; - - var historyArray = historyProvider.GetHistory(requests, TimeZones.Utc).ToArray(); - foreach (var slice in historyArray) + Assert.IsNull(history); + return; + } + + Assert.IsNotNull(history); + + foreach (var data in history) { - if (resolution == Resolution.Tick) + if (data is Tick tick) { - foreach (var tick in slice.Ticks[symbol]) - { - Log.Trace("{0}: {1} - {2} / {3}", tick.Time.ToStringInvariant("yyyy-MM-dd HH:mm:ss.fff"), - tick.Symbol, tick.BidPrice, tick.AskPrice); - } + Log.Trace("{0}: {1} - {2} / {3}", tick.Time.ToStringInvariant("yyyy-MM-dd HH:mm:ss.fff"), + tick.Symbol, tick.BidPrice, tick.AskPrice); } - else if (slice.QuoteBars.TryGetValue(symbol, out var quoteBar)) + else if (data is QuoteBar quoteBar) { Log.Trace($"QuoteBar: {quoteBar}"); } - else if (slice.Bars.TryGetValue(symbol, out var bar)) + else if (data is TradeBar bar) { Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High, bar.Low, bar.Close); } } - if (emptyData) - { - Assert.Zero(historyProvider.DataPointCount); - } - else - { - Assert.Greater(historyProvider.DataPointCount, 0); - } + Assert.Greater(history.Count, 0); - if (historyProvider.DataPointCount > 0) - { - // Ordered by time - Assert.That(historyArray, Is.Ordered.By("Time")); + // Ordered by time + Assert.That(history, Is.Ordered.By("Time")); + var timesArray = history.Select(x => x.Time).ToArray(); + if (resolution != Resolution.Tick) + { // No repeating bars - var timesArray = historyArray.Select(x => x.Time).ToArray(); Assert.AreEqual(timesArray.Length, timesArray.Distinct().Count()); + } - foreach (var slice in historyArray) - { - var data = slice.AllData[0]; - Assert.AreEqual(symbol, data.Symbol); + foreach (var data in history) + { + Assert.AreEqual(symbol, data.Symbol); - if (data.DataType != MarketDataType.Tick) - { - Assert.AreEqual(resolution.ToTimeSpan(), data.EndTime - data.Time); - } + if (data.DataType != MarketDataType.Tick) + { + Assert.AreEqual(resolution.ToTimeSpan(), data.EndTime - data.Time); } + } - // No missing bars - if (resolution != Resolution.Tick && historyProvider.DataPointCount >= 2) + // No missing bars + if (resolution != Resolution.Tick && history.Count >= 2) + { + var diff = resolution.ToTimeSpan(); + for (var i = 1; i < timesArray.Length; i++) { - var diff = resolution.ToTimeSpan(); - for (var i = 1; i < timesArray.Length; i++) - { - Assert.AreEqual(diff, timesArray[i] - timesArray[i - 1]); - } + Assert.AreEqual(diff, timesArray[i] - timesArray[i - 1]); } } - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); + Log.Trace("Data points retrieved: " + history.Count); } private Brokerage CreateBrokerage() diff --git a/QuantConnect.BybitBrokerage.Tests/BybitFuturesBrokerageHistoryProviderTests.cs b/QuantConnect.BybitBrokerage.Tests/BybitFuturesBrokerageHistoryProviderTests.cs index e8de3fd..c4584ae 100644 --- a/QuantConnect.BybitBrokerage.Tests/BybitFuturesBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BybitBrokerage.Tests/BybitFuturesBrokerageHistoryProviderTests.cs @@ -48,9 +48,9 @@ public override void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan } [Ignore("The brokerage is shared between different product categories, therefore this test is only required in the base class")] - public override void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) + public override void ReturnsNullOnInvalidHistoryRequest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) { - base.GetEmptyHistory(symbol, resolution, period, tickType); + base.ReturnsNullOnInvalidHistoryRequest(symbol, resolution, period, tickType); } } } \ No newline at end of file diff --git a/QuantConnect.BybitBrokerage.ToolBox/BybitBrokerageDownloader.cs b/QuantConnect.BybitBrokerage.ToolBox/BybitBrokerageDownloader.cs index 8267456..648ed66 100644 --- a/QuantConnect.BybitBrokerage.ToolBox/BybitBrokerageDownloader.cs +++ b/QuantConnect.BybitBrokerage.ToolBox/BybitBrokerageDownloader.cs @@ -65,17 +65,6 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa var endUtc = dataDownloaderGetParameters.EndUtc; var tickType = dataDownloaderGetParameters.TickType; - if (tickType is not (TickType.Trade or TickType.OpenInterest)) - { - return Enumerable.Empty(); - } - - if (!_symbolMapper.IsKnownLeanSymbol(symbol)) - throw new ArgumentException($"The ticker {symbol.Value} is not available."); - - if (endUtc < startUtc) - throw new ArgumentException("The end date must be greater or equal than the start date."); - var historyRequest = new HistoryRequest( startUtc, endUtc, @@ -91,8 +80,7 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa tickType); var brokerage = CreateBrokerage(); - var data = brokerage.GetHistory(historyRequest); - return data; + return brokerage.GetHistory(historyRequest); } /// @@ -145,6 +133,10 @@ public static void DownloadHistory(List tickers, string resolution, stri // Download the data var symbol = downloader.GetSymbol(ticker, securityTypeEnum); var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, fromDate, toDate, tickType: tickTypeEnum)); + if (data == null) + { + continue; + } // todo how to write open interest data // Save the data (single resolution) diff --git a/QuantConnect.BybitBrokerage/BybitBrokerage.cs b/QuantConnect.BybitBrokerage/BybitBrokerage.cs index ed889eb..9f6eaa6 100644 --- a/QuantConnect.BybitBrokerage/BybitBrokerage.cs +++ b/QuantConnect.BybitBrokerage/BybitBrokerage.cs @@ -63,6 +63,12 @@ public partial class BybitBrokerage : BaseWebsocketsBrokerage, IDataQueueHandler private Lazy _apiClientLazy; private BrokerageConcurrentMessageHandler _messageHandler; + private bool _unsupportedAssetHistoryLogged; + private bool _unsupportedResolutionHistoryLogged; + private bool _unsupportedTickTypeHistoryLogged; + private bool _unsupportedResolutionOpenInterestHistoryLogged; + private bool _invalidTimeRangeHistoryLogged; + /// /// Order provider /// @@ -143,32 +149,52 @@ public BybitBrokerage(string apiKey, string apiSecret, string restApiUrl, string /// An enumerable of bars or ticks covering the span specified in the request public override IEnumerable GetHistory(HistoryRequest request) { - if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol)) + if (!CanSubscribe(request.Symbol)) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol", - $"Unknown symbol: {request.Symbol.Value}, no history returned")); - return Array.Empty(); + if (!_unsupportedAssetHistoryLogged) + { + _unsupportedAssetHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol", + $"Invalid symbol: {request.Symbol.Value}, no history returned")); + } + + return null; } if (request.Resolution == Resolution.Second) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", - $"{request.Resolution} resolution is not supported, no history returned")); - return Array.Empty(); + if (!_unsupportedResolutionHistoryLogged) + { + _unsupportedResolutionHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", + $"{request.Resolution} resolution is not supported, no history returned")); + } + + return null; } if (request.TickType is not (TickType.OpenInterest or TickType.Trade)) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", - $"{request.TickType} tick type not supported, no history returned")); - return Array.Empty(); + if (!_unsupportedTickTypeHistoryLogged) + { + _unsupportedTickTypeHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", + $"{request.TickType} tick type not supported, no history returned")); + } + + return null; } - if (!IsSupported(request.Symbol)) + if (request.StartTimeUtc >= request.EndTimeUtc) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType", - $"{request.Symbol.SecurityType} security type not supported, no history returned")); - return Array.Empty(); + if (!_invalidTimeRangeHistoryLogged) + { + _invalidTimeRangeHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange", + "The history request start date must precede the end date, no history returned")); + } + + return null; } var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(request.Symbol); @@ -303,7 +329,10 @@ private Dictionary FetchSymbolWeights(BybitApi client, BybitProduct /// returns true if brokerage supports the specified symbol; otherwise false protected virtual bool CanSubscribe(Symbol symbol) { - var baseCanSubscribe = !symbol.Value.Contains("UNIVERSE") && IsSupported(symbol) && symbol.ID.Market == MarketName; + var baseCanSubscribe = !symbol.Value.Contains("UNIVERSE") && + IsSupported(symbol) && + symbol.ID.Market == MarketName && + _symbolMapper.IsKnownLeanSymbol(symbol); if (baseCanSubscribe && symbol.SecurityType == SecurityType.CryptoFuture) { @@ -387,11 +416,21 @@ private IEnumerable GetOpenInterestHistory(string brokerageSymbol, { if (request.Resolution is not (Resolution.Hour or Resolution.Daily)) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", - $"Only hourly and daily resolutions are supported for open interest history. No history returned")); - yield break; + if (!_unsupportedResolutionOpenInterestHistoryLogged) + { + _unsupportedResolutionOpenInterestHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", + $"Only hourly and daily resolutions are supported for open interest history. No history returned")); + } + + return null; } + return GetOpenInterestHistoryImpl(brokerageSymbol, request); + } + + private IEnumerable GetOpenInterestHistoryImpl(string brokerageSymbol, HistoryRequest request) + { var usingTempClient = MakeTempApiClient( () => GetApiClient(_symbolMapper, null, Config.Get("bybit-api-url", "https://api.bybit.com"), null, null, BybitVIPLevel.VIP0), out var client);