Implementation.

MMBotDownloader.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.31729.503
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MMBotDownloader", "MMBotDownloader\MMBotDownloader.csproj", "{02B7C7A2-64A8-4201-AB9F-EC99B0EBAF91}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{02B7C7A2-64A8-4201-AB9F-EC99B0EBAF91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{02B7C7A2-64A8-4201-AB9F-EC99B0EBAF91}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{02B7C7A2-64A8-4201-AB9F-EC99B0EBAF91}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{02B7C7A2-64A8-4201-AB9F-EC99B0EBAF91}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {888216A5-C180-48B0-B263-DD3D51BC4AF1}
+	EndGlobalSection
+EndGlobal

MMBotDownloader/Configuration/ConfigProvider.cs

@@ -0,0 +1,21 @@
+using MMBotDownloader.Utils;
+
+namespace MMBotDownloader.Configuration
+{
+    internal class ConfigProvider : IConfigProvider
+    {
+        private readonly UserInterface _ui;
+
+        public ConfigProvider(UserInterface ui)
+        {
+            _ui = ui;
+        }
+
+        public Configuration GetConfig()
+        {
+            var selection = _ui.Prompt("Load settings from Config.json? (y/N): ").Trim().ToLower();
+            IConfigProvider provider = selection == "a" || selection == "y" ? new FileConfigProvider() : new UserPromptConfigProvider(_ui);
+            return provider.GetConfig();
+        }
+    }
+}

MMBotDownloader/Configuration/Configuration.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace MMBotDownloader.Configuration
+{
+    internal class Configuration
+    {
+        public List<ConfigurationEntry> Pairs { get; set; }
+    }
+}

MMBotDownloader/Configuration/ConfigurationEntry.cs

@@ -0,0 +1,10 @@
+namespace MMBotDownloader.Configuration
+{
+    internal class ConfigurationEntry
+    {
+        public string TradingPair { get; set; }
+        public string StartDate { get; set; }
+        public string StopDate { get; set; }
+        public string Exchange { get; set; }
+    }
+}

MMBotDownloader/Configuration/ConfigurationEntryExtensions.cs

@@ -0,0 +1,18 @@
+using MMBotDownloader.Core;
+using System;
+using System.Globalization;
+
+namespace MMBotDownloader.Configuration
+{
+    internal static class ConfigurationEntryExtensions
+    {
+        public static DownloadTask ToDownloadTask(this ConfigurationEntry configuration)
+        {
+            return new DownloadTask(
+                configuration.Exchange,
+                configuration.TradingPair,
+                DateTime.Parse(configuration.StartDate, null, DateTimeStyles.AssumeUniversal),
+                DateTime.Parse(configuration.StopDate, null, DateTimeStyles.AssumeUniversal));
+        }
+    }
+}

MMBotDownloader/Configuration/ConfigurationExtensions.cs

@@ -0,0 +1,21 @@
+using MMBotDownloader.Utils;
+
+namespace MMBotDownloader.Configuration
+{
+    internal static class ConfigurationExtensions
+    {
+        public static void PrintTo(this Configuration configuration, UserInterface ui)
+        {
+            ui.WriteLine();
+            ui.WriteLine("Selected options:");
+            foreach (var option in configuration.Pairs)
+            {
+                ui.WriteSelection("Exchange:", option.Exchange);
+                ui.WriteSelection("Symbol:", option.TradingPair);
+                ui.WriteSelection("Start:", option.StartDate);
+                ui.WriteSelection("End:", option.StopDate);
+                ui.WriteLine();
+            }
+        }
+    }
+}

MMBotDownloader/Configuration/FileConfigProvider.cs

@@ -0,0 +1,21 @@
+using System.IO;
+using Newtonsoft.Json;
+
+namespace MMBotDownloader.Configuration
+{
+    internal class FileConfigProvider : IConfigProvider
+    {
+        private readonly string _path;
+
+        public FileConfigProvider(string path = "Config.json")
+        {
+            _path = path;
+        }
+
+        public Configuration GetConfig()
+        {
+            var content = File.ReadAllText(_path);
+            return JsonConvert.DeserializeObject<Configuration>(content);
+        }
+    }
+}

MMBotDownloader/Configuration/IConfigProvider.cs

@@ -0,0 +1,7 @@
+namespace MMBotDownloader.Configuration
+{
+    internal interface IConfigProvider
+    {
+        Configuration GetConfig();
+    }
+}

MMBotDownloader/Configuration/UserPromptConfigProvider.cs

@@ -0,0 +1,32 @@
+using MMBotDownloader.Utils;
+using System.Collections.Generic;
+
+namespace MMBotDownloader.Configuration
+{
+    internal class UserPromptConfigProvider : IConfigProvider
+    {
+        private readonly UserInterface _ui;
+
+        public UserPromptConfigProvider(UserInterface ui)
+        {
+            _ui = ui;
+        }
+
+        public Configuration GetConfig()
+        {
+            return new Configuration
+            {
+                Pairs = new List<ConfigurationEntry>
+                {
+                    new ConfigurationEntry
+                    {
+                        Exchange = _ui.Prompt("Exchange:"),
+                        TradingPair = _ui.Prompt("Symbol:"),
+                        StartDate = _ui.Prompt("Start date (1.1.2021):"),
+                        StopDate = _ui.Prompt("End date (1.2.2021):")
+                    }
+                }
+            };
+        }
+    }
+}

MMBotDownloader/Core/DownloadOrchestrator.cs

@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using System.Linq;
+using System.Threading;
+using System.IO;
+using System.Diagnostics;
+using System.Net;
+using MMBotDownloader.Utils;
+
+namespace MMBotDownloader.Core
+{
+    internal class DownloadOrchestrator
+    {
+        private readonly UserInterface _ui;
+        private readonly int _degreeOfParallelism;
+        private readonly IGenericDownloader[] _downloaders;
+
+        public DownloadOrchestrator(UserInterface ui, IGenericDownloader[] downloaders, int degreeOfParallelism = 10)
+        {
+            _ui = ui;
+            _degreeOfParallelism = degreeOfParallelism;
+            _downloaders = downloaders;
+            ServicePointManager.DefaultConnectionLimit = degreeOfParallelism;
+        }
+
+        public void PrintExchanges()
+        {
+            _ui.WriteLine("Available exchanges (name / pair example):");
+            foreach (var downloader in _downloaders)
+            {
+                _ui.WriteSelection($"{downloader.Name}:", downloader.SymbolExample);
+            }
+            _ui.WriteLine();
+        }
+
+        public void Download(DownloadTask downloadTask)
+        {
+            foreach (var downloader in _downloaders.Where(x => x.Name == downloadTask.Exchange))
+            {
+                downloader.DownloadWith(this, downloadTask);
+            }
+        }
+
+        public void Download<T>(IDownloader<T> downloader, DownloadTask downloadTask) where T : struct
+        {
+            var fileName = $"{downloadTask.Symbol}_{downloadTask.Start.ToShortDateString()}_{downloadTask.End.ToShortDateString()}.csv";
+            foreach (var ch in Path.GetInvalidFileNameChars())
+            {
+                fileName = fileName.Replace(ch.ToString(), string.Empty);
+            }
+
+            _ui.WriteSelection("Downloading:", downloadTask.ToString());
+
+            if (File.Exists(fileName))
+            {
+                _ui.WriteLine($"{fileName} already exists, skipping.");
+                return;
+            }
+
+            var chunks = downloader.PrepareChunks(downloadTask).ToList();
+            var results = new IList<string>[chunks.Count];
+            var tasks = chunks.Select<T, Action>((x, i) => () =>
+            {
+                var stopwatch = new Stopwatch();
+                stopwatch.Restart();
+                results[i] = downloader.DownloadLinesAsync(x).GetAwaiter().GetResult().ToList();
+                stopwatch.Stop();
+                _ui.WriteSelection($"Downloaded in {stopwatch.ElapsedMilliseconds} ms:", x.ToString());
+            });
+            var queue = new Queue<Action>(tasks);
+            var awaits = new List<Task>();
+            using var semaphore = new SemaphoreSlim(_degreeOfParallelism);
+
+            _ui.WriteSelection("Number of chunks to download:", queue.Count.ToString());
+            while (queue.Any())
+            {
+                semaphore.Wait();
+                var action = queue.Dequeue();
+                awaits.Add(Task.Run(() =>
+                {
+                    action();
+                    semaphore.Release();
+                }));
+            }
+            Task.WaitAll(awaits.ToArray());
+
+            _ui.WriteSelection("Writing:", fileName);
+
+            using var writer = new StreamWriter(fileName, true);
+            foreach (var line in results.Where(x => x != null).SelectMany(x => x))
+            {
+                writer.WriteLine(line);
+            }
+        }
+    }
+}

MMBotDownloader/Core/DownloadTask.cs

@@ -0,0 +1,25 @@
+using System;
+
+namespace MMBotDownloader.Core
+{
+    internal struct DownloadTask
+    {
+        public string Exchange { get; private set; }
+        public string Symbol { get; private set; }
+        public DateTime Start { get; private set; }
+        public DateTime End { get; private set; }
+
+        public DownloadTask(string exchange, string symbol, DateTime start, DateTime end)
+        {
+            Exchange = exchange;
+            Symbol = symbol;
+            Start = start;
+            End = end;
+        }
+
+        public override string ToString()
+        {
+            return $"{Exchange}/{Symbol} ({Start.ToShortDateString()}-{End.ToShortDateString()})";
+        }
+    }
+}

MMBotDownloader/Core/IDownloader.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace MMBotDownloader.Core
+{
+    internal interface IDownloader<T> : IGenericDownloader where T : struct
+    {
+        IEnumerable<T> PrepareChunks(DownloadTask downloadTask);
+
+        Task<IEnumerable<string>> DownloadLinesAsync(T chunk);
+    }
+}

MMBotDownloader/Core/IGenericDownloader.cs

@@ -0,0 +1,9 @@
+namespace MMBotDownloader.Core
+{
+    internal interface IGenericDownloader
+    {
+        string Name { get; }
+        string SymbolExample { get; }
+        void DownloadWith(DownloadOrchestrator orchestrator, DownloadTask downloadTask);
+    }
+}

MMBotDownloader/Exchange/Binance/BinanceChunk.cs

@@ -0,0 +1,24 @@
+using MMBotDownloader.Utils;
+using System;
+
+namespace MMBotDownloader.Exchange.Binance
+{
+    internal struct BinanceChunk
+    {
+        public string Symbol { get; private set; }
+        public long StartTimeMs { get; private set; }
+        private readonly DateTime _startTime;
+
+        public BinanceChunk(string symbol, long startTimeMs)
+        {
+            Symbol = symbol;
+            StartTimeMs = startTimeMs;
+            _startTime = UnixEpoch.GetDateTimeMs(startTimeMs);
+        }
+
+        public override string ToString()
+        {
+            return _startTime.ToString();
+        }
+    }
+}

MMBotDownloader/Exchange/Binance/BinanceDownloader.cs

@@ -0,0 +1,48 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using System.Threading.Tasks;
+using System.Linq;
+using System.Net.Http;
+using MMBotDownloader.Core;
+using MMBotDownloader.Utils;
+
+namespace MMBotDownloader.Exchange.Binance
+{
+    internal class BinanceDownloader : IDownloader<BinanceChunk>
+    {
+        private readonly HttpClient _client;
+
+        public string Name => "BINANCE";
+        public string SymbolExample => "ADAUSDT";
+
+        public BinanceDownloader(HttpClient client)
+        {
+            _client = client;
+        }
+
+        public IEnumerable<BinanceChunk> PrepareChunks(DownloadTask downloadTask)
+        {
+            var startMs = UnixEpoch.GetEpochMs(downloadTask.Start);
+            var endMs = UnixEpoch.GetEpochMs(downloadTask.End);
+            var increment = 60000 * 1000L;
+            var count = (int)((endMs - startMs) / increment);
+
+            return Enumerable
+                .Range(0, count)
+                .Select(i => new BinanceChunk(downloadTask.Symbol, startMs + i * increment));
+        }
+
+        public async Task<IEnumerable<string>> DownloadLinesAsync(BinanceChunk chunk)
+        {
+            var url = $"https://api.binance.com/api/v3/klines?symbol={chunk.Symbol}&interval=1m&startTime={chunk.StartTimeMs}&limit=1000";
+            var dataString = await _client.GetStringAsync(url);
+            var data = JsonConvert.DeserializeObject<IList<IList<object>>>(dataString);
+            return data.Select(x => x[4].ToString());
+        }
+
+        public void DownloadWith(DownloadOrchestrator orchestrator, DownloadTask downloadTask)
+        {
+            orchestrator.Download(this, downloadTask);
+        }
+    }
+}