Merge pull request #6 from YPermitin/develop

Улучшение операций скачивания, распаковки и очистки файлов ФИАС + реф…
YPermitin · Jul 7, 2023 · c4bb162 · c4bb162
2 parents 8cc8188 + f622d8a
commit c4bb162
Show file tree

Hide file tree

Showing 12 changed files with 598 additions and 68 deletions.
diff --git a/Apps/YPermitin.FIASToolSet.DistributionDownloader/Program.cs b/Apps/YPermitin.FIASToolSet.DistributionDownloader/Program.cs
@@ -1,47 +1,55 @@
-// See https://aka.ms/new-console-template for more information
-
+using Microsoft.Extensions.Configuration;
 using YPermitin.FIASToolSet.DistributionBrowser;
 using YPermitin.FIASToolSet.DistributionBrowser.Enums;
 using YPermitin.FIASToolSet.DistributionBrowser.Models;
 
 Console.WriteLine("Тестовая загрузка полного дистрибутива ФИАС.");
 
-IFIASDistributionBrowser loader = new FIASDistributionBrowser();
+IConfiguration configuration = new ConfigurationBuilder()
+    .SetBasePath(Directory.GetCurrentDirectory())
+    .AddJsonFile("appsettings.json", optional: false, reloadOnChange: false)
+    .Build();
+
+var options = new FIASDistributionBrowserOptions(
+    workingDirectory: configuration.GetValue("FIASToolSet:WorkingDirectory", string.Empty));
+IFIASDistributionBrowser loader = new FIASDistributionBrowser(options);
 var lastInfo = await loader.GetLastDistributionInfo();
-var xmlFullUrl = lastInfo?.GetUrlByFileType(DistributionFileType.GARFIASXmlComplete);
-
-var tempFileToDownload = Path.Combine(
-    Path.GetTempPath(),
-    "FIAS_FULL.zip");
 
 DateTime downloadStarted = DateTime.UtcNow;
 DateTime lastProgressShow = DateTime.MinValue;
 double lastProgressPercentage = 0;
-await lastInfo?.DownloadDistributionByFileTypeAsync(
-    DistributionFileType.GARFIASXmlComplete,
-    tempFileToDownload,
-    (args) =>
-    {
-        double progress = Math.Round(args.ProgressPercentage, 2);
-
-        double progressChanged = lastProgressPercentage - progress;
-        
-        var showProgressTimeLeft = DateTime.UtcNow - lastProgressShow;
-        if (args.State != DownloadDistributionFileProgressChangedEventType.Downloading
-            || showProgressTimeLeft.TotalSeconds > 10
-            || progressChanged >= 1)
+if (lastInfo != null)
+{
+    await lastInfo.DownloadDistributionByFileTypeAsync(
+        DistributionFileType.GARFIASXmlComplete,
+        (args) =>
         {
-            var downloadDuration = DateTime.UtcNow - downloadStarted;
-            Console.WriteLine($"[{DateTime.UtcNow}] {args.State}. Progress: {progress} %. Seconds left: {downloadDuration.TotalSeconds}");
-            if (args.ErrorInfo != null)
+            double progress = Math.Round(args.ProgressPercentage, 2);
+
+            double progressChanged = lastProgressPercentage - progress;
+
+            var showProgressTimeLeft = DateTime.UtcNow - lastProgressShow;
+            if (args.State != DownloadDistributionFileProgressChangedEventType.Downloading
+                || showProgressTimeLeft.TotalSeconds > 10
+                || progressChanged >= 1)
             {
-                Console.WriteLine(args.ErrorInfo.GetBaseException().Message);
-            }
+                lastProgressPercentage = progress;
+                lastProgressShow = DateTime.UtcNow;
 
-            lastProgressPercentage = progress;
-            lastProgressShow = DateTime.UtcNow;
-        }
-    });
+                var downloadDuration = DateTime.UtcNow - downloadStarted;
+                Console.WriteLine(
+                    $"[{DateTime.UtcNow}] {args.State}. Progress: {progress} %. Seconds left: {Math.Round(downloadDuration.TotalSeconds, 2)}");
+                if (args.ErrorInfo != null)
+                {
+                    Console.WriteLine(args.ErrorInfo.GetBaseException().Message);
+                }
+            }
+        });
+
+    lastInfo.ExtractDistributionFile(
+        DistributionFileType.GARFIASXmlComplete,
+        true);
+}
 
 Console.WriteLine("Для выхода нажмите любую клавишу...");
 Console.ReadKey();
diff --git a/...in.FIASToolSet.DistributionDownloader/YPermitin.FIASToolSet.DistributionDownloader.csproj b/...in.FIASToolSet.DistributionDownloader/YPermitin.FIASToolSet.DistributionDownloader.csproj
@@ -11,4 +11,16 @@
       <ProjectReference Include="..\..\Libs\YPermitin.FIASToolSet.DistributionBrowser\YPermitin.FIASToolSet.DistributionBrowser.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
+      <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <None Update="appsettings.json">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
 </Project>
diff --git a/Libs/YPermitin.FIASToolSet.DistributionBrowser/API/APIHelper.cs b/Libs/YPermitin.FIASToolSet.DistributionBrowser/API/APIHelper.cs
@@ -8,22 +8,47 @@ namespace YPermitin.FIASToolSet.DistributionBrowser.API
 {
     internal class APIHelper : IAPIHelper
     {
-        private static readonly string DefaultUserAgent = GeneralResources.DefaultUserAgent;
         private static readonly HttpClient APIClient;
-        private static readonly DownloadConfiguration DownloaderConfiguration = new DownloadConfiguration()
+        private static readonly string DefaultUserAgent = GeneralResources.DefaultUserAgent;
+        private static readonly DownloadConfiguration DefaultDownloaderConfiguration;
+        private static DownloadConfiguration GenerateDownloaderConfiguration(
+            long maximumDownloadSpeedBytesPerSecond = long.MinValue)
         {
-            ChunkCount = 8,
-            ParallelDownload = true,
-            RequestConfiguration = new RequestConfiguration()
+            return new DownloadConfiguration()
             {
-                UserAgent = DefaultUserAgent
-            }
-        };
+                MaximumBytesPerSecond = maximumDownloadSpeedBytesPerSecond,
+                ChunkCount = 8,
+                ParallelDownload = true,
+                RequestConfiguration = new RequestConfiguration()
+                {
+                    UserAgent = DefaultUserAgent
+                }
+            };
+        }
+
+        private readonly DownloadConfiguration _downloaderConfiguration;
+
         static APIHelper()
         {
             APIClient = new HttpClient();
             APIClient.DefaultRequestHeaders
                 .UserAgent.TryParseAdd(DefaultUserAgent);
+            DefaultDownloaderConfiguration = GenerateDownloaderConfiguration();
+        }
+
+        public APIHelper()
+        {
+            _downloaderConfiguration = DefaultDownloaderConfiguration;
+        }
+
+        public APIHelper(long maximumDownloadSpeedBytesPerSecond) : this()
+        {
+            if (_downloaderConfiguration.MaximumBytesPerSecond != maximumDownloadSpeedBytesPerSecond
+                && maximumDownloadSpeedBytesPerSecond != 0
+                && maximumDownloadSpeedBytesPerSecond != long.MaxValue)
+            {
+                _downloaderConfiguration = GenerateDownloaderConfiguration(maximumDownloadSpeedBytesPerSecond);
+            }
         }
 
         /// <summary>
@@ -36,7 +61,7 @@ static APIHelper()
         public async Task DownloadFileAsync(Uri uriFile, string savePath,
             Action<DownloadDistributionFileProgressChangedEventArgs> onDownloadFileProgressChangedEvent = null)
         {
-            var downloader = new DownloadService(DownloaderConfiguration);
+            var downloader = new DownloadService(_downloaderConfiguration);
             downloader.DownloadStarted += (sender, args) =>
             {
                 var eventArgs = new DownloadDistributionFileProgressChangedEventArgs(

diff --git a/Libs/YPermitin.FIASToolSet.DistributionBrowser/FIASDistributionBrowser.cs b/Libs/YPermitin.FIASToolSet.DistributionBrowser/FIASDistributionBrowser.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -16,15 +17,35 @@ namespace YPermitin.FIASToolSet.DistributionBrowser
     public sealed class FIASDistributionBrowser : IFIASDistributionBrowser
     {
         private const string APIBaseUrl = "http://fias.nalog.ru/WebServices/Public";
-        private readonly IAPIHelper _apiHelper;
+
+        private IAPIHelper _apiHelperObject;
+        private IAPIHelper _apiHelper
+        {
+            get
+            {
+                return _apiHelperObject ??= new APIHelper(Options.MaximumDownloadSpeedBytesPerSecond);
+            }
+        }
+
         private readonly JsonSerializerOptions _serializerOptions;
-
+
+        public FIASDistributionBrowserOptions Options { get; }
+
         public FIASDistributionBrowser()
         {
-            _apiHelper = new APIHelper();
+            Options ??= FIASDistributionBrowserOptions.Default;
+
             _serializerOptions = new JsonSerializerOptions();
             _serializerOptions.Converters.Add(new InternalDateTimeConverter("dd.MM.yyyy"));
         }
+
+        public FIASDistributionBrowser(FIASDistributionBrowserOptions options) : this()
+        {
+            if (options != null)
+            {
+                Options = options;
+            }
+        }
 
         /// <summary>
         /// Информацию о последней версии файлов дистрибутива, доступных для скачивания
@@ -36,7 +57,7 @@ public async Task<FIASDistributionInfo> GetLastDistributionInfo()
             string contentDownloadFileInfo = await _apiHelper.GetContentAsStringAsync(methodUri);
             DownloadFileInfo lastFileInfo = JsonSerializer.Deserialize<DownloadFileInfo>(contentDownloadFileInfo, _serializerOptions);
 
-            return new FIASDistributionInfo(lastFileInfo);
+            return new FIASDistributionInfo(lastFileInfo, Options, _apiHelper);
         }
 
         /// <summary>
@@ -52,9 +73,20 @@ public async Task<IReadOnlyList<FIASDistributionInfo>> GetAllDistributionInfo()
             if(allFileInfo == null)
                 return new List<FIASDistributionInfo>();
 
-            return allFileInfo.Select(e => new FIASDistributionInfo(e)).ToList();
+            return allFileInfo.Select(e => new FIASDistributionInfo(e, Options, _apiHelper)).ToList();
         }
 
+        /// <summary>
+        /// Очистка всех рабочих каталогов, связанных с браузером дистрибутивов
+        /// </summary>
+        public void ClearAllWorkingDirectories()
+        {
+            if (Directory.Exists(Options.WorkingDirectory))
+            {
+                Directory.Delete(Options.WorkingDirectory, true);
+            }
+        }
+
         private class InternalDateTimeConverter : JsonConverter<DateTime>
         {
             private readonly string _format;

diff --git a/Libs/YPermitin.FIASToolSet.DistributionBrowser/GeneralResources.resx b/Libs/YPermitin.FIASToolSet.DistributionBrowser/GeneralResources.resx
@@ -19,6 +19,6 @@
         <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
     </resheader>
     <data name="DefaultUserAgent" xml:space="preserve">
-        <value>Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 YaBrowser/23.5.1.800 Yowser/2.5 Safari/537.36</value>
+        <value>Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0</value>
     </data>
 </root>
diff --git a/...SToolSet.DistributionBrowser/ZipHelper.cs → ....DistributionBrowser/Helpers/ZipHelper.cs b/...SToolSet.DistributionBrowser/ZipHelper.cs → ....DistributionBrowser/Helpers/ZipHelper.cs
@@ -1,7 +1,7 @@
 using System.IO;
 using System.IO.Compression;
 
-namespace YPermitin.FIASToolSet.DistributionBrowser;
+namespace YPermitin.FIASToolSet.DistributionBrowser.Helpers;
 
 public static class ZipHelper
 {

diff --git a/Libs/YPermitin.FIASToolSet.DistributionBrowser/IFIASDistributionBrowser.cs b/Libs/YPermitin.FIASToolSet.DistributionBrowser/IFIASDistributionBrowser.cs
@@ -9,6 +9,8 @@ namespace YPermitin.FIASToolSet.DistributionBrowser
     /// </summary>
     public interface IFIASDistributionBrowser
     {
+        FIASDistributionBrowserOptions Options { get; }
+
         /// <summary>
         /// Информацию о последней версии файлов дистрибутива, доступных для скачивания
         /// </summary>
@@ -20,5 +22,10 @@ public interface IFIASDistributionBrowser
         /// </summary>
         /// <returns>Коллекция доступных дистрибутивов классификатора ФИАС</returns>
         Task<IReadOnlyList<FIASDistributionInfo>> GetAllDistributionInfo();
+
+        /// <summary>
+        /// Очистка всех рабочих каталогов, связанных с браузером дистрибутивов
+        /// </summary>
+        void ClearAllWorkingDirectories();
     }
 }
diff --git a/...IASToolSet.DistributionBrowser/Models/DownloadDistributionFileProgressChangedEventType.cs b/...IASToolSet.DistributionBrowser/Models/DownloadDistributionFileProgressChangedEventType.cs
@@ -6,5 +6,6 @@ public enum DownloadDistributionFileProgressChangedEventType
     Downloading,
     Compleated,
     Failure,
-    Canceled
+    Canceled,
+    AlreadyExists
 }
diff --git a/Libs/YPermitin.FIASToolSet.DistributionBrowser/Models/FIASDistributionBrowserOptions.cs b/Libs/YPermitin.FIASToolSet.DistributionBrowser/Models/FIASDistributionBrowserOptions.cs
@@ -0,0 +1,49 @@
+using System.IO;
+
+namespace YPermitin.FIASToolSet.DistributionBrowser.Models;
+
+/// <summary>
+/// Параметры обозревателя дистрибутивов ФИАС
+/// </summary>
+public class FIASDistributionBrowserOptions
+{
+    public static readonly FIASDistributionBrowserOptions Default;
+
+    static FIASDistributionBrowserOptions()
+    {
+        string workingDirectory = Path.Combine(
+            Path.GetTempPath(),
+            "FIASToolSet");
+        if (!Directory.Exists(workingDirectory))
+        {
+            Directory.CreateDirectory(workingDirectory);
+        }
+
+        Default = new FIASDistributionBrowserOptions(
+            workingDirectory: workingDirectory);
+    }
+
+    /// <summary>
+    /// Рабочий каталог
+    ///
+    /// Используется для хранения служебных и временных файлов.
+    /// </summary>
+    public readonly string WorkingDirectory;
+
+    /// <summary>
+    /// Максимальная скороть загрузки дистрибутивов ФИАС (байт в секунду).
+    ///
+    /// Значение 0 означает отсутствие ограничений. Используется по умолчанию.
+    /// </summary>
+    public readonly long MaximumDownloadSpeedBytesPerSecond;
+
+    public FIASDistributionBrowserOptions(
+        string workingDirectory = null, 
+        long maximumDownloadSpeedBytesPerSecond = long.MinValue)
+    {
+        workingDirectory ??= Default.WorkingDirectory;
+
+        WorkingDirectory = workingDirectory;
+        MaximumDownloadSpeedBytesPerSecond = maximumDownloadSpeedBytesPerSecond;
+    }
+}