Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Улучшения загрузки и исправление ошибок #28

Merged
merged 1 commit into from
Nov 26, 2023
Merged

Улучшения загрузки и исправление ошибок #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 85 additions & 12 deletions Libs/YPermitin.FIASToolSet.DistributionLoader/FIASDistributionLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public FIASDistributionLoader(
_classifierDataRepository = classifierDataRepository;
}

#region Manage

public async Task<bool> ActiveInstallationExists()
{
var activeInstallations = await _fiasInstallationManagerService.GetInstallations(
Expand All @@ -63,19 +65,12 @@ public async Task<List<FIASVersionInstallation>> FixStuckInstallationExists()

foreach (var activeInstallation in activeInstallations)
{
if (activeInstallation.StartDate != null)
{
var startTimeLeft = DateTime.UtcNow - (DateTime)activeInstallation.StartDate;
if (startTimeLeft.TotalHours >= 4)
{
activeInstallation.StartDate = null;
activeInstallation.FinishDate = null;
activeInstallation.StatusId = FIASVersionInstallationStatus.New;
_fiasInstallationManagerService.UpdateInstallation(activeInstallation);
activeInstallation.StartDate = null;
activeInstallation.FinishDate = null;
activeInstallation.StatusId = FIASVersionInstallationStatus.New;
_fiasInstallationManagerService.UpdateInstallation(activeInstallation);

stuckInstallations.Add(activeInstallation);
}
}
stuckInstallations.Add(activeInstallation);
}

await _fiasInstallationManagerService.SaveAsync();
Expand Down Expand Up @@ -240,6 +235,7 @@ public async Task SetInstallationToStatusNew()
_installation.FinishDate = null;
_fiasInstallationManagerService.UpdateInstallation(_installation);
await _fiasInstallationManagerService.SaveAsync();
_fiasInstallationManagerService.ClearChangeTracking();
}

public async Task SetInstallationToStatusInstalling()
Expand All @@ -248,6 +244,7 @@ public async Task SetInstallationToStatusInstalling()
_installation.StartDate = DateTime.UtcNow;
_fiasInstallationManagerService.UpdateInstallation(_installation);
await _fiasInstallationManagerService.SaveAsync();
_fiasInstallationManagerService.ClearChangeTracking();
}

public async Task SetInstallationToStatusInstalled()
Expand All @@ -256,8 +253,80 @@ public async Task SetInstallationToStatusInstalled()
_installation.FinishDate = DateTime.UtcNow;
_fiasInstallationManagerService.UpdateInstallation(_installation);
await _fiasInstallationManagerService.SaveAsync();
_fiasInstallationManagerService.ClearChangeTracking();
}

/// <summary>
/// Проверка был ли регион уже загружен
/// </summary>
/// <returns>Истина, если регион уже загружен. Ложь в противном случае.</returns>
public async Task<bool> RegionWasLoaded(int regionCode)
{
var regionInstallation = await _fiasInstallationManagerService
.GetVersionInstallationRegion(_installation.Id, regionCode);

if (regionInstallation == null)
{
return false;
}

return regionInstallation.StatusId == FIASVersionInstallationStatus.Installed;
}

/// <summary>
/// Установка статуса установки региона на "Устанавливается"
/// </summary>
public async Task SetRegionInstallationStatusToInstalling(int regionCode)
{
var regionInstallation = await _fiasInstallationManagerService
.GetVersionInstallationRegion(_installation.Id, regionCode);

if (regionInstallation == null)
{
regionInstallation = new FIASVersionInstallationRegion();
regionInstallation.FIASVersionInstallationId = _installation.Id;
regionInstallation.RegionCode = regionCode;
_fiasInstallationManagerService.AddInstallationRegion(regionInstallation);
}
else
{
_fiasInstallationManagerService.UpdateInstallationRegion(regionInstallation);
}

regionInstallation.StatusId = FIASVersionInstallationStatus.Installing;

await _fiasInstallationManagerService.SaveAsync();
_fiasInstallationManagerService.ClearChangeTracking();
}

/// <summary>
/// Установка статуса установки региона на "Установлено"
/// </summary>
public async Task SetRegionInstallationStatusToInstalled(int regionCode)
{
var regionInstallation = await _fiasInstallationManagerService
.GetVersionInstallationRegion(_installation.Id, regionCode);

if (regionInstallation == null)
{
regionInstallation = new FIASVersionInstallationRegion();
regionInstallation.FIASVersionInstallationId = _installation.Id;
regionInstallation.RegionCode = regionCode;
_fiasInstallationManagerService.AddInstallationRegion(regionInstallation);
}
else
{
_fiasInstallationManagerService.UpdateInstallationRegion(regionInstallation);
}

regionInstallation.StatusId = FIASVersionInstallationStatus.Installed;

await _fiasInstallationManagerService.SaveAsync();
_fiasInstallationManagerService.ClearChangeTracking();
}

#endregion

#region BaseCatalogs

public async Task LoadAddressObjectTypes()
Expand Down Expand Up @@ -2847,6 +2916,8 @@ public async Task LoadObjectsRegistry(Region region)

#endregion

#region Service

private IFIASDistributionReader GetDistributionReader()
{
if (_distributionDirectory == null || !Directory.Exists(_distributionDirectory))
Expand Down Expand Up @@ -3731,4 +3802,6 @@ private async Task SaveObjectsRegistryPortion(List<DistributionReader.Models.Cla
await _classifierDataRepository.SaveBulkAsync();
currentPortion.Clear();
}

#endregion
}
16 changes: 16 additions & 0 deletions Libs/YPermitin.FIASToolSet.DistributionLoader/IFIASDistributionLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,22 @@ Task DownloadAndExtractDistribution(
/// <returns>Объект асинхронной операции</returns>
Task SetInstallationToStatusInstalled();

/// <summary>
/// Проверка был ли регион уже загружен
/// </summary>
/// <returns>Истина, если регион уже загружен. Ложь в противном случае.</returns>
Task<bool> RegionWasLoaded(int regionCode);

/// <summary>
/// Установка статуса установки региона на "Устанавливается"
/// </summary>
Task SetRegionInstallationStatusToInstalling(int regionCode);

/// <summary>
/// Установка статуса установки региона на "Установлено"
/// </summary>
Task SetRegionInstallationStatusToInstalled(int regionCode);

#region BaseCatalogs

/// <summary>
Expand Down
32 changes: 32 additions & 0 deletions Libs/YPermitin.FIASToolSet.DistributionLoader/YPermitin.FIASToolSet.DistributionLoader.xml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 13 additions & 2 deletions Libs/YPermitin.FIASToolSet.Jobs/JobItems/InstallAndUpdateFIASJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace YPermitin.FIASToolSet.Jobs.JobItems;
[DisallowConcurrentExecution]
public class InstallAndUpdateFIASJob : IJob
{
private readonly ILogger<ActualizeFIASVersionHistoryJob> _logger;
private readonly ILogger<InstallAndUpdateFIASJob> _logger;
private readonly IServiceProvider _provider;
private readonly IConfiguration _configuration;

Expand All @@ -25,7 +25,7 @@ public class InstallAndUpdateFIASJob : IJob

public InstallAndUpdateFIASJob(
IServiceProvider provider,
ILogger<ActualizeFIASVersionHistoryJob> logger,
ILogger<InstallAndUpdateFIASJob> logger,
IConfiguration configuration)
{
_logger = logger;
Expand Down Expand Up @@ -148,6 +148,13 @@ await loader.DownloadAndExtractDistribution(args =>

foreach (var availableRegion in availableRegions)
{
bool regionWasLoaded = await loader.RegionWasLoaded(availableRegion.Code);
if (regionWasLoaded)
{
_logger.LogWarning($"Регион с кодом \"{availableRegion.Code}\" уже был загружен. Пропускаем обработки файлов данных.");
continue;
}

try
{
loader.ExtractDataForRegion(availableRegion);
Expand All @@ -157,6 +164,8 @@ await loader.DownloadAndExtractDistribution(args =>
_logger.LogError($"Не найден регион с кодом ${availableRegion.Code} среди доступных регионов в дистрибутиве ФИАС. Загрузка пропущена.");
continue;
}

await loader.SetRegionInstallationStatusToInstalling(availableRegion.Code);

await loader.LoadNormativeDocuments(availableRegion);
await loader.LoadAddressObjects(availableRegion);
Expand All @@ -177,6 +186,8 @@ await loader.DownloadAndExtractDistribution(args =>
await loader.LoadChangeHistory(availableRegion);
await loader.LoadObjectsRegistry(availableRegion);

await loader.SetRegionInstallationStatusToInstalled(availableRegion.Code);

if (_removeExtractedDistributionFiles)
{
loader.RemoveDistributionRegionDirectory(availableRegion);
Expand Down
39 changes: 39 additions & 0 deletions Libs/YPermitin.FIASToolSet.Storage.Core/Models/Versions/FIASVersionInstallationRegion.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
using System.ComponentModel.DataAnnotations.Schema;

namespace YPermitin.FIASToolSet.Storage.Core.Models.Versions;

/// <summary>
/// Информация о состоянии установки / обновления региона дистрибутива ФИАС
/// </summary>
public class FIASVersionInstallationRegion
{
/// <summary>
/// Идентификатор шага
/// </summary>
public Guid Id { get; set; }

/// <summary>
/// Установка версия ФИАС
/// </summary>
[ForeignKey("FIASVersionInstallationId")]
public FIASVersionInstallation FIASVersionInstallation { get; set; }
/// <summary>
/// Идентификатор установки версии ФИАС
/// </summary>
public Guid FIASVersionInstallationId { get; set; }

/// <summary>
/// Статус установки данных региона дистрибутива ФИАС
/// </summary>
[ForeignKey("StatusId")]
public FIASVersionInstallationStatus Status { get; set; }
/// <summary>
/// Идентификатор статуса установки данных региона дистрибутива ФИАС
/// </summary>
public Guid StatusId { get; set; }

/// <summary>
/// Код региона
/// </summary>
public int RegionCode { get; set; }
}
13 changes: 13 additions & 0 deletions Libs/YPermitin.FIASToolSet.Storage.Core/Services/IFIASInstallationManagerRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,21 @@ Task<List<FIASVersionInstallation>> GetInstallations(Guid? statusId = null, Guid

#endregion

#region FIASVersionInstallationRegion

Task<List<FIASVersionInstallationRegion>> GetVersionInstallationRegions(Guid installationId);

Task<FIASVersionInstallationRegion> GetVersionInstallationRegion(Guid installationId, int regionCode);

void AddInstallationRegion(FIASVersionInstallationRegion installationRegion);

void UpdateInstallationRegion(FIASVersionInstallationRegion installationRegion);

#endregion

Task<bool> BeginTransactionAsync();
Task CommitTransactionAsync();
Task RollbackTransactionAsync();
Task<bool> SaveAsync();
void ClearChangeTracking();
}
35 changes: 35 additions & 0 deletions Libs/YPermitin.FIASToolSet.Storage.Core/YPermitin.FIASToolSet.Storage.Core.xml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 12 additions & 1 deletion Libs/YPermitin.FIASToolSet.Storage.PostgreSQL/DbContexts/FIASToolSetServiceContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ public class FIASToolSetServiceContext : DbContext
public DbSet<FIASVersionInstallation> FIASVersionInstallations { get; set; }
public DbSet<FIASVersionInstallationType> FIASVersionInstallationsTypes { get; set; }
public DbSet<FIASVersionInstallationStep> FIASVersionInstallationSteps { get; set; }

public DbSet<FIASVersionInstallationRegion> FIASVersionInstallationRegions { get; set; }

#endregion

#region Notification
Expand Down Expand Up @@ -228,6 +229,16 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
.HasMaxLength(512);

#endregion

#region FIASVersionInstallationRegion

modelBuilder.Entity<FIASVersionInstallationRegion>()
.HasKey(e => e.Id);
modelBuilder.Entity<FIASVersionInstallationRegion>()
.HasIndex(e => new { e.RegionCode, e.FIASVersionInstallationId })
.IsUnique();

#endregion

#region AddressObjectType

Expand Down
Loading
Loading