Skip to content

Commit

Permalink
Улучшения и оптимизации
Browse files Browse the repository at this point in the history
- Автоматическое применение миграций при старте приложения.
- Для сущностей истории изменений и объектов реестра добавлено поле хэша, чтобы однозначно определять запись.
- Для сущностей истории изменений и объектов реестра добавлено поле Id с уникальным хэшем элемента (binary(16) и оптимизированы методы работы с элементами по этому ID. Старые ключи больше не используются, как и доп. индексы.
- Обновлены миграции PostgreSQL / SQL Server
- При загрузке истории изменений и объектов реестра теперь исполоьзуются уникальные ключи для каждой записи.
- Расширены unit-тесты для библиотеки чтения данных ФИАС.
- Небольшой рефакторинг заданий
  • Loading branch information
YPermitin committed Dec 29, 2023
1 parent b061f03 commit d247c78
Show file tree
Hide file tree
Showing 32 changed files with 7,387 additions and 500 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3838,18 +3838,13 @@ private async Task SaveNormativeDocumentsPortion(List<DistributionReader.Models.

private async Task SaveChangeHistoryPortion(List<DistributionReader.Models.ClassifierData.ChangeHistory> currentPortion)
{
var existsNormativeDocuments = await _classifierDataRepository
.GetChangeHistoryItems(keys: currentPortion.Select(e => new ChangeHistory.ChangeHistoryItemKey()
{
ChangeId = e.ChangeId,
AddressObjectGuid = e.AddressObjectGuid,
ObjectId = e.ObjectId
}).ToList());
var existsChangeHistoryItems = await _classifierDataRepository
.GetChangeHistoryItems(ids: currentPortion.Select(e => e.HashMD5).ToList());

var itemsToProceed = currentPortion.AsQueryable()
.LeftJoin(existsNormativeDocuments.AsQueryable(),
o => new { o.ObjectId, o.AddressObjectGuid, o.ChangeId },
i => new { i.ObjectId, i.AddressObjectGuid, i.ChangeId },
.LeftJoin(existsChangeHistoryItems.AsQueryable(),
o => o.HashMD5,
i => i.Id,
(r) => new
{
SourceItem = r.Outer,
Expand All @@ -3863,9 +3858,7 @@ private async Task SaveChangeHistoryPortion(List<DistributionReader.Models.Class
if (itemToProceed.DatabaseItem == null)
{
changeHistory = new ChangeHistory();
changeHistory.ObjectId = itemToProceed.SourceItem.ObjectId;
changeHistory.AddressObjectGuid = itemToProceed.SourceItem.AddressObjectGuid;
changeHistory.ChangeId = itemToProceed.SourceItem.ChangeId;
changeHistory.Id = itemToProceed.SourceItem.HashMD5;
_classifierDataRepository.AddChangeHistory(changeHistory);
}
else
Expand All @@ -3874,6 +3867,9 @@ private async Task SaveChangeHistoryPortion(List<DistributionReader.Models.Class
_classifierDataRepository.UpdateChangeHistory(changeHistory);
}

changeHistory.ObjectId = itemToProceed.SourceItem.ObjectId;
changeHistory.AddressObjectGuid = itemToProceed.SourceItem.AddressObjectGuid;
changeHistory.ChangeId = itemToProceed.SourceItem.ChangeId;
changeHistory.ObjectId = itemToProceed.SourceItem.ObjectId;
changeHistory.AddressObjectGuid = itemToProceed.SourceItem.AddressObjectGuid;
changeHistory.ChangeId = itemToProceed.SourceItem.ChangeId;
Expand All @@ -3889,17 +3885,12 @@ private async Task SaveChangeHistoryPortion(List<DistributionReader.Models.Class
private async Task SaveObjectsRegistryPortion(List<DistributionReader.Models.ClassifierData.ObjectRegistry> currentPortion)
{
var existsObjectsRegistry = await _classifierDataRepository
.GetObjectRegistryItems(keys: currentPortion.Select(e => new ObjectRegistry.ObjectRegistryItemKey()
{
ChangeId = e.ChangeId,
ObjectGuid = e.ObjectGuid,
ObjectId = e.ObjectId
}).ToList());

.GetObjectRegistryItems(ids: currentPortion.Select(e => e.HashMD5).ToList());

var itemsToProceed = currentPortion.AsQueryable()
.LeftJoin(existsObjectsRegistry.AsQueryable(),
o => new { o.ObjectId, o.ObjectGuid, o.ChangeId },
i => new { i.ObjectId, i.ObjectGuid, i.ChangeId },
o => o.HashMD5,
i => i.Id,
(r) => new
{
SourceItem = r.Outer,
Expand All @@ -3913,9 +3904,7 @@ private async Task SaveObjectsRegistryPortion(List<DistributionReader.Models.Cla
if (itemToProceed.DatabaseItem == null)
{
objectRegistry = new ObjectRegistry();
objectRegistry.ObjectId = itemToProceed.SourceItem.ObjectId;
objectRegistry.ObjectGuid = itemToProceed.SourceItem.ObjectGuid;
objectRegistry.ChangeId = itemToProceed.SourceItem.ChangeId;
objectRegistry.Id = itemToProceed.SourceItem.HashMD5;
_classifierDataRepository.AddObjectRegistry(objectRegistry);
}
else
Expand All @@ -3924,6 +3913,9 @@ private async Task SaveObjectsRegistryPortion(List<DistributionReader.Models.Cla
_classifierDataRepository.UpdateObjectRegistry(objectRegistry);
}

objectRegistry.ObjectId = itemToProceed.SourceItem.ObjectId;
objectRegistry.ObjectGuid = itemToProceed.SourceItem.ObjectGuid;
objectRegistry.ChangeId = itemToProceed.SourceItem.ChangeId;
objectRegistry.ObjectId = itemToProceed.SourceItem.ObjectId;
objectRegistry.ObjectGuid = itemToProceed.SourceItem.ObjectGuid;
objectRegistry.ChangeId = itemToProceed.SourceItem.ChangeId;
Expand Down
118 changes: 118 additions & 0 deletions Libs/YPermitin.FIASToolSet.DistributionReader/Helpers/MD5Builder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
using System.Security.Cryptography;
using System.Text;

namespace YPermitin.FIASToolSet.DistributionReader.Helpers;

public class MD5Builder : IDisposable
{
private readonly MD5 _md5;

public MD5Builder()
{
_md5 = MD5.Create();
}

public MD5Builder Add(int value)
{
AddIntToHash(_md5, value);
return this;
}

public MD5Builder Add(long value)
{
AddLongToHash(_md5, value);
return this;
}

public MD5Builder Add(DateTime value)
{
AddDateTimeToHash(_md5, value);
return this;
}

public MD5Builder Add(string value)
{
AddStringToHash(_md5, value);
return this;
}

public MD5Builder Add(Guid value)
{
AddGuidToHash(_md5, value);
return this;
}

public MD5Builder Add(bool value)
{
AddBoolToHash(_md5, value);
return this;
}

public byte[] Build()
{
_md5.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
return _md5.Hash;
}

public string BuildAsString()
{
var hash = Build();
return ConvertByteArrayToString(hash);
}

public void Dispose()
{
_md5?.Dispose();
}

#region Service

private void AddStringToHash(ICryptoTransform cryptoTransform, string textToHash)
{
var inputBuffer = Encoding.UTF8.GetBytes(textToHash);
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private void AddIntToHash(ICryptoTransform cryptoTransform, int intToHash)
{
var inputBuffer = BitConverter.GetBytes(intToHash);
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private void AddLongToHash(ICryptoTransform cryptoTransform, long longToHash)
{
var inputBuffer = BitConverter.GetBytes(longToHash);
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private void AddDateTimeToHash(ICryptoTransform cryptoTransform, DateTime dateTimeToHash)
{
var inputBuffer = BitConverter.GetBytes(dateTimeToHash.ToBinary());
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private void AddGuidToHash(ICryptoTransform cryptoTransform, Guid guidToHash)
{
var inputBuffer = guidToHash.ToByteArray();
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private void AddBoolToHash(ICryptoTransform cryptoTransform, bool boolToHash)
{
var inputBuffer = BitConverter.GetBytes(boolToHash);
cryptoTransform.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0);
}

private string ConvertByteArrayToString(byte[] bytes)
{
var sb = new StringBuilder();
foreach (var b in bytes)
{
sb.Append(b.ToString("X2"));
}

return sb.ToString();
}

#endregion
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using YPermitin.FIASToolSet.DistributionReader.Helpers;
using YPermitin.FIASToolSet.DistributionReader.Models.BaseCatalogs;

namespace YPermitin.FIASToolSet.DistributionReader.Models.ClassifierData;
Expand Down Expand Up @@ -37,6 +38,33 @@ public class ChangeHistory
/// Дата изменения
/// </summary>
public readonly DateOnly ChangeDate;

public byte[] _hashMD5;
/// <summary>
/// MD5-хэш элемента
/// </summary>
public byte[] HashMD5
{
get
{
if (_hashMD5 == null)
{
using (var md5Builder = new MD5Builder())
{
_hashMD5 = md5Builder
.Add(ChangeId)
.Add(ObjectId)
.Add(AddressObjectGuid)
.Add(ChangeDate.ToDateTime(TimeOnly.MinValue))
.Add(OperationTypeId)
.Add(NormativeDocId)
.Build();
}
}

return _hashMD5;
}
}

public ChangeHistory(int objectId, Guid addressObjectGuid, int changeId, int operationTypeId,
int normativeDocId, DateOnly changeDate)
Expand All @@ -48,4 +76,6 @@ public ChangeHistory(int objectId, Guid addressObjectGuid, int changeId, int ope
NormativeDocId = normativeDocId;
ChangeDate = changeDate;
}


}
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using YPermitin.FIASToolSet.DistributionReader.Helpers;

namespace YPermitin.FIASToolSet.DistributionReader.Models.ClassifierData;

/// <summary>
Expand Down Expand Up @@ -41,6 +43,34 @@ public class ObjectRegistry
/// </summary>
public readonly DateOnly UpdateDate;

public byte[] _hashMD5;
/// <summary>
/// MD5-хэш элемента
/// </summary>
public byte[] HashMD5
{
get
{
if (_hashMD5 == null)
{
using (var md5Builder = new MD5Builder())
{
_hashMD5 = md5Builder
.Add(ObjectId)
.Add(ObjectGuid)
.Add(ChangeId)
.Add(IsActive)
.Add(LevelId)
.Add(CreateDate.ToDateTime(TimeOnly.MinValue))
.Add(UpdateDate.ToDateTime(TimeOnly.MinValue))
.Build();
}
}

return _hashMD5;
}
}

public ObjectRegistry(int objectId, Guid objectGuid, int changeId, bool isActive,
int levelId, DateOnly createDate, DateOnly updateDate)
{
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d247c78

Please sign in to comment.