From ad86a507ed652e27533bdf9d028e86af10f6a5e8 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 15:59:47 +0900 Subject: [PATCH 01/12] Bump lib9c main --- lib9c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib9c b/lib9c index fef54f5..95968df 160000 --- a/lib9c +++ b/lib9c @@ -1 +1 @@ -Subproject commit fef54f5fc2043a82cc11f8aae91507c6e2f0553a +Subproject commit 95968dfc2482cdaf21ad5fc71cf40b5f191b6d39 From 57783bd2fc0c2cb1afc969792267503406a1053b Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 16:00:25 +0900 Subject: [PATCH 02/12] Publish docker image --- .github/workflows/publish_docker_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_docker_image.yaml b/.github/workflows/publish_docker_image.yaml index 7ee34df..3ea5a65 100644 --- a/.github/workflows/publish_docker_image.yaml +++ b/.github/workflows/publish_docker_image.yaml @@ -6,7 +6,7 @@ on: - rc-* - hotfix/* - release/* - - upgrade/libplanet-4.0 + - feature/issue-117 env: DOCKER_REPO: planetariumhq/market-service jobs: From a35e043340920da66e06f4cf865560eef945c7ab Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 16:00:40 +0900 Subject: [PATCH 03/12] Update product by action render --- MarketService/MarketService.csproj | 1 + MarketService/Program.cs | 11 ++ MarketService/Receiver.cs | 23 ++++- MarketService/RpcClient.cs | 161 ++++++++++++++++++++++++++++- 4 files changed, 194 insertions(+), 2 deletions(-) diff --git a/MarketService/MarketService.csproj b/MarketService/MarketService.csproj index 33b45f5..ec29c26 100644 --- a/MarketService/MarketService.csproj +++ b/MarketService/MarketService.csproj @@ -31,6 +31,7 @@ + diff --git a/MarketService/Program.cs b/MarketService/Program.cs index bfd764b..bf57587 100644 --- a/MarketService/Program.cs +++ b/MarketService/Program.cs @@ -1,4 +1,8 @@ using System.Text.Json.Serialization; +using Lib9c.Formatters; +using Lib9c.Renderers; +using MessagePack; +using MessagePack.Resolvers; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Diagnostics; @@ -68,6 +72,13 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddHostedService(); WorkerOptions workerOptions = new(); + services.AddSingleton(); + var resolver = MessagePack.Resolvers.CompositeResolver.Create( + NineChroniclesResolver.Instance, + StandardResolver.Instance + ); + var options = MessagePackSerializerOptions.Standard.WithResolver(resolver); + MessagePackSerializer.DefaultOptions = options; Configuration.GetSection(WorkerOptions.WorkerConfig) .Bind(workerOptions); if (workerOptions.SyncShop) diff --git a/MarketService/Receiver.cs b/MarketService/Receiver.cs index 2514d46..1affc00 100644 --- a/MarketService/Receiver.cs +++ b/MarketService/Receiver.cs @@ -1,6 +1,10 @@ +using System.IO.Compression; using Bencodex; using Bencodex.Types; +using Lib9c.Renderers; using Libplanet.Types.Blocks; +using MessagePack; +using Nekoyume.Action; using Nekoyume.Shared.Hubs; namespace MarketService; @@ -11,14 +15,31 @@ public class Receiver : IActionEvaluationHubReceiver public Block PreviousTip; private readonly ILogger _logger; private readonly Codec _codec = new Codec(); + private readonly ActionRenderer _actionRenderer; - public Receiver(ILogger logger) + public Receiver(ILogger logger, ActionRenderer actionRenderer) { _logger = logger; + _actionRenderer = actionRenderer; } public void OnRender(byte[] evaluation) { + using (var cp = new MemoryStream(evaluation)) + { + using (var decompressed = new MemoryStream()) + { + using (var df = new DeflateStream(cp, CompressionMode.Decompress)) + { + df.CopyTo(decompressed); + decompressed.Seek(0, SeekOrigin.Begin); + var dec = decompressed.ToArray(); + var ev = MessagePackSerializer.Deserialize(dec) + .ToActionEvaluation(); + _actionRenderer.ActionRenderSubject.OnNext(ev); + } + } + } } public void OnUnrender(byte[] evaluation) diff --git a/MarketService/RpcClient.cs b/MarketService/RpcClient.cs index 7a3e71c..d83b33d 100644 --- a/MarketService/RpcClient.cs +++ b/MarketService/RpcClient.cs @@ -1,10 +1,13 @@ using System.Collections.Concurrent; using System.Diagnostics; +using System.Reactive.Subjects; using Bencodex; using Bencodex.Types; using Grpc.Core; using Grpc.Net.Client; using Lib9c.Model.Order; +using Lib9c.Renderers; +using Libplanet.Action; using Libplanet.Action.State; using Libplanet.Crypto; using Libplanet.Types.Blocks; @@ -13,6 +16,8 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using Nekoyume; +using Nekoyume.Action; +using Nekoyume.Action.Guild.Migration; using Nekoyume.Model.Item; using Nekoyume.Model.Market; using Nekoyume.Model.State; @@ -61,9 +66,10 @@ public class RpcClient public Block Tip => _receiver.Tip; public Block PreviousTip => _receiver.PreviousTip; + private readonly ActionRenderer _actionRenderer; public RpcClient(IOptions options, ILogger logger, Receiver receiver, - IDbContextFactory contextFactory) + IDbContextFactory contextFactory, ActionRenderer actionRenderer) { _logger = logger; _address = new PrivateKey().Address; @@ -82,6 +88,149 @@ public RpcClient(IOptions options, ILogger logger, ); _receiver = receiver; _contextFactory = contextFactory; + _actionRenderer = actionRenderer; + _actionRenderer.ActionRenderSubject.Subscribe(RenderAction); + } + + public async void RenderAction(ActionEvaluation ev) + { + if (ev.Exception is null) + { + var seed = ev.RandomSeed; + var random = new LocalRandom(seed); + var stateRootHash = ev.OutputState; + var hashBytes = stateRootHash.ToByteArray(); + switch (ev.Action) + { + // Insert new product + case RegisterProduct registerProduct: + { + var crystalEquipmentGrindingSheet = await GetSheet(hashBytes); + var crystalMonsterCollectionMultiplierSheet = + await GetSheet(hashBytes); + var costumeStatSheet = await GetSheet(hashBytes); + var products = new List(); + var productIds = registerProduct.RegisterInfos.Select(_ => random.GenerateRandomGuid()).ToList(); + var states = await GetProductStates(productIds, hashBytes); + foreach (var kv in states) + { + if (kv.Value is List deserialized) + { + products.Add(ProductFactory.DeserializeProduct(deserialized)); + } + } + + await InsertProducts(products, costumeStatSheet, crystalEquipmentGrindingSheet, crystalMonsterCollectionMultiplierSheet); + break; + } + // Update product exist = false + case BuyProduct buyProduct: + { + var orderIds = new List(); + var productIds = new List(); + foreach (var productInfo in buyProduct.ProductInfos) + { + if (productInfo is ItemProductInfo {Legacy: true} _) + { + orderIds.Add(productInfo.ProductId); + } + else + { + productIds.Add(productInfo.ProductId); + } + } + + var marketContext = await _contextFactory.CreateDbContextAsync(); + if (orderIds.Any()) + { + await UpdateProducts(productIds, marketContext, true); + } + + if (productIds.Any()) + { + await UpdateProducts(productIds, marketContext, false); + } + + break; + } + case CancelProductRegistration cancelProductRegistration: + { + var orderIds = new List(); + var productIds = new List(); + foreach (var productInfo in cancelProductRegistration.ProductInfos) + { + if (productInfo is ItemProductInfo {Legacy: true} _) + { + orderIds.Add(productInfo.ProductId); + } + else + { + productIds.Add(productInfo.ProductId); + } + } + + var marketContext = await _contextFactory.CreateDbContextAsync(); + if (orderIds.Any()) + { + await UpdateProducts(productIds, marketContext, true); + } + + if (productIds.Any()) + { + await UpdateProducts(productIds, marketContext, false); + } + + break; + } + // Insert new product and Update product exist = false + case ReRegisterProduct reRegisterProduct: + { + var deletedOrderIds = new List(); + var deletedProductIds = new List(); + var productIds = new List(); + foreach (var (productInfo, _) in reRegisterProduct.ReRegisterInfos) + { + if (productInfo is ItemProductInfo {Legacy: true} _) + { + deletedOrderIds.Add(productInfo.ProductId); + } + else + { + deletedProductIds.Add(productInfo.ProductId); + } + productIds.Add(random.GenerateRandomGuid()); + } + var crystalEquipmentGrindingSheet = await GetSheet(hashBytes); + var crystalMonsterCollectionMultiplierSheet = + await GetSheet(hashBytes); + var costumeStatSheet = await GetSheet(hashBytes); + var products = new List(); + var states = await GetProductStates(productIds, hashBytes); + foreach (var kv in states) + { + // check db all product ids avoid already synced products + if (kv.Value is List deserialized) + { + products.Add(ProductFactory.DeserializeProduct(deserialized)); + } + } + + await InsertProducts(products, costumeStatSheet, crystalEquipmentGrindingSheet, crystalMonsterCollectionMultiplierSheet); + var marketContext = await _contextFactory.CreateDbContextAsync(); + if (deletedOrderIds.Any()) + { + await UpdateProducts(productIds, marketContext, true); + } + + if (deletedProductIds.Any()) + { + await UpdateProducts(productIds, marketContext, false); + } + + break; + } + } + } } public async Task StartAsync(CancellationToken stoppingToken) @@ -758,4 +907,14 @@ public async Task> GetOrderDigests(List
avatarAddress }); return orderDigests.ToList(); } + + internal class LocalRandom : System.Random, IRandom + { + public int Seed { get; } + + public LocalRandom(int seed) : base(seed) + { + Seed = seed; + } + } } From 6142dcc843d717cc035dddae9100ca76822b69bd Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 18:06:31 +0900 Subject: [PATCH 04/12] Increase worker interval --- MarketService/ProductWorker.cs | 2 +- MarketService/ShopWorker.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MarketService/ProductWorker.cs b/MarketService/ProductWorker.cs index 59a6d36..04d9229 100644 --- a/MarketService/ProductWorker.cs +++ b/MarketService/ProductWorker.cs @@ -54,7 +54,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) stopWatch.Stop(); var ts = stopWatch.Elapsed; _logger.LogInformation("[ProductWorker]Complete sync product on {BlockIndex}. {TotalElapsed}", _rpcClient.Tip.Index, ts); - await Task.Delay(8000, stoppingToken); + await Task.Delay(6000 * 5, stoppingToken); } } } diff --git a/MarketService/ShopWorker.cs b/MarketService/ShopWorker.cs index 6ee2fd9..21e0735 100644 --- a/MarketService/ShopWorker.cs +++ b/MarketService/ShopWorker.cs @@ -54,7 +54,7 @@ await _rpcClient.SyncOrder(hashBytes, stopWatch.Stop(); var ts = stopWatch.Elapsed; _logger.LogInformation("Complete sync shop on {BlockIndex}. {TotalElapsed}", _rpcClient.Tip, ts); - await Task.Delay(8000, stoppingToken); + await Task.Delay(6000 * 5, stoppingToken); } } } From 457de2e2d84ef884a23bd45061357d9f33bd26fb Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 18:09:01 +0900 Subject: [PATCH 05/12] Fix test --- MarketService.Tests/RpcClientTest.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/MarketService.Tests/RpcClientTest.cs b/MarketService.Tests/RpcClientTest.cs index a565771..8291c84 100644 --- a/MarketService.Tests/RpcClientTest.cs +++ b/MarketService.Tests/RpcClientTest.cs @@ -8,6 +8,7 @@ using Bencodex.Types; using Grpc.Core; using Lib9c.Model.Order; +using Lib9c.Renderers; using Libplanet.Action.State; using Libplanet.Crypto; using Libplanet.Types.Assets; @@ -329,10 +330,10 @@ public RpcClientTest(ITestOutputHelper output) .UseLowerCaseNamingConvention().Options, new DbContextFactorySource()); #pragma warning restore EF1001 var rpcConfigOptions = new RpcConfigOptions {Host = "localhost", Port = 5000}; - var receiver = new Receiver(new Logger(new LoggerFactory())); + var receiver = new Receiver(new Logger(new LoggerFactory()), new ActionRenderer()); using var logger = _output.BuildLoggerFor(); _client = new TestClient(new OptionsWrapper(rpcConfigOptions), - logger, receiver, _contextFactory, _testService); + logger, receiver, _contextFactory, _testService, new ActionRenderer()); } [Theory] @@ -668,8 +669,8 @@ public async Task UpdateProducts(bool legacy) private class TestClient : RpcClient { public TestClient(IOptions options, ILogger logger, Receiver receiver, - IDbContextFactory contextFactory, TestService service) : base(options, logger, receiver, - contextFactory) + IDbContextFactory contextFactory, TestService service, ActionRenderer renderer) : base(options, logger, receiver, + contextFactory, renderer) { Service = service; var path = "../../../genesis"; From 131a5abbc77aa6e3d0f9a087e3cd68b321c53fc2 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 2 Sep 2024 18:36:12 +0900 Subject: [PATCH 06/12] Fix delay --- MarketService/ProductWorker.cs | 2 +- MarketService/ShopWorker.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MarketService/ProductWorker.cs b/MarketService/ProductWorker.cs index 04d9229..a62870a 100644 --- a/MarketService/ProductWorker.cs +++ b/MarketService/ProductWorker.cs @@ -54,7 +54,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) stopWatch.Stop(); var ts = stopWatch.Elapsed; _logger.LogInformation("[ProductWorker]Complete sync product on {BlockIndex}. {TotalElapsed}", _rpcClient.Tip.Index, ts); - await Task.Delay(6000 * 5, stoppingToken); + await Task.Delay(60000 * 5, stoppingToken); } } } diff --git a/MarketService/ShopWorker.cs b/MarketService/ShopWorker.cs index 21e0735..8da82b2 100644 --- a/MarketService/ShopWorker.cs +++ b/MarketService/ShopWorker.cs @@ -54,7 +54,7 @@ await _rpcClient.SyncOrder(hashBytes, stopWatch.Stop(); var ts = stopWatch.Elapsed; _logger.LogInformation("Complete sync shop on {BlockIndex}. {TotalElapsed}", _rpcClient.Tip, ts); - await Task.Delay(6000 * 5, stoppingToken); + await Task.Delay(60000 * 5, stoppingToken); } } } From 00b476446df004c0d7216de8eb1fb936a720096c Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Tue, 3 Sep 2024 11:13:04 +0900 Subject: [PATCH 07/12] Enable render action only write node --- MarketService.Tests/RpcClientTest.cs | 7 ++++++- MarketService/Receiver.cs | 26 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/MarketService.Tests/RpcClientTest.cs b/MarketService.Tests/RpcClientTest.cs index 8291c84..3e9e582 100644 --- a/MarketService.Tests/RpcClientTest.cs +++ b/MarketService.Tests/RpcClientTest.cs @@ -330,7 +330,12 @@ public RpcClientTest(ITestOutputHelper output) .UseLowerCaseNamingConvention().Options, new DbContextFactorySource()); #pragma warning restore EF1001 var rpcConfigOptions = new RpcConfigOptions {Host = "localhost", Port = 5000}; - var receiver = new Receiver(new Logger(new LoggerFactory()), new ActionRenderer()); + var workerOptions = new WorkerOptions + { + SyncProduct = false, + SyncShop = false, + }; + var receiver = new Receiver(new Logger(new LoggerFactory()), new ActionRenderer(), new OptionsWrapper(workerOptions)); using var logger = _output.BuildLoggerFor(); _client = new TestClient(new OptionsWrapper(rpcConfigOptions), logger, receiver, _contextFactory, _testService, new ActionRenderer()); diff --git a/MarketService/Receiver.cs b/MarketService/Receiver.cs index 1affc00..c0008aa 100644 --- a/MarketService/Receiver.cs +++ b/MarketService/Receiver.cs @@ -4,6 +4,7 @@ using Lib9c.Renderers; using Libplanet.Types.Blocks; using MessagePack; +using Microsoft.Extensions.Options; using Nekoyume.Action; using Nekoyume.Shared.Hubs; @@ -16,27 +17,32 @@ public class Receiver : IActionEvaluationHubReceiver private readonly ILogger _logger; private readonly Codec _codec = new Codec(); private readonly ActionRenderer _actionRenderer; + private readonly WorkerOptions _workerOptions; - public Receiver(ILogger logger, ActionRenderer actionRenderer) + public Receiver(ILogger logger, ActionRenderer actionRenderer, IOptions options) { _logger = logger; _actionRenderer = actionRenderer; + _workerOptions = options.Value; } public void OnRender(byte[] evaluation) { - using (var cp = new MemoryStream(evaluation)) + if (_workerOptions.SyncProduct || _workerOptions.SyncShop) { - using (var decompressed = new MemoryStream()) + using (var cp = new MemoryStream(evaluation)) { - using (var df = new DeflateStream(cp, CompressionMode.Decompress)) + using (var decompressed = new MemoryStream()) { - df.CopyTo(decompressed); - decompressed.Seek(0, SeekOrigin.Begin); - var dec = decompressed.ToArray(); - var ev = MessagePackSerializer.Deserialize(dec) - .ToActionEvaluation(); - _actionRenderer.ActionRenderSubject.OnNext(ev); + using (var df = new DeflateStream(cp, CompressionMode.Decompress)) + { + df.CopyTo(decompressed); + decompressed.Seek(0, SeekOrigin.Begin); + var dec = decompressed.ToArray(); + var ev = MessagePackSerializer.Deserialize(dec) + .ToActionEvaluation(); + _actionRenderer.ActionRenderSubject.OnNext(ev); + } } } } From de52e34410d594af15475a7af80b13f530c5bb8e Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Tue, 3 Sep 2024 13:36:56 +0900 Subject: [PATCH 08/12] Enable rpc client on write node --- MarketService/Program.cs | 57 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/MarketService/Program.cs b/MarketService/Program.cs index bf57587..2e1f2bd 100644 --- a/MarketService/Program.cs +++ b/MarketService/Program.cs @@ -68,38 +68,43 @@ public void ConfigureServices(IServiceCollection services) .UseLowerCaseNamingConvention() .ConfigureWarnings(w => w.Throw(RelationalEventId.MultipleCollectionIncludeWarning)) ); - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(); + services.AddMvc() + .AddJsonOptions( + options => { options.JsonSerializerOptions.ReferenceHandler = ReferenceHandler.IgnoreCycles; } + ); + var healthChecksBuilder = services.AddHealthChecks() + .AddDbContextCheck(); + WorkerOptions workerOptions = new(); - services.AddSingleton(); - var resolver = MessagePack.Resolvers.CompositeResolver.Create( - NineChroniclesResolver.Instance, - StandardResolver.Instance - ); - var options = MessagePackSerializerOptions.Standard.WithResolver(resolver); - MessagePackSerializer.DefaultOptions = options; Configuration.GetSection(WorkerOptions.WorkerConfig) .Bind(workerOptions); - if (workerOptions.SyncShop) + var writeMode = workerOptions.SyncProduct || workerOptions.SyncShop; + if (writeMode) { - services.AddHostedService(); - } + services + .AddHostedService() + .AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddSingleton(); + var resolver = MessagePack.Resolvers.CompositeResolver.Create( + NineChroniclesResolver.Instance, + StandardResolver.Instance + ); + var options = MessagePackSerializerOptions.Standard.WithResolver(resolver); + MessagePackSerializer.DefaultOptions = options; + if (workerOptions.SyncShop) + { + services.AddHostedService(); + } - if (workerOptions.SyncProduct) - { - services.AddHostedService(); + if (workerOptions.SyncProduct) + { + services.AddHostedService(); + } + healthChecksBuilder.AddCheck(nameof(RpcNodeHealthCheck)); } - services.AddMvc() - .AddJsonOptions( - options => { options.JsonSerializerOptions.ReferenceHandler = ReferenceHandler.IgnoreCycles; } - ); - services - .AddHostedService() - .AddSingleton(); - services.AddHealthChecks() - .AddDbContextCheck() - .AddCheck(nameof(RpcNodeHealthCheck)); } public void Configure(IApplicationBuilder app, IWebHostEnvironment env) From 06425e3eb15485c5a32f7a44eb6f688e9f44a60d Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Tue, 3 Sep 2024 14:58:58 +0900 Subject: [PATCH 09/12] document for product sync --- MarketService/RpcClient.cs | 127 ++++++++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 30 deletions(-) diff --git a/MarketService/RpcClient.cs b/MarketService/RpcClient.cs index d83b33d..a1125ed 100644 --- a/MarketService/RpcClient.cs +++ b/MarketService/RpcClient.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Diagnostics; -using System.Reactive.Subjects; using Bencodex; using Bencodex.Types; using Grpc.Core; @@ -17,7 +16,6 @@ using Microsoft.Extensions.Options; using Nekoyume; using Nekoyume.Action; -using Nekoyume.Action.Guild.Migration; using Nekoyume.Model.Item; using Nekoyume.Model.Market; using Nekoyume.Model.State; @@ -33,6 +31,9 @@ public class RpcClient { private const int MaxDegreeOfParallelism = 8; + /// + /// for + /// private static readonly List ShardedSubTypes = new() { ItemSubType.Weapon, @@ -92,6 +93,10 @@ public RpcClient(IOptions options, ILogger logger, _actionRenderer.ActionRenderSubject.Subscribe(RenderAction); } + /// + /// Insert or Update by Market related actions. + /// + /// public async void RenderAction(ActionEvaluation ev) { if (ev.Exception is null) @@ -284,6 +289,12 @@ public async Task StopAsync(CancellationToken cancellationToken) await _hub.LeaveAsync(); } + /// + /// Get of for get registered agent addresses + /// + /// + /// + /// public async Task> GetOrderDigests(ItemSubType itemSubType, byte[] hashBytes) { while (Tip is null) await Task.Delay(100); @@ -294,7 +305,7 @@ public async Task> GetOrderDigests(ItemSubType itemSubType, by var addressList = GetShopAddress(itemSubType); var result = await Service.GetBulkStateByStateRootHash(hashBytes, ReservedAddresses.LegacyAccount.ToByteArray(), addressList); - var shopStates = GetShopStates(result); + var shopStates = DeserializeShopStates(result); foreach (var shopState in shopStates) foreach (var orderDigest in shopState.OrderDigestList) { @@ -309,6 +320,13 @@ public async Task> GetOrderDigests(ItemSubType itemSubType, by return orderDigestList; } + /// + /// Insert and Update from + /// + /// byte array from + /// + /// + /// public async Task SyncOrder(byte[] hashBytes, CrystalEquipmentGrindingSheet crystalEquipmentGrindingSheet, CrystalMonsterCollectionMultiplierSheet crystalMonsterCollectionMultiplierSheet, @@ -465,6 +483,13 @@ await InsertOrders(hashBytes, orderIds.ToList(), tradableIds, marketContext, ord _logger.LogDebug("RestoreProducts: {Ts}", sw.Elapsed); } + /// + /// Set exist = false + /// + /// + /// + /// + /// public async Task UpdateProducts(List deletedIds, MarketContext marketContext, bool legacy, bool exist = false) { @@ -480,6 +505,13 @@ await marketContext.Database.ExecuteSqlRawAsync( } } + /// + /// Create from + /// + /// byte array from + /// + /// + /// public async Task InsertOrders(byte[] hashBytes, List orderIds, List tradableIds, MarketContext marketContext, List orderDigestList, CrystalEquipmentGrindingSheet crystalEquipmentGrindingSheet, @@ -526,6 +558,13 @@ public async Task InsertOrders(byte[] hashBytes, List orderIds, List } } + /// + /// Insert and Update ProductModel from + /// + /// byte array from + /// + /// + /// public async Task SyncProduct(byte[] hashBytes, CrystalEquipmentGrindingSheet crystalEquipmentGrindingSheet, CrystalMonsterCollectionMultiplierSheet crystalMonsterCollectionMultiplierSheet, CostumeStatSheet costumeStatSheet) @@ -565,7 +604,7 @@ public async Task SyncProduct(byte[] hashBytes, CrystalEquipmentGrindingSheet cr sw.Stop(); _logger.LogDebug("[ProductWorker]Get ChunkedStates: {Elapsed}", sw.Elapsed); sw.Restart(); - var productLists = GetProductsState(productListResult); + var productLists = DeserializeProductsState(productListResult); sw.Stop(); _logger.LogDebug("[ProductWorker]Get ProductsState: {Elapsed}", sw.Elapsed); sw.Restart(); @@ -624,6 +663,13 @@ private async Task> GetProductStates(List product return result; } + /// + /// Insert from + /// + /// List of + /// + /// + /// public async Task InsertProducts(List products, CostumeStatSheet costumeStatSheet, CrystalEquipmentGrindingSheet crystalEquipmentGrindingSheet, CrystalMonsterCollectionMultiplierSheet crystalMonsterCollectionMultiplierSheet) @@ -713,31 +759,7 @@ public async Task GetBlockStateRootHashBytes() return _receiver.Tip.StateRootHash.ToByteArray(); } - public async Task> GetProductStates(IEnumerable
avatarAddressList, - byte[] hashBytes) - { - var productListAddresses = avatarAddressList.Select(a => ProductsState.DeriveAddress(a).ToByteArray()).ToList(); - var productListResult = - await GetChunkedStates(hashBytes, ReservedAddresses.LegacyAccount.ToByteArray(), productListAddresses); - var productLists = GetProductsState(productListResult); - var productIdList = productLists.SelectMany(p => p.ProductIds).ToList(); - var productIds = new Dictionary(); - foreach (var productId in productIdList) productIds[Product.DeriveAddress(productId)] = productId; - var productResult = await GetChunkedStates( - hashBytes, - ReservedAddresses.LegacyAccount.ToByteArray(), - productIds.Keys.Select(a => a.ToByteArray()).ToList()); - var result = new Dictionary(); - foreach (var kv in productResult) - { - var productId = productIds[kv.Key]; - result[productId] = kv.Value; - } - - return result; - } - - public List GetProductsState(Dictionary queryResult) + public List DeserializeProductsState(Dictionary queryResult) { var result = new List(); foreach (var kv in queryResult) @@ -759,7 +781,12 @@ public IEnumerable GetShopAddress(ItemSubType itemSubType) return new[] {ShardedShopStateV2.DeriveAddress(itemSubType, "").ToByteArray()}; } - public IEnumerable GetShopStates(Dictionary queryResult) + /// + /// Get of for listing + /// + /// + /// + public IEnumerable DeserializeShopStates(Dictionary queryResult) { var result = new List(); foreach (var kv in queryResult) @@ -771,6 +798,14 @@ public IEnumerable GetShopStates(Dictionary return result; } + /// + /// Get of for . + /// + /// + /// + /// + /// + /// public async Task> GetOrders(IEnumerable orderIds, byte[] hashBytes) { var orderAddressList = orderIds.Select(i => Order.DeriveAddress(i).ToByteArray()).ToList(); @@ -793,6 +828,14 @@ await Parallel.ForEachAsync(chunks, _parallelOptions, async (chunk, token) => return orderBag.ToList(); } + /// + /// Get of for . + /// + /// + /// + /// + /// + /// public async Task> GetItems(IEnumerable tradableIds, byte[] hashBytes) { var itemAddressList = tradableIds.Select(i => Addresses.GetItemAddress(i).ToByteArray()).ToList(); @@ -830,6 +873,13 @@ public async Task> GetStates(byte[] hashBytes, byte[ return result.ToDictionary(kv => kv.Key, kv => kv.Value); } + /// + /// GetBulkState with chunking size 1000 + /// + /// + /// + /// + /// public async Task> GetChunkedStates(byte[] hashBytes, byte[] accountBytes, List addressList) { var result = new ConcurrentDictionary(); @@ -847,6 +897,12 @@ await Parallel.ForEachAsync(chunks, _parallelOptions, async (chunk, token) => return result.ToDictionary(kv => kv.Key, kv => kv.Value); } + /// + /// Get for listing avatar addresses. + /// + /// + /// + /// public async Task> GetAgentStates(byte[] hashBytes, List addressList) { var result = new ConcurrentDictionary(); @@ -870,6 +926,11 @@ public async Task> GetAgentStates(byte[] hashByt return result.ToDictionary(kv => kv.Key, kv => kv.Value); } + /// + /// Get for listing avatar addresses. + /// + /// + /// public async Task GetMarket(byte[] hashBytes) { var marketResult = await Service.GetStateByStateRootHash( @@ -882,6 +943,12 @@ public async Task GetMarket(byte[] hashBytes) return new MarketState(); } + /// + /// Get of from avatar addresses. + /// + /// + /// + /// of public async Task> GetOrderDigests(List
avatarAddresses, byte[] hashBytes) { var digestListStateAddresses = From 0c7b5f89bbf24ce9524189d4f56033e20995d903 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Tue, 3 Sep 2024 21:02:04 +0900 Subject: [PATCH 10/12] Fix typo --- MarketService/RpcClient.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MarketService/RpcClient.cs b/MarketService/RpcClient.cs index a1125ed..b74fce5 100644 --- a/MarketService/RpcClient.cs +++ b/MarketService/RpcClient.cs @@ -224,12 +224,12 @@ public async void RenderAction(ActionEvaluation ev) var marketContext = await _contextFactory.CreateDbContextAsync(); if (deletedOrderIds.Any()) { - await UpdateProducts(productIds, marketContext, true); + await UpdateProducts(deletedOrderIds, marketContext, true); } if (deletedProductIds.Any()) { - await UpdateProducts(productIds, marketContext, false); + await UpdateProducts(deletedProductIds, marketContext, false); } break; From 46591353ad8f25b5f8923d424a1b9c19e69ca2ca Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Mon, 9 Sep 2024 20:20:59 +0900 Subject: [PATCH 11/12] Delete not exist products --- .github/workflows/publish_docker_image.yaml | 2 +- MarketService.Tests/RpcClientTest.cs | 14 +++++-------- MarketService/RpcClient.cs | 22 +++++++++++---------- lib9c | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/publish_docker_image.yaml b/.github/workflows/publish_docker_image.yaml index 3ea5a65..df0d1d5 100644 --- a/.github/workflows/publish_docker_image.yaml +++ b/.github/workflows/publish_docker_image.yaml @@ -6,7 +6,7 @@ on: - rc-* - hotfix/* - release/* - - feature/issue-117 + - feature/issue-123 env: DOCKER_REPO: planetariumhq/market-service jobs: diff --git a/MarketService.Tests/RpcClientTest.cs b/MarketService.Tests/RpcClientTest.cs index 3e9e582..eac68b8 100644 --- a/MarketService.Tests/RpcClientTest.cs +++ b/MarketService.Tests/RpcClientTest.cs @@ -414,9 +414,7 @@ await _client.SyncOrder(null!, _crystalEquipmentGrindingSheet, #pragma warning disable EF1001 var nextContext = await _contextFactory.CreateDbContextAsync(ct); #pragma warning restore EF1001 - var updatedProductModel = Assert.Single(nextContext.Products); - Assert.Equal(productModel.ProductId, updatedProductModel.ProductId); - Assert.False(updatedProductModel.Exist); + Assert.Empty(nextContext.Products); } [Theory] @@ -493,11 +491,8 @@ await _client.SyncOrder(null!, _crystalEquipmentGrindingSheet, #pragma warning disable EF1001 var nextContext = await _contextFactory.CreateDbContextAsync(ct); #pragma warning restore EF1001 - Assert.Equal(2, nextContext.Products.Count()); - var oldProduct = nextContext.Products.Single(p => p.ProductId == order.OrderId); - Assert.Equal(1, oldProduct.Price); - Assert.False(oldProduct.Exist); - var newProduct = nextContext.Products.Single(p => p.ProductId == order2.OrderId); + Assert.Empty(nextContext.Products.Where(p => p.ProductId == order.OrderId)); + var newProduct = Assert.Single(nextContext.Products); Assert.Equal(2, newProduct.Price); Assert.True(newProduct.Exist); } @@ -575,8 +570,9 @@ await _client.SyncProduct(null!, _crystalEquipmentGrindingSheet, _crystalMonster var nextContext = await _contextFactory.CreateDbContextAsync(ct); #pragma warning restore EF1001 var nextProducts = nextContext.Products.AsNoTracking().ToList(); + Assert.Equal(90, nextProducts.Count); Assert.Equal(90, nextProducts.Count(p => p.Exist)); - Assert.Equal(10, nextProducts.Count(p => !p.Exist)); + Assert.Equal(0, nextProducts.Count(p => !p.Exist)); } [Fact] diff --git a/MarketService/RpcClient.cs b/MarketService/RpcClient.cs index b74fce5..9bd5a95 100644 --- a/MarketService/RpcClient.cs +++ b/MarketService/RpcClient.cs @@ -474,13 +474,7 @@ await InsertOrders(hashBytes, orderIds.ToList(), tradableIds, marketContext, ord _logger.LogDebug("InsertOrders: {Ts}", sw.Elapsed); sw.Restart(); - await UpdateProducts(deletedIds.ToList(), marketContext, true); - sw.Stop(); - _logger.LogDebug("DeleteProducts: {Ts}", sw.Elapsed); - sw.Restart(); - await UpdateProducts(restoreIds.ToList(), marketContext, true, true); - sw.Stop(); - _logger.LogDebug("RestoreProducts: {Ts}", sw.Elapsed); + await DeleteProducts(deletedIds.ToList(), marketContext); } /// @@ -570,7 +564,6 @@ public async Task SyncProduct(byte[] hashBytes, CrystalEquipmentGrindingSheet cr CostumeStatSheet costumeStatSheet) { while (Tip is null) await Task.Delay(100); - try { var sw = new Stopwatch(); @@ -636,7 +629,7 @@ await InsertProducts(products, costumeStatSheet, crystalEquipmentGrindingSheet, sw.Stop(); _logger.LogDebug("[ProductWorker]Insert Products: {Elapsed}", sw.Elapsed); sw.Restart(); - await UpdateProducts(deletedIds, marketContext, false); + await DeleteProducts(deletedIds, marketContext); sw.Stop(); _logger.LogDebug("[ProductWorker]Update Products: {Elapsed}", sw.Elapsed); } @@ -975,7 +968,16 @@ public async Task> GetOrderDigests(List
avatarAddress return orderDigests.ToList(); } - internal class LocalRandom : System.Random, IRandom + public async Task DeleteProducts(List deletedIds, MarketContext marketContext) + { + // 등록취소, 판매된 경우 해당 row를 삭제함 + if (deletedIds.Any()) + { + await marketContext.Products.Where(p => deletedIds.Contains(p.ProductId)).ExecuteDeleteAsync(); + } + } + + internal class LocalRandom : Random, IRandom { public int Seed { get; } diff --git a/lib9c b/lib9c index 95968df..fef54f5 160000 --- a/lib9c +++ b/lib9c @@ -1 +1 @@ -Subproject commit 95968dfc2482cdaf21ad5fc71cf40b5f191b6d39 +Subproject commit fef54f5fc2043a82cc11f8aae91507c6e2f0553a From a72f06b2704994f32dc6822d2d14e38a12714623 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Sat, 21 Sep 2024 13:49:56 +0900 Subject: [PATCH 12/12] Delete product in action render --- MarketService/RpcClient.cs | 73 ++++++-------------------------------- 1 file changed, 11 insertions(+), 62 deletions(-) diff --git a/MarketService/RpcClient.cs b/MarketService/RpcClient.cs index 9bd5a95..511f816 100644 --- a/MarketService/RpcClient.cs +++ b/MarketService/RpcClient.cs @@ -128,81 +128,39 @@ public async void RenderAction(ActionEvaluation ev) await InsertProducts(products, costumeStatSheet, crystalEquipmentGrindingSheet, crystalMonsterCollectionMultiplierSheet); break; } - // Update product exist = false + // delete product case BuyProduct buyProduct: { - var orderIds = new List(); - var productIds = new List(); + var deletedIds = new List(); foreach (var productInfo in buyProduct.ProductInfos) { - if (productInfo is ItemProductInfo {Legacy: true} _) - { - orderIds.Add(productInfo.ProductId); - } - else - { - productIds.Add(productInfo.ProductId); - } + deletedIds.Add(productInfo.ProductId); } var marketContext = await _contextFactory.CreateDbContextAsync(); - if (orderIds.Any()) - { - await UpdateProducts(productIds, marketContext, true); - } - - if (productIds.Any()) - { - await UpdateProducts(productIds, marketContext, false); - } - + await DeleteProducts(deletedIds, marketContext); break; } case CancelProductRegistration cancelProductRegistration: { - var orderIds = new List(); - var productIds = new List(); + var deletedIds = new List(); foreach (var productInfo in cancelProductRegistration.ProductInfos) { - if (productInfo is ItemProductInfo {Legacy: true} _) - { - orderIds.Add(productInfo.ProductId); - } - else - { - productIds.Add(productInfo.ProductId); - } + deletedIds.Add(productInfo.ProductId); } var marketContext = await _contextFactory.CreateDbContextAsync(); - if (orderIds.Any()) - { - await UpdateProducts(productIds, marketContext, true); - } - - if (productIds.Any()) - { - await UpdateProducts(productIds, marketContext, false); - } - + await DeleteProducts(deletedIds, marketContext); break; } - // Insert new product and Update product exist = false + // Insert new product and delete product case ReRegisterProduct reRegisterProduct: { - var deletedOrderIds = new List(); - var deletedProductIds = new List(); var productIds = new List(); + var deletedIds = new List(); foreach (var (productInfo, _) in reRegisterProduct.ReRegisterInfos) { - if (productInfo is ItemProductInfo {Legacy: true} _) - { - deletedOrderIds.Add(productInfo.ProductId); - } - else - { - deletedProductIds.Add(productInfo.ProductId); - } + deletedIds.Add(productInfo.ProductId); productIds.Add(random.GenerateRandomGuid()); } var crystalEquipmentGrindingSheet = await GetSheet(hashBytes); @@ -222,16 +180,7 @@ public async void RenderAction(ActionEvaluation ev) await InsertProducts(products, costumeStatSheet, crystalEquipmentGrindingSheet, crystalMonsterCollectionMultiplierSheet); var marketContext = await _contextFactory.CreateDbContextAsync(); - if (deletedOrderIds.Any()) - { - await UpdateProducts(deletedOrderIds, marketContext, true); - } - - if (deletedProductIds.Any()) - { - await UpdateProducts(deletedProductIds, marketContext, false); - } - + await DeleteProducts(deletedIds, marketContext); break; } }