Skip to content

Commit

Permalink
Merge pull request #736 from planetarium/main
Browse files Browse the repository at this point in the history 
Main
  • Loading branch information
@area363
area363 authored Jul 30, 2024
2 parents 8d0cfc6 + e745ab6 commit d797568
Show file tree
Hide file tree
Showing 8 changed files with 258 additions and 96 deletions.
172 changes: 164 additions & 8 deletions NineChronicles.DataProvider.Executable/Commands/MySqlMigration.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
using Nekoyume.Action.AdventureBoss;
using Nekoyume.Model.EnumType;
using NineChronicles.DataProvider.DataRendering.AdventureBoss;
using NineChronicles.DataProvider.Store.Models.AdventureBoss;

namespace NineChronicles.DataProvider.Executable.Commands
{
Expand Down Expand Up @@ -43,6 +46,12 @@ namespace NineChronicles.DataProvider.Executable.Commands

public class MySqlMigration
{
private readonly Dictionary<long, AdventureBossSeasonModel> _adventureBossSeasonDict = new ();
private readonly List<AdventureBossWantedModel> _adventureBossWantedList = new ();
private readonly List<AdventureBossChallengeModel> _adventureBossChallengeList = new ();
private readonly List<AdventureBossRushModel> _adventureBossRushList = new ();
private readonly List<AdventureBossUnlockFloorModel> _adventureBossUnlockFloorList = new ();
private readonly List<AdventureBossClaimRewardModel> _adventureBossClaimRewardList = new ();
private string _connectionString;
private IStore _baseStore;
private BlockChain _baseChain;
Expand Down Expand Up @@ -93,7 +102,7 @@ public class MySqlMigration
private List<RequestPledgeModel> _requestPledgeList;

[Command(Description = "Migrate action data in rocksdb store to mysql db.")]
public void Migration(
public async Task Migration(
[Option('o', Description = "Rocksdb path to migrate.")]
string storePath,
[Option(
Expand Down Expand Up @@ -191,7 +200,9 @@ public void Migration(
_ => blockPolicy.BlockAction,
baseStateStore,
new NCActionLoader());
_baseChain = new BlockChain(blockPolicy, stagePolicy, _baseStore, baseStateStore, genesis, blockChainStates, actionEvaluator);
_baseChain = new BlockChain(
blockPolicy, stagePolicy, _baseStore, baseStateStore, genesis, blockChainStates, actionEvaluator
);

// Check offset and limit value based on chain height
long height = _baseChain.Tip.Index;
Expand Down Expand Up @@ -273,7 +284,11 @@ public void Migration(
}

foreach (var item in
_baseStore.IterateIndexes(_baseChain.Id, offset + offsetIdx ?? 0 + offsetIdx, limitInterval).Select((value, i) => new { i, value }))
_baseStore.IterateIndexes(
_baseChain.Id,
offset + offsetIdx ?? 0 + offsetIdx,
limitInterval
).Select((value, i) => new { i, value }))
{
var block = _baseStore.GetBlock(item.value);
_blockList.Add(BlockData.GetBlockInfo(block));
Expand All @@ -292,12 +307,20 @@ public void Migration(
}
}

taskArray[item.i] = Task.Factory.StartNew(() =>
try
{
List<ICommittedActionEvaluation> actionEvaluations = EvaluateBlock(block);
Console.WriteLine($"Block progress: #{block.Index}/{remainingCount}");
return actionEvaluations;
});
taskArray[item.i] = Task.Factory.StartNew(() =>
{
List<ICommittedActionEvaluation> actionEvaluations = EvaluateBlock(block);
Console.WriteLine($"Block progress: #{block.Index}/{remainingCount}");
return actionEvaluations;
});
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
}

if (interval < remainingCount)
Expand Down Expand Up @@ -362,10 +385,46 @@ public void Migration(
_mySqlStore.StorePetEnhancementList(_petEnhancementList);
_mySqlStore.StoreTransferAssetList(_transferAssetList);
_mySqlStore.StoreRequestPledgeList(_requestPledgeList);
await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossSeasonDict.Count} Season");
await _mySqlStore.StoreAdventureBossSeasonList(_adventureBossSeasonDict.Values.ToList());
});

await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossWantedList.Count} Wanted");
await _mySqlStore.StoreAdventureBossWantedList(_adventureBossWantedList);
});

await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossChallengeList.Count} Challenge");
await _mySqlStore.StoreAdventureBossChallengeList(_adventureBossChallengeList);
});

await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossRushList.Count} Rush");
await _mySqlStore.StoreAdventureBossRushList(_adventureBossRushList);
});

await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossUnlockFloorList.Count} Unlock");
await _mySqlStore.StoreAdventureBossUnlockFloorList(_adventureBossUnlockFloorList);
});

await Task.Run(async () =>
{
Console.WriteLine($"[Adventure Boss] {_adventureBossClaimRewardList.Count} claim");
await _mySqlStore.StoreAdventureBossClaimRewardList(_adventureBossClaimRewardList);
});
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}

DateTimeOffset end = DateTimeOffset.UtcNow;
Expand Down Expand Up @@ -1141,6 +1200,103 @@ private void ProcessTasks(Task<List<ICommittedActionEvaluation>>[] taskArray, IB
Console.WriteLine(
"Stored RequestPledge action in block #{0}. Time Taken: {1} ms.", ae.InputContext.BlockIndex, (end - start).Milliseconds);
}

switch (action)
{
// avatarNames will be stored as "N/A" for optimization
case Wanted wanted:
_avatarList.Add(AvatarData.GetAvatarInfo(
outputState,
ae.InputContext.Signer,
wanted.AvatarAddress,
_blockTimeOffset,
BattleType.Adventure
));
_adventureBossWantedList.Add(AdventureBossWantedData.GetWantedInfo(
outputState, _blockIndex, _blockTimeOffset, wanted
));
Console.WriteLine(
$"[Adventure Boss] Wanted added : {_adventureBossWantedList.Count}");

// Update season info
_adventureBossSeasonDict[wanted.Season] =
AdventureBossSeasonData.GetAdventureBossSeasonInfo(
outputState, wanted.Season, _blockTimeOffset
);
Console.WriteLine(
$"[Adventure Boss] Season added : {_adventureBossSeasonDict.Count}");
break;
case ExploreAdventureBoss challenge:
_avatarList.Add(AvatarData.GetAvatarInfo(
outputState,
ae.InputContext.Signer,
challenge.AvatarAddress,
_blockTimeOffset,
BattleType.Adventure
));
_adventureBossChallengeList.Add(AdventureBossChallengeData.GetChallengeInfo(
inputState, outputState, _blockIndex, _blockTimeOffset, challenge
));
Console.WriteLine(
$"[Adventure Boss] Challenge added : {_adventureBossChallengeList.Count}");
break;
case SweepAdventureBoss rush:
_avatarList.Add(AvatarData.GetAvatarInfo(
outputState,
ae.InputContext.Signer,
rush.AvatarAddress,
_blockTimeOffset,
BattleType.Adventure
));
_adventureBossRushList.Add(AdventureBossRushData.GetRushInfo(
inputState, outputState, _blockIndex, _blockTimeOffset, rush
));
Console.WriteLine(
$"[Adventure Boss] Rush added : {_adventureBossRushList.Count}");
break;
case UnlockFloor unlock:
_avatarList.Add(AvatarData.GetAvatarInfo(
outputState,
ae.InputContext.Signer,
unlock.AvatarAddress,
_blockTimeOffset,
BattleType.Adventure
));
_adventureBossUnlockFloorList.Add(AdventureBossUnlockFloorData.GetUnlockInfo(
inputState, outputState, _blockIndex, _blockTimeOffset, unlock
));
Console.WriteLine(
$"[Adventure Boss] Unlock added : {_adventureBossUnlockFloorList.Count}");
break;
case ClaimAdventureBossReward claim:
{
_avatarList.Add(AvatarData.GetAvatarInfo(
outputState,
ae.InputContext.Signer,
claim.AvatarAddress,
_blockTimeOffset,
BattleType.Adventure
));
_adventureBossClaimRewardList.Add(AdventureBossClaimRewardData.GetClaimInfo(
inputState, _blockIndex, _blockTimeOffset, claim
));
Console.WriteLine(
$"[Adventure Boss] Claim added : {_adventureBossClaimRewardList.Count}");

// Update season info
var latestSeason = inputState.GetLatestAdventureBossSeason();
var season = latestSeason.EndBlockIndex <= _blockIndex
? latestSeason.Season // New season not started
: latestSeason.Season - 1; // New season started
_adventureBossSeasonDict[season] =
AdventureBossSeasonData.GetAdventureBossSeasonInfo(
outputState, season, _blockTimeOffset
);
Console.WriteLine(
$"[Adventure Boss] Season updated : {_adventureBossSeasonDict.Count}");
break;
}
}
}
}
}
Expand Down
101 changes: 60 additions & 41 deletions NineChronicles.DataProvider.Executable/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Linq;
using System.Net.Http;
using Libplanet.Headless.Hosting;
using Microsoft.Extensions.Logging;
using Nekoyume.Action.Loader;
using IPAddress = System.Net.IPAddress;

Expand Down Expand Up @@ -38,51 +39,12 @@ public class Program : CoconaLiteConsoleAppBase
{
public static async Task Main(string[] args)
{
var host = CreateHostBuilder(args).Build();
await MigrateDatabaseAsync(host);
await CoconaLiteApp.CreateHostBuilder()
.RunAsync<Program>(args);
}

// EF Core uses this method at design time to access the DbContext
public static IHostBuilder CreateHostBuilder(string[] args)
=> Host.CreateDefaultBuilder(args)
.ConfigureServices(services =>
{
services.AddDbContextFactory<NineChroniclesContext>(options =>
{
// Get configuration from appsettings or env
var configurationBuilder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables("NC_");
IConfiguration config = configurationBuilder.Build();
var headlessConfig = new Configuration();
config.Bind(headlessConfig);
if (headlessConfig.MySqlConnectionString != string.Empty)
{
args = new[] { headlessConfig.MySqlConnectionString };
}
if (args.Length == 1)
{
options.UseMySql(
args[0],
ServerVersion.AutoDetect(
args[0]),
b =>
{
b.MigrationsAssembly("NineChronicles.DataProvider.Executable");
b.CommandTimeout(600000);
});
}
else
{
options.UseSqlite(
@"Data Source=9c.gg.db",
b => b.MigrationsAssembly("NineChronicles.DataProvider.Executable")
);
}
});
});

[PrimaryCommand]
public async Task Run(
[Option(Description = "The path of the appsettings JSON file.")] string? configPath = null)
Expand Down Expand Up @@ -276,5 +238,62 @@ IActionEvaluatorConfiguration GetActionEvaluatorConfiguration(IConfiguration con

await hostBuilder.RunConsoleAsync(token);
}

// EF Core uses this method at design time to access the DbContext
private static IHostBuilder CreateHostBuilder(string[] args)
=> Host.CreateDefaultBuilder(args)
.ConfigureServices(services =>
{
services.AddDbContextFactory<NineChroniclesContext>(options =>
{
// Get configuration from appsettings or env
var configurationBuilder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables("NC_");
IConfiguration config = configurationBuilder.Build();
var headlessConfig = new Configuration();
config.Bind(headlessConfig);
if (headlessConfig.MySqlConnectionString != string.Empty)
{
args = new[] { headlessConfig.MySqlConnectionString };
}
if (args.Length == 1)
{
options.UseMySql(
args[0],
ServerVersion.AutoDetect(
args[0]),
b =>
{
b.MigrationsAssembly("NineChronicles.DataProvider.Executable");
b.CommandTimeout(600000);
});
}
else
{
options.UseSqlite(
@"Data Source=9c.gg.db",
b => b.MigrationsAssembly("NineChronicles.DataProvider.Executable")
);
}
});
});

private static async Task MigrateDatabaseAsync(IHost host)
{
using var scope = host.Services.CreateScope();
var services = scope.ServiceProvider;
try
{
var context = ServiceProviderServiceExtensions.GetRequiredService<NineChroniclesContext>(services);
await context.Database.MigrateAsync();
}
catch (Exception ex)
{
var logger = ServiceProviderServiceExtensions.GetRequiredService<ILogger<Program>>(services);
logger.LogError(ex, "An error occurred while migrating the database.");
}
}
}
}
2 changes: 1 addition & 1 deletion NineChronicles.DataProvider.Tests/NineChronicles.DataProvider.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

<ItemGroup>
<ProjectReference Include="..\NineChronicles.DataProvider\NineChronicles.DataProvider.csproj" />
<ProjectReference Include="..\NineChronicles.Headless\Lib9c\.Libplanet\Libplanet.Mocks\Libplanet.Mocks.csproj" />
<ProjectReference Include="..\NineChronicles.Headless\Lib9c\.Libplanet\test\Libplanet.Mocks\Libplanet.Mocks.csproj" />
</ItemGroup>

</Project>
Loading

0 comments on commit d797568

Please sign in to comment.