From 4df93ed069109e42bfb7018e35a067bd00065db6 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 24 Sep 2024 22:29:45 +0800 Subject: [PATCH 01/36] Seems nice --- .../Synchronization/MergeSynchronizer.cs | 56 ++-- .../FastBlocks/BodiesSyncFeed.cs | 5 +- .../FastBlocks/ReceiptsSyncFeed.cs | 3 +- .../Synchronizer.cs | 261 ++++++++++-------- 4 files changed, 186 insertions(+), 139 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 38e65be39e7..46485d5efa3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -18,18 +19,13 @@ using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; -using Nethermind.Trie; -using Nethermind.Trie.Pruning; namespace Nethermind.Merge.Plugin.Synchronization; public class MergeSynchronizer : Synchronizer { - private readonly IPoSSwitcher _poSSwitcher; - private readonly IMergeConfig _mergeConfig; - private readonly IInvalidChainTracker _invalidChainTracker; - private BeaconHeadersSyncFeed _beaconHeadersFeed = null!; private readonly IBeaconSyncStrategy _beaconSync; + private readonly ServiceProvider _beaconServiceProvider; public override ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( SyncProgressResolver, @@ -76,10 +72,38 @@ public MergeSynchronizer( stateReader, logManager) { - _invalidChainTracker = invalidChainTracker; - _poSSwitcher = poSSwitcher; - _mergeConfig = mergeConfig; _beaconSync = beaconSync; + + IServiceCollection beaconServiceCollection = new ServiceCollection(); + beaconServiceCollection + .AddSingleton(blockTree) + .AddSingleton(invalidChainTracker) + .AddSingleton(poSSwitcher) + .AddSingleton(mergeConfig) + .AddSingleton(dbProvider) + .AddSingleton(nodeStorage) + .AddSingleton(peerPool) + .AddSingleton(logManager) + .AddSingleton(specProvider) + .AddSingleton(receiptStorage) + .AddSingleton(pivot) + .AddKeyedSingleton(DbNames.Metadata, (sp, _) => dbProvider.MetadataDb) + .AddKeyedSingleton(DbNames.Code, (sp, _) => dbProvider.CodeDb) + .AddSingleton(_syncReport) + .AddSingleton(syncConfig); + + RegisterBeaconHeaderSyncComponent(beaconServiceCollection); + _beaconServiceProvider = beaconServiceCollection.BuildServiceProvider(); + } + + private static void RegisterBeaconHeaderSyncComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton, BeaconHeadersSyncFeed>() + .AddSingleton, BeaconHeadersSyncDownloader>() + .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); + + RegisterDispatcher(serviceCollection); } public override void Start() @@ -96,16 +120,8 @@ public override void Start() private void StartBeaconHeadersComponents() { - FastBlocksPeerAllocationStrategyFactory fastFactory = new(); - _beaconHeadersFeed = - new(_poSSwitcher, _blockTree, _syncPeerPool, _syncConfig, _syncReport, _pivot, _mergeConfig, _invalidChainTracker, _logManager); - BeaconHeadersSyncDownloader beaconHeadersDownloader = new(_logManager); - - SyncDispatcher dispatcher = CreateDispatcher( - _beaconHeadersFeed!, - beaconHeadersDownloader, - fastFactory - ); + SyncDispatcher dispatcher = + _beaconServiceProvider.GetRequiredService>(); dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -122,6 +138,6 @@ private void StartBeaconHeadersComponents() private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_beaconHeadersFeed); + WireFeedWithModeSelector(_beaconServiceProvider.GetRequiredService>()); } } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 9b972b68d67..30c4225f6c3 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus.Validators; @@ -50,8 +51,8 @@ public BodiesSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - IDbMeta blocksDb, - IDb metadataDb, + [FromKeyedServices(DbNames.Blocks)] IDbMeta blocksDb, + [FromKeyedServices(DbNames.Metadata)] IDb metadataDb, ILogManager logManager, long flushDbInterval = DefaultFlushDbInterval) : base(metadataDb, specProvider, logManager.GetClassLogger()) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index e621dc56795..f7fc6afe321 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -57,7 +58,7 @@ public ReceiptsSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - IDb metadataDb, + [FromKeyedServices(DbNames.Metadata)] IDb metadataDb, ILogManager logManager) : base(metadataDb, specProvider, logManager?.GetClassLogger() ?? default) { diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index ff69883e6f6..d541225240a 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -19,15 +20,13 @@ using Nethermind.Stats.Model; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.DbTuner; -using Nethermind.Synchronization.FastBlocks - ; +using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.StateSync; -using Nethermind.Trie; namespace Nethermind.Synchronization { @@ -38,18 +37,17 @@ public class Synchronizer : ISynchronizer private static MallocTrimmer? s_trimmer; private static SyncDbTuner? s_dbTuner; - private readonly ISpecProvider _specProvider; private readonly IReceiptStorage _receiptStorage; private readonly IBlockDownloaderFactory _blockDownloaderFactory; private readonly INodeStatsManager _nodeStatsManager; protected readonly ILogger _logger; - protected readonly IBlockTree _blockTree; + private readonly IBlockTree _blockTree; protected readonly ISyncConfig _syncConfig; protected readonly ISyncPeerPool _syncPeerPool; protected readonly ILogManager _logManager; protected readonly ISyncReport _syncReport; - protected readonly IPivot _pivot; + private readonly IPivot _pivot; protected CancellationTokenSource? _syncCancellation = new(); @@ -58,41 +56,16 @@ public class Synchronizer : ISynchronizer private readonly IDbProvider _dbProvider; private FastSyncFeed? _fastSyncFeed; - private StateSyncFeed? _stateSyncFeed; private FullSyncFeed? _fullSyncFeed; private readonly IProcessExitSource _exitSource; protected IBetterPeerStrategy _betterPeerStrategy; private readonly ChainSpec _chainSpec; - public ISnapProvider SnapProvider { get; } - - private HeadersSyncFeed? _headersSyncFeed; - private HeadersSyncFeed? HeadersSyncFeed => _headersSyncFeed ??= CreateHeadersSyncFeed(); - - private ReceiptsSyncFeed? _receiptsSyncFeed; - private ReceiptsSyncFeed? ReceiptsSyncFeed => _receiptsSyncFeed ??= CreateReceiptsSyncFeed(); - - private BodiesSyncFeed? _bodiesSyncFeed; - private BodiesSyncFeed? BodiesSyncFeed => _bodiesSyncFeed ??= CreateBodiesSyncFeed(); - - private SnapSyncFeed? _snapSyncFeed; - private SnapSyncFeed? SnapSyncFeed => _snapSyncFeed ??= CreateSnapSyncFeed(); - - private ISyncProgressResolver? _syncProgressResolver; - public ISyncProgressResolver SyncProgressResolver => _syncProgressResolver ??= new SyncProgressResolver( - _blockTree, - new FullStateFinder(_blockTree, _stateReader), - _syncConfig, - HeadersSyncFeed, - BodiesSyncFeed, - ReceiptsSyncFeed, - SnapSyncFeed, - _logManager); + public ISyncProgressResolver SyncProgressResolver => _serviceProvider.GetRequiredService(); protected ISyncModeSelector? _syncModeSelector; private readonly IStateReader _stateReader; - private readonly INodeStorage _nodeStorage; - private readonly ProgressTracker _progressTracker; + private readonly ServiceProvider _serviceProvider; public virtual ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( SyncProgressResolver, @@ -121,9 +94,7 @@ public Synchronizer( ILogManager logManager) { _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); - _nodeStorage = nodeStorage ?? throw new ArgumentNullException(nameof(nodeStorage)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); @@ -139,12 +110,121 @@ public Synchronizer( _syncReport = new SyncReport(_syncPeerPool!, nodeStatsManager!, _syncConfig, _pivot, logManager); - _progressTracker = new( - blockTree, - dbProvider.StateDb, - logManager, - _syncConfig.SnapSyncAccountRangePartitionCount); - SnapProvider = new SnapProvider(_progressTracker, dbProvider.CodeDb, nodeStorage, logManager); + var serviceCollection = new ServiceCollection() + .AddSingleton(blockTree) + .AddSingleton(dbProvider) + .AddSingleton(nodeStorage) + .AddSingleton(peerPool) + .AddSingleton(logManager) + .AddSingleton(specProvider) + .AddSingleton(receiptStorage) + .AddSingleton(stateReader) + .AddSingleton() + .AddSingleton() + .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) + .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) + .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb) + .AddSingleton(_syncReport) + .AddSingleton(syncConfig); + + if (_syncConfig.FastSync && _syncConfig.SnapSync) RegisterSnapComponent(serviceCollection); + + if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync) RegisterHeaderSyncComponent(serviceCollection); + + if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync && + _syncConfig.DownloadReceiptsInFastSync) + RegisterReceiptSyncComponent(serviceCollection); + + if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync) + RegisterBodiesSyncComponent(serviceCollection); + + RegisterStateSyncComponent(serviceCollection); + _serviceProvider = serviceCollection.BuildServiceProvider(); + } + + protected static void RegisterDispatcher(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton>((sp) => new( + sp.GetRequiredService().MaxProcessingThreads, + sp.GetRequiredService>(), + sp.GetRequiredService>(), + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService())); + } + + private static void RegisterSnapComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, SnapSyncDownloader>() + .AddSingleton, SnapSyncAllocationStrategyFactory>() + .AddSingleton((sp) => new ProgressTracker( + sp.GetRequiredService(), + sp.GetRequiredService().StateDb, + sp.GetRequiredService(), + sp.GetRequiredService().SnapSyncAccountRangePartitionCount + )) + .AddSingleton((sp) => new SnapProvider( + sp.GetRequiredService(), + sp.GetRequiredService().CodeDb, + sp.GetRequiredService(), + sp.GetRequiredService() + )); + + RegisterDispatcher(serviceCollection); + } + + private static void RegisterHeaderSyncComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, HeadersSyncDownloader>() + .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); + + RegisterDispatcher(serviceCollection); + } + + private static void RegisterReceiptSyncComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, ReceiptsSyncDispatcher>() + .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); + + RegisterDispatcher(serviceCollection); + } + + private static void RegisterBodiesSyncComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, BodiesSyncDownloader>() + .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); + + RegisterDispatcher(serviceCollection); + } + + private static void RegisterStateSyncComponent(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton(sp => new( + SyncMode.StateNodes, + sp.GetService().CodeDb, + sp.GetService(), + sp.GetService(), + sp.GetService())) + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, StateSyncDownloader>() + .AddSingleton, StateSyncAllocationStrategyFactory>(); + + RegisterDispatcher(serviceCollection); } public virtual void Start() @@ -186,37 +266,13 @@ public virtual void Start() SyncModeSelector.Changed += _syncReport.SyncModeSelectorOnChanged; } - private HeadersSyncFeed? CreateHeadersSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync) return null; - return new HeadersSyncFeed(_blockTree, _syncPeerPool, _syncConfig, _syncReport, _logManager); - } - - private BodiesSyncFeed? CreateBodiesSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync) return null; - return new BodiesSyncFeed(_specProvider, _blockTree, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.BlocksDb, _dbProvider.MetadataDb, _logManager); - } - - private ReceiptsSyncFeed? CreateReceiptsSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync || !_syncConfig.DownloadReceiptsInFastSync) return null; - return new ReceiptsSyncFeed(_specProvider, _blockTree, _receiptStorage, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.MetadataDb, _logManager); - } - - private SnapSyncFeed? CreateSnapSyncFeed() - { - if (!_syncConfig.FastSync || !_syncConfig.SnapSync) return null; - return new SnapSyncFeed(SnapProvider, _logManager); - } - private void SetupDbOptimizer() { s_dbTuner ??= new SyncDbTuner( _syncConfig, - SnapSyncFeed, - BodiesSyncFeed, - ReceiptsSyncFeed, + _serviceProvider.GetService(), + _serviceProvider.GetService(), + _serviceProvider.GetService(), _dbProvider.StateDb as ITunableDb, _dbProvider.CodeDb as ITunableDb, _dbProvider.BlocksDb as ITunableDb, @@ -275,13 +331,7 @@ private void StartFastSyncComponents() private void StartStateSyncComponents() { - TreeSync treeSync = new(SyncMode.StateNodes, _dbProvider.CodeDb, _nodeStorage, _blockTree, _logManager); - _stateSyncFeed = new StateSyncFeed(treeSync, _logManager); - SyncDispatcher stateSyncDispatcher = CreateDispatcher( - _stateSyncFeed, - new StateSyncDownloader(_logManager), - new StateSyncAllocationStrategyFactory() - ); + SyncDispatcher stateSyncDispatcher = _serviceProvider.GetRequiredService>(); Task syncDispatcherTask = stateSyncDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -296,13 +346,10 @@ private void StartStateSyncComponents() }); } + private void StartSnapSyncComponents() { - SyncDispatcher dispatcher = CreateDispatcher( - SnapSyncFeed, - new SnapSyncDownloader(_logManager), - new SnapSyncAllocationStrategyFactory() - ); + SyncDispatcher dispatcher = _serviceProvider.GetRequiredService>(); Task _ = dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -319,12 +366,7 @@ private void StartSnapSyncComponents() private void StartFastBlocksComponents() { - FastBlocksPeerAllocationStrategyFactory fastFactory = new(); - SyncDispatcher headersDispatcher = CreateDispatcher( - HeadersSyncFeed, - new HeadersSyncDownloader(_logManager), - fastFactory - ); + SyncDispatcher headersDispatcher = _serviceProvider.GetService>(); Task headersTask = headersDispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -342,12 +384,8 @@ private void StartFastBlocksComponents() { if (_syncConfig.DownloadBodiesInFastSync) { - - SyncDispatcher bodiesDispatcher = CreateDispatcher( - BodiesSyncFeed!, - new BodiesSyncDownloader(_logManager), - fastFactory - ); + SyncDispatcher bodiesDispatcher = + _serviceProvider.GetRequiredService>(); Task bodiesTask = bodiesDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -364,11 +402,8 @@ private void StartFastBlocksComponents() if (_syncConfig.DownloadReceiptsInFastSync) { - SyncDispatcher receiptsDispatcher = CreateDispatcher( - ReceiptsSyncFeed!, - new ReceiptsSyncDispatcher(_logManager), - fastFactory - ); + SyncDispatcher receiptsDispatcher = + _serviceProvider.GetService>(); Task receiptsTask = receiptsDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -385,7 +420,7 @@ private void StartFastBlocksComponents() } } - protected SyncDispatcher CreateDispatcher(ISyncFeed feed, ISyncDownloader downloader, IPeerAllocationStrategyFactory peerAllocationStrategyFactory) + private SyncDispatcher CreateDispatcher(ISyncFeed feed, ISyncDownloader downloader, IPeerAllocationStrategyFactory peerAllocationStrategyFactory) { return new( _syncConfig.MaxProcessingThreads, @@ -422,23 +457,23 @@ public Task StopAsync() Task.Delay(FeedsTerminationTimeout), Task.WhenAll( _fastSyncFeed?.FeedTask ?? Task.CompletedTask, - _stateSyncFeed?.FeedTask ?? Task.CompletedTask, - SnapSyncFeed?.FeedTask ?? Task.CompletedTask, + _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, + _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, _fullSyncFeed?.FeedTask ?? Task.CompletedTask, - HeadersSyncFeed?.FeedTask ?? Task.CompletedTask, - BodiesSyncFeed?.FeedTask ?? Task.CompletedTask, - ReceiptsSyncFeed?.FeedTask ?? Task.CompletedTask)); + _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, + _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, + _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask)); } private void WireMultiSyncModeSelector() { WireFeedWithModeSelector(_fastSyncFeed); - WireFeedWithModeSelector(_stateSyncFeed); - WireFeedWithModeSelector(SnapSyncFeed); + WireFeedWithModeSelector(_serviceProvider.GetService()); + WireFeedWithModeSelector(_serviceProvider.GetService()); WireFeedWithModeSelector(_fullSyncFeed); - WireFeedWithModeSelector(HeadersSyncFeed); - WireFeedWithModeSelector(BodiesSyncFeed); - WireFeedWithModeSelector(ReceiptsSyncFeed); + WireFeedWithModeSelector(_serviceProvider.GetService()); + WireFeedWithModeSelector(_serviceProvider.GetService()); + WireFeedWithModeSelector(_serviceProvider.GetService()); } protected void WireFeedWithModeSelector(ISyncFeed? feed) @@ -456,15 +491,9 @@ public void Dispose() CancellationTokenExtensions.CancelDisposeAndClear(ref _syncCancellation); _syncReport.Dispose(); _fastSyncFeed?.Dispose(); - _stateSyncFeed?.Dispose(); - _stateSyncFeed = null; - SnapSyncFeed?.Dispose(); - _snapSyncFeed = null; _fullSyncFeed?.Dispose(); - HeadersSyncFeed?.Dispose(); - BodiesSyncFeed?.Dispose(); - ReceiptsSyncFeed?.Dispose(); - _progressTracker.Dispose(); + + _serviceProvider.Dispose(); } } } From 4b3cc69062a98d924498dd36682f2b11efe08e75 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 24 Sep 2024 22:42:56 +0800 Subject: [PATCH 02/36] Snap provider --- .../SnapSync/ProgressTracker.cs | 7 +++++++ .../SnapSync/SnapProvider.cs | 3 ++- .../Synchronizer.cs | 21 +++++++------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index eda6709e66e..06286e3abf5 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -8,7 +8,9 @@ using System.Linq; using System.Text; using System.Threading; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; +using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -58,6 +60,11 @@ public class ProgressTracker : IDisposable private readonly Pivot _pivot; + public ProgressTracker(IBlockTree blockTree, [FromKeyedServices(DbNames.State)] IDb db, ILogManager logManager, ISyncConfig syncConfig) + : this(blockTree, db, logManager, syncConfig.MaxProcessingThreads) + { + } + public ProgressTracker(IBlockTree blockTree, IDb db, ILogManager logManager, int accountRangePartitionCount = 8) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index afb19297673..f3c864c06f1 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.ObjectPool; using Nethermind.Core; using Nethermind.Core.Caching; @@ -33,7 +34,7 @@ public class SnapProvider : ISnapProvider // This is actually close to 97% effective. private readonly ClockKeyCache _codeExistKeyCache = new(1024 * 16); - public SnapProvider(ProgressTracker progressTracker, IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) + public SnapProvider(ProgressTracker progressTracker, [FromKeyedServices(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) { _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index d541225240a..65d7d513bbf 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -123,13 +123,16 @@ public Synchronizer( .AddSingleton() .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) + .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb) .AddSingleton(_syncReport) .AddSingleton(syncConfig); - if (_syncConfig.FastSync && _syncConfig.SnapSync) RegisterSnapComponent(serviceCollection); + if (_syncConfig.FastSync && _syncConfig.SnapSync) + RegisterSnapComponent(serviceCollection); - if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync) RegisterHeaderSyncComponent(serviceCollection); + if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync) + RegisterHeaderSyncComponent(serviceCollection); if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync && _syncConfig.DownloadReceiptsInFastSync) @@ -161,18 +164,8 @@ private static void RegisterSnapComponent(IServiceCollection serviceCollection) .AddSingleton>(sp => sp.GetRequiredService()) .AddSingleton, SnapSyncDownloader>() .AddSingleton, SnapSyncAllocationStrategyFactory>() - .AddSingleton((sp) => new ProgressTracker( - sp.GetRequiredService(), - sp.GetRequiredService().StateDb, - sp.GetRequiredService(), - sp.GetRequiredService().SnapSyncAccountRangePartitionCount - )) - .AddSingleton((sp) => new SnapProvider( - sp.GetRequiredService(), - sp.GetRequiredService().CodeDb, - sp.GetRequiredService(), - sp.GetRequiredService() - )); + .AddSingleton() + .AddSingleton(); RegisterDispatcher(serviceCollection); } From 479a3f8493a3e14bec45c2b5e89aff866e3ba899 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 24 Sep 2024 22:45:37 +0800 Subject: [PATCH 03/36] Sync dispatcher --- .../ParallelSync/SyncDispatcher.cs | 13 +++++++++++++ .../SnapSync/ProgressTracker.cs | 2 +- .../Nethermind.Synchronization/Synchronizer.cs | 9 +-------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index 50c38f3384b..f7066d9dfc4 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Nethermind.Blockchain.Synchronization; using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -25,6 +26,18 @@ public class SyncDispatcher private readonly SemaphoreSlim _concurrentProcessingSemaphore; + public SyncDispatcher( + ISyncConfig syncConfig, + ISyncFeed? syncFeed, + ISyncDownloader? downloader, + ISyncPeerPool? syncPeerPool, + IPeerAllocationStrategyFactory? peerAllocationStrategy, + ILogManager? logManager) + :this (syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) + { + + } + public SyncDispatcher( int maxNumberOfProcessingThread, ISyncFeed? syncFeed, diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index 06286e3abf5..064675f421f 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -61,7 +61,7 @@ public class ProgressTracker : IDisposable private readonly Pivot _pivot; public ProgressTracker(IBlockTree blockTree, [FromKeyedServices(DbNames.State)] IDb db, ILogManager logManager, ISyncConfig syncConfig) - : this(blockTree, db, logManager, syncConfig.MaxProcessingThreads) + : this(blockTree, db, logManager, syncConfig.SnapSyncAccountRangePartitionCount) { } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 65d7d513bbf..0004b4fbc95 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -147,14 +147,7 @@ public Synchronizer( protected static void RegisterDispatcher(IServiceCollection serviceCollection) { - serviceCollection - .AddSingleton>((sp) => new( - sp.GetRequiredService().MaxProcessingThreads, - sp.GetRequiredService>(), - sp.GetRequiredService>(), - sp.GetRequiredService(), - sp.GetRequiredService>(), - sp.GetRequiredService())); + serviceCollection.AddSingleton>(); } private static void RegisterSnapComponent(IServiceCollection serviceCollection) From 0a52825668ce9b6737f7a0806893c512301075fa Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 24 Sep 2024 22:55:58 +0800 Subject: [PATCH 04/36] Dedup --- .../FastSync/TreeSync.cs | 7 +++ .../Synchronizer.cs | 53 ++++++------------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 4cc449cb47b..552ce8316db 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Caching; @@ -74,6 +75,12 @@ public class TreeSync private long _blockNumber; private readonly SyncMode _syncMode; + public TreeSync([FromKeyedServices(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) + : this (SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) + { + + } + public TreeSync(SyncMode syncMode, IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) { _syncMode = syncMode; diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 0004b4fbc95..3aa039928c7 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -153,64 +153,43 @@ protected static void RegisterDispatcher(IServiceCollection serviceCollection private static void RegisterSnapComponent(IServiceCollection serviceCollection) { serviceCollection - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, SnapSyncDownloader>() - .AddSingleton, SnapSyncAllocationStrategyFactory>() .AddSingleton() .AddSingleton(); - RegisterDispatcher(serviceCollection); + RegisterSyncFeed(serviceCollection); } private static void RegisterHeaderSyncComponent(IServiceCollection serviceCollection) { - serviceCollection - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, HeadersSyncDownloader>() - .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); - - RegisterDispatcher(serviceCollection); + RegisterSyncFeed(serviceCollection); } private static void RegisterReceiptSyncComponent(IServiceCollection serviceCollection) { - serviceCollection - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, ReceiptsSyncDispatcher>() - .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); - - RegisterDispatcher(serviceCollection); + RegisterSyncFeed(serviceCollection); } private static void RegisterBodiesSyncComponent(IServiceCollection serviceCollection) { - serviceCollection - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, BodiesSyncDownloader>() - .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); + RegisterSyncFeed(serviceCollection); + } - RegisterDispatcher(serviceCollection); + private static void RegisterSyncFeed(IServiceCollection serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory + { + serviceCollection + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, TDownloader>() + .AddSingleton, TAllocationStrategy>() + .AddSingleton>(); } private static void RegisterStateSyncComponent(IServiceCollection serviceCollection) { serviceCollection - .AddSingleton(sp => new( - SyncMode.StateNodes, - sp.GetService().CodeDb, - sp.GetService(), - sp.GetService(), - sp.GetService())) - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, StateSyncDownloader>() - .AddSingleton, StateSyncAllocationStrategyFactory>(); - - RegisterDispatcher(serviceCollection); + .AddSingleton(); + + RegisterSyncFeed(serviceCollection); } public virtual void Start() From 3144f3b65bfbae8aa3bdffc51107336e275ad474 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 08:46:39 +0800 Subject: [PATCH 05/36] Multi sync mode selector --- .../Steps/InitializeNetwork.cs | 1 + .../Synchronization/MergeSynchronizer.cs | 26 +++++++----- .../OldStyleFullSynchronizerTests.cs | 1 + .../SyncThreadTests.cs | 1 + .../SynchronizerTests.cs | 1 + .../ParallelSync/MultiSyncModeSelector.cs | 6 ++- .../Synchronizer.cs | 42 +++++++++++++------ 7 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 69a066e9b0a..d018d62571a 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -148,6 +148,7 @@ private async Task Initialize(CancellationToken cancellationToken) _api.BetterPeerStrategy, _api.ChainSpec, _api.StateReader!, + No.BeaconSync, _api.LogManager); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 46485d5efa3..9019d7d20b2 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -24,17 +24,8 @@ namespace Nethermind.Merge.Plugin.Synchronization; public class MergeSynchronizer : Synchronizer { - private readonly IBeaconSyncStrategy _beaconSync; private readonly ServiceProvider _beaconServiceProvider; - public override ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( - SyncProgressResolver, - _syncPeerPool, - _syncConfig, - _beaconSync, - _betterPeerStrategy!, - _logManager); - public MergeSynchronizer( IDbProvider dbProvider, INodeStorage nodeStorage, @@ -70,10 +61,9 @@ public MergeSynchronizer( betterPeerStrategy, chainSpec, stateReader, + beaconSync, logManager) { - _beaconSync = beaconSync; - IServiceCollection beaconServiceCollection = new ServiceCollection(); beaconServiceCollection .AddSingleton(blockTree) @@ -96,6 +86,20 @@ public MergeSynchronizer( _beaconServiceProvider = beaconServiceCollection.BuildServiceProvider(); } + protected override void ConfigureServiceCollection(IServiceCollection serviceCollection) + { + base.ConfigureServiceCollection(serviceCollection); + + // It does not set the last parameter + serviceCollection.AddSingleton(sp => new MultiSyncModeSelector( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService())); + } + private static void RegisterBeaconHeaderSyncComponent(IServiceCollection serviceCollection) { serviceCollection diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index d5e3adcbeae..c0b2e7986fd 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -85,6 +85,7 @@ public async Task Setup() bestPeerStrategy, new ChainSpec(), stateReader, + No.BeaconSync, LimboLogs.Instance); _syncServer = new SyncServer( trieStore.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index fb3f5927289..85a00d803ec 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -377,6 +377,7 @@ private SyncTestContext CreateSyncManager(int index) bestPeerStrategy, new ChainSpec(), stateReader, + No.BeaconSync, logManager); ISyncModeSelector selector = synchronizer.SyncModeSelector; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 6509942731e..b4abd18714e 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -398,6 +398,7 @@ ISyncConfig GetSyncConfig() => bestPeerStrategy, new ChainSpec(), reader, + No.BeaconSync, _logManager); } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index e9eff01f553..10e6539e471 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -7,6 +7,7 @@ using System.ComponentModel; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; using Nethermind.Int256; using Nethermind.Logging; @@ -45,6 +46,8 @@ public class MultiSyncModeSelector : ISyncModeSelector /// private const int StickyStateNodesDelta = 32; + public const string ParameterNeedToWaitForHeader = "needToWaitForHeaders"; + private readonly ISyncProgressResolver _syncProgressResolver; private readonly ISyncPeerPool _syncPeerPool; private readonly ISyncConfig _syncConfig; @@ -81,7 +84,7 @@ public MultiSyncModeSelector( IBeaconSyncStrategy beaconSyncStrategy, IBetterPeerStrategy betterPeerStrategy, ILogManager logManager, - bool needToWaitForHeaders = false) + [FromKeyedServices(ParameterNeedToWaitForHeader)] bool needToWaitForHeaders = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); @@ -102,6 +105,7 @@ public MultiSyncModeSelector( _ = StartAsync(_cancellation.Token); } + private async Task StartAsync(CancellationToken cancellationToken) { PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_syncConfig.MultiSyncModeSelectorLoopTimerMs)); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 3aa039928c7..311b38f051d 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -58,23 +58,13 @@ public class Synchronizer : ISynchronizer private FastSyncFeed? _fastSyncFeed; private FullSyncFeed? _fullSyncFeed; private readonly IProcessExitSource _exitSource; - protected IBetterPeerStrategy _betterPeerStrategy; - private readonly ChainSpec _chainSpec; public ISyncProgressResolver SyncProgressResolver => _serviceProvider.GetRequiredService(); - protected ISyncModeSelector? _syncModeSelector; private readonly IStateReader _stateReader; private readonly ServiceProvider _serviceProvider; - public virtual ISyncModeSelector SyncModeSelector => _syncModeSelector ??= new MultiSyncModeSelector( - SyncProgressResolver, - _syncPeerPool!, - _syncConfig, - No.BeaconSync, - _betterPeerStrategy!, - _logManager, - _chainSpec?.SealEngineType == SealEngineType.Clique); + public ISyncModeSelector SyncModeSelector => _serviceProvider.GetRequiredService(); public Synchronizer( IDbProvider dbProvider, @@ -91,6 +81,7 @@ public Synchronizer( IBetterPeerStrategy betterPeerStrategy, ChainSpec chainSpec, IStateReader stateReader, + IBeaconSyncStrategy beaconSyncStrategy, ILogManager logManager) { _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); @@ -104,8 +95,6 @@ public Synchronizer( _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _chainSpec = chainSpec ?? throw new ArgumentNullException(nameof(chainSpec)); _stateReader = stateReader ?? throw new ArgumentNullException(nameof(_stateReader)); _syncReport = new SyncReport(_syncPeerPool!, nodeStatsManager!, _syncConfig, _pivot, logManager); @@ -119,6 +108,10 @@ public Synchronizer( .AddSingleton(specProvider) .AddSingleton(receiptStorage) .AddSingleton(stateReader) + .AddSingleton(chainSpec) + .AddSingleton(betterPeerStrategy) + // .AddSingleton(No.BeaconSync) + .AddSingleton(beaconSyncStrategy) .AddSingleton() .AddSingleton() .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) @@ -128,6 +121,8 @@ public Synchronizer( .AddSingleton(_syncReport) .AddSingleton(syncConfig); + ConfigureServiceCollection(serviceCollection); + if (_syncConfig.FastSync && _syncConfig.SnapSync) RegisterSnapComponent(serviceCollection); @@ -145,6 +140,27 @@ public Synchronizer( _serviceProvider = serviceCollection.BuildServiceProvider(); } + protected virtual void ConfigureServiceCollection(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton() + .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) + .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) + .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) + .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb) + .AddSingleton(No.BeaconSync) + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton(sp => new MultiSyncModeSelector( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)); + } + protected static void RegisterDispatcher(IServiceCollection serviceCollection) { serviceCollection.AddSingleton>(); From c85f3e874ec228a92fffd45ccad5c111caf598b7 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 09:52:22 +0800 Subject: [PATCH 06/36] Fast and full sync --- .../IServiceCollectionExtensions.cs | 30 +++ .../Nethermind.Core/Nethermind.Core.csproj | 5 + .../Synchronization/MergeSynchronizer.cs | 33 +-- .../Synchronizer.cs | 190 ++++++++++-------- 4 files changed, 145 insertions(+), 113 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs new file mode 100644 index 00000000000..fceb55d622f --- /dev/null +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace Nethermind.Core; + +public static class IServiceCollectionExtensions +{ + public static IServiceCollection ForwardServiceAsSingleton(this IServiceCollection configuration, IServiceProvider baseServiceProvider) where T : class + { + T? theService = baseServiceProvider.GetService(); + if (theService != null) + { + configuration.AddSingleton(baseServiceProvider.GetRequiredService()); + } + else + { + // It could be that this is in a test where the service was not registered and any dependency will be + // replaced anyway. While using a factory function like this seems like it would have the same behaviour + // as getting it directly first, it has one critical difference. When the final IServiceProvider is + // disposed, it would also call the Dispose function of the service as it assume that it created and + // therefore owned the service. + configuration.AddSingleton((sp) => baseServiceProvider.GetRequiredService()); + } + + return configuration; + } +} diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index de2832e6ea5..8d4a5a8e1ae 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -16,4 +16,9 @@ + + + ..\..\..\..\..\.nuget\packages\microsoft.extensions.dependencyinjection.abstractions\8.0.1\lib\net8.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll + + diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 9019d7d20b2..1fb2531209c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -24,7 +24,7 @@ namespace Nethermind.Merge.Plugin.Synchronization; public class MergeSynchronizer : Synchronizer { - private readonly ServiceProvider _beaconServiceProvider; + private readonly ServiceProvider _beaconScope; public MergeSynchronizer( IDbProvider dbProvider, @@ -64,26 +64,8 @@ public MergeSynchronizer( beaconSync, logManager) { - IServiceCollection beaconServiceCollection = new ServiceCollection(); - beaconServiceCollection - .AddSingleton(blockTree) - .AddSingleton(invalidChainTracker) - .AddSingleton(poSSwitcher) - .AddSingleton(mergeConfig) - .AddSingleton(dbProvider) - .AddSingleton(nodeStorage) - .AddSingleton(peerPool) - .AddSingleton(logManager) - .AddSingleton(specProvider) - .AddSingleton(receiptStorage) - .AddSingleton(pivot) - .AddKeyedSingleton(DbNames.Metadata, (sp, _) => dbProvider.MetadataDb) - .AddKeyedSingleton(DbNames.Code, (sp, _) => dbProvider.CodeDb) - .AddSingleton(_syncReport) - .AddSingleton(syncConfig); - - RegisterBeaconHeaderSyncComponent(beaconServiceCollection); - _beaconServiceProvider = beaconServiceCollection.BuildServiceProvider(); + RegisterBeaconHeaderSyncComponent(_serviceCollection); + _beaconScope = _serviceCollection.BuildServiceProvider(); } protected override void ConfigureServiceCollection(IServiceCollection serviceCollection) @@ -105,9 +87,8 @@ private static void RegisterBeaconHeaderSyncComponent(IServiceCollection service serviceCollection .AddSingleton, BeaconHeadersSyncFeed>() .AddSingleton, BeaconHeadersSyncDownloader>() - .AddSingleton, FastBlocksPeerAllocationStrategyFactory>(); - - RegisterDispatcher(serviceCollection); + .AddSingleton, FastBlocksPeerAllocationStrategyFactory>() + .AddSingleton>(); } public override void Start() @@ -125,7 +106,7 @@ public override void Start() private void StartBeaconHeadersComponents() { SyncDispatcher dispatcher = - _beaconServiceProvider.GetRequiredService>(); + _beaconScope.GetRequiredService>(); dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -142,6 +123,6 @@ private void StartBeaconHeadersComponents() private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_beaconServiceProvider.GetRequiredService>()); + WireFeedWithModeSelector(_beaconScope.GetRequiredService>()); } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 311b38f051d..12c55398b09 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -37,17 +37,11 @@ public class Synchronizer : ISynchronizer private static MallocTrimmer? s_trimmer; private static SyncDbTuner? s_dbTuner; - private readonly IReceiptStorage _receiptStorage; - private readonly IBlockDownloaderFactory _blockDownloaderFactory; private readonly INodeStatsManager _nodeStatsManager; protected readonly ILogger _logger; - private readonly IBlockTree _blockTree; protected readonly ISyncConfig _syncConfig; - protected readonly ISyncPeerPool _syncPeerPool; protected readonly ILogManager _logManager; - protected readonly ISyncReport _syncReport; - private readonly IPivot _pivot; protected CancellationTokenSource? _syncCancellation = new(); @@ -55,16 +49,18 @@ public class Synchronizer : ISynchronizer public event EventHandler? SyncEvent; private readonly IDbProvider _dbProvider; - private FastSyncFeed? _fastSyncFeed; - private FullSyncFeed? _fullSyncFeed; private readonly IProcessExitSource _exitSource; - public ISyncProgressResolver SyncProgressResolver => _serviceProvider.GetRequiredService(); + public ISyncProgressResolver SyncProgressResolver => _mainScope.GetRequiredService(); - private readonly IStateReader _stateReader; - private readonly ServiceProvider _serviceProvider; + private readonly ServiceProvider _mainScope; + private readonly ServiceProvider _fastSyncScope; + private readonly ServiceProvider _fullSyncScope; - public ISyncModeSelector SyncModeSelector => _serviceProvider.GetRequiredService(); + // Used by subclass for beacon header + protected readonly IServiceCollection _serviceCollection; + + public ISyncModeSelector SyncModeSelector => _mainScope.GetRequiredService(); public Synchronizer( IDbProvider dbProvider, @@ -86,18 +82,10 @@ public Synchronizer( { _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); - _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); - _blockDownloaderFactory = blockDownloaderFactory ?? throw new ArgumentNullException(nameof(blockDownloaderFactory)); - _pivot = pivot ?? throw new ArgumentNullException(nameof(pivot)); - _syncPeerPool = peerPool ?? throw new ArgumentNullException(nameof(peerPool)); _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _stateReader = stateReader ?? throw new ArgumentNullException(nameof(_stateReader)); - - _syncReport = new SyncReport(_syncPeerPool!, nodeStatsManager!, _syncConfig, _pivot, logManager); var serviceCollection = new ServiceCollection() .AddSingleton(blockTree) @@ -110,16 +98,15 @@ public Synchronizer( .AddSingleton(stateReader) .AddSingleton(chainSpec) .AddSingleton(betterPeerStrategy) - // .AddSingleton(No.BeaconSync) .AddSingleton(beaconSyncStrategy) - .AddSingleton() - .AddSingleton() + .AddSingleton(blockDownloaderFactory) + .AddSingleton(syncConfig) + .AddSingleton(pivot) + .AddSingleton(nodeStatsManager) .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) - .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb) - .AddSingleton(_syncReport) - .AddSingleton(syncConfig); + .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb); ConfigureServiceCollection(serviceCollection); @@ -137,19 +124,47 @@ public Synchronizer( RegisterBodiesSyncComponent(serviceCollection); RegisterStateSyncComponent(serviceCollection); - _serviceProvider = serviceCollection.BuildServiceProvider(); + + _mainScope = serviceCollection.BuildServiceProvider(); + + ConfigureBlocksDownloader(serviceCollection); + + ConfigureFastSync(serviceCollection); + _fastSyncScope = serviceCollection.BuildServiceProvider(); + + ConfigureFullSync(serviceCollection); + _fullSyncScope = serviceCollection.BuildServiceProvider(); + + _serviceCollection = serviceCollection; + } + + private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) + { + // Now we configure blocks downloader + serviceCollection + // These three line make sure that these components are not duplicated + .ForwardServiceAsSingleton(_mainScope) + .ForwardServiceAsSingleton(_mainScope) + .ForwardServiceAsSingleton(_mainScope) + + .AddSingleton(sp => sp.GetRequiredService() + .Create(sp.GetRequiredService>(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService()) + ) + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton(sp => sp .GetRequiredService() + .CreateAllocationStrategyFactory()); } protected virtual void ConfigureServiceCollection(IServiceCollection serviceCollection) { serviceCollection .AddSingleton() + .AddSingleton() .AddSingleton() - .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) - .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) - .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) - .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb) - .AddSingleton(No.BeaconSync) .AddSingleton(sp => sp.GetRequiredService()) .AddSingleton(sp => new MultiSyncModeSelector( sp.GetRequiredService(), @@ -158,14 +173,36 @@ protected virtual void ConfigureServiceCollection(IServiceCollection serviceColl sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), - sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)); + // Need to set this + sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)) + .AddSingleton(sp => new SyncDbTuner( + sp.GetRequiredService(), + + // These are all optional so need to set manually + sp.GetService(), + sp.GetService(), + sp.GetService(), + + sp.GetRequiredService().StateDb as ITunableDb, + sp.GetRequiredService().CodeDb as ITunableDb, + sp.GetRequiredService().BlocksDb as ITunableDb, + sp.GetRequiredService().ReceiptsDb as ITunableDb)) + .AddSingleton>(); } - protected static void RegisterDispatcher(IServiceCollection serviceCollection) + private static void ConfigureFullSync(IServiceCollection serviceCollection) { - serviceCollection.AddSingleton>(); + serviceCollection + .AddSingleton() + .AddSingleton, FullSyncFeed>(sp => sp.GetRequiredService()); } + private static void ConfigureFastSync(IServiceCollection serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton, FastSyncFeed>(sp => sp.GetRequiredService()); + } private static void RegisterSnapComponent(IServiceCollection serviceCollection) { serviceCollection @@ -244,16 +281,16 @@ public virtual void Start() WireMultiSyncModeSelector(); s_trimmer ??= new MallocTrimmer(SyncModeSelector, TimeSpan.FromSeconds(_syncConfig.MallocTrimIntervalSec), _logManager); - SyncModeSelector.Changed += _syncReport.SyncModeSelectorOnChanged; + SyncModeSelector.Changed += _mainScope.GetRequiredService().SyncModeSelectorOnChanged; } private void SetupDbOptimizer() { s_dbTuner ??= new SyncDbTuner( _syncConfig, - _serviceProvider.GetService(), - _serviceProvider.GetService(), - _serviceProvider.GetService(), + _mainScope.GetService(), + _mainScope.GetService(), + _mainScope.GetService(), _dbProvider.StateDb as ITunableDb, _dbProvider.CodeDb as ITunableDb, _dbProvider.BlocksDb as ITunableDb, @@ -262,15 +299,10 @@ private void SetupDbOptimizer() private void StartFullSyncComponents() { - _fullSyncFeed = new FullSyncFeed(); - BlockDownloader fullSyncBlockDownloader = _blockDownloaderFactory.Create(_fullSyncFeed, _blockTree, _receiptStorage, _syncPeerPool, _syncReport); + BlockDownloader fullSyncBlockDownloader = _fullSyncScope.GetRequiredService(); fullSyncBlockDownloader.SyncEvent += DownloaderOnSyncEvent; - SyncDispatcher dispatcher = CreateDispatcher( - _fullSyncFeed, - fullSyncBlockDownloader, - _blockDownloaderFactory.CreateAllocationStrategyFactory() - ); + SyncDispatcher dispatcher = _fastSyncScope.GetRequiredService>(); dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -287,15 +319,11 @@ private void StartFullSyncComponents() private void StartFastSyncComponents() { - _fastSyncFeed = new FastSyncFeed(_syncConfig); - BlockDownloader downloader = _blockDownloaderFactory.Create(_fastSyncFeed, _blockTree, _receiptStorage, _syncPeerPool, _syncReport); + BlockDownloader downloader = _fastSyncScope.GetRequiredService(); downloader.SyncEvent += DownloaderOnSyncEvent; - SyncDispatcher dispatcher = CreateDispatcher( - _fastSyncFeed, - downloader, - _blockDownloaderFactory.CreateAllocationStrategyFactory() - ); + SyncDispatcher dispatcher = + _fastSyncScope.GetRequiredService>(); dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -312,7 +340,7 @@ private void StartFastSyncComponents() private void StartStateSyncComponents() { - SyncDispatcher stateSyncDispatcher = _serviceProvider.GetRequiredService>(); + SyncDispatcher stateSyncDispatcher = _mainScope.GetRequiredService>(); Task syncDispatcherTask = stateSyncDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -330,7 +358,7 @@ private void StartStateSyncComponents() private void StartSnapSyncComponents() { - SyncDispatcher dispatcher = _serviceProvider.GetRequiredService>(); + SyncDispatcher dispatcher = _mainScope.GetRequiredService>(); Task _ = dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -347,7 +375,7 @@ private void StartSnapSyncComponents() private void StartFastBlocksComponents() { - SyncDispatcher headersDispatcher = _serviceProvider.GetService>(); + SyncDispatcher headersDispatcher = _mainScope.GetService>(); Task headersTask = headersDispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { @@ -366,7 +394,7 @@ private void StartFastBlocksComponents() if (_syncConfig.DownloadBodiesInFastSync) { SyncDispatcher bodiesDispatcher = - _serviceProvider.GetRequiredService>(); + _mainScope.GetRequiredService>(); Task bodiesTask = bodiesDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -384,7 +412,7 @@ private void StartFastBlocksComponents() if (_syncConfig.DownloadReceiptsInFastSync) { SyncDispatcher receiptsDispatcher = - _serviceProvider.GetService>(); + _mainScope.GetService>(); Task receiptsTask = receiptsDispatcher.Start(_syncCancellation.Token).ContinueWith(t => { @@ -401,17 +429,6 @@ private void StartFastBlocksComponents() } } - private SyncDispatcher CreateDispatcher(ISyncFeed feed, ISyncDownloader downloader, IPeerAllocationStrategyFactory peerAllocationStrategyFactory) - { - return new( - _syncConfig.MaxProcessingThreads, - feed!, - downloader, - _syncPeerPool, - peerAllocationStrategyFactory, - _logManager); - } - private static NodeStatsEventType Convert(SyncEvent syncEvent) { return syncEvent switch @@ -437,24 +454,24 @@ public Task StopAsync() return Task.WhenAny( Task.Delay(FeedsTerminationTimeout), Task.WhenAll( - _fastSyncFeed?.FeedTask ?? Task.CompletedTask, - _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, - _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, - _fullSyncFeed?.FeedTask ?? Task.CompletedTask, - _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, - _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask, - _serviceProvider.GetService()?.FeedTask ?? Task.CompletedTask)); + _fastSyncScope.GetService()?.FeedTask ?? Task.CompletedTask, + _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, + _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, + _fastSyncScope.GetService()?.FeedTask ?? Task.CompletedTask, + _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, + _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, + _mainScope.GetService()?.FeedTask ?? Task.CompletedTask)); } private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_fastSyncFeed); - WireFeedWithModeSelector(_serviceProvider.GetService()); - WireFeedWithModeSelector(_serviceProvider.GetService()); - WireFeedWithModeSelector(_fullSyncFeed); - WireFeedWithModeSelector(_serviceProvider.GetService()); - WireFeedWithModeSelector(_serviceProvider.GetService()); - WireFeedWithModeSelector(_serviceProvider.GetService()); + WireFeedWithModeSelector(_fastSyncScope.GetService()); + WireFeedWithModeSelector(_mainScope.GetService()); + WireFeedWithModeSelector(_mainScope.GetService()); + WireFeedWithModeSelector(_fullSyncScope.GetService()); + WireFeedWithModeSelector(_mainScope.GetService()); + WireFeedWithModeSelector(_mainScope.GetService()); + WireFeedWithModeSelector(_mainScope.GetService()); } protected void WireFeedWithModeSelector(ISyncFeed? feed) @@ -470,11 +487,10 @@ protected void WireFeedWithModeSelector(ISyncFeed? feed) public void Dispose() { CancellationTokenExtensions.CancelDisposeAndClear(ref _syncCancellation); - _syncReport.Dispose(); - _fastSyncFeed?.Dispose(); - _fullSyncFeed?.Dispose(); - _serviceProvider.Dispose(); + _fullSyncScope.Dispose(); + _fastSyncScope.Dispose(); + _mainScope.Dispose(); } } } From c126f95e16a71f05471548585936a8c2d135ba46 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 12:05:29 +0800 Subject: [PATCH 07/36] The merge synchronizer --- .../Nethermind.Api/IApiWithBlockchain.cs | 10 ++++++ .../Nethermind.Api/IApiWithNetwork.cs | 11 ++++++ .../Nethermind.Api/IApiWithStores.cs | 9 +++++ src/Nethermind/Nethermind.Api/IBasicApi.cs | 31 ++++++++++++++++ .../IServiceCollectionExtensions.cs | 15 ++++++++ .../Steps/InitializeNetwork.cs | 19 ++++------ .../Nethermind.Merge.Plugin/MergePlugin.cs | 26 ++++++-------- .../Synchronization/MergeSynchronizer.cs | 35 ++----------------- .../Nethermind.Optimism/OptimismPlugin.cs | 28 +++++++-------- .../Synchronizer.cs | 24 ++++++++----- 10 files changed, 125 insertions(+), 83 deletions(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index a3a510e7c89..6c32bd3d745 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only #nullable enable +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; using Nethermind.Blockchain.FullPruning; @@ -99,5 +100,14 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory INodeStorageFactory NodeStorageFactory { get; set; } BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } + + public IServiceCollection ConfigureServiceCollectionWithBlockchain() + { + return ConfigureApiWithStoreServices() + .AddSingletonIfNotNull(BlockTree) + .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)) + .AddSingletonIfNotNull(StateReader); + + } } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 0f5d2262535..127fd51002e 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Consensus; +using Nethermind.Core; using Nethermind.Core.PubSub; using Nethermind.Grpc; using Nethermind.JsonRpc; @@ -48,5 +50,14 @@ public interface IApiWithNetwork : IApiWithBlockchain ISyncServer? SyncServer { get; set; } IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } + + public IServiceCollection CreateServiceCollectionForSynchronizer() + { + return ConfigureServiceCollectionWithBlockchain() + .AddSingletonIfNotNull(SyncPeerPool) + .AddSingletonIfNotNull(Pivot) + .AddSingletonIfNotNull(PoSSwitcher) + .AddSingletonIfNotNull(NodeStatsManager); + } } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index f6e02313adc..2b950093a4a 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus; +using Nethermind.Core; using Nethermind.Crypto; using Nethermind.Db.Blooms; using Nethermind.State.Repositories; @@ -29,5 +31,12 @@ public interface IApiWithStores : IBasicApi IReceiptMonitor? ReceiptMonitor { get; set; } IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } + + public IServiceCollection ConfigureApiWithStoreServices() + { + return ConfigureBasicApiServices() + .AddSingletonIfNotNull(BlockTree!) + .AddSingletonIfNotNull(ReceiptStorage!); + } } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 3b78f16a865..66af46dbd7e 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -1,11 +1,14 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using System.IO.Abstractions; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Abi; using Nethermind.Api.Extensions; +using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Specs; @@ -57,5 +60,33 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); + + public IServiceCollection ConfigureBasicApiServices() + { + IServiceCollection sc = new ServiceCollection() + .AddSingleton(SpecProvider!) + .AddSingleton(DbProvider!) + .AddSingleton(ChainSpec) + .AddSingletonIfNotNull(BetterPeerStrategy) + .AddSingleton(ConfigProvider.GetConfig()) + .AddSingleton(LogManager); + + string[] dbNames = [DbNames.State, DbNames.Code, DbNames.Metadata, DbNames.Blocks]; + foreach (string dbName in dbNames) + { + sc.AddKeyedSingleton(dbName, DbProvider!.GetDb(dbName)); + sc.AddKeyedSingleton(dbName, DbProvider!.GetDb(dbName)); + } + + sc.AddSingleton>(DbProvider!.GetColumnDb(DbNames.Receipts)); + + foreach (var kv in DbProvider.GetAllDbMeta()) + { + // The key here is large case for some reason... + sc.AddKeyedSingleton(kv.Key, kv.Value); + } + + return sc; + } } } diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index fceb55d622f..23792dd0059 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -27,4 +27,19 @@ public static IServiceCollection ForwardServiceAsSingleton(this IServiceColle return configuration; } + + /// + /// Used for initialization. Some property in INethermindApi may be null at the time of configuration, + /// so this just make it slightly easier to use IServiceCollection. + /// + /// + /// + /// + /// + public static IServiceCollection AddSingletonIfNotNull(this IServiceCollection configuration, T? service) where T : class + { + if (service != null) + return configuration.AddSingleton(service); + return configuration; + } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index d018d62571a..a168b1ac8f6 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Blockchain.Synchronization; @@ -126,29 +127,23 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.Synchronizer is null) { - BlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( + IBlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( _api.SpecProvider!, _api.BlockValidator!, _api.SealValidator!, _api.BetterPeerStrategy!, _api.LogManager); + IServiceCollection serviceCollection = _api.CreateServiceCollectionForSynchronizer() + .AddSingleton(No.BeaconSync) + .AddSingleton(blockDownloaderFactory); + _api.Synchronizer ??= new Synchronizer( + serviceCollection, _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree, - _api.ReceiptStorage!, - _api.SyncPeerPool, _api.NodeStatsManager!, _syncConfig, - blockDownloaderFactory, - _api.Pivot, _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _api.StateReader!, - No.BeaconSync, _api.LogManager); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 283ce8231aa..17f3cab3b30 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -6,6 +6,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Blockchain; @@ -30,6 +31,8 @@ using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; +using Nethermind.Synchronization; +using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.TxPool; @@ -446,25 +449,18 @@ public Task InitSynchronization() new FullStateFinder(_api.BlockTree, _api.StateReader), _api.LogManager); + IServiceCollection serviceCollection = _api.CreateServiceCollectionForSynchronizer() + .AddSingleton(_beaconSync) + .AddSingleton(_mergeConfig) + .AddSingleton(_invalidChainTracker) + .AddSingleton(blockDownloaderFactory); + MergeSynchronizer synchronizer = new MergeSynchronizer( + serviceCollection, _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree!, - _api.ReceiptStorage!, - _api.SyncPeerPool, - _api.NodeStatsManager!, + _api.NodeStatsManager, _syncConfig, - blockDownloaderFactory, - _beaconPivot, - _poSSwitcher, - _mergeConfig, - _invalidChainTracker, _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _beaconSync, - _api.StateReader, _api.LogManager ); _api.Synchronizer = synchronizer; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 1fb2531209c..1d5f1af8029 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -2,20 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using Microsoft.Extensions.DependencyInjection; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; -using Nethermind.Consensus; -using Nethermind.Core.Specs; using Nethermind.Db; using Nethermind.Logging; -using Nethermind.Merge.Plugin.InvalidChainTracker; -using Nethermind.Specs.ChainSpecStyle; -using Nethermind.State; using Nethermind.Stats; using Nethermind.Synchronization; -using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -27,41 +19,18 @@ public class MergeSynchronizer : Synchronizer private readonly ServiceProvider _beaconScope; public MergeSynchronizer( + IServiceCollection serviceCollection, IDbProvider dbProvider, - INodeStorage nodeStorage, - ISpecProvider specProvider, - IBlockTree blockTree, - IReceiptStorage receiptStorage, - ISyncPeerPool peerPool, INodeStatsManager nodeStatsManager, ISyncConfig syncConfig, - IBlockDownloaderFactory blockDownloaderFactory, - IPivot pivot, - IPoSSwitcher poSSwitcher, - IMergeConfig mergeConfig, - IInvalidChainTracker invalidChainTracker, IProcessExitSource exitSource, - IBetterPeerStrategy betterPeerStrategy, - ChainSpec chainSpec, - IBeaconSyncStrategy beaconSync, - IStateReader stateReader, ILogManager logManager) : base( + serviceCollection, dbProvider, - nodeStorage, - specProvider, - blockTree, - receiptStorage, - peerPool, nodeStatsManager, syncConfig, - blockDownloaderFactory, - pivot, exitSource, - betterPeerStrategy, - chainSpec, - stateReader, - beaconSync, logManager) { RegisterBeaconHeaderSyncComponent(_serviceCollection); diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 0f45fe0ddbf..f034da4d4a4 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; using Nethermind.Consensus; @@ -30,6 +31,8 @@ using Nethermind.Core; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Merge.Plugin.handlers; +using Nethermind.Synchronization; +using Nethermind.Synchronization.Blocks; namespace Nethermind.Optimism; @@ -158,25 +161,20 @@ public Task InitSynchronization() new FullStateFinder(_api.BlockTree, _api.StateReader!), _api.LogManager); + IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionForSynchronizer() + .AddSingleton(_beaconSync) + .AddSingleton(_beaconPivot) + .AddSingleton(_api.PoSSwitcher) + .AddSingleton(_mergeConfig) + .AddSingleton(_invalidChainTracker) + .AddSingleton(blockDownloaderFactory); + _api.Synchronizer = new MergeSynchronizer( + serviceCollection, _api.DbProvider, - _api.NodeStorageFactory.WrapKeyValueStore(_api.DbProvider.StateDb), - _api.SpecProvider!, - _api.BlockTree!, - _api.ReceiptStorage!, - _api.SyncPeerPool, - _api.NodeStatsManager!, + _api.NodeStatsManager, _syncConfig, - blockDownloaderFactory, - _beaconPivot, - _api.PoSSwitcher, - _mergeConfig, - _invalidChainTracker, _api.ProcessExit!, - _api.BetterPeerStrategy, - _api.ChainSpec, - _beaconSync, - _api.StateReader!, _api.LogManager ); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 12c55398b09..6a6ee867f99 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -63,6 +63,21 @@ public class Synchronizer : ISynchronizer public ISyncModeSelector SyncModeSelector => _mainScope.GetRequiredService(); public Synchronizer( + IServiceCollection serviceCollection, + IDbProvider dbProvider, + INodeStatsManager nodeStatsManager, + ISyncConfig syncConfig, + IProcessExitSource processExitSource, + ILogManager logManager) + { + _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); + _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); + _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); + _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + + /* IDbProvider dbProvider, INodeStorage nodeStorage, ISpecProvider specProvider, @@ -79,14 +94,6 @@ public Synchronizer( IStateReader stateReader, IBeaconSyncStrategy beaconSyncStrategy, ILogManager logManager) - { - _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); - _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); - _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - var serviceCollection = new ServiceCollection() .AddSingleton(blockTree) .AddSingleton(dbProvider) @@ -107,6 +114,7 @@ public Synchronizer( .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb); + */ ConfigureServiceCollection(serviceCollection); From 5d218261a43230691680aa999baebaecef77aa97 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 12:25:38 +0800 Subject: [PATCH 08/36] Use reflection --- .../Nethermind.Api/IApiWithBlockchain.cs | 9 +++-- .../Nethermind.Api/IApiWithNetwork.cs | 9 ++--- .../Nethermind.Api/IApiWithStores.cs | 7 ++-- src/Nethermind/Nethermind.Api/IBasicApi.cs | 13 +++---- .../IServiceCollectionExtensions.cs | 35 +++++++++++++++---- .../Steps/InitializeNetwork.cs | 2 +- .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- .../Nethermind.Optimism/OptimismPlugin.cs | 2 +- 8 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 6c32bd3d745..80578464f71 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -101,12 +101,11 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } - public IServiceCollection ConfigureServiceCollectionWithBlockchain() + public IServiceCollection CreateServiceCollectionFromApiWithBlockchain() { - return ConfigureApiWithStoreServices() - .AddSingletonIfNotNull(BlockTree) - .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)) - .AddSingletonIfNotNull(StateReader); + return CreateServiceCollectionFromApiWithStores() + .AddPropertiesFrom(this) + .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 127fd51002e..8af4473b363 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -51,13 +51,10 @@ public interface IApiWithNetwork : IApiWithBlockchain IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } - public IServiceCollection CreateServiceCollectionForSynchronizer() + public IServiceCollection CreateServiceCollectionFromApiWithNetwork() { - return ConfigureServiceCollectionWithBlockchain() - .AddSingletonIfNotNull(SyncPeerPool) - .AddSingletonIfNotNull(Pivot) - .AddSingletonIfNotNull(PoSSwitcher) - .AddSingletonIfNotNull(NodeStatsManager); + return CreateServiceCollectionFromApiWithBlockchain() + .AddPropertiesFrom(this); } } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 2b950093a4a..a5c0a2371e3 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -32,11 +32,10 @@ public interface IApiWithStores : IBasicApi IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } - public IServiceCollection ConfigureApiWithStoreServices() + public IServiceCollection CreateServiceCollectionFromApiWithStores() { - return ConfigureBasicApiServices() - .AddSingletonIfNotNull(BlockTree!) - .AddSingletonIfNotNull(ReceiptStorage!); + return CreateServiceCollectionFromBasicApi() + .AddPropertiesFrom(this); } } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 66af46dbd7e..a0ff85fe084 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -41,6 +41,7 @@ public interface IBasicApi ILogManager LogManager { get; set; } ProtectedPrivateKey? OriginalSignerKey { get; set; } IReadOnlyList Plugins { get; } + [SkipServiceCollection] string SealEngineType { get; set; } ISpecProvider? SpecProvider { get; set; } ISyncModeSelector SyncModeSelector { get; set; } @@ -61,15 +62,11 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); - public IServiceCollection ConfigureBasicApiServices() + public IServiceCollection CreateServiceCollectionFromBasicApi() { IServiceCollection sc = new ServiceCollection() - .AddSingleton(SpecProvider!) - .AddSingleton(DbProvider!) - .AddSingleton(ChainSpec) - .AddSingletonIfNotNull(BetterPeerStrategy) - .AddSingleton(ConfigProvider.GetConfig()) - .AddSingleton(LogManager); + .AddPropertiesFrom(this) + .AddSingleton(ConfigProvider.GetConfig()); string[] dbNames = [DbNames.State, DbNames.Code, DbNames.Metadata, DbNames.Blocks]; foreach (string dbName in dbNames) @@ -80,7 +77,7 @@ public IServiceCollection ConfigureBasicApiServices() sc.AddSingleton>(DbProvider!.GetColumnDb(DbNames.Receipts)); - foreach (var kv in DbProvider.GetAllDbMeta()) + foreach (KeyValuePair kv in DbProvider.GetAllDbMeta()) { // The key here is large case for some reason... sc.AddKeyedSingleton(kv.Key, kv.Value); diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index 23792dd0059..11ce08832fe 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -2,6 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; using Microsoft.Extensions.DependencyInjection; namespace Nethermind.Core; @@ -29,17 +32,37 @@ public static IServiceCollection ForwardServiceAsSingleton(this IServiceColle } /// - /// Used for initialization. Some property in INethermindApi may be null at the time of configuration, - /// so this just make it slightly easier to use IServiceCollection. + /// Add all properties as singleton. It get them ahead of time instead of lazily to prevent the final service provider + /// from disposing it. To prevent a property from being included, use . /// /// - /// + /// /// /// - public static IServiceCollection AddSingletonIfNotNull(this IServiceCollection configuration, T? service) where T : class + public static IServiceCollection AddPropertiesFrom(this IServiceCollection configuration, T source) where T : class { - if (service != null) - return configuration.AddSingleton(service); + Type t = typeof(T); + + IEnumerable properties = t + .GetProperties(BindingFlags.GetProperty | BindingFlags.Instance | BindingFlags.Public | BindingFlags.DeclaredOnly) + .Where(p => p.GetCustomAttribute() == null); + + foreach (PropertyInfo propertyInfo in properties) + { + object? val = propertyInfo.GetValue(source); + if (val != null) + { + configuration = configuration.AddSingleton(propertyInfo.PropertyType, val); + } + } + return configuration; } } + +/// +/// Mark a property so that it is not picked up by `AddPropertiesFrom`. +/// +public class SkipServiceCollectionAttribute: Attribute +{ +} diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index a168b1ac8f6..cd7239977ad 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -134,7 +134,7 @@ private async Task Initialize(CancellationToken cancellationToken) _api.BetterPeerStrategy!, _api.LogManager); - IServiceCollection serviceCollection = _api.CreateServiceCollectionForSynchronizer() + IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() .AddSingleton(No.BeaconSync) .AddSingleton(blockDownloaderFactory); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 17f3cab3b30..ec2095783b1 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -449,7 +449,7 @@ public Task InitSynchronization() new FullStateFinder(_api.BlockTree, _api.StateReader), _api.LogManager); - IServiceCollection serviceCollection = _api.CreateServiceCollectionForSynchronizer() + IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() .AddSingleton(_beaconSync) .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index f034da4d4a4..ac53c641d01 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -161,7 +161,7 @@ public Task InitSynchronization() new FullStateFinder(_api.BlockTree, _api.StateReader!), _api.LogManager); - IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionForSynchronizer() + IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork() .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) From 3d9023693e4884048aa51cfb4377624872a68e14 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 14:16:08 +0800 Subject: [PATCH 09/36] Fic test --- src/Nethermind/Directory.Packages.props | 1 + src/Nethermind/Nethermind.Api/IBasicApi.cs | 15 +---- .../Nethermind.Core/Nethermind.Core.csproj | 1 + src/Nethermind/Nethermind.Db/IDbProvider.cs | 31 ++++++++++ .../OldStyleFullSynchronizerTests.cs | 31 ++++++---- .../SyncThreadTests.cs | 35 +++++++---- .../SynchronizerTests.cs | 58 ++++++++++--------- .../ParallelSync/MultiSyncModeSelector.cs | 4 +- .../ParallelSync/SyncDispatcher.cs | 2 +- .../Synchronizer.cs | 17 +++++- 10 files changed, 126 insertions(+), 69 deletions(-) diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 6b405f92b29..bf307b4c4a7 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -36,6 +36,7 @@ + diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index a0ff85fe084..d4df0e0870a 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -68,20 +68,7 @@ public IServiceCollection CreateServiceCollectionFromBasicApi() .AddPropertiesFrom(this) .AddSingleton(ConfigProvider.GetConfig()); - string[] dbNames = [DbNames.State, DbNames.Code, DbNames.Metadata, DbNames.Blocks]; - foreach (string dbName in dbNames) - { - sc.AddKeyedSingleton(dbName, DbProvider!.GetDb(dbName)); - sc.AddKeyedSingleton(dbName, DbProvider!.GetDb(dbName)); - } - - sc.AddSingleton>(DbProvider!.GetColumnDb(DbNames.Receipts)); - - foreach (KeyValuePair kv in DbProvider.GetAllDbMeta()) - { - // The key here is large case for some reason... - sc.AddKeyedSingleton(kv.Key, kv.Value); - } + DbProvider!.ConfigureServiceCollection(sc); return sc; } diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index 8d4a5a8e1ae..a130d95696d 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -6,6 +6,7 @@ + diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 7786271bc74..74900ee9bf1 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; namespace Nethermind.Db { @@ -30,5 +31,35 @@ public interface IDbProvider : IDisposable void RegisterDb(string dbName, T db) where T : class, IDb; void RegisterColumnDb(string dbName, IColumnsDb db); IEnumerable> GetAllDbMeta(); + + void ConfigureServiceCollection(IServiceCollection sc) + { + sc.AddSingleton(this); + + string[] dbNames = [ + DbNames.State, + DbNames.Code, + DbNames.Metadata, + DbNames.Blocks, + DbNames.Headers, + DbNames.BlockInfos, + DbNames.BadBlocks, + DbNames.Bloom, + DbNames.Metadata, + ]; + foreach (string dbName in dbNames) + { + sc.AddKeyedSingleton(dbName, GetDb(dbName)); + sc.AddKeyedSingleton(dbName, GetDb(dbName)); + } + + sc.AddSingleton>(GetColumnDb(DbNames.Receipts)); + + foreach (KeyValuePair kv in GetAllDbMeta()) + { + // The key here is large case for some reason... + sc.AddKeyedSingleton(kv.Key, kv.Value); + } + } } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index c0b2e7986fd..daacc50d73a 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; @@ -70,22 +71,30 @@ public async Task Setup() IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); + IServiceCollection serviceCollection = new ServiceCollection() + .AddSingleton(dbProvider) + .AddSingleton(nodeStorage) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(_blockTree) + .AddSingleton(_receiptStorage) + .AddSingleton(_pool) + .AddSingleton(stats) + .AddSingleton(syncConfig) + .AddSingleton(blockDownloaderFactory) + .AddSingleton(pivot) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(stateReader) + .AddSingleton(No.BeaconSync) + .AddSingleton(LimboLogs.Instance); + _synchronizer = new Synchronizer( + serviceCollection, dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - _blockTree, - _receiptStorage, - _pool, stats, syncConfig, - blockDownloaderFactory, - pivot, Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - stateReader, - No.BeaconSync, LimboLogs.Instance); _syncServer = new SyncServer( trieStore.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 85a00d803ec..7b22b50d068 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.BeaconBlockRoot; using Nethermind.Blockchain.Blocks; @@ -42,6 +43,7 @@ using BlockTree = Nethermind.Blockchain.BlockTree; using Nethermind.Synchronization.SnapSync; using Nethermind.Config; +using Nethermind.Core.Specs; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Trie; @@ -362,22 +364,33 @@ private SyncTestContext CreateSyncManager(int index) sealValidator, new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), logManager); + + IServiceCollection serviceCollection = new ServiceCollection() + .AddSingleton(dbProvider) + .AddSingleton(new NodeStorage(dbProvider.StateDb)) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(tree) + .AddSingleton(NullReceiptStorage.Instance) + .AddSingleton(syncPeerPool) + .AddSingleton(nodeStatsManager) + .AddSingleton(syncConfig) + .AddSingleton(blockDownloaderFactory) + .AddSingleton(pivot) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(stateReader) + .AddSingleton(receiptStorage) + .AddSingleton(No.BeaconSync) + .AddSingleton(logManager); + dbProvider.ConfigureServiceCollection(serviceCollection); + Synchronizer synchronizer = new( + serviceCollection, dbProvider, - new NodeStorage(dbProvider.StateDb), - MainnetSpecProvider.Instance, - tree, - NullReceiptStorage.Instance, - syncPeerPool, nodeStatsManager, syncConfig, - blockDownloaderFactory, - pivot, Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - stateReader, - No.BeaconSync, logManager); ISyncModeSelector selector = synchronizer.SyncModeSelector; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index b4abd18714e..5a4dd3dee77 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -17,6 +18,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Core.Timers; using Nethermind.Db; @@ -308,7 +310,7 @@ ISyncConfig GetSyncConfig() => syncConfig.MultiSyncModeSelectorLoopTimerMs = 1; IDbProvider dbProvider = TestMemDbProvider.Init(); - IDb stateDb = new MemDb(); + IDb stateDb = dbProvider.StateDb; IDb codeDb = dbProvider.CodeDb; BlockTree = Build.A.BlockTree() .WithSpecProvider(new TestSingleReleaseSpecProvider(Constantinople.Instance)) @@ -340,6 +342,30 @@ ISyncConfig GetSyncConfig() => Pivot pivot = new(syncConfig); IInvalidChainTracker invalidChainTracker = new NoopInvalidChainTracker(); + + IServiceCollection serviceCollection = new ServiceCollection(); + dbProvider.ConfigureServiceCollection(serviceCollection); + + serviceCollection + .AddSingleton(dbProvider) + .AddSingleton(nodeStorage) + .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(BlockTree) + .AddSingleton(NullReceiptStorage.Instance) + .AddSingleton(SyncPeerPool) + .AddSingleton(stats) + .AddSingleton(syncConfig) + .AddSingleton(pivot) + .AddSingleton(poSSwitcher) + .AddSingleton(mergeConfig) + .AddSingleton(invalidChainTracker) + .AddSingleton(Substitute.For()) + .AddSingleton(bestPeerStrategy) + .AddSingleton(new ChainSpec()) + .AddSingleton(No.BeaconSync) + .AddSingleton(reader) + .AddSingleton(_logManager); + if (IsMerge(synchronizerType)) { IBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( @@ -353,25 +379,14 @@ ISyncConfig GetSyncConfig() => fullStateFinder, _logManager ); + serviceCollection.AddSingleton(blockDownloaderFactory); + Synchronizer = new MergeSynchronizer( + serviceCollection, dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - BlockTree, - NullReceiptStorage.Instance, - SyncPeerPool, stats, syncConfig, - blockDownloaderFactory, - pivot, - poSSwitcher, - mergeConfig, - invalidChainTracker, Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - No.BeaconSync, - reader, _logManager); } else @@ -382,23 +397,14 @@ ISyncConfig GetSyncConfig() => Always.Valid, new TotalDifficultyBetterPeerStrategy(_logManager), _logManager); + serviceCollection.AddSingleton(blockDownloaderFactory); Synchronizer = new Synchronizer( + serviceCollection, dbProvider, - nodeStorage, - MainnetSpecProvider.Instance, - BlockTree, - NullReceiptStorage.Instance, - SyncPeerPool, stats, syncConfig, - blockDownloaderFactory, - pivot, Substitute.For(), - bestPeerStrategy, - new ChainSpec(), - reader, - No.BeaconSync, _logManager); } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index 10e6539e471..e131fa95c37 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -46,8 +46,6 @@ public class MultiSyncModeSelector : ISyncModeSelector /// private const int StickyStateNodesDelta = 32; - public const string ParameterNeedToWaitForHeader = "needToWaitForHeaders"; - private readonly ISyncProgressResolver _syncProgressResolver; private readonly ISyncPeerPool _syncPeerPool; private readonly ISyncConfig _syncConfig; @@ -84,7 +82,7 @@ public MultiSyncModeSelector( IBeaconSyncStrategy beaconSyncStrategy, IBetterPeerStrategy betterPeerStrategy, ILogManager logManager, - [FromKeyedServices(ParameterNeedToWaitForHeader)] bool needToWaitForHeaders = false) + bool needToWaitForHeaders = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index f7066d9dfc4..d8f38e626f2 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -33,7 +33,7 @@ public SyncDispatcher( ISyncPeerPool? syncPeerPool, IPeerAllocationStrategyFactory? peerAllocationStrategy, ILogManager? logManager) - :this (syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) + : this (syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) { } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 6a6ee867f99..971a8566d4d 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -131,7 +131,8 @@ public Synchronizer( if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync) RegisterBodiesSyncComponent(serviceCollection); - RegisterStateSyncComponent(serviceCollection); + if (_syncConfig.FastSync) + RegisterStateSyncComponent(serviceCollection); _mainScope = serviceCollection.BuildServiceProvider(); @@ -170,7 +171,17 @@ private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) protected virtual void ConfigureServiceCollection(IServiceCollection serviceCollection) { serviceCollection - .AddSingleton() + .AddSingleton(sp => + new SyncProgressResolver( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetService>(), + sp.GetService>(), + sp.GetService>(), + sp.GetService>(), + sp.GetRequiredService() + )) .AddSingleton() .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) @@ -310,7 +321,7 @@ private void StartFullSyncComponents() BlockDownloader fullSyncBlockDownloader = _fullSyncScope.GetRequiredService(); fullSyncBlockDownloader.SyncEvent += DownloaderOnSyncEvent; - SyncDispatcher dispatcher = _fastSyncScope.GetRequiredService>(); + SyncDispatcher dispatcher = _fullSyncScope.GetRequiredService>(); dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { From b9dac974fdaeb583bba2d838c0bd486e878edfaf Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 14:18:40 +0800 Subject: [PATCH 10/36] Minor cleanup --- .../Synchronizer.cs | 52 +------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 971a8566d4d..894180a3a11 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -77,45 +77,6 @@ public Synchronizer( _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - /* - IDbProvider dbProvider, - INodeStorage nodeStorage, - ISpecProvider specProvider, - IBlockTree blockTree, - IReceiptStorage receiptStorage, - ISyncPeerPool peerPool, - INodeStatsManager nodeStatsManager, - ISyncConfig syncConfig, - IBlockDownloaderFactory blockDownloaderFactory, - IPivot pivot, - IProcessExitSource processExitSource, - IBetterPeerStrategy betterPeerStrategy, - ChainSpec chainSpec, - IStateReader stateReader, - IBeaconSyncStrategy beaconSyncStrategy, - ILogManager logManager) - var serviceCollection = new ServiceCollection() - .AddSingleton(blockTree) - .AddSingleton(dbProvider) - .AddSingleton(nodeStorage) - .AddSingleton(peerPool) - .AddSingleton(logManager) - .AddSingleton(specProvider) - .AddSingleton(receiptStorage) - .AddSingleton(stateReader) - .AddSingleton(chainSpec) - .AddSingleton(betterPeerStrategy) - .AddSingleton(beaconSyncStrategy) - .AddSingleton(blockDownloaderFactory) - .AddSingleton(syncConfig) - .AddSingleton(pivot) - .AddSingleton(nodeStatsManager) - .AddKeyedSingleton(DbNames.Metadata, (sp, _) => _dbProvider.MetadataDb) - .AddKeyedSingleton(DbNames.Code, (sp, _) => _dbProvider.CodeDb) - .AddKeyedSingleton(DbNames.State, (sp, _) => _dbProvider.StateDb) - .AddKeyedSingleton(DbNames.Blocks, (sp, _) => _dbProvider.BlocksDb); - */ - ConfigureServiceCollection(serviceCollection); if (_syncConfig.FastSync && _syncConfig.SnapSync) @@ -176,6 +137,7 @@ protected virtual void ConfigureServiceCollection(IServiceCollection serviceColl sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), + // These are optional, thats why this need to be set manually like this. sp.GetService>(), sp.GetService>(), sp.GetService>(), @@ -194,18 +156,6 @@ protected virtual void ConfigureServiceCollection(IServiceCollection serviceColl sp.GetRequiredService(), // Need to set this sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)) - .AddSingleton(sp => new SyncDbTuner( - sp.GetRequiredService(), - - // These are all optional so need to set manually - sp.GetService(), - sp.GetService(), - sp.GetService(), - - sp.GetRequiredService().StateDb as ITunableDb, - sp.GetRequiredService().CodeDb as ITunableDb, - sp.GetRequiredService().BlocksDb as ITunableDb, - sp.GetRequiredService().ReceiptsDb as ITunableDb)) .AddSingleton>(); } From 0f5343dcd08fbd3a4228dc75f805006ecee9bb34 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 14:41:52 +0800 Subject: [PATCH 11/36] Remote block downloader --- .../Steps/InitializeNetwork.cs | 10 +-- .../Nethermind.Merge.Plugin/MergePlugin.cs | 15 +--- .../MergeBlockDownloaderFactory.cs | 81 ------------------- .../Synchronization/MergeSynchronizer.cs | 23 ++++-- .../Nethermind.Optimism/OptimismPlugin.cs | 14 +--- .../OldStyleFullSynchronizerTests.cs | 12 +-- .../SyncThreadTests.cs | 9 +-- .../SynchronizerTests.cs | 26 ++---- .../Blocks/IBlockDownloaderFactory.cs | 65 --------------- .../Synchronizer.cs | 20 ++--- 10 files changed, 37 insertions(+), 238 deletions(-) delete mode 100644 src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs delete mode 100644 src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index cd7239977ad..7e51076d45f 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -127,16 +127,8 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.Synchronizer is null) { - IBlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( - _api.SpecProvider!, - _api.BlockValidator!, - _api.SealValidator!, - _api.BetterPeerStrategy!, - _api.LogManager); - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() - .AddSingleton(No.BeaconSync) - .AddSingleton(blockDownloaderFactory); + .AddSingleton(No.BeaconSync); _api.Synchronizer ??= new Synchronizer( serviceCollection, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index ec2095783b1..258cf4b3ae8 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -438,22 +438,11 @@ public Task InitSynchronization() _api.Pivot = _beaconPivot; - MergeBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - _poSSwitcher, - _beaconPivot, - _api.SpecProvider, - _api.BlockValidator!, - _api.SealValidator!, - _syncConfig, - _api.BetterPeerStrategy!, - new FullStateFinder(_api.BlockTree, _api.StateReader), - _api.LogManager); - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() .AddSingleton(_beaconSync) + .AddSingleton(_beaconPivot) .AddSingleton(_mergeConfig) - .AddSingleton(_invalidChainTracker) - .AddSingleton(blockDownloaderFactory); + .AddSingleton(_invalidChainTracker); MergeSynchronizer synchronizer = new MergeSynchronizer( serviceCollection, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs deleted file mode 100644 index 8c3b99555bf..00000000000 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloaderFactory.cs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; -using Nethermind.Blockchain.Synchronization; -using Nethermind.Consensus; -using Nethermind.Consensus.Validators; -using Nethermind.Core.Specs; -using Nethermind.Logging; -using Nethermind.Synchronization; -using Nethermind.Synchronization.Blocks; -using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; - - -namespace Nethermind.Merge.Plugin.Synchronization -{ - public class MergeBlockDownloaderFactory : IBlockDownloaderFactory - { - private readonly IPoSSwitcher _poSSwitcher; - private readonly IBeaconPivot _beaconPivot; - private readonly ISpecProvider _specProvider; - private readonly IBlockValidator _blockValidator; - private readonly ISealValidator _sealValidator; - private readonly IBetterPeerStrategy _betterPeerStrategy; - private readonly ILogManager _logManager; - private readonly IFullStateFinder _fullStateFinder; - private readonly ISyncConfig _syncConfig; - - public MergeBlockDownloaderFactory( - IPoSSwitcher poSSwitcher, - IBeaconPivot beaconPivot, - ISpecProvider specProvider, - IBlockValidator blockValidator, - ISealValidator sealValidator, - ISyncConfig syncConfig, - IBetterPeerStrategy betterPeerStrategy, - IFullStateFinder fullStateFinder, - ILogManager logManager) - { - _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - _beaconPivot = beaconPivot ?? throw new ArgumentNullException(nameof(beaconPivot)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); - _sealValidator = sealValidator ?? throw new ArgumentNullException(nameof(sealValidator)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - _fullStateFinder = fullStateFinder ?? throw new ArgumentNullException(nameof(fullStateFinder)); ; - _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); ; - } - - public BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, - ISyncPeerPool syncPeerPool, ISyncReport syncReport) - { - ChainLevelHelper chainLevelHelper = new ChainLevelHelper(blockTree, _beaconPivot, _syncConfig, _logManager); - return new MergeBlockDownloader( - _poSSwitcher, - _beaconPivot, - syncFeed, - syncPeerPool, - blockTree, - _blockValidator, - _sealValidator, - syncReport, - receiptStorage, - _specProvider, - _betterPeerStrategy, - chainLevelHelper, - _fullStateFinder, - _logManager); - } - - public IPeerAllocationStrategyFactory CreateAllocationStrategyFactory() - { - return new MergeBlocksSyncPeerAllocationStrategyFactory(_poSSwitcher, _beaconPivot, _logManager); - } - } -} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 1d5f1af8029..4c1aee53db2 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -8,6 +8,7 @@ using Nethermind.Logging; using Nethermind.Stats; using Nethermind.Synchronization; +using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -41,14 +42,20 @@ protected override void ConfigureServiceCollection(IServiceCollection serviceCol { base.ConfigureServiceCollection(serviceCollection); - // It does not set the last parameter - serviceCollection.AddSingleton(sp => new MultiSyncModeSelector( - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService())); + serviceCollection + // It does not set the last parameter + .AddSingleton(sp => new MultiSyncModeSelector( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService())) + + // Replace block downloader + .AddSingleton() + .AddSingleton() + .AddSingleton, MergeBlocksSyncPeerAllocationStrategyFactory>(); } private static void RegisterBeaconHeaderSyncComponent(IServiceCollection serviceCollection) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index ac53c641d01..b098bf470f0 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -150,24 +150,12 @@ public Task InitSynchronization() _api.BetterPeerStrategy = new MergeBetterPeerStrategy(null!, _api.PoSSwitcher, _beaconPivot, _api.LogManager); _api.Pivot = _beaconPivot; - MergeBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - _api.PoSSwitcher, - _beaconPivot, - _api.SpecProvider, - _api.BlockValidator!, - _api.SealValidator!, - _syncConfig, - _api.BetterPeerStrategy!, - new FullStateFinder(_api.BlockTree, _api.StateReader!), - _api.LogManager); - IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork() .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) .AddSingleton(_mergeConfig) - .AddSingleton(_invalidChainTracker) - .AddSingleton(blockDownloaderFactory); + .AddSingleton(_invalidChainTracker); _api.Synchronizer = new MergeSynchronizer( serviceCollection, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index daacc50d73a..37fc4dee70f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -16,6 +16,7 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Core.Timers; using Nethermind.Db; @@ -62,25 +63,20 @@ public async Task Setup() TrieStore trieStore = new(nodeStorage, LimboLogs.Instance); TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); - BlockDownloaderFactory blockDownloaderFactory = new( - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), - LimboLogs.Instance); IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); IServiceCollection serviceCollection = new ServiceCollection() .AddSingleton(dbProvider) .AddSingleton(nodeStorage) - .AddSingleton(MainnetSpecProvider.Instance) + .AddSingleton(MainnetSpecProvider.Instance) .AddSingleton(_blockTree) .AddSingleton(_receiptStorage) .AddSingleton(_pool) .AddSingleton(stats) .AddSingleton(syncConfig) - .AddSingleton(blockDownloaderFactory) + .AddSingleton(Always.Valid) + .AddSingleton(Always.Valid) .AddSingleton(pivot) .AddSingleton(Substitute.For()) .AddSingleton(bestPeerStrategy) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 7b22b50d068..9be5b8fafd1 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -358,12 +358,6 @@ private SyncTestContext CreateSyncManager(int index) TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); - BlockDownloaderFactory blockDownloaderFactory = new( - MainnetSpecProvider.Instance, - blockValidator, - sealValidator, - new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), - logManager); IServiceCollection serviceCollection = new ServiceCollection() .AddSingleton(dbProvider) @@ -374,7 +368,8 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(syncPeerPool) .AddSingleton(nodeStatsManager) .AddSingleton(syncConfig) - .AddSingleton(blockDownloaderFactory) + .AddSingleton(blockValidator) + .AddSingleton(sealValidator) .AddSingleton(pivot) .AddSingleton(Substitute.For()) .AddSingleton(bestPeerStrategy) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 5a4dd3dee77..57b85a42721 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -335,7 +335,6 @@ ISyncConfig GetSyncConfig() => : totalDifficultyBetterPeerStrategy; StateReader reader = new StateReader(trieStore, codeDb, LimboLogs.Instance); - FullStateFinder fullStateFinder = new FullStateFinder(BlockTree, reader); INodeStorage nodeStorage = new NodeStorage(dbProvider.StateDb); SyncPeerPool = new SyncPeerPool(BlockTree, stats, bestPeerStrategy, _logManager, 25); @@ -364,23 +363,13 @@ ISyncConfig GetSyncConfig() => .AddSingleton(new ChainSpec()) .AddSingleton(No.BeaconSync) .AddSingleton(reader) + .AddSingleton(Always.Valid) + .AddSingleton(Always.Valid) + .AddSingleton(beaconPivot) .AddSingleton(_logManager); if (IsMerge(synchronizerType)) { - IBlockDownloaderFactory blockDownloaderFactory = new MergeBlockDownloaderFactory( - poSSwitcher, - beaconPivot, - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - syncConfig, - bestPeerStrategy, - fullStateFinder, - _logManager - ); - serviceCollection.AddSingleton(blockDownloaderFactory); - Synchronizer = new MergeSynchronizer( serviceCollection, dbProvider, @@ -391,13 +380,8 @@ ISyncConfig GetSyncConfig() => } else { - IBlockDownloaderFactory blockDownloaderFactory = new BlockDownloaderFactory( - MainnetSpecProvider.Instance, - Always.Valid, - Always.Valid, - new TotalDifficultyBetterPeerStrategy(_logManager), - _logManager); - serviceCollection.AddSingleton(blockDownloaderFactory); + serviceCollection + .AddSingleton(new TotalDifficultyBetterPeerStrategy(_logManager)); Synchronizer = new Synchronizer( serviceCollection, diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs deleted file mode 100644 index 64afb7b1061..00000000000 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/IBlockDownloaderFactory.cs +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; -using Nethermind.Consensus; -using Nethermind.Consensus.Validators; -using Nethermind.Core.Specs; -using Nethermind.Logging; -using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; - -namespace Nethermind.Synchronization.Blocks -{ - public class BlockDownloaderFactory : IBlockDownloaderFactory - { - private readonly ISpecProvider _specProvider; - private readonly IBlockValidator _blockValidator; - private readonly ISealValidator _sealValidator; - private readonly IBetterPeerStrategy _betterPeerStrategy; - private readonly ILogManager _logManager; - - public BlockDownloaderFactory( - ISpecProvider specProvider, - IBlockValidator blockValidator, - ISealValidator sealValidator, - IBetterPeerStrategy betterPeerStrategy, - ILogManager logManager) - { - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); - _sealValidator = sealValidator ?? throw new ArgumentNullException(nameof(sealValidator)); - _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - } - - public BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, ISyncPeerPool peerPool, ISyncReport syncReport) - { - return new( - syncFeed, - peerPool, - blockTree, - _blockValidator, - _sealValidator, - syncReport, - receiptStorage, - _specProvider, - _betterPeerStrategy, - _logManager); - } - - public IPeerAllocationStrategyFactory CreateAllocationStrategyFactory() - { - return new BlocksSyncPeerAllocationStrategyFactory(); - } - } - - public interface IBlockDownloaderFactory - { - BlockDownloader Create(ISyncFeed syncFeed, IBlockTree blockTree, IReceiptStorage receiptStorage, ISyncPeerPool syncPeerPool, ISyncReport syncReport); - IPeerAllocationStrategyFactory CreateAllocationStrategyFactory(); - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 894180a3a11..cb7de637da9 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -115,18 +115,7 @@ private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) // These three line make sure that these components are not duplicated .ForwardServiceAsSingleton(_mainScope) .ForwardServiceAsSingleton(_mainScope) - .ForwardServiceAsSingleton(_mainScope) - - .AddSingleton(sp => sp.GetRequiredService() - .Create(sp.GetRequiredService>(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService()) - ) - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton(sp => sp .GetRequiredService() - .CreateAllocationStrategyFactory()); + .ForwardServiceAsSingleton(_mainScope); } protected virtual void ConfigureServiceCollection(IServiceCollection serviceCollection) @@ -156,7 +145,12 @@ protected virtual void ConfigureServiceCollection(IServiceCollection serviceColl sp.GetRequiredService(), // Need to set this sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)) - .AddSingleton>(); + .AddSingleton>() + + // These are here so that MergeSynchronizer can replace them + .AddSingleton() + .AddSingleton>(sp => sp.GetRequiredService()) + .AddSingleton, BlocksSyncPeerAllocationStrategyFactory>(); } private static void ConfigureFullSync(IServiceCollection serviceCollection) From 1181068d3889cf6a7a024b9e3464a0b4c1f2ac7a Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 15:02:11 +0800 Subject: [PATCH 12/36] Reduce constructor parameter --- .../Steps/InitializeNetwork.cs | 8 +-- .../Nethermind.Merge.Plugin/MergePlugin.cs | 9 +-- .../Synchronization/MergeSynchronizer.cs | 22 +------ .../Nethermind.Optimism/OptimismPlugin.cs | 9 +-- .../OldStyleFullSynchronizerTests.cs | 11 +--- .../SyncThreadTests.cs | 8 +-- .../SynchronizerTests.cs | 16 +---- .../Synchronizer.cs | 60 ++++++++----------- 8 files changed, 37 insertions(+), 106 deletions(-) diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 7e51076d45f..e9607a698ba 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -130,13 +130,7 @@ private async Task Initialize(CancellationToken cancellationToken) IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() .AddSingleton(No.BeaconSync); - _api.Synchronizer ??= new Synchronizer( - serviceCollection, - _api.DbProvider, - _api.NodeStatsManager!, - _syncConfig, - _api.ProcessExit!, - _api.LogManager); + _api.Synchronizer ??= new Synchronizer(serviceCollection, _syncConfig); } _api.SyncModeSelector = _api.Synchronizer.SyncModeSelector; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 258cf4b3ae8..5e3d43fab02 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -444,14 +444,7 @@ public Task InitSynchronization() .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - MergeSynchronizer synchronizer = new MergeSynchronizer( - serviceCollection, - _api.DbProvider, - _api.NodeStatsManager, - _syncConfig, - _api.ProcessExit!, - _api.LogManager - ); + MergeSynchronizer synchronizer = new MergeSynchronizer(serviceCollection, _syncConfig); _api.Synchronizer = synchronizer; PivotUpdator pivotUpdator = new( diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 4c1aee53db2..98611de9177 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -3,10 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; -using Nethermind.Config; -using Nethermind.Db; using Nethermind.Logging; -using Nethermind.Stats; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; @@ -19,28 +16,15 @@ public class MergeSynchronizer : Synchronizer { private readonly ServiceProvider _beaconScope; - public MergeSynchronizer( - IServiceCollection serviceCollection, - IDbProvider dbProvider, - INodeStatsManager nodeStatsManager, - ISyncConfig syncConfig, - IProcessExitSource exitSource, - ILogManager logManager) - : base( - serviceCollection, - dbProvider, - nodeStatsManager, - syncConfig, - exitSource, - logManager) + public MergeSynchronizer(IServiceCollection serviceCollection, ISyncConfig syncConfig) : base(serviceCollection, syncConfig) { RegisterBeaconHeaderSyncComponent(_serviceCollection); _beaconScope = _serviceCollection.BuildServiceProvider(); } - protected override void ConfigureServiceCollection(IServiceCollection serviceCollection) + protected override void ConfigureSynchronizerServiceCollection(IServiceCollection serviceCollection) { - base.ConfigureServiceCollection(serviceCollection); + base.ConfigureSynchronizerServiceCollection(serviceCollection); serviceCollection // It does not set the last parameter diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index b098bf470f0..ebe66c03d1b 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -157,14 +157,7 @@ public Task InitSynchronization() .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - _api.Synchronizer = new MergeSynchronizer( - serviceCollection, - _api.DbProvider, - _api.NodeStatsManager, - _syncConfig, - _api.ProcessExit!, - _api.LogManager - ); + _api.Synchronizer = new MergeSynchronizer(serviceCollection, _syncConfig); _ = new PivotUpdator( _api.BlockTree, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 37fc4dee70f..f94213f0f2c 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -67,7 +67,6 @@ public async Task Setup() IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); IServiceCollection serviceCollection = new ServiceCollection() - .AddSingleton(dbProvider) .AddSingleton(nodeStorage) .AddSingleton(MainnetSpecProvider.Instance) .AddSingleton(_blockTree) @@ -84,14 +83,10 @@ public async Task Setup() .AddSingleton(stateReader) .AddSingleton(No.BeaconSync) .AddSingleton(LimboLogs.Instance); + dbProvider.ConfigureServiceCollection(serviceCollection); + + _synchronizer = new Synchronizer(serviceCollection, syncConfig); - _synchronizer = new Synchronizer( - serviceCollection, - dbProvider, - stats, - syncConfig, - Substitute.For(), - LimboLogs.Instance); _syncServer = new SyncServer( trieStore.TrieNodeRlpStore, _codeDb, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 9be5b8fafd1..54f82208d57 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -380,13 +380,7 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(logManager); dbProvider.ConfigureServiceCollection(serviceCollection); - Synchronizer synchronizer = new( - serviceCollection, - dbProvider, - nodeStatsManager, - syncConfig, - Substitute.For(), - logManager); + Synchronizer synchronizer = new(serviceCollection, syncConfig); ISyncModeSelector selector = synchronizer.SyncModeSelector; SyncServer syncServer = new( diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 57b85a42721..455a7a8e50c 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -370,26 +370,14 @@ ISyncConfig GetSyncConfig() => if (IsMerge(synchronizerType)) { - Synchronizer = new MergeSynchronizer( - serviceCollection, - dbProvider, - stats, - syncConfig, - Substitute.For(), - _logManager); + Synchronizer = new MergeSynchronizer(serviceCollection, syncConfig); } else { serviceCollection .AddSingleton(new TotalDifficultyBetterPeerStrategy(_logManager)); - Synchronizer = new Synchronizer( - serviceCollection, - dbProvider, - stats, - syncConfig, - Substitute.For(), - _logManager); + Synchronizer = new Synchronizer(serviceCollection, syncConfig); } SyncServer = new SyncServer( diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index cb7de637da9..0561bfbbbae 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -6,16 +6,13 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Extensions; -using Nethermind.Core.Specs; using Nethermind.Db; using Nethermind.Logging; using Nethermind.Specs.ChainSpecStyle; -using Nethermind.State; using Nethermind.Stats; using Nethermind.Stats.Model; using Nethermind.Synchronization.Blocks; @@ -62,40 +59,33 @@ public class Synchronizer : ISynchronizer public ISyncModeSelector SyncModeSelector => _mainScope.GetRequiredService(); - public Synchronizer( - IServiceCollection serviceCollection, - IDbProvider dbProvider, - INodeStatsManager nodeStatsManager, - ISyncConfig syncConfig, - IProcessExitSource processExitSource, - ILogManager logManager) + public Synchronizer(IServiceCollection serviceCollection, ISyncConfig config) { - _dbProvider = dbProvider ?? throw new ArgumentNullException(nameof(dbProvider)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); - _nodeStatsManager = nodeStatsManager ?? throw new ArgumentNullException(nameof(nodeStatsManager)); - _exitSource = processExitSource ?? throw new ArgumentNullException(nameof(processExitSource)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - - ConfigureServiceCollection(serviceCollection); + _syncConfig = config; + ConfigureSynchronizerServiceCollection(serviceCollection); if (_syncConfig.FastSync && _syncConfig.SnapSync) - RegisterSnapComponent(serviceCollection); + ConfigureSnapComponent(serviceCollection); if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync) - RegisterHeaderSyncComponent(serviceCollection); + ConfigureHeaderSyncComponent(serviceCollection); if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync && _syncConfig.DownloadReceiptsInFastSync) - RegisterReceiptSyncComponent(serviceCollection); + ConfigureReceiptSyncComponent(serviceCollection); if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync) - RegisterBodiesSyncComponent(serviceCollection); + ConfigureBodiesSyncComponent(serviceCollection); if (_syncConfig.FastSync) - RegisterStateSyncComponent(serviceCollection); + ConfigureStateSyncComponent(serviceCollection); _mainScope = serviceCollection.BuildServiceProvider(); + _dbProvider = _mainScope.GetRequiredService(); + _logManager = _mainScope.GetRequiredService(); + _logger = _logManager.GetClassLogger(); + _nodeStatsManager = _mainScope.GetRequiredService(); + _exitSource = _mainScope.GetRequiredService(); ConfigureBlocksDownloader(serviceCollection); @@ -118,7 +108,7 @@ private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) .ForwardServiceAsSingleton(_mainScope); } - protected virtual void ConfigureServiceCollection(IServiceCollection serviceCollection) + protected virtual void ConfigureSynchronizerServiceCollection(IServiceCollection serviceCollection) { serviceCollection .AddSingleton(sp => @@ -166,31 +156,31 @@ private static void ConfigureFastSync(IServiceCollection serviceCollection) .AddSingleton() .AddSingleton, FastSyncFeed>(sp => sp.GetRequiredService()); } - private static void RegisterSnapComponent(IServiceCollection serviceCollection) + private static void ConfigureSnapComponent(IServiceCollection serviceCollection) { serviceCollection .AddSingleton() .AddSingleton(); - RegisterSyncFeed(serviceCollection); + ConfigureSyncFeed(serviceCollection); } - private static void RegisterHeaderSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureHeaderSyncComponent(IServiceCollection serviceCollection) { - RegisterSyncFeed(serviceCollection); + ConfigureSyncFeed(serviceCollection); } - private static void RegisterReceiptSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureReceiptSyncComponent(IServiceCollection serviceCollection) { - RegisterSyncFeed(serviceCollection); + ConfigureSyncFeed(serviceCollection); } - private static void RegisterBodiesSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureBodiesSyncComponent(IServiceCollection serviceCollection) { - RegisterSyncFeed(serviceCollection); + ConfigureSyncFeed(serviceCollection); } - private static void RegisterSyncFeed(IServiceCollection serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory + private static void ConfigureSyncFeed(IServiceCollection serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory { serviceCollection .AddSingleton() @@ -200,12 +190,12 @@ private static void RegisterSyncFeed>(); } - private static void RegisterStateSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureStateSyncComponent(IServiceCollection serviceCollection) { serviceCollection .AddSingleton(); - RegisterSyncFeed(serviceCollection); + ConfigureSyncFeed(serviceCollection); } public virtual void Start() From 54facc1411ef09a292e2475d866ed2cc0d028e28 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 15:35:17 +0800 Subject: [PATCH 13/36] Add unit tests --- .../ServiceCollectionExtensionsTests.cs | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs diff --git a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs new file mode 100644 index 00000000000..3d4aba2f1c8 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; + +namespace Nethermind.Core.Test; + +public class ServiceCollectionExtensionsTests +{ + [Test] + public void AddPropertiesFrom_CanAddProperties() + { + ITestInterface interfaceImplementation = new InterfaceImplementation(); + IServiceProvider sp = new ServiceCollection() + .AddPropertiesFrom(interfaceImplementation) + .BuildServiceProvider(); + + sp.GetService().Should().NotBeNull(); + sp.GetService().Should().BeNull(); + sp.GetService().Should().BeNull(); + sp.GetService().Should().BeNull(); + } + + [Test] + public void TestForwardDependency_ShouldNotDispose() + { + ServiceProvider sp1 = new ServiceCollection() + .AddSingleton() + .BuildServiceProvider(); + + ServiceProvider sp2 = new ServiceCollection() + .ForwardServiceAsSingleton(sp1) + .BuildServiceProvider(); + + DisposableService disposableService = sp2.GetRequiredService(); + disposableService.WasDisposed.Should().BeFalse(); + + sp2.Dispose(); + disposableService.WasDisposed.Should().BeFalse(); + + sp1.Dispose(); + disposableService.WasDisposed.Should().BeTrue(); + } + + private class InterfaceImplementation: ITestInterface + { + public DeclaredService TheService { get; set; } = new DeclaredService(); + public DeclaredButNullService? NullService { get; set; } = null; + public Ignored IgnoredService { get; set; } = new Ignored(); + public DeclaredInBase BaseService { get; set; } = new DeclaredInBase(); + } + + private interface ITestInterface: ITestInterfaceBase + { + DeclaredService TheService { get; set; } + DeclaredButNullService? NullService { get; set; } + + [SkipServiceCollection] + Ignored IgnoredService { get; set; } + } + + private interface ITestInterfaceBase + { + DeclaredInBase BaseService { get; set; } + } + + private class DeclaredInBase {} + private class DeclaredService {} + private class DeclaredButNullService {} + private class Ignored {} + + private class DisposableService : IDisposable + { + public bool WasDisposed { get; set; } = false; + + public void Dispose() + { + WasDisposed = true; + } + } +} From 67fe00403ba991806eda0965a787cb17fa1e197d Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 15:46:50 +0800 Subject: [PATCH 14/36] Cleanup --- src/Nethermind/Nethermind.Core/Nethermind.Core.csproj | 5 ----- src/Nethermind/Nethermind.Db/IDbProvider.cs | 6 ------ .../Nethermind.Synchronization.Test/SynchronizerTests.cs | 3 --- .../ParallelSync/MultiSyncModeSelector.cs | 2 -- 4 files changed, 16 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index a130d95696d..f0471e5f190 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -17,9 +17,4 @@ - - - ..\..\..\..\..\.nuget\packages\microsoft.extensions.dependencyinjection.abstractions\8.0.1\lib\net8.0\Microsoft.Extensions.DependencyInjection.Abstractions.dll - - diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 74900ee9bf1..3f7c9f5a55b 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -54,12 +54,6 @@ void ConfigureServiceCollection(IServiceCollection sc) } sc.AddSingleton>(GetColumnDb(DbNames.Receipts)); - - foreach (KeyValuePair kv in GetAllDbMeta()) - { - // The key here is large case for some reason... - sc.AddKeyedSingleton(kv.Key, kv.Value); - } } } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 455a7a8e50c..45273fd6e3d 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -374,9 +374,6 @@ ISyncConfig GetSyncConfig() => } else { - serviceCollection - .AddSingleton(new TotalDifficultyBetterPeerStrategy(_logManager)); - Synchronizer = new Synchronizer(serviceCollection, syncConfig); } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index e131fa95c37..e9eff01f553 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -7,7 +7,6 @@ using System.ComponentModel; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; using Nethermind.Int256; using Nethermind.Logging; @@ -103,7 +102,6 @@ public MultiSyncModeSelector( _ = StartAsync(_cancellation.Token); } - private async Task StartAsync(CancellationToken cancellationToken) { PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_syncConfig.MultiSyncModeSelectorLoopTimerMs)); From aecaea1c00fc736f83b165d1a4e7609382fd5211 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Wed, 25 Sep 2024 16:37:29 +0800 Subject: [PATCH 15/36] Whitespace --- .../ServiceCollectionExtensionsTests.cs | 12 ++++++------ .../Nethermind.Core/IServiceCollectionExtensions.cs | 2 +- .../Nethermind.Synchronization/FastSync/TreeSync.cs | 2 +- .../ParallelSync/SyncDispatcher.cs | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs index 3d4aba2f1c8..74901dd3254 100644 --- a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs @@ -45,7 +45,7 @@ public void TestForwardDependency_ShouldNotDispose() disposableService.WasDisposed.Should().BeTrue(); } - private class InterfaceImplementation: ITestInterface + private class InterfaceImplementation : ITestInterface { public DeclaredService TheService { get; set; } = new DeclaredService(); public DeclaredButNullService? NullService { get; set; } = null; @@ -53,7 +53,7 @@ private class InterfaceImplementation: ITestInterface public DeclaredInBase BaseService { get; set; } = new DeclaredInBase(); } - private interface ITestInterface: ITestInterfaceBase + private interface ITestInterface : ITestInterfaceBase { DeclaredService TheService { get; set; } DeclaredButNullService? NullService { get; set; } @@ -67,10 +67,10 @@ private interface ITestInterfaceBase DeclaredInBase BaseService { get; set; } } - private class DeclaredInBase {} - private class DeclaredService {} - private class DeclaredButNullService {} - private class Ignored {} + private class DeclaredInBase { } + private class DeclaredService { } + private class DeclaredButNullService { } + private class Ignored { } private class DisposableService : IDisposable { diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index 11ce08832fe..1d3cd802434 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -63,6 +63,6 @@ public static IServiceCollection AddPropertiesFrom(this IServiceCollection co /// /// Mark a property so that it is not picked up by `AddPropertiesFrom`. /// -public class SkipServiceCollectionAttribute: Attribute +public class SkipServiceCollectionAttribute : Attribute { } diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 552ce8316db..dd0fcecce11 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -76,7 +76,7 @@ public class TreeSync private readonly SyncMode _syncMode; public TreeSync([FromKeyedServices(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) - : this (SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) + : this(SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) { } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index d8f38e626f2..9b8f368a196 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -33,7 +33,7 @@ public SyncDispatcher( ISyncPeerPool? syncPeerPool, IPeerAllocationStrategyFactory? peerAllocationStrategy, ILogManager? logManager) - : this (syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) + : this(syncConfig.MaxProcessingThreads, syncFeed, downloader, syncPeerPool, peerAllocationStrategy, logManager) { } From 867f1f38c01cc7e1ef7d213b4cddc7651da3addc Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 08:42:53 +0800 Subject: [PATCH 16/36] Pass through the servide collection --- src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs | 4 ++-- src/Nethermind/Nethermind.Api/IApiWithNetwork.cs | 4 ++-- src/Nethermind/Nethermind.Api/IApiWithStores.cs | 4 ++-- src/Nethermind/Nethermind.Api/IBasicApi.cs | 4 ++-- src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs | 2 +- src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 80578464f71..c255ad0c506 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -101,9 +101,9 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithBlockchain() + public IServiceCollection CreateServiceCollectionFromApiWithBlockchain(IServiceCollection serviceCollection) { - return CreateServiceCollectionFromApiWithStores() + return CreateServiceCollectionFromApiWithStores(serviceCollection) .AddPropertiesFrom(this) .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 8af4473b363..c936c7c406f 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -51,9 +51,9 @@ public interface IApiWithNetwork : IApiWithBlockchain IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithNetwork() + public IServiceCollection CreateServiceCollectionFromApiWithNetwork(IServiceCollection serviceCollection) { - return CreateServiceCollectionFromApiWithBlockchain() + return CreateServiceCollectionFromApiWithBlockchain(serviceCollection) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index a5c0a2371e3..950dcd34d47 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -32,9 +32,9 @@ public interface IApiWithStores : IBasicApi IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithStores() + public IServiceCollection CreateServiceCollectionFromApiWithStores(IServiceCollection serviceCollection) { - return CreateServiceCollectionFromBasicApi() + return CreateServiceCollectionFromBasicApi(serviceCollection) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index d4df0e0870a..641c55b6b19 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -62,9 +62,9 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); - public IServiceCollection CreateServiceCollectionFromBasicApi() + public IServiceCollection CreateServiceCollectionFromBasicApi(IServiceCollection serviceCollection) { - IServiceCollection sc = new ServiceCollection() + IServiceCollection sc = serviceCollection .AddPropertiesFrom(this) .AddSingleton(ConfigProvider.GetConfig()); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index e9607a698ba..54d9bad893c 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -127,7 +127,7 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.Synchronizer is null) { - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() + IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) .AddSingleton(No.BeaconSync); _api.Synchronizer ??= new Synchronizer(serviceCollection, _syncConfig); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 5e3d43fab02..8308107bfa9 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -438,7 +438,7 @@ public Task InitSynchronization() _api.Pivot = _beaconPivot; - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork() + IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_mergeConfig) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index ebe66c03d1b..9dac412e651 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -150,7 +150,7 @@ public Task InitSynchronization() _api.BetterPeerStrategy = new MergeBetterPeerStrategy(null!, _api.PoSSwitcher, _beaconPivot, _api.LogManager); _api.Pivot = _beaconPivot; - IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork() + IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) From cb82a56f24390ed5ff8713d733cd54d990235974 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 08:47:14 +0800 Subject: [PATCH 17/36] Use a need to wait for header config --- .../Synchronization/ISyncConfig.cs | 3 +++ .../Synchronization/SyncConfig.cs | 1 + .../Nethermind.Init/Steps/InitializeNetwork.cs | 3 +++ .../Synchronization/MergeSynchronizer.cs | 11 ----------- .../MultiSyncModeSelectorTests.Scenario.cs | 5 ++--- .../ParallelSync/MultiSyncModeSelector.cs | 7 ++++--- .../Nethermind.Synchronization/Synchronizer.cs | 10 +--------- 7 files changed, 14 insertions(+), 26 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index 35c6c3fc1ac..cfee2a225fd 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -143,4 +143,7 @@ public interface ISyncConfig : IConfig [ConfigItem(Description = "_Technical._ MultiSyncModeSelector sync mode timer loop interval. Used for testing.", DefaultValue = "1000", HiddenFromDocs = true)] int MultiSyncModeSelectorLoopTimerMs { get; set; } + + [ConfigItem(Description = "_Technical._ MultiSyncModeSelector will wait for header to completely sync first.", DefaultValue = "false", HiddenFromDocs = true)] + bool NeedToWaitForHeader { get; set; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index d2645336586..c467650ca7a 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -65,6 +65,7 @@ public string? PivotHash public int MallocTrimIntervalSec { get; set; } = 300; public bool? SnapServingEnabled { get; set; } = null; public int MultiSyncModeSelectorLoopTimerMs { get; set; } = 1000; + public bool NeedToWaitForHeader { get; set; } public bool TrieHealing { get; set; } = true; public override string ToString() diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 54d9bad893c..50ffb2d8750 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -130,6 +130,9 @@ private async Task Initialize(CancellationToken cancellationToken) IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) .AddSingleton(No.BeaconSync); + if (_api.ChainSpec.SealEngineType == SealEngineType.Clique) + _syncConfig.NeedToWaitForHeader = true; // Should this be in chainspec itself? + _api.Synchronizer ??= new Synchronizer(serviceCollection, _syncConfig); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 98611de9177..8986a7884b4 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -3,12 +3,10 @@ using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; -using Nethermind.Logging; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; namespace Nethermind.Merge.Plugin.Synchronization; @@ -27,15 +25,6 @@ protected override void ConfigureSynchronizerServiceCollection(IServiceCollectio base.ConfigureSynchronizerServiceCollection(serviceCollection); serviceCollection - // It does not set the last parameter - .AddSingleton(sp => new MultiSyncModeSelector( - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService())) - // Replace block downloader .AddSingleton() .AddSingleton() diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs index f44d95d9c8c..87ab10fe7bd 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ParallelSync/MultiSyncModeSelectorTests.Scenario.cs @@ -145,7 +145,6 @@ public class ScenarioBuilder private readonly List _overwrites = new(); private readonly List _peers = new(); - private bool _needToWaitForHeaders; public ISyncPeerPool SyncPeerPool { get; set; } = null!; @@ -812,7 +811,7 @@ void Test() } TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); - MultiSyncModeSelector selector = new(SyncProgressResolver, SyncPeerPool, SyncConfig, BeaconSyncStrategy, bestPeerStrategy, LimboLogs.Instance, _needToWaitForHeaders); + MultiSyncModeSelector selector = new(SyncProgressResolver, SyncPeerPool, SyncConfig, BeaconSyncStrategy, bestPeerStrategy, LimboLogs.Instance); selector.Stop(); selector.Update(); selector.Current.Should().Be(syncMode); @@ -840,7 +839,7 @@ void Test() public ScenarioBuilder WhenConsensusRequiresToWaitForHeaders(bool needToWaitForHeaders) { - _needToWaitForHeaders = needToWaitForHeaders; + SyncConfig.NeedToWaitForHeader = needToWaitForHeaders; return this; } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index e9eff01f553..7cfc08deca8 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -8,8 +8,10 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; using Nethermind.Int256; using Nethermind.Logging; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.State.Snap; using Nethermind.Synchronization.Peers; @@ -80,8 +82,7 @@ public MultiSyncModeSelector( ISyncConfig syncConfig, IBeaconSyncStrategy beaconSyncStrategy, IBetterPeerStrategy betterPeerStrategy, - ILogManager logManager, - bool needToWaitForHeaders = false) + ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); @@ -89,7 +90,7 @@ public MultiSyncModeSelector( _syncPeerPool = syncPeerPool ?? throw new ArgumentNullException(nameof(syncPeerPool)); _betterPeerStrategy = betterPeerStrategy ?? throw new ArgumentNullException(nameof(betterPeerStrategy)); _syncProgressResolver = syncProgressResolver ?? throw new ArgumentNullException(nameof(syncProgressResolver)); - _needToWaitForHeaders = needToWaitForHeaders; + _needToWaitForHeaders = syncConfig.NeedToWaitForHeader; if (syncConfig.FastSyncCatchUpHeightDelta <= FastSyncLag) { diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 0561bfbbbae..1ec55555946 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -126,15 +126,7 @@ protected virtual void ConfigureSynchronizerServiceCollection(IServiceCollection .AddSingleton() .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) - .AddSingleton(sp => new MultiSyncModeSelector( - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - // Need to set this - sp.GetRequiredService()?.SealEngineType == SealEngineType.Clique)) + .AddSingleton() .AddSingleton>() // These are here so that MergeSynchronizer can replace them From 7b2b7ec4f18fd64723c128faada0b793cbda2e32 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 09:57:13 +0800 Subject: [PATCH 18/36] Remove more manual construction --- src/Nethermind/Nethermind.Db/IDbProvider.cs | 20 +++- src/Nethermind/Nethermind.Db/ITunableDb.cs | 9 +- .../DbTuner/SyncDbTunerTests.cs | 9 -- .../DbTuner/SyncDbOptimizer.cs | 15 +-- .../MallocTrimmer.cs | 9 ++ .../ParallelSync/ISyncFeed.cs | 2 +- .../ParallelSync/NoopSyncFeed.cs | 48 ++++++++ .../ParallelSync/SyncProgressResolver.cs | 22 ++-- .../Synchronizer.cs | 108 +++++++++--------- 9 files changed, 154 insertions(+), 88 deletions(-) create mode 100644 src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 3f7c9f5a55b..4faf741ead6 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -49,11 +49,25 @@ void ConfigureServiceCollection(IServiceCollection sc) ]; foreach (string dbName in dbNames) { - sc.AddKeyedSingleton(dbName, GetDb(dbName)); - sc.AddKeyedSingleton(dbName, GetDb(dbName)); + var db = GetDb(dbName); + sc.AddKeyedSingleton(dbName, db); + sc.AddKeyedSingleton(dbName, db); + if (db is ITunableDb tunableDb) + { + sc.AddKeyedSingleton(dbName, tunableDb); + } + else + { + sc.AddKeyedSingleton(dbName, new NoopTunableDb()); + } } - sc.AddSingleton>(GetColumnDb(DbNames.Receipts)); + IColumnsDb receiptColumnDb = GetColumnDb(DbNames.Receipts); + sc.AddSingleton>(receiptColumnDb); + if (receiptColumnDb is ITunableDb tunableDb2) + sc.AddKeyedSingleton(DbNames.Receipts, tunableDb2); + else + sc.AddKeyedSingleton(DbNames.Receipts, new NoopTunableDb()); } } } diff --git a/src/Nethermind/Nethermind.Db/ITunableDb.cs b/src/Nethermind/Nethermind.Db/ITunableDb.cs index 7584364badd..a8f3e727e36 100644 --- a/src/Nethermind/Nethermind.Db/ITunableDb.cs +++ b/src/Nethermind/Nethermind.Db/ITunableDb.cs @@ -3,7 +3,7 @@ namespace Nethermind.Db; -public interface ITunableDb : IDb +public interface ITunableDb { public void Tune(TuneType type); @@ -18,3 +18,10 @@ enum TuneType HashDb } } + +public class NoopTunableDb: ITunableDb +{ + public void Tune(ITunableDb.TuneType type) + { + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs index 8c7954d8b92..ffc2251face 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/DbTuner/SyncDbTunerTests.cs @@ -53,15 +53,6 @@ public void Setup() _receiptDb); } - [TearDown] - public void TearDown() - { - _blockDb?.Dispose(); - _codeDb?.Dispose(); - _receiptDb?.Dispose(); - _stateDb?.Dispose(); - } - [Test] public void WhenSnapIsOn_TriggerStateDbTune() { diff --git a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs index e9f5c7c10d5..58ca69b6047 100644 --- a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; using Nethermind.Db; using Nethermind.Synchronization.FastBlocks; @@ -24,26 +25,26 @@ public SyncDbTuner( ISyncFeed? snapSyncFeed, ISyncFeed? bodiesSyncFeed, ISyncFeed? receiptSyncFeed, - ITunableDb? stateDb, - ITunableDb? codeDb, - ITunableDb? blockDb, - ITunableDb? receiptDb + [FromKeyedServices(DbNames.State)] ITunableDb? stateDb, + [FromKeyedServices(DbNames.Code)] ITunableDb? codeDb, + [FromKeyedServices(DbNames.Blocks)] ITunableDb? blockDb, + [FromKeyedServices(DbNames.Receipts)] ITunableDb? receiptDb ) { // Only these three make sense as they are write heavy // Headers is used everywhere, so slowing read might slow the whole sync. // Statesync is read heavy, Forward sync is just plain too slow to saturate IO. - if (snapSyncFeed is not null) + if (snapSyncFeed is not NoopSyncFeed) { snapSyncFeed.StateChanged += SnapStateChanged; } - if (bodiesSyncFeed is not null) + if (bodiesSyncFeed is not NoopSyncFeed) { bodiesSyncFeed.StateChanged += BodiesStateChanged; } - if (receiptSyncFeed is not null) + if (receiptSyncFeed is not NoopSyncFeed) { receiptSyncFeed.StateChanged += ReceiptsStateChanged; } diff --git a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs index 63c6e85e36c..5752af568d9 100644 --- a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs +++ b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using Nethermind.Blockchain.Synchronization; using Nethermind.Core.Extensions; using Nethermind.Core.Memory; using Nethermind.Logging; @@ -16,6 +17,14 @@ public class MallocTrimmer private readonly MallocHelper _mallocHelper; private readonly ILogger _logger; + public MallocTrimmer( + ISyncModeSelector syncModeSelector, + ISyncConfig syncConfig, + ILogManager logManager + ): this(syncModeSelector, TimeSpan.FromSeconds(syncConfig.MallocTrimIntervalSec), logManager) + { + } + public MallocTrimmer( ISyncModeSelector syncModeSelector, TimeSpan interval, diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs index 64ac3f4687e..1f87736fb08 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/ISyncFeed.cs @@ -30,6 +30,6 @@ public interface ISyncFeed /// Return true if not finished. May not run even if return true if MultiSyncModeSelector said no, probably /// because it's waiting for other sync or something. - bool IsFinished { get; } + bool IsFinished => false; } } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs new file mode 100644 index 00000000000..8ba1236ad42 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Synchronization.Peers; + +namespace Nethermind.Synchronization.ParallelSync; + +public class NoopSyncFeed: ISyncFeed +{ + public SyncFeedState CurrentState { get; } + + #pragma warning disable CS0067 + public event EventHandler? StateChanged; + #pragma warning disable + + public Task PrepareRequest(CancellationToken token = default) + { + throw new InvalidOperationException("null sync feed should not be used"); + } + + public SyncResponseHandlingResult HandleResponse(T response, PeerInfo? peer = null) + { + throw new InvalidOperationException("null sync feed should not be used"); + } + + public bool IsMultiFeed { get; } + public AllocationContexts Contexts { get; } + public void Activate() + { + throw new InvalidOperationException("null sync feed should not be used"); + } + + public void Finish() + { + throw new InvalidOperationException("null sync feed should not be used"); + } + + public Task FeedTask => Task.CompletedTask; + public void SyncModeSelectorOnChanged(SyncMode current) + { + throw new InvalidOperationException("null sync feed should not be used"); + } + + public bool IsFinished { get; } = true; +} diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs index 73a325d3149..1c61ce82fa8 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs @@ -23,19 +23,19 @@ public class SyncProgressResolver : ISyncProgressResolver // ReSharper disable once NotAccessedField.Local private readonly ILogger _logger; - private readonly ISyncFeed? _headersSyncFeed; - private readonly ISyncFeed? _bodiesSyncFeed; - private readonly ISyncFeed? _receiptsSyncFeed; - private readonly ISyncFeed? _snapSyncFeed; + private readonly ISyncFeed _headersSyncFeed; + private readonly ISyncFeed _bodiesSyncFeed; + private readonly ISyncFeed _receiptsSyncFeed; + private readonly ISyncFeed _snapSyncFeed; public SyncProgressResolver( IBlockTree blockTree, IFullStateFinder fullStateFinder, ISyncConfig syncConfig, - ISyncFeed? headersSyncFeed, - ISyncFeed? bodiesSyncFeed, - ISyncFeed? receiptsSyncFeed, - ISyncFeed? snapSyncFeed, + ISyncFeed headersSyncFeed, + ISyncFeed bodiesSyncFeed, + ISyncFeed receiptsSyncFeed, + ISyncFeed snapSyncFeed, ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); @@ -83,11 +83,11 @@ public long FindBestFullState() return _blockTree.FindHeader(blockHash)?.TotalDifficulty == 0 ? null : _blockTree.FindHeader(blockHash)?.TotalDifficulty; } - public bool IsFastBlocksHeadersFinished() => !IsFastBlocks() || !_syncConfig.DownloadHeadersInFastSync || _headersSyncFeed?.IsFinished == true; + public bool IsFastBlocksHeadersFinished() => !IsFastBlocks() || !_syncConfig.DownloadHeadersInFastSync || _headersSyncFeed.IsFinished; - public bool IsFastBlocksBodiesFinished() => !IsFastBlocks() || !_syncConfig.DownloadBodiesInFastSync || _bodiesSyncFeed?.IsFinished == true; + public bool IsFastBlocksBodiesFinished() => !IsFastBlocks() || !_syncConfig.DownloadBodiesInFastSync || _bodiesSyncFeed.IsFinished; - public bool IsFastBlocksReceiptsFinished() => !IsFastBlocks() || !_syncConfig.DownloadReceiptsInFastSync || _receiptsSyncFeed?.IsFinished == true; + public bool IsFastBlocksReceiptsFinished() => !IsFastBlocks() || !_syncConfig.DownloadReceiptsInFastSync || _receiptsSyncFeed.IsFinished; public bool IsSnapGetRangesFinished() => _snapSyncFeed?.IsFinished ?? true; diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 1ec55555946..29e6d0e2267 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -12,7 +12,6 @@ using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Logging; -using Nethermind.Specs.ChainSpecStyle; using Nethermind.Stats; using Nethermind.Stats.Model; using Nethermind.Synchronization.Blocks; @@ -20,7 +19,6 @@ using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.StateSync; @@ -31,9 +29,6 @@ public class Synchronizer : ISynchronizer { private const int FeedsTerminationTimeout = 5_000; - private static MallocTrimmer? s_trimmer; - private static SyncDbTuner? s_dbTuner; - private readonly INodeStatsManager _nodeStatsManager; protected readonly ILogger _logger; @@ -45,7 +40,6 @@ public class Synchronizer : ISynchronizer /* sync events are used mainly for managing sync peers reputation */ public event EventHandler? SyncEvent; - private readonly IDbProvider _dbProvider; private readonly IProcessExitSource _exitSource; public ISyncProgressResolver SyncProgressResolver => _mainScope.GetRequiredService(); @@ -63,25 +57,13 @@ public Synchronizer(IServiceCollection serviceCollection, ISyncConfig config) { _syncConfig = config; ConfigureSynchronizerServiceCollection(serviceCollection); - - if (_syncConfig.FastSync && _syncConfig.SnapSync) - ConfigureSnapComponent(serviceCollection); - - if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync) - ConfigureHeaderSyncComponent(serviceCollection); - - if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync && - _syncConfig.DownloadReceiptsInFastSync) - ConfigureReceiptSyncComponent(serviceCollection); - - if (_syncConfig.FastSync && _syncConfig.DownloadHeadersInFastSync && _syncConfig.DownloadBodiesInFastSync) - ConfigureBodiesSyncComponent(serviceCollection); - - if (_syncConfig.FastSync) - ConfigureStateSyncComponent(serviceCollection); + ConfigureSnapComponent(serviceCollection); + ConfigureHeaderSyncComponent(serviceCollection); + ConfigureReceiptSyncComponent(serviceCollection); + ConfigureBodiesSyncComponent(serviceCollection); + ConfigureStateSyncComponent(serviceCollection); _mainScope = serviceCollection.BuildServiceProvider(); - _dbProvider = _mainScope.GetRequiredService(); _logManager = _mainScope.GetRequiredService(); _logger = _logManager.GetClassLogger(); _nodeStatsManager = _mainScope.GetRequiredService(); @@ -89,6 +71,8 @@ public Synchronizer(IServiceCollection serviceCollection, ISyncConfig config) ConfigureBlocksDownloader(serviceCollection); + // These to use a separate IServiceProvider because the both expose and use an + // ISyncFeed,Synchronizer,ISyncDownloader where T is BlocksRequest. ConfigureFastSync(serviceCollection); _fastSyncScope = serviceCollection.BuildServiceProvider(); @@ -111,23 +95,14 @@ private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) protected virtual void ConfigureSynchronizerServiceCollection(IServiceCollection serviceCollection) { serviceCollection - .AddSingleton(sp => - new SyncProgressResolver( - sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService(), - // These are optional, thats why this need to be set manually like this. - sp.GetService>(), - sp.GetService>(), - sp.GetService>(), - sp.GetService>(), - sp.GetRequiredService() - )) + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) .AddSingleton() .AddSingleton>() + .AddSingleton() + .AddSingleton() // These are here so that MergeSynchronizer can replace them .AddSingleton() @@ -142,14 +117,20 @@ private static void ConfigureFullSync(IServiceCollection serviceCollection) .AddSingleton, FullSyncFeed>(sp => sp.GetRequiredService()); } - private static void ConfigureFastSync(IServiceCollection serviceCollection) + private void ConfigureFastSync(IServiceCollection serviceCollection) { serviceCollection .AddSingleton() .AddSingleton, FastSyncFeed>(sp => sp.GetRequiredService()); } - private static void ConfigureSnapComponent(IServiceCollection serviceCollection) + private void ConfigureSnapComponent(IServiceCollection serviceCollection) { + if (!_syncConfig.FastSync || !_syncConfig.SnapSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + return; + } + serviceCollection .AddSingleton() .AddSingleton(); @@ -157,18 +138,40 @@ private static void ConfigureSnapComponent(IServiceCollection serviceCollection) ConfigureSyncFeed(serviceCollection); } - private static void ConfigureHeaderSyncComponent(IServiceCollection serviceCollection) + private void ConfigureHeaderSyncComponent(IServiceCollection serviceCollection) { + if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + return; + } + ConfigureSyncFeed(serviceCollection); } - private static void ConfigureReceiptSyncComponent(IServiceCollection serviceCollection) + private void ConfigureReceiptSyncComponent(IServiceCollection serviceCollection) { + + if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || + !_syncConfig.DownloadBodiesInFastSync || + !_syncConfig.DownloadReceiptsInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + return; + } + ConfigureSyncFeed(serviceCollection); } - private static void ConfigureBodiesSyncComponent(IServiceCollection serviceCollection) + private void ConfigureBodiesSyncComponent(IServiceCollection serviceCollection) { + if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || + !_syncConfig.DownloadBodiesInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + return; + } + ConfigureSyncFeed(serviceCollection); } @@ -182,8 +185,14 @@ private static void ConfigureSyncFeed>(); } - private static void ConfigureStateSyncComponent(IServiceCollection serviceCollection) + private void ConfigureStateSyncComponent(IServiceCollection serviceCollection) { + if (!_syncConfig.FastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + return; + } + serviceCollection .AddSingleton(); @@ -215,7 +224,7 @@ public virtual void Start() if (_syncConfig.TuneDbMode != ITunableDb.TuneType.Default || _syncConfig.BlocksDbTuneDbMode != ITunableDb.TuneType.Default) { - SetupDbOptimizer(); + _mainScope.GetRequiredService(); } if (_syncConfig.ExitOnSynced) @@ -225,23 +234,10 @@ public virtual void Start() WireMultiSyncModeSelector(); - s_trimmer ??= new MallocTrimmer(SyncModeSelector, TimeSpan.FromSeconds(_syncConfig.MallocTrimIntervalSec), _logManager); + _mainScope.GetRequiredService(); SyncModeSelector.Changed += _mainScope.GetRequiredService().SyncModeSelectorOnChanged; } - private void SetupDbOptimizer() - { - s_dbTuner ??= new SyncDbTuner( - _syncConfig, - _mainScope.GetService(), - _mainScope.GetService(), - _mainScope.GetService(), - _dbProvider.StateDb as ITunableDb, - _dbProvider.CodeDb as ITunableDb, - _dbProvider.BlocksDb as ITunableDb, - _dbProvider.ReceiptsDb as ITunableDb); - } - private void StartFullSyncComponents() { BlockDownloader fullSyncBlockDownloader = _fullSyncScope.GetRequiredService(); From 13e167f4b6586e7702d5afb54375ef88c2647022 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 09:58:52 +0800 Subject: [PATCH 19/36] Whitespace --- src/Nethermind/Nethermind.Db/ITunableDb.cs | 2 +- src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs | 2 +- .../Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/ITunableDb.cs b/src/Nethermind/Nethermind.Db/ITunableDb.cs index a8f3e727e36..00b0edf0f0c 100644 --- a/src/Nethermind/Nethermind.Db/ITunableDb.cs +++ b/src/Nethermind/Nethermind.Db/ITunableDb.cs @@ -19,7 +19,7 @@ enum TuneType } } -public class NoopTunableDb: ITunableDb +public class NoopTunableDb : ITunableDb { public void Tune(ITunableDb.TuneType type) { diff --git a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs index 5752af568d9..0010bc111db 100644 --- a/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs +++ b/src/Nethermind/Nethermind.Synchronization/MallocTrimmer.cs @@ -21,7 +21,7 @@ public MallocTrimmer( ISyncModeSelector syncModeSelector, ISyncConfig syncConfig, ILogManager logManager - ): this(syncModeSelector, TimeSpan.FromSeconds(syncConfig.MallocTrimIntervalSec), logManager) + ) : this(syncModeSelector, TimeSpan.FromSeconds(syncConfig.MallocTrimIntervalSec), logManager) { } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs index 8ba1236ad42..d8739a19124 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs @@ -8,13 +8,13 @@ namespace Nethermind.Synchronization.ParallelSync; -public class NoopSyncFeed: ISyncFeed +public class NoopSyncFeed : ISyncFeed { public SyncFeedState CurrentState { get; } - #pragma warning disable CS0067 +#pragma warning disable CS0067 public event EventHandler? StateChanged; - #pragma warning disable +#pragma warning disable public Task PrepareRequest(CancellationToken token = default) { From 9b0cf3d845399b0b1a3797ce781bdabdeba63d59 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 16:10:05 +0800 Subject: [PATCH 20/36] Do this more properly --- src/Nethermind/Directory.Packages.props | 2 + .../IServiceCollectionExtensions.cs | 96 ++++++ .../Nethermind.Core/Nethermind.Core.csproj | 2 + .../Steps/InitializeNetwork.cs | 13 +- .../Nethermind.Merge.Plugin/MergePlugin.cs | 12 +- .../Synchronization/MergeSynchronizer.cs | 77 +++-- .../Nethermind.Optimism/OptimismPlugin.cs | 11 +- .../OldStyleFullSynchronizerTests.cs | 9 +- .../SyncThreadTests.cs | 9 +- .../SynchronizerTests.cs | 17 +- .../DbTuner/SyncDbOptimizer.cs | 6 + .../FeedComponent.cs | 25 ++ .../ParallelSync/NoopSyncFeed.cs | 1 - .../ParallelSync/SyncDispatcher.cs | 2 + .../ParallelSync/SyncProgressResolver.cs | 3 +- .../Synchronizer.cs | 321 +++++++----------- 16 files changed, 377 insertions(+), 229 deletions(-) create mode 100644 src/Nethermind/Nethermind.Synchronization/FeedComponent.cs diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 6fea4fedcd1..be8b30f6139 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -6,6 +6,8 @@ + + diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index 1d3cd802434..4c0a0993975 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -5,6 +5,9 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using Autofac; +using Autofac.Core.Activators.Reflection; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; namespace Nethermind.Core; @@ -58,6 +61,84 @@ public static IServiceCollection AddPropertiesFrom(this IServiceCollection co return configuration; } + + public static void ForwardNamedTypeFromLifetime(this ContainerBuilder builder, string name, params Type[] types) + { + foreach (Type type in types) + { + builder.Register(ctx => + { + var theLifetime = ctx.ResolveNamed(name); + Console.Error.WriteLine($"Attempting to resolve {type.Name} from {name} scope"); + return theLifetime.Resolve(type); + }).Named(name, type); + } + } + + public static void ForwardNamedTypeFromLifetimeButNotAsNamed(this ContainerBuilder builder, string name) where T : notnull + { + builder.Register(ctx => ctx.ResolveNamed(name).Resolve()).As(); + } + + public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where T : notnull + { + builder.RegisterType() + .As() + .FindConstructorsWith(new UseThisPleaseConstructorFinder()) + .WithAttributeFiltering() + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where TImpl : notnull where T : notnull + { + builder.RegisterType() + .As() + .AsSelf() + .FindConstructorsWith(new UseThisPleaseConstructorFinder()) + .WithAttributeFiltering() + .SingleInstance(); + + return builder; + } + + public static ContainerBuilder AddScoped(this ContainerBuilder builder) where T : notnull + { + builder.RegisterType() + .As() + .AsSelf() + .FindConstructorsWith(new UseThisPleaseConstructorFinder()) + .InstancePerLifetimeScope(); + + return builder; + } + + public static ContainerBuilder AddScoped(this ContainerBuilder builder) where TImpl : notnull where T : notnull + { + builder.RegisterType() + .As() + .FindConstructorsWith(new UseThisPleaseConstructorFinder()) + .InstancePerLifetimeScope(); + + return builder; + } + + /// + /// A convenient way of creating a service whose member can be configured indipendent of other instance of the same + /// type (assuming the type is of lifetime scope). This is useful for same type with multiple configuration + /// or a graph of multiple same type. The T is expected to be of a main container of sort that contains the + /// main service of interest. + /// Note: The T should dispose an injected ILifetimeScope as ILifetimeScope is not automatically disposed. + /// TODO: Double check this behaviour + /// + public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this ContainerBuilder builder, string name, Action configurator) where T : notnull + { + builder.Register(ctx => ctx.BeginLifetimeScope(configurator).Resolve()) + .Named(name); + + return builder; + } } /// @@ -66,3 +147,18 @@ public static IServiceCollection AddPropertiesFrom(this IServiceCollection co public class SkipServiceCollectionAttribute : Attribute { } + +public class UseThisPleaseAttribute : Attribute +{ +} + +public class UseThisPleaseConstructorFinder : IConstructorFinder +{ + public ConstructorInfo[] FindConstructors(Type targetType) + { + ConstructorInfo[] constructors = targetType.GetConstructors(); + ConstructorInfo? specifiedConstructor = constructors.FirstOrDefault(it => it.GetCustomAttribute() != null); + if (specifiedConstructor != null) return [specifiedConstructor]; + return constructors; + } +} diff --git a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj index f0471e5f190..930e96abe48 100644 --- a/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj +++ b/src/Nethermind/Nethermind.Core/Nethermind.Core.csproj @@ -5,6 +5,8 @@ + + diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 50ffb2d8750..4461a912ff4 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -3,9 +3,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; @@ -133,7 +136,13 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.ChainSpec.SealEngineType == SealEngineType.Clique) _syncConfig.NeedToWaitForHeader = true; // Should this be in chainspec itself? - _api.Synchronizer ??= new Synchronizer(serviceCollection, _syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); + IContainer container = builder.Build(); + + _api.Synchronizer ??= container.Resolve(); + _api.DisposeStack.Append(container); } _api.SyncModeSelector = _api.Synchronizer.SyncModeSelector; @@ -143,7 +152,7 @@ private async Task Initialize(CancellationToken cancellationToken) _api.SyncModeSelector, _api.SyncProgressResolver, _api.LogManager); _api.TxGossipPolicy.Policies.Add(new SyncedTxGossipPolicy(_api.SyncModeSelector)); _api.DisposeStack.Push(_api.SyncModeSelector); - _api.DisposeStack.Push(_api.Synchronizer); + // _api.DisposeStack.Push(_api.Synchronizer); ISyncServer syncServer = _api.SyncServer = new SyncServer( _api.TrieStore!.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 6b59ecfe66f..84f00bce6f9 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -6,6 +6,9 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Core; +using Autofac.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; @@ -443,8 +446,15 @@ public Task InitSynchronization() .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - MergeSynchronizer synchronizer = new MergeSynchronizer(serviceCollection, _syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); + MergeSynchronizer.ConfigureMergeComponent(builder); + IContainer container = builder.Build(); + + MergeSynchronizer synchronizer = container.Resolve(); _api.Synchronizer = synchronizer; + _api.DisposeStack.Append(container); PivotUpdator pivotUpdator = new( _api.BlockTree, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 8986a7884b4..2e6a680df54 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -1,8 +1,14 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Microsoft.Extensions.DependencyInjection; +using System; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; +using Nethermind.Logging; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.FastBlocks; @@ -10,54 +16,62 @@ namespace Nethermind.Merge.Plugin.Synchronization; -public class MergeSynchronizer : Synchronizer +public class MergeSynchronizer( + [KeyFilter(nameof(BeaconHeadersSyncFeed))] FeedComponent beaconHeaderComponent, + ISyncConfig syncConfig, + Synchronizer baseSynchronizer, + ILogManager logManager) + : ISynchronizer { - private readonly ServiceProvider _beaconScope; + private readonly CancellationTokenSource? _syncCancellation = new(); + private readonly ILogger _logger = logManager.GetClassLogger(); - public MergeSynchronizer(IServiceCollection serviceCollection, ISyncConfig syncConfig) : base(serviceCollection, syncConfig) + public static void ConfigureMergeComponent(ContainerBuilder serviceCollection) { - RegisterBeaconHeaderSyncComponent(_serviceCollection); - _beaconScope = _serviceCollection.BuildServiceProvider(); - } - - protected override void ConfigureSynchronizerServiceCollection(IServiceCollection serviceCollection) - { - base.ConfigureSynchronizerServiceCollection(serviceCollection); - serviceCollection - // Replace block downloader + .AddSingleton() + .AddSingleton() - .AddSingleton() - .AddSingleton, MergeBlocksSyncPeerAllocationStrategyFactory>(); + .AddScoped() + .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() + + .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), + scopeConfig => scopeConfig + .AddScoped, BeaconHeadersSyncFeed>() + .AddScoped, BeaconHeadersSyncDownloader>()); } - private static void RegisterBeaconHeaderSyncComponent(IServiceCollection serviceCollection) + public event EventHandler? SyncEvent { - serviceCollection - .AddSingleton, BeaconHeadersSyncFeed>() - .AddSingleton, BeaconHeadersSyncDownloader>() - .AddSingleton, FastBlocksPeerAllocationStrategyFactory>() - .AddSingleton>(); + add => baseSynchronizer.SyncEvent += value; + remove => baseSynchronizer.SyncEvent -= value; } - public override void Start() + public void Start() { - if (!_syncConfig.SynchronizationEnabled) + if (!syncConfig.SynchronizationEnabled) { return; } - base.Start(); + baseSynchronizer.Start(); StartBeaconHeadersComponents(); WireMultiSyncModeSelector(); } - private void StartBeaconHeadersComponents() + public Task StopAsync() { - SyncDispatcher dispatcher = - _beaconScope.GetRequiredService>(); + _syncCancellation?.Cancel(); + return baseSynchronizer.StopAsync(); + } + + public ISyncProgressResolver SyncProgressResolver => baseSynchronizer.SyncProgressResolver; - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + public ISyncModeSelector SyncModeSelector => baseSynchronizer.SyncModeSelector; + + private void StartBeaconHeadersComponents() + { + beaconHeaderComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -72,6 +86,11 @@ private void StartBeaconHeadersComponents() private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_beaconScope.GetRequiredService>()); + baseSynchronizer.WireFeedWithModeSelector(beaconHeaderComponent.Feed); + } + + public void Dispose() + { + baseSynchronizer.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index f6848c69dc8..fe47dc49d69 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -2,7 +2,10 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Linq; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Nethermind.Api; using Nethermind.Api.Extensions; @@ -152,7 +155,13 @@ public Task InitSynchronization() .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - _api.Synchronizer = new MergeSynchronizer(serviceCollection, _syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); + MergeSynchronizer.ConfigureMergeComponent(builder); + IContainer container = builder.Build(); + _api.Synchronizer = container.Resolve(); + _api.DisposeStack.Append(container); _ = new PivotUpdator( _api.BlockTree, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index f94213f0f2c..95dc8674c64 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -4,6 +4,8 @@ using System; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; @@ -85,7 +87,12 @@ public async Task Setup() .AddSingleton(LimboLogs.Instance); dbProvider.ConfigureServiceCollection(serviceCollection); - _synchronizer = new Synchronizer(serviceCollection, syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + IContainer container = builder.Build(); + + _synchronizer = container.Resolve(); _syncServer = new SyncServer( trieStore.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 54f82208d57..0d3da39a02b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.BeaconBlockRoot; @@ -380,7 +382,12 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(logManager); dbProvider.ConfigureServiceCollection(serviceCollection); - Synchronizer synchronizer = new(serviceCollection, syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + IContainer container = builder.Build(); + + Synchronizer synchronizer = container.Resolve(); ISyncModeSelector selector = synchronizer.SyncModeSelector; SyncServer syncServer = new( diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 45273fd6e3d..87add14018a 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -8,6 +8,8 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; @@ -370,11 +372,22 @@ ISyncConfig GetSyncConfig() => if (IsMerge(synchronizerType)) { - Synchronizer = new MergeSynchronizer(serviceCollection, syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + MergeSynchronizer.ConfigureMergeComponent(builder); + IContainer container = builder.Build(); + + Synchronizer = container.Resolve(); } else { - Synchronizer = new Synchronizer(serviceCollection, syncConfig); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + IContainer container = builder.Build(); + + Synchronizer = container.Resolve(); } SyncServer = new SyncServer( diff --git a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs index 58ca69b6047..6e1137c25f9 100644 --- a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs @@ -31,6 +31,12 @@ public SyncDbTuner( [FromKeyedServices(DbNames.Receipts)] ITunableDb? receiptDb ) { + if (syncConfig.TuneDbMode == ITunableDb.TuneType.Default && syncConfig.BlocksDbTuneDbMode == ITunableDb.TuneType.Default) + { + // Do nothing. + return; + } + // Only these three make sense as they are write heavy // Headers is used everywhere, so slowing read might slow the whole sync. // Statesync is read heavy, Forward sync is just plain too slow to saturate IO. diff --git a/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs b/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs new file mode 100644 index 00000000000..b321a48b9e3 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Autofac; +using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.ParallelSync; + +public class FeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope): IDisposable, IAsyncDisposable +{ + public ISyncFeed Feed => feed.Value; + public SyncDispatcher Dispatcher => dispatcher.Value; + public BlockDownloader BlockDownloader => (BlockDownloader)blockDownloader.Value; + + public void Dispose() + { + lifetimeScope.Dispose(); + } + + public async ValueTask DisposeAsync() + { + await lifetimeScope.DisposeAsync(); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs index d8739a19124..05f07fd5015 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs @@ -41,7 +41,6 @@ public void Finish() public Task FeedTask => Task.CompletedTask; public void SyncModeSelectorOnChanged(SyncMode current) { - throw new InvalidOperationException("null sync feed should not be used"); } public bool IsFinished { get; } = true; diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index 9b8f368a196..597a8569019 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain.Synchronization; +using Nethermind.Core; using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -26,6 +27,7 @@ public class SyncDispatcher private readonly SemaphoreSlim _concurrentProcessingSemaphore; + [UseThisPlease] public SyncDispatcher( ISyncConfig syncConfig, ISyncFeed? syncFeed, diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs index 1c61ce82fa8..705611e0f79 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncProgressResolver.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; @@ -32,7 +33,7 @@ public SyncProgressResolver( IBlockTree blockTree, IFullStateFinder fullStateFinder, ISyncConfig syncConfig, - ISyncFeed headersSyncFeed, + [KeyFilter(nameof(HeadersSyncFeed))] ISyncFeed headersSyncFeed, ISyncFeed bodiesSyncFeed, ISyncFeed receiptsSyncFeed, ISyncFeed snapSyncFeed, diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 29e6d0e2267..2af2a32bdcb 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -4,13 +4,12 @@ using System; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Nethermind.Blockchain; +using Autofac; +using Autofac.Features.AttributeFilters; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Extensions; -using Nethermind.Db; using Nethermind.Logging; using Nethermind.Stats; using Nethermind.Stats.Model; @@ -25,196 +24,169 @@ namespace Nethermind.Synchronization { - public class Synchronizer : ISynchronizer + public class Synchronizer( + ISyncProgressResolver syncProgressResolver, + ISyncModeSelector syncModeSelector, + ISyncReport syncReport, + ISyncConfig syncConfig, + ILogManager logManager, + INodeStatsManager nodeStatsManager, + [KeyFilter(nameof(FullSyncFeed))] FeedComponent fullSyncComponent, + [KeyFilter(nameof(FastSyncFeed))] FeedComponent fastSyncComponent, + FeedComponent stateSyncComponent, + FeedComponent snapSyncComponent, + [KeyFilter(nameof(HeadersSyncFeed))] FeedComponent fastHeaderComponent, + FeedComponent oldBodiesComponent, + FeedComponent oldReceiptsComponent, +#pragma warning disable CS9113 // Parameter is unread. But it need to be instantiated to function + SyncDbTuner syncDbTuner, + MallocTrimmer mallocTrimmer, +#pragma warning restore CS9113 // Parameter is unread. + IProcessExitSource exitSource) + : ISynchronizer { private const int FeedsTerminationTimeout = 5_000; - private readonly INodeStatsManager _nodeStatsManager; + private readonly ILogger _logger = logManager.GetClassLogger(); - protected readonly ILogger _logger; - protected readonly ISyncConfig _syncConfig; - protected readonly ILogManager _logManager; - - protected CancellationTokenSource? _syncCancellation = new(); + private CancellationTokenSource? _syncCancellation = new(); /* sync events are used mainly for managing sync peers reputation */ public event EventHandler? SyncEvent; - private readonly IProcessExitSource _exitSource; - - public ISyncProgressResolver SyncProgressResolver => _mainScope.GetRequiredService(); - - private readonly ServiceProvider _mainScope; - private readonly ServiceProvider _fastSyncScope; - private readonly ServiceProvider _fullSyncScope; + public ISyncProgressResolver SyncProgressResolver => syncProgressResolver; - // Used by subclass for beacon header - protected readonly IServiceCollection _serviceCollection; + public ISyncModeSelector SyncModeSelector => syncModeSelector; - public ISyncModeSelector SyncModeSelector => _mainScope.GetRequiredService(); - - public Synchronizer(IServiceCollection serviceCollection, ISyncConfig config) + public static void ConfigureContainerBuilder(ContainerBuilder builder, ISyncConfig syncConfig) { - _syncConfig = config; - ConfigureSynchronizerServiceCollection(serviceCollection); - ConfigureSnapComponent(serviceCollection); - ConfigureHeaderSyncComponent(serviceCollection); - ConfigureReceiptSyncComponent(serviceCollection); - ConfigureBodiesSyncComponent(serviceCollection); - ConfigureStateSyncComponent(serviceCollection); - - _mainScope = serviceCollection.BuildServiceProvider(); - _logManager = _mainScope.GetRequiredService(); - _logger = _logManager.GetClassLogger(); - _nodeStatsManager = _mainScope.GetRequiredService(); - _exitSource = _mainScope.GetRequiredService(); - - ConfigureBlocksDownloader(serviceCollection); - - // These to use a separate IServiceProvider because the both expose and use an - // ISyncFeed,Synchronizer,ISyncDownloader where T is BlocksRequest. - ConfigureFastSync(serviceCollection); - _fastSyncScope = serviceCollection.BuildServiceProvider(); - - ConfigureFullSync(serviceCollection); - _fullSyncScope = serviceCollection.BuildServiceProvider(); - - _serviceCollection = serviceCollection; - } - - private void ConfigureBlocksDownloader(IServiceCollection serviceCollection) - { - // Now we configure blocks downloader - serviceCollection - // These three line make sure that these components are not duplicated - .ForwardServiceAsSingleton(_mainScope) - .ForwardServiceAsSingleton(_mainScope) - .ForwardServiceAsSingleton(_mainScope); - } + builder + .AddSingleton() - protected virtual void ConfigureSynchronizerServiceCollection(IServiceCollection serviceCollection) - { - serviceCollection + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(sp => sp.GetRequiredService()) - .AddSingleton() - .AddSingleton>() .AddSingleton() .AddSingleton() - // These are here so that MergeSynchronizer can replace them - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, BlocksSyncPeerAllocationStrategyFactory>(); - } - - private static void ConfigureFullSync(IServiceCollection serviceCollection) - { - serviceCollection - .AddSingleton() - .AddSingleton, FullSyncFeed>(sp => sp.GetRequiredService()); + // For blocks. There are two block scope, Fast and Full + .AddScoped>() + .AddScoped, BlockDownloader>() + .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() + .AddScoped>() + + // For headers. There are two header scope, Fast and Beacon + .AddScoped>() + .AddScoped, HeadersSyncDownloader>() + .AddScoped, FastBlocksPeerAllocationStrategyFactory>() + .AddScoped>() + + // SyncProgress resolver need one header sync batch feed, which is the fast header one. + .Register(ctx => ctx + .ResolveNamed>(nameof(HeadersSyncFeed)) + .Feed) + .Named>(nameof(HeadersSyncFeed)); + + ConfigureSnapComponent(builder, syncConfig); + ConfigureReceiptSyncComponent(builder, syncConfig); + ConfigureBodiesSyncComponent(builder, syncConfig); + ConfigureStateSyncComponent(builder, syncConfig); + + builder + .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), + scopeConfig => + { + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + else + { + scopeConfig.AddScoped, HeadersSyncFeed>(); + } + }) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), + scopeConfig => scopeConfig.AddScoped, FastSyncFeed>()) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), + scopeConfig => scopeConfig.AddScoped, FullSyncFeed>()); } - private void ConfigureFastSync(IServiceCollection serviceCollection) + private static void ConfigureSnapComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) { - serviceCollection - .AddSingleton() - .AddSingleton, FastSyncFeed>(sp => sp.GetRequiredService()); - } - private void ConfigureSnapComponent(IServiceCollection serviceCollection) - { - if (!_syncConfig.FastSync || !_syncConfig.SnapSync) - { - serviceCollection.AddSingleton, NoopSyncFeed>(); - return; - } - serviceCollection .AddSingleton() .AddSingleton(); - ConfigureSyncFeed(serviceCollection); - } + ConfigureSingletonSyncFeed(serviceCollection); - private void ConfigureHeaderSyncComponent(IServiceCollection serviceCollection) - { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync) + if (!syncConfig.FastSync || !syncConfig.SnapSync) { - serviceCollection.AddSingleton, NoopSyncFeed>(); - return; + serviceCollection.AddSingleton, NoopSyncFeed>(); } - - ConfigureSyncFeed(serviceCollection); } - private void ConfigureReceiptSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureReceiptSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) { + ConfigureSingletonSyncFeed(serviceCollection); - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || - !_syncConfig.DownloadBodiesInFastSync || - !_syncConfig.DownloadReceiptsInFastSync) + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync || + !syncConfig.DownloadReceiptsInFastSync) { serviceCollection.AddSingleton, NoopSyncFeed>(); - return; } - - ConfigureSyncFeed(serviceCollection); } - private void ConfigureBodiesSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureBodiesSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) { - if (!_syncConfig.FastSync || !_syncConfig.DownloadHeadersInFastSync || - !_syncConfig.DownloadBodiesInFastSync) + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync) { serviceCollection.AddSingleton, NoopSyncFeed>(); - return; } - ConfigureSyncFeed(serviceCollection); } - private static void ConfigureSyncFeed(IServiceCollection serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory + private static void ConfigureStateSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) { serviceCollection - .AddSingleton() - .AddSingleton>(sp => sp.GetRequiredService()) - .AddSingleton, TDownloader>() - .AddSingleton, TAllocationStrategy>() - .AddSingleton>(); + .AddSingleton(); + + ConfigureSingletonSyncFeed(serviceCollection); + + // Disable it by setting noop + if (!syncConfig.FastSync) serviceCollection.AddSingleton, NoopSyncFeed>(); } - private void ConfigureStateSyncComponent(IServiceCollection serviceCollection) + private static void ConfigureSingletonSyncFeed(ContainerBuilder serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory { - if (!_syncConfig.FastSync) - { - serviceCollection.AddSingleton, NoopSyncFeed>(); - return; - } - serviceCollection - .AddSingleton(); - - ConfigureSyncFeed(serviceCollection); + .AddSingleton, TFeed>() + .AddSingleton>() + .AddSingleton, TDownloader>() + .AddSingleton, TAllocationStrategy>() + .AddSingleton>(); } public virtual void Start() { - if (!_syncConfig.SynchronizationEnabled) + if (!syncConfig.SynchronizationEnabled) { return; } StartFullSyncComponents(); - if (_syncConfig.FastSync) + if (syncConfig.FastSync) { StartFastBlocksComponents(); StartFastSyncComponents(); - if (_syncConfig.SnapSync) + if (syncConfig.SnapSync) { StartSnapSyncComponents(); } @@ -222,30 +194,20 @@ public virtual void Start() StartStateSyncComponents(); } - if (_syncConfig.TuneDbMode != ITunableDb.TuneType.Default || _syncConfig.BlocksDbTuneDbMode != ITunableDb.TuneType.Default) - { - _mainScope.GetRequiredService(); - } - - if (_syncConfig.ExitOnSynced) + if (syncConfig.ExitOnSynced) { - _exitSource.WatchForExit(SyncModeSelector, _logManager, TimeSpan.FromSeconds(_syncConfig.ExitOnSyncedWaitTimeSec)); + exitSource.WatchForExit(SyncModeSelector, logManager, TimeSpan.FromSeconds(syncConfig.ExitOnSyncedWaitTimeSec)); } WireMultiSyncModeSelector(); - _mainScope.GetRequiredService(); - SyncModeSelector.Changed += _mainScope.GetRequiredService().SyncModeSelectorOnChanged; + SyncModeSelector.Changed += syncReport.SyncModeSelectorOnChanged; } private void StartFullSyncComponents() { - BlockDownloader fullSyncBlockDownloader = _fullSyncScope.GetRequiredService(); - fullSyncBlockDownloader.SyncEvent += DownloaderOnSyncEvent; - - SyncDispatcher dispatcher = _fullSyncScope.GetRequiredService>(); - - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + fullSyncComponent.BlockDownloader.SyncEvent += DownloaderOnSyncEvent; + fullSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -260,13 +222,8 @@ private void StartFullSyncComponents() private void StartFastSyncComponents() { - BlockDownloader downloader = _fastSyncScope.GetRequiredService(); - downloader.SyncEvent += DownloaderOnSyncEvent; - - SyncDispatcher dispatcher = - _fastSyncScope.GetRequiredService>(); - - dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + fastSyncComponent.BlockDownloader.SyncEvent += DownloaderOnSyncEvent; + fastSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -281,9 +238,7 @@ private void StartFastSyncComponents() private void StartStateSyncComponents() { - SyncDispatcher stateSyncDispatcher = _mainScope.GetRequiredService>(); - - Task syncDispatcherTask = stateSyncDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task syncDispatcherTask = stateSyncComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -299,9 +254,7 @@ private void StartStateSyncComponents() private void StartSnapSyncComponents() { - SyncDispatcher dispatcher = _mainScope.GetRequiredService>(); - - Task _ = dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + Task _ = snapSyncComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -316,9 +269,7 @@ private void StartSnapSyncComponents() private void StartFastBlocksComponents() { - SyncDispatcher headersDispatcher = _mainScope.GetService>(); - - Task headersTask = headersDispatcher.Start(_syncCancellation!.Token).ContinueWith(t => + Task headersTask = fastHeaderComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -330,14 +281,11 @@ private void StartFastBlocksComponents() } }); - if (_syncConfig.DownloadHeadersInFastSync) + if (syncConfig.DownloadHeadersInFastSync) { - if (_syncConfig.DownloadBodiesInFastSync) + if (syncConfig.DownloadBodiesInFastSync) { - SyncDispatcher bodiesDispatcher = - _mainScope.GetRequiredService>(); - - Task bodiesTask = bodiesDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task bodiesTask = oldBodiesComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -350,12 +298,9 @@ private void StartFastBlocksComponents() }); } - if (_syncConfig.DownloadReceiptsInFastSync) + if (syncConfig.DownloadReceiptsInFastSync) { - SyncDispatcher receiptsDispatcher = - _mainScope.GetService>(); - - Task receiptsTask = receiptsDispatcher.Start(_syncCancellation.Token).ContinueWith(t => + Task receiptsTask = oldReceiptsComponent.Dispatcher.Start(_syncCancellation.Token).ContinueWith(t => { if (t.IsFaulted) { @@ -384,7 +329,7 @@ private static NodeStatsEventType Convert(SyncEvent syncEvent) private void DownloaderOnSyncEvent(object? sender, SyncEventArgs e) { - _nodeStatsManager.ReportSyncEvent(e.Peer.Node, Convert(e.SyncEvent)); + nodeStatsManager.ReportSyncEvent(e.Peer.Node, Convert(e.SyncEvent)); SyncEvent?.Invoke(this, e); } @@ -395,27 +340,27 @@ public Task StopAsync() return Task.WhenAny( Task.Delay(FeedsTerminationTimeout), Task.WhenAll( - _fastSyncScope.GetService()?.FeedTask ?? Task.CompletedTask, - _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, - _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, - _fastSyncScope.GetService()?.FeedTask ?? Task.CompletedTask, - _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, - _mainScope.GetService()?.FeedTask ?? Task.CompletedTask, - _mainScope.GetService()?.FeedTask ?? Task.CompletedTask)); + fullSyncComponent.Feed.FeedTask, + fastSyncComponent.Feed.FeedTask, + stateSyncComponent.Feed.FeedTask, + snapSyncComponent.Feed.FeedTask, + fastHeaderComponent.Feed.FeedTask, + oldBodiesComponent.Feed.FeedTask, + oldReceiptsComponent.Feed.FeedTask)); } private void WireMultiSyncModeSelector() { - WireFeedWithModeSelector(_fastSyncScope.GetService()); - WireFeedWithModeSelector(_mainScope.GetService()); - WireFeedWithModeSelector(_mainScope.GetService()); - WireFeedWithModeSelector(_fullSyncScope.GetService()); - WireFeedWithModeSelector(_mainScope.GetService()); - WireFeedWithModeSelector(_mainScope.GetService()); - WireFeedWithModeSelector(_mainScope.GetService()); + WireFeedWithModeSelector(fullSyncComponent.Feed); + WireFeedWithModeSelector(fastSyncComponent.Feed); + WireFeedWithModeSelector(stateSyncComponent.Feed); + WireFeedWithModeSelector(snapSyncComponent.Feed); + WireFeedWithModeSelector(fastHeaderComponent.Feed); + WireFeedWithModeSelector(oldBodiesComponent.Feed); + WireFeedWithModeSelector(oldReceiptsComponent.Feed); } - protected void WireFeedWithModeSelector(ISyncFeed? feed) + public void WireFeedWithModeSelector(ISyncFeed? feed) { if (feed is null) return; SyncModeSelector.Changed += ((sender, args) => @@ -428,10 +373,6 @@ protected void WireFeedWithModeSelector(ISyncFeed? feed) public void Dispose() { CancellationTokenExtensions.CancelDisposeAndClear(ref _syncCancellation); - - _fullSyncScope.Dispose(); - _fastSyncScope.Dispose(); - _mainScope.Dispose(); } } } From b06b200a90f2704d667afc642ac4c9d6bf4b5277 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 16:18:46 +0800 Subject: [PATCH 21/36] Set sync mode selector from container directly --- .../Steps/InitializeNetwork.cs | 7 +++--- .../Nethermind.Merge.Plugin/MergePlugin.cs | 4 +++- .../Synchronization/MergeSynchronizer.cs | 4 ---- .../Nethermind.Optimism/OptimismPlugin.cs | 5 +++- .../OldStyleFullSynchronizerTests.cs | 3 ++- .../SynchronizerTests.cs | 23 ++++++------------- .../ISynchronizer.cs | 3 --- .../Synchronizer.cs | 3 --- 8 files changed, 19 insertions(+), 33 deletions(-) diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 4461a912ff4..910880c0ed8 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -142,14 +142,13 @@ private async Task Initialize(CancellationToken cancellationToken) IContainer container = builder.Build(); _api.Synchronizer ??= container.Resolve(); + _api.SyncModeSelector ??= container.Resolve(); + _api.SyncProgressResolver ??= container.Resolve(); _api.DisposeStack.Append(container); } - _api.SyncModeSelector = _api.Synchronizer.SyncModeSelector; - _api.SyncProgressResolver = _api.Synchronizer.SyncProgressResolver; - _api.EthSyncingInfo = new EthSyncingInfo(_api.BlockTree, _api.ReceiptStorage!, _syncConfig, - _api.SyncModeSelector, _api.SyncProgressResolver, _api.LogManager); + _api.SyncModeSelector, _api.SyncProgressResolver!, _api.LogManager); _api.TxGossipPolicy.Policies.Add(new SyncedTxGossipPolicy(_api.SyncModeSelector)); _api.DisposeStack.Push(_api.SyncModeSelector); // _api.DisposeStack.Push(_api.Synchronizer); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 84f00bce6f9..bf53e96a5c6 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -454,11 +454,13 @@ public Task InitSynchronization() MergeSynchronizer synchronizer = container.Resolve(); _api.Synchronizer = synchronizer; + _api.SyncModeSelector = container.Resolve(); + _api.SyncProgressResolver = container.Resolve(); _api.DisposeStack.Append(container); PivotUpdator pivotUpdator = new( _api.BlockTree, - synchronizer.SyncModeSelector, + _api.SyncModeSelector, _api.SyncPeerPool, _syncConfig, _blockCacheService, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 2e6a680df54..0302082d857 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -65,10 +65,6 @@ public Task StopAsync() return baseSynchronizer.StopAsync(); } - public ISyncProgressResolver SyncProgressResolver => baseSynchronizer.SyncProgressResolver; - - public ISyncModeSelector SyncModeSelector => baseSynchronizer.SyncModeSelector; - private void StartBeaconHeadersComponents() { beaconHeaderComponent.Dispatcher.Start(_syncCancellation!.Token).ContinueWith(t => diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index fe47dc49d69..25bc3b85a84 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -31,6 +31,7 @@ using Nethermind.Serialization.Rlp; using Nethermind.Optimism.Rpc; using Nethermind.Synchronization; +using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Optimism; @@ -161,11 +162,13 @@ public Task InitSynchronization() MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); _api.Synchronizer = container.Resolve(); + _api.SyncModeSelector = container.Resolve(); + _api.SyncProgressResolver = container.Resolve(); _api.DisposeStack.Append(container); _ = new PivotUpdator( _api.BlockTree, - _api.Synchronizer.SyncModeSelector, + _api.SyncModeSelector, _api.SyncPeerPool, _syncConfig, _blockCacheService, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 95dc8674c64..800a1233755 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -28,6 +28,7 @@ using Nethermind.State; using Nethermind.Stats; using Nethermind.Synchronization.Blocks; +using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; @@ -102,7 +103,7 @@ public async Task Setup() Always.Valid, Always.Valid, _pool, - _synchronizer.SyncModeSelector, + container.Resolve(), quickConfig, Policy.FullGossip, MainnetSpecProvider.Instance, diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 87add14018a..c2e1927ac1b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -370,26 +370,17 @@ ISyncConfig GetSyncConfig() => .AddSingleton(beaconPivot) .AddSingleton(_logManager); + ContainerBuilder builder = new ContainerBuilder(); + builder.Populate(serviceCollection); + Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + if (IsMerge(synchronizerType)) { - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); - Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); - IContainer container = builder.Build(); - - Synchronizer = container.Resolve(); - } - else - { - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); - Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); - IContainer container = builder.Build(); - - Synchronizer = container.Resolve(); } + IContainer container = builder.Build(); + Synchronizer = container.Resolve(); SyncServer = new SyncServer( trieStore.TrieNodeRlpStore, codeDb, @@ -398,7 +389,7 @@ ISyncConfig GetSyncConfig() => Always.Valid, Always.Valid, SyncPeerPool, - Synchronizer.SyncModeSelector, + container.Resolve(), syncConfig, Policy.FullGossip, MainnetSpecProvider.Instance, diff --git a/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs b/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs index 512b0a767a5..011659c5238 100644 --- a/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/ISynchronizer.cs @@ -15,8 +15,5 @@ public interface ISynchronizer : IDisposable void Start(); Task StopAsync(); - - ISyncProgressResolver SyncProgressResolver { get; } - ISyncModeSelector SyncModeSelector { get; } } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 2af2a32bdcb..a3a788cc157 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -25,7 +25,6 @@ namespace Nethermind.Synchronization { public class Synchronizer( - ISyncProgressResolver syncProgressResolver, ISyncModeSelector syncModeSelector, ISyncReport syncReport, ISyncConfig syncConfig, @@ -54,8 +53,6 @@ public class Synchronizer( /* sync events are used mainly for managing sync peers reputation */ public event EventHandler? SyncEvent; - public ISyncProgressResolver SyncProgressResolver => syncProgressResolver; - public ISyncModeSelector SyncModeSelector => syncModeSelector; public static void ConfigureContainerBuilder(ContainerBuilder builder, ISyncConfig syncConfig) From 8034622597fea6c36ea013f64693802c4a19f229 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 16:28:41 +0800 Subject: [PATCH 22/36] Get component directly from container --- src/Nethermind/Nethermind.Api/IApiWithNetwork.cs | 8 +++++++- src/Nethermind/Nethermind.Api/IBasicApi.cs | 2 -- src/Nethermind/Nethermind.Api/NethermindApi.cs | 9 ++++++--- .../Nethermind.Core/IServiceCollectionExtensions.cs | 9 +++++++++ .../Nethermind.Init/Steps/InitializeNetwork.cs | 9 ++------- .../Nethermind.Merge.Plugin/MergePlugin.cs | 6 +----- .../Synchronization/MergeSynchronizer.cs | 2 +- .../Nethermind.Optimism/OptimismPlugin.cs | 4 +--- .../Ethereum/ContextWithMocks.cs | 13 +++++++++---- .../Nethermind.Synchronization/Synchronizer.cs | 2 +- 10 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index c936c7c406f..2c2d5e1f129 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using Autofac; using Microsoft.Extensions.DependencyInjection; using Nethermind.Consensus; using Nethermind.Core; @@ -18,6 +19,7 @@ using Nethermind.Synchronization; using Nethermind.Synchronization.Peers; using Nethermind.Sockets; +using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Api { @@ -43,7 +45,9 @@ public interface IApiWithNetwork : IApiWithBlockchain IJsonRpcLocalStats? JsonRpcLocalStats { get; set; } ISessionMonitor? SessionMonitor { get; set; } IStaticNodesManager? StaticNodesManager { get; set; } - ISynchronizer? Synchronizer { get; set; } + ISynchronizer? Synchronizer { get; } + ISyncModeSelector SyncModeSelector { get; } + ISyncProgressResolver? SyncProgressResolver { get; } IPivot? Pivot { get; set; } ISyncPeerPool? SyncPeerPool { get; set; } IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; set; } @@ -51,6 +55,8 @@ public interface IApiWithNetwork : IApiWithBlockchain IWebSocketsManager WebSocketsManager { get; set; } ISubscriptionFactory? SubscriptionFactory { get; set; } + IContainer? ApiWithNetworkServiceContainer { get; set; } + public IServiceCollection CreateServiceCollectionFromApiWithNetwork(IServiceCollection serviceCollection) { return CreateServiceCollectionFromApiWithBlockchain(serviceCollection) diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 641c55b6b19..a4762835981 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -44,8 +44,6 @@ public interface IBasicApi [SkipServiceCollection] string SealEngineType { get; set; } ISpecProvider? SpecProvider { get; set; } - ISyncModeSelector SyncModeSelector { get; set; } - ISyncProgressResolver? SyncProgressResolver { get; set; } IBetterPeerStrategy? BetterPeerStrategy { get; set; } ITimestamper Timestamper { get; } ITimerFactory TimerFactory { get; } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index ceee96957d1..624d8d99344 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO.Abstractions; +using Autofac; using Nethermind.Abi; using Nethermind.Api.Extensions; using Nethermind.Blockchain; @@ -185,14 +186,14 @@ public ISealEngine SealEngine public ISessionMonitor? SessionMonitor { get; set; } public ISpecProvider? SpecProvider { get; set; } public IPoSSwitcher PoSSwitcher { get; set; } = NoPoS.Instance; - public ISyncModeSelector SyncModeSelector { get; set; } = null!; + public ISyncModeSelector SyncModeSelector => ApiWithNetworkServiceContainer?.Resolve()!; - public ISyncProgressResolver? SyncProgressResolver { get; set; } + public ISyncProgressResolver? SyncProgressResolver => ApiWithNetworkServiceContainer?.Resolve(); public IBetterPeerStrategy? BetterPeerStrategy { get; set; } public IPivot? Pivot { get; set; } public ISyncPeerPool? SyncPeerPool { get; set; } public IPeerDifficultyRefreshPool? PeerDifficultyRefreshPool { get; set; } - public ISynchronizer? Synchronizer { get; set; } + public ISynchronizer? Synchronizer => ApiWithNetworkServiceContainer?.Resolve(); public ISyncServer? SyncServer { get; set; } public IWorldState? WorldState { get; set; } public IReadOnlyStateProvider? ChainHeadStateProvider { get; set; } @@ -241,5 +242,7 @@ public ISealEngine SealEngine public CompositePruningTrigger PruningTrigger { get; } = new(); public IProcessExitSource? ProcessExit { get; set; } public CompositeTxGossipPolicy TxGossipPolicy { get; } = new(); + + public IContainer? ApiWithNetworkServiceContainer { get; set; } } } diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index 4c0a0993975..bca4cd2622c 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -91,6 +91,15 @@ public static ContainerBuilder AddSingleton(this ContainerBuilder builder) wh return builder; } + public static ContainerBuilder AddSingleton(this ContainerBuilder builder, T instance) where T : class + { + builder.RegisterInstance(instance) + .As() + .SingleInstance(); + + return builder; + } + public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where TImpl : notnull where T : notnull { builder.RegisterType() diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 910880c0ed8..5605223d2d7 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -140,18 +140,13 @@ private async Task Initialize(CancellationToken cancellationToken) builder.Populate(serviceCollection); Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); IContainer container = builder.Build(); - - _api.Synchronizer ??= container.Resolve(); - _api.SyncModeSelector ??= container.Resolve(); - _api.SyncProgressResolver ??= container.Resolve(); + _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); } _api.EthSyncingInfo = new EthSyncingInfo(_api.BlockTree, _api.ReceiptStorage!, _syncConfig, - _api.SyncModeSelector, _api.SyncProgressResolver!, _api.LogManager); + _api.SyncModeSelector!, _api.SyncProgressResolver!, _api.LogManager); _api.TxGossipPolicy.Policies.Add(new SyncedTxGossipPolicy(_api.SyncModeSelector)); - _api.DisposeStack.Push(_api.SyncModeSelector); - // _api.DisposeStack.Push(_api.Synchronizer); ISyncServer syncServer = _api.SyncServer = new SyncServer( _api.TrieStore!.TrieNodeRlpStore, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index bf53e96a5c6..c7bd1266359 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -451,11 +451,7 @@ public Task InitSynchronization() Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); - - MergeSynchronizer synchronizer = container.Resolve(); - _api.Synchronizer = synchronizer; - _api.SyncModeSelector = container.Resolve(); - _api.SyncProgressResolver = container.Resolve(); + _api.ApiWithNetworkServiceContainer = builder.Build(); _api.DisposeStack.Append(container); PivotUpdator pivotUpdator = new( diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 0302082d857..015b8f72ddb 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -29,7 +29,7 @@ public class MergeSynchronizer( public static void ConfigureMergeComponent(ContainerBuilder serviceCollection) { serviceCollection - .AddSingleton() + .AddSingleton() .AddSingleton() .AddScoped() diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 25bc3b85a84..9f6c059b483 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -161,9 +161,7 @@ public Task InitSynchronization() Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); - _api.Synchronizer = container.Resolve(); - _api.SyncModeSelector = container.Resolve(); - _api.SyncProgressResolver = container.Resolve(); + _api.ApiWithNetworkServiceContainer = builder.Build(); _api.DisposeStack.Append(container); _ = new PivotUpdator( diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs index 73c0c925c45..7478cfed5d5 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.IO.Abstractions; +using Autofac; using Nethermind.Api; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; @@ -46,6 +47,7 @@ using Nethermind.Trie; using NSubstitute; using Nethermind.Blockchain.Blocks; +using Nethermind.Core; namespace Nethermind.Runner.Test.Ethereum { @@ -75,7 +77,6 @@ public static NethermindApi ContextWithMocks() StaticNodesManager = Substitute.For(), BloomStorage = Substitute.For(), Sealer = Substitute.For(), - Synchronizer = Substitute.For(), BlockchainProcessor = Substitute.For(), BlockProducer = Substitute.For(), DiscoveryApp = Substitute.For(), @@ -102,7 +103,6 @@ public static NethermindApi ContextWithMocks() EngineSignerStore = Substitute.For(), NodeStatsManager = Substitute.For(), RpcModuleProvider = Substitute.For(), - SyncModeSelector = Substitute.For(), SyncPeerPool = Substitute.For(), PeerDifficultyRefreshPool = Substitute.For(), WebSocketsManager = Substitute.For(), @@ -116,10 +116,15 @@ public static NethermindApi ContextWithMocks() TxValidator = new TxValidator(MainnetSpecProvider.Instance.ChainId), UnclesValidator = Substitute.For(), BlockProductionPolicy = Substitute.For(), - SyncProgressResolver = Substitute.For(), BetterPeerStrategy = Substitute.For(), ReceiptMonitor = Substitute.For(), - BadBlocksStore = Substitute.For() + BadBlocksStore = Substitute.For(), + + ApiWithNetworkServiceContainer = new ContainerBuilder() + .AddSingleton(Substitute.For()) + .AddSingleton(Substitute.For()) + .AddSingleton(Substitute.For()) + .Build(), }; api.WorldStateManager = new ReadOnlyWorldStateManager(api.DbProvider, Substitute.For(), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index a3a788cc157..e1d81daabcc 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -58,7 +58,7 @@ public class Synchronizer( public static void ConfigureContainerBuilder(ContainerBuilder builder, ISyncConfig syncConfig) { builder - .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() From 915047e7e5a10ca106732bce780103381926c557 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 16:40:07 +0800 Subject: [PATCH 23/36] Remove unnecessary code --- .../IServiceCollectionExtensions.cs | 38 ------------------- .../FeedComponent.cs | 2 +- .../ParallelSync/SyncDispatcher.cs | 1 - 3 files changed, 1 insertion(+), 40 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs index bca4cd2622c..d68a0fe41ca 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Reflection; using Autofac; -using Autofac.Core.Activators.Reflection; using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; @@ -62,29 +61,10 @@ public static IServiceCollection AddPropertiesFrom(this IServiceCollection co return configuration; } - public static void ForwardNamedTypeFromLifetime(this ContainerBuilder builder, string name, params Type[] types) - { - foreach (Type type in types) - { - builder.Register(ctx => - { - var theLifetime = ctx.ResolveNamed(name); - Console.Error.WriteLine($"Attempting to resolve {type.Name} from {name} scope"); - return theLifetime.Resolve(type); - }).Named(name, type); - } - } - - public static void ForwardNamedTypeFromLifetimeButNotAsNamed(this ContainerBuilder builder, string name) where T : notnull - { - builder.Register(ctx => ctx.ResolveNamed(name).Resolve()).As(); - } - public static ContainerBuilder AddSingleton(this ContainerBuilder builder) where T : notnull { builder.RegisterType() .As() - .FindConstructorsWith(new UseThisPleaseConstructorFinder()) .WithAttributeFiltering() .SingleInstance(); @@ -105,7 +85,6 @@ public static ContainerBuilder AddSingleton(this ContainerBuilder buil builder.RegisterType() .As() .AsSelf() - .FindConstructorsWith(new UseThisPleaseConstructorFinder()) .WithAttributeFiltering() .SingleInstance(); @@ -117,7 +96,6 @@ public static ContainerBuilder AddScoped(this ContainerBuilder builder) where builder.RegisterType() .As() .AsSelf() - .FindConstructorsWith(new UseThisPleaseConstructorFinder()) .InstancePerLifetimeScope(); return builder; @@ -127,7 +105,6 @@ public static ContainerBuilder AddScoped(this ContainerBuilder builder { builder.RegisterType() .As() - .FindConstructorsWith(new UseThisPleaseConstructorFinder()) .InstancePerLifetimeScope(); return builder; @@ -156,18 +133,3 @@ public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this Co public class SkipServiceCollectionAttribute : Attribute { } - -public class UseThisPleaseAttribute : Attribute -{ -} - -public class UseThisPleaseConstructorFinder : IConstructorFinder -{ - public ConstructorInfo[] FindConstructors(Type targetType) - { - ConstructorInfo[] constructors = targetType.GetConstructors(); - ConstructorInfo? specifiedConstructor = constructors.FirstOrDefault(it => it.GetCustomAttribute() != null); - if (specifiedConstructor != null) return [specifiedConstructor]; - return constructors; - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs b/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs index b321a48b9e3..fb27fe8c7f1 100644 --- a/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs +++ b/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs @@ -7,7 +7,7 @@ using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; -public class FeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope): IDisposable, IAsyncDisposable +public class FeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope) : IDisposable, IAsyncDisposable { public ISyncFeed Feed => feed.Value; public SyncDispatcher Dispatcher => dispatcher.Value; diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs index 597a8569019..2bbdb3e50d4 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/SyncDispatcher.cs @@ -27,7 +27,6 @@ public class SyncDispatcher private readonly SemaphoreSlim _concurrentProcessingSemaphore; - [UseThisPlease] public SyncDispatcher( ISyncConfig syncConfig, ISyncFeed? syncFeed, From e4b3baafc23052f28fdc5df2bb9e3368f439c182 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 16:59:14 +0800 Subject: [PATCH 24/36] I thought I specifically fixed this before --- src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index c7bd1266359..8bc0ddb80dc 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -451,7 +451,7 @@ public Task InitSynchronization() Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); - _api.ApiWithNetworkServiceContainer = builder.Build(); + _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); PivotUpdator pivotUpdator = new( diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 9f6c059b483..7a91f8d58ef 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -161,7 +161,7 @@ public Task InitSynchronization() Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); - _api.ApiWithNetworkServiceContainer = builder.Build(); + _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); _ = new PivotUpdator( From 1c54c1db77a1c8acbcec376e38da30ff9f748dd7 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 20:28:03 +0800 Subject: [PATCH 25/36] Well.. might as well make everything autofac --- .../Nethermind.Api/IApiWithBlockchain.cs | 5 +-- .../Nethermind.Api/IApiWithNetwork.cs | 4 +-- .../Nethermind.Api/IApiWithStores.cs | 5 +-- src/Nethermind/Nethermind.Api/IBasicApi.cs | 9 ++--- .../ServiceCollectionExtensionsTests.cs | 34 ++++-------------- ...sions.cs => ContainerBuilderExtensions.cs} | 36 +++++++------------ src/Nethermind/Nethermind.Db/IDbProvider.cs | 4 ++- .../Steps/InitializeNetwork.cs | 7 ++-- .../Nethermind.Merge.Plugin/MergePlugin.cs | 8 ++--- .../Nethermind.Optimism/OptimismPlugin.cs | 7 ++-- .../OldStyleFullSynchronizerTests.cs | 8 ++--- .../SyncThreadTests.cs | 8 ++--- .../SynchronizerTests.cs | 8 ++--- 13 files changed, 54 insertions(+), 89 deletions(-) rename src/Nethermind/Nethermind.Core/{IServiceCollectionExtensions.cs => ContainerBuilderExtensions.cs} (73%) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index c255ad0c506..6fa5d248d00 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only #nullable enable +using Autofac; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; @@ -101,9 +102,9 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithBlockchain(IServiceCollection serviceCollection) + public ContainerBuilder CreateServiceCollectionFromApiWithBlockchain(ContainerBuilder builder) { - return CreateServiceCollectionFromApiWithStores(serviceCollection) + return CreateServiceCollectionFromApiWithStores(builder) .AddPropertiesFrom(this) .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 2c2d5e1f129..771e242a2e2 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -57,9 +57,9 @@ public interface IApiWithNetwork : IApiWithBlockchain IContainer? ApiWithNetworkServiceContainer { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithNetwork(IServiceCollection serviceCollection) + public ContainerBuilder CreateServiceCollectionFromApiWithNetwork(ContainerBuilder builder) { - return CreateServiceCollectionFromApiWithBlockchain(serviceCollection) + return CreateServiceCollectionFromApiWithBlockchain(builder) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 950dcd34d47..8e63cdac496 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Autofac; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; @@ -32,9 +33,9 @@ public interface IApiWithStores : IBasicApi IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } - public IServiceCollection CreateServiceCollectionFromApiWithStores(IServiceCollection serviceCollection) + public ContainerBuilder CreateServiceCollectionFromApiWithStores(ContainerBuilder builder) { - return CreateServiceCollectionFromBasicApi(serviceCollection) + return CreateServiceCollectionFromBasicApi(builder) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index a4762835981..131660307ad 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO.Abstractions; using System.Linq; +using Autofac; using Microsoft.Extensions.DependencyInjection; using Nethermind.Abi; using Nethermind.Api.Extensions; @@ -60,15 +61,15 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); - public IServiceCollection CreateServiceCollectionFromBasicApi(IServiceCollection serviceCollection) + public ContainerBuilder CreateServiceCollectionFromBasicApi(ContainerBuilder builder) { - IServiceCollection sc = serviceCollection + builder .AddPropertiesFrom(this) .AddSingleton(ConfigProvider.GetConfig()); - DbProvider!.ConfigureServiceCollection(sc); + DbProvider!.ConfigureServiceCollection(builder); - return sc; + return builder; } } } diff --git a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs index 74901dd3254..3bf75f3bb68 100644 --- a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Autofac; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; @@ -14,35 +15,14 @@ public class ServiceCollectionExtensionsTests public void AddPropertiesFrom_CanAddProperties() { ITestInterface interfaceImplementation = new InterfaceImplementation(); - IServiceProvider sp = new ServiceCollection() + IContainer sp = new ContainerBuilder() .AddPropertiesFrom(interfaceImplementation) - .BuildServiceProvider(); + .Build(); - sp.GetService().Should().NotBeNull(); - sp.GetService().Should().BeNull(); - sp.GetService().Should().BeNull(); - sp.GetService().Should().BeNull(); - } - - [Test] - public void TestForwardDependency_ShouldNotDispose() - { - ServiceProvider sp1 = new ServiceCollection() - .AddSingleton() - .BuildServiceProvider(); - - ServiceProvider sp2 = new ServiceCollection() - .ForwardServiceAsSingleton(sp1) - .BuildServiceProvider(); - - DisposableService disposableService = sp2.GetRequiredService(); - disposableService.WasDisposed.Should().BeFalse(); - - sp2.Dispose(); - disposableService.WasDisposed.Should().BeFalse(); - - sp1.Dispose(); - disposableService.WasDisposed.Should().BeTrue(); + sp.Resolve().Should().NotBeNull(); + sp.Resolve().Should().BeNull(); + sp.Resolve().Should().BeNull(); + sp.Resolve().Should().BeNull(); } private class InterfaceImplementation : ITestInterface diff --git a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs similarity index 73% rename from src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs rename to src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index d68a0fe41ca..207e024fc08 100644 --- a/src/Nethermind/Nethermind.Core/IServiceCollectionExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -7,32 +7,11 @@ using System.Reflection; using Autofac; using Autofac.Features.AttributeFilters; -using Microsoft.Extensions.DependencyInjection; namespace Nethermind.Core; -public static class IServiceCollectionExtensions +public static class ContainerBuilderExtensions { - public static IServiceCollection ForwardServiceAsSingleton(this IServiceCollection configuration, IServiceProvider baseServiceProvider) where T : class - { - T? theService = baseServiceProvider.GetService(); - if (theService != null) - { - configuration.AddSingleton(baseServiceProvider.GetRequiredService()); - } - else - { - // It could be that this is in a test where the service was not registered and any dependency will be - // replaced anyway. While using a factory function like this seems like it would have the same behaviour - // as getting it directly first, it has one critical difference. When the final IServiceProvider is - // disposed, it would also call the Dispose function of the service as it assume that it created and - // therefore owned the service. - configuration.AddSingleton((sp) => baseServiceProvider.GetRequiredService()); - } - - return configuration; - } - /// /// Add all properties as singleton. It get them ahead of time instead of lazily to prevent the final service provider /// from disposing it. To prevent a property from being included, use . @@ -41,7 +20,7 @@ public static IServiceCollection ForwardServiceAsSingleton(this IServiceColle /// /// /// - public static IServiceCollection AddPropertiesFrom(this IServiceCollection configuration, T source) where T : class + public static ContainerBuilder AddPropertiesFrom(this ContainerBuilder configuration, T source) where T : class { Type t = typeof(T); @@ -54,7 +33,7 @@ public static IServiceCollection AddPropertiesFrom(this IServiceCollection co object? val = propertyInfo.GetValue(source); if (val != null) { - configuration = configuration.AddSingleton(propertyInfo.PropertyType, val); + configuration.RegisterInstance(val).As(propertyInfo.PropertyType); } } @@ -91,6 +70,15 @@ public static ContainerBuilder AddSingleton(this ContainerBuilder buil return builder; } + public static ContainerBuilder AddKeyedSingleton(this ContainerBuilder builder, string key, T instance) where T : class + { + builder.RegisterInstance(instance) + .Named(key) + .SingleInstance(); + + return builder; + } + public static ContainerBuilder AddScoped(this ContainerBuilder builder) where T : notnull { builder.RegisterType() diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 4faf741ead6..5c6c4b7fa50 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; +using Autofac; using Microsoft.Extensions.DependencyInjection; +using Nethermind.Core; namespace Nethermind.Db { @@ -32,7 +34,7 @@ public interface IDbProvider : IDisposable void RegisterColumnDb(string dbName, IColumnsDb db); IEnumerable> GetAllDbMeta(); - void ConfigureServiceCollection(IServiceCollection sc) + void ConfigureServiceCollection(ContainerBuilder sc) { sc.AddSingleton(this); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 5605223d2d7..783c7f3b086 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -130,16 +130,15 @@ private async Task Initialize(CancellationToken cancellationToken) if (_api.Synchronizer is null) { - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) - .AddSingleton(No.BeaconSync); - if (_api.ChainSpec.SealEngineType == SealEngineType.Clique) _syncConfig.NeedToWaitForHeader = true; // Should this be in chainspec itself? ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); + _api.CreateServiceCollectionFromApiWithNetwork(builder) + .AddSingleton(No.BeaconSync); Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); IContainer container = builder.Build(); + _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 8bc0ddb80dc..c8ab4d71c10 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -440,16 +440,16 @@ public Task InitSynchronization() _api.Pivot = _beaconPivot; - IServiceCollection serviceCollection = _api.CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) + ContainerBuilder builder = new ContainerBuilder(); + + _api.CreateServiceCollectionFromApiWithNetwork(builder) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); + IContainer container = builder.Build(); _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 6897d923ea6..411cd068a03 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -152,18 +152,17 @@ public Task InitSynchronization() _api.BetterPeerStrategy = new MergeBetterPeerStrategy(null!, _api.PoSSwitcher, _beaconPivot, _api.LogManager); _api.Pivot = _beaconPivot; - IServiceCollection serviceCollection = ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork(new ServiceCollection()) + ContainerBuilder builder = new ContainerBuilder(); + ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork(builder) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); MergeSynchronizer.ConfigureMergeComponent(builder); IContainer container = builder.Build(); + _api.ApiWithNetworkServiceContainer = container; _api.DisposeStack.Append(container); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 800a1233755..70e958b6c53 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -69,7 +69,7 @@ public async Task Setup() IStateReader stateReader = new StateReader(trieStore, _codeDb, LimboLogs.Instance); - IServiceCollection serviceCollection = new ServiceCollection() + ContainerBuilder builder = new ContainerBuilder() .AddSingleton(nodeStorage) .AddSingleton(MainnetSpecProvider.Instance) .AddSingleton(_blockTree) @@ -86,11 +86,9 @@ public async Task Setup() .AddSingleton(stateReader) .AddSingleton(No.BeaconSync) .AddSingleton(LimboLogs.Instance); - dbProvider.ConfigureServiceCollection(serviceCollection); - - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); + dbProvider.ConfigureServiceCollection(builder); Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + IContainer container = builder.Build(); _synchronizer = container.Resolve(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 0d3da39a02b..f6fac1169d0 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -361,7 +361,8 @@ private SyncTestContext CreateSyncManager(int index) TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); - IServiceCollection serviceCollection = new ServiceCollection() + ContainerBuilder builder = new ContainerBuilder(); + builder .AddSingleton(dbProvider) .AddSingleton(new NodeStorage(dbProvider.StateDb)) .AddSingleton(MainnetSpecProvider.Instance) @@ -380,10 +381,7 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(receiptStorage) .AddSingleton(No.BeaconSync) .AddSingleton(logManager); - dbProvider.ConfigureServiceCollection(serviceCollection); - - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); + dbProvider.ConfigureServiceCollection(builder); Synchronizer.ConfigureContainerBuilder(builder, syncConfig); IContainer container = builder.Build(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index c2e1927ac1b..753d0dc9e5f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -344,10 +344,10 @@ ISyncConfig GetSyncConfig() => IInvalidChainTracker invalidChainTracker = new NoopInvalidChainTracker(); - IServiceCollection serviceCollection = new ServiceCollection(); - dbProvider.ConfigureServiceCollection(serviceCollection); + ContainerBuilder builder = new ContainerBuilder(); + dbProvider.ConfigureServiceCollection(builder); - serviceCollection + builder .AddSingleton(dbProvider) .AddSingleton(nodeStorage) .AddSingleton(MainnetSpecProvider.Instance) @@ -370,8 +370,6 @@ ISyncConfig GetSyncConfig() => .AddSingleton(beaconPivot) .AddSingleton(_logManager); - ContainerBuilder builder = new ContainerBuilder(); - builder.Populate(serviceCollection); Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); if (IsMerge(synchronizerType)) From e7abfba1ebf8f38b4e371c17e75e71b48eec301a Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 21:37:46 +0800 Subject: [PATCH 26/36] Fix test --- .../ServiceCollectionExtensionsTests.cs | 8 ++++---- src/Nethermind/Nethermind.Db/IDbProvider.cs | 1 + .../DbTuner/SyncDbOptimizer.cs | 9 +++++---- .../FastBlocks/BodiesSyncFeed.cs | 5 +++-- .../FastBlocks/ReceiptsSyncFeed.cs | 3 ++- .../Nethermind.Synchronization/FastSync/TreeSync.cs | 3 ++- .../SnapSync/ProgressTracker.cs | 3 ++- .../Nethermind.Synchronization/SnapSync/SnapProvider.cs | 3 ++- 8 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs index 3bf75f3bb68..413dc8b0363 100644 --- a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs @@ -19,10 +19,10 @@ public void AddPropertiesFrom_CanAddProperties() .AddPropertiesFrom(interfaceImplementation) .Build(); - sp.Resolve().Should().NotBeNull(); - sp.Resolve().Should().BeNull(); - sp.Resolve().Should().BeNull(); - sp.Resolve().Should().BeNull(); + sp.ResolveOptional().Should().NotBeNull(); + sp.ResolveOptional().Should().BeNull(); + sp.ResolveOptional().Should().BeNull(); + sp.ResolveOptional().Should().BeNull(); } private class InterfaceImplementation : ITestInterface diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 5c6c4b7fa50..38d2a09a813 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -38,6 +38,7 @@ void ConfigureServiceCollection(ContainerBuilder sc) { sc.AddSingleton(this); + // TODO: Have hooks that automatically get these string[] dbNames = [ DbNames.State, DbNames.Code, diff --git a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs index 6e1137c25f9..13a8ecbb4a6 100644 --- a/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/DbTuner/SyncDbOptimizer.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain.Synchronization; using Nethermind.Db; @@ -25,10 +26,10 @@ public SyncDbTuner( ISyncFeed? snapSyncFeed, ISyncFeed? bodiesSyncFeed, ISyncFeed? receiptSyncFeed, - [FromKeyedServices(DbNames.State)] ITunableDb? stateDb, - [FromKeyedServices(DbNames.Code)] ITunableDb? codeDb, - [FromKeyedServices(DbNames.Blocks)] ITunableDb? blockDb, - [FromKeyedServices(DbNames.Receipts)] ITunableDb? receiptDb + [KeyFilter(DbNames.State)] ITunableDb? stateDb, + [KeyFilter(DbNames.Code)] ITunableDb? codeDb, + [KeyFilter(DbNames.Blocks)] ITunableDb? blockDb, + [KeyFilter(DbNames.Receipts)] ITunableDb? receiptDb ) { if (syncConfig.TuneDbMode == ITunableDb.TuneType.Default && syncConfig.BlocksDbTuneDbMode == ITunableDb.TuneType.Default) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 30c4225f6c3..1152f416054 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; @@ -51,8 +52,8 @@ public BodiesSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - [FromKeyedServices(DbNames.Blocks)] IDbMeta blocksDb, - [FromKeyedServices(DbNames.Metadata)] IDb metadataDb, + [KeyFilter(DbNames.Blocks)] IDbMeta blocksDb, + [KeyFilter(DbNames.Metadata)] IDb metadataDb, ILogManager logManager, long flushDbInterval = DefaultFlushDbInterval) : base(metadataDb, specProvider, logManager.GetClassLogger()) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index f7fc6afe321..cbfce46215c 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -6,6 +6,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; @@ -58,7 +59,7 @@ public ReceiptsSyncFeed( ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, - [FromKeyedServices(DbNames.Metadata)] IDb metadataDb, + [KeyFilter(DbNames.Metadata)] IDb metadataDb, ILogManager logManager) : base(metadataDb, specProvider, logManager?.GetClassLogger() ?? default) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index dd0fcecce11..e10a80a62a1 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Core; @@ -75,7 +76,7 @@ public class TreeSync private long _blockNumber; private readonly SyncMode _syncMode; - public TreeSync([FromKeyedServices(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) + public TreeSync([KeyFilter(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, IBlockTree blockTree, ILogManager logManager) : this(SyncMode.StateNodes, codeDb, nodeStorage, blockTree, logManager) { diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index 064675f421f..b6dba820c0c 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Text; using System.Threading; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; @@ -60,7 +61,7 @@ public class ProgressTracker : IDisposable private readonly Pivot _pivot; - public ProgressTracker(IBlockTree blockTree, [FromKeyedServices(DbNames.State)] IDb db, ILogManager logManager, ISyncConfig syncConfig) + public ProgressTracker(IBlockTree blockTree, [KeyFilter(DbNames.State)] IDb db, ILogManager logManager, ISyncConfig syncConfig) : this(blockTree, db, logManager, syncConfig.SnapSyncAccountRangePartitionCount) { } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index f3c864c06f1..51548bfa9f6 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading; +using Autofac.Features.AttributeFilters; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.ObjectPool; using Nethermind.Core; @@ -34,7 +35,7 @@ public class SnapProvider : ISnapProvider // This is actually close to 97% effective. private readonly ClockKeyCache _codeExistKeyCache = new(1024 * 16); - public SnapProvider(ProgressTracker progressTracker, [FromKeyedServices(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) + public SnapProvider(ProgressTracker progressTracker, [KeyFilter(DbNames.Code)] IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) { _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); _progressTracker = progressTracker ?? throw new ArgumentNullException(nameof(progressTracker)); From 0b58c9cff1c693516a04ece671a25b88a8324221 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 21:58:02 +0800 Subject: [PATCH 27/36] Cleanup --- .../Nethermind.Api/IApiWithBlockchain.cs | 5 +- .../Nethermind.Api/IApiWithNetwork.cs | 5 +- .../Nethermind.Api/IApiWithStores.cs | 5 +- src/Nethermind/Nethermind.Api/IBasicApi.cs | 4 +- ....cs => ContainerBuilderExtensionsTests.cs} | 53 ++++++++++++++++++- .../ContainerBuilderExtensions.cs | 14 ++--- .../Steps/InitializeNetwork.cs | 2 +- .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- .../Nethermind.Optimism/OptimismPlugin.cs | 2 +- 9 files changed, 68 insertions(+), 24 deletions(-) rename src/Nethermind/Nethermind.Core.Test/{ServiceCollectionExtensionsTests.cs => ContainerBuilderExtensionsTests.cs} (53%) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 6fa5d248d00..07dc3ae4e8e 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -3,7 +3,6 @@ #nullable enable using Autofac; -using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Filters; using Nethermind.Blockchain.FullPruning; @@ -102,9 +101,9 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory BackgroundTaskScheduler BackgroundTaskScheduler { get; set; } CensorshipDetector CensorshipDetector { get; set; } - public ContainerBuilder CreateServiceCollectionFromApiWithBlockchain(ContainerBuilder builder) + public ContainerBuilder ConfigureContainerBuilderFromApiWithBlockchain(ContainerBuilder builder) { - return CreateServiceCollectionFromApiWithStores(builder) + return ConfigureContainerBuilderFromApiWithStores(builder) .AddPropertiesFrom(this) .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 771e242a2e2..93bb0769d2f 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using Autofac; -using Microsoft.Extensions.DependencyInjection; using Nethermind.Consensus; using Nethermind.Core; using Nethermind.Core.PubSub; @@ -57,9 +56,9 @@ public interface IApiWithNetwork : IApiWithBlockchain IContainer? ApiWithNetworkServiceContainer { get; set; } - public ContainerBuilder CreateServiceCollectionFromApiWithNetwork(ContainerBuilder builder) + public ContainerBuilder ConfigureContainerBuilderFromApiWithNetwork(ContainerBuilder builder) { - return CreateServiceCollectionFromApiWithBlockchain(builder) + return ConfigureContainerBuilderFromApiWithBlockchain(builder) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 8e63cdac496..22b2b0eb9e9 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using Autofac; -using Microsoft.Extensions.DependencyInjection; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Find; @@ -33,9 +32,9 @@ public interface IApiWithStores : IBasicApi IWallet? Wallet { get; set; } IBlockStore? BadBlocksStore { get; set; } - public ContainerBuilder CreateServiceCollectionFromApiWithStores(ContainerBuilder builder) + public ContainerBuilder ConfigureContainerBuilderFromApiWithStores(ContainerBuilder builder) { - return CreateServiceCollectionFromBasicApi(builder) + return ConfigureContainerBuilderFromBasicApi(builder) .AddPropertiesFrom(this); } } diff --git a/src/Nethermind/Nethermind.Api/IBasicApi.cs b/src/Nethermind/Nethermind.Api/IBasicApi.cs index 131660307ad..1ba0adb91f9 100644 --- a/src/Nethermind/Nethermind.Api/IBasicApi.cs +++ b/src/Nethermind/Nethermind.Api/IBasicApi.cs @@ -6,7 +6,6 @@ using System.IO.Abstractions; using System.Linq; using Autofac; -using Microsoft.Extensions.DependencyInjection; using Nethermind.Abi; using Nethermind.Api.Extensions; using Nethermind.Blockchain.Synchronization; @@ -21,7 +20,6 @@ using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; using Nethermind.Synchronization; -using Nethermind.Synchronization.ParallelSync; namespace Nethermind.Api { @@ -61,7 +59,7 @@ public IEnumerable GetConsensusWrapperPlugins() => public IEnumerable GetSynchronizationPlugins() => Plugins.OfType(); - public ContainerBuilder CreateServiceCollectionFromBasicApi(ContainerBuilder builder) + public ContainerBuilder ConfigureContainerBuilderFromBasicApi(ContainerBuilder builder) { builder .AddPropertiesFrom(this) diff --git a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs b/src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs similarity index 53% rename from src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs rename to src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs index 413dc8b0363..28e890c8499 100644 --- a/src/Nethermind/Nethermind.Core.Test/ServiceCollectionExtensionsTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/ContainerBuilderExtensionsTests.cs @@ -4,12 +4,11 @@ using System; using Autofac; using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; namespace Nethermind.Core.Test; -public class ServiceCollectionExtensionsTests +public class ContainerBuilderExtensionsTests { [Test] public void AddPropertiesFrom_CanAddProperties() @@ -25,6 +24,56 @@ public void AddPropertiesFrom_CanAddProperties() sp.ResolveOptional().Should().BeNull(); } + [Test] + public void TestRegisterNamedComponent() + { + IContainer sp = new ContainerBuilder() + .AddScoped() + .AddScoped() + .RegisterNamedComponentInItsOwnLifetime("custom", cfg => + { + // Override it in custom + cfg.AddScoped(); + }) + .Build(); + + using (ILifetimeScope scope = sp.BeginLifetimeScope()) + { + scope.Resolve().Property.Should().BeOfType(); + } + + MainComponentDependency customMainComponentDependency = sp.ResolveNamed("custom").Property; + sp.ResolveNamed("custom").Property.Should().BeOfType(); + + sp.Dispose(); + + customMainComponentDependency.WasDisposed.Should().BeTrue(); + } + + private class MainComponent(MainComponentDependency mainComponentDependency, ILifetimeScope scope) : IDisposable + { + public MainComponentDependency Property => mainComponentDependency; + + public void Dispose() + { + scope.Dispose(); + } + } + + private class MainComponentDependency : IDisposable + { + public bool WasDisposed { get; set; } + + public void Dispose() + { + WasDisposed = true; + } + } + + private class MainComponentDependencySubClass : MainComponentDependency + { + } + private class InterfaceImplementation : ITestInterface { public DeclaredService TheService { get; set; } = new DeclaredService(); diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index 207e024fc08..77e7a93db34 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -30,11 +30,9 @@ public static ContainerBuilder AddPropertiesFrom(this ContainerBuilder config foreach (PropertyInfo propertyInfo in properties) { - object? val = propertyInfo.GetValue(source); - if (val != null) - { - configuration.RegisterInstance(val).As(propertyInfo.PropertyType); - } + configuration.Register(_ => propertyInfo.GetValue(source)!) + .ExternallyOwned() // Don't disposeee this + .As(propertyInfo.PropertyType); } return configuration; @@ -84,6 +82,7 @@ public static ContainerBuilder AddScoped(this ContainerBuilder builder) where builder.RegisterType() .As() .AsSelf() + .WithAttributeFiltering() .InstancePerLifetimeScope(); return builder; @@ -93,6 +92,7 @@ public static ContainerBuilder AddScoped(this ContainerBuilder builder { builder.RegisterType() .As() + .WithAttributeFiltering() .InstancePerLifetimeScope(); return builder; @@ -103,8 +103,8 @@ public static ContainerBuilder AddScoped(this ContainerBuilder builder /// type (assuming the type is of lifetime scope). This is useful for same type with multiple configuration /// or a graph of multiple same type. The T is expected to be of a main container of sort that contains the /// main service of interest. - /// Note: The T should dispose an injected ILifetimeScope as ILifetimeScope is not automatically disposed. - /// TODO: Double check this behaviour + /// Note: The T should dispose an injected ILifetimeScope on dispose as ILifetimeScope is not automatically disposed + /// when parent scope is disposed. /// public static ContainerBuilder RegisterNamedComponentInItsOwnLifetime(this ContainerBuilder builder, string name, Action configurator) where T : notnull { diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 783c7f3b086..59687c9ba9f 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -134,7 +134,7 @@ private async Task Initialize(CancellationToken cancellationToken) _syncConfig.NeedToWaitForHeader = true; // Should this be in chainspec itself? ContainerBuilder builder = new ContainerBuilder(); - _api.CreateServiceCollectionFromApiWithNetwork(builder) + _api.ConfigureContainerBuilderFromApiWithNetwork(builder) .AddSingleton(No.BeaconSync); Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); IContainer container = builder.Build(); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index c8ab4d71c10..06d52c6d48c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -442,7 +442,7 @@ public Task InitSynchronization() ContainerBuilder builder = new ContainerBuilder(); - _api.CreateServiceCollectionFromApiWithNetwork(builder) + _api.ConfigureContainerBuilderFromApiWithNetwork(builder) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_mergeConfig) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 411cd068a03..56afe08a256 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -153,7 +153,7 @@ public Task InitSynchronization() _api.Pivot = _beaconPivot; ContainerBuilder builder = new ContainerBuilder(); - ((INethermindApi)_api).CreateServiceCollectionFromApiWithNetwork(builder) + ((INethermindApi)_api).ConfigureContainerBuilderFromApiWithNetwork(builder) .AddSingleton(_beaconSync) .AddSingleton(_beaconPivot) .AddSingleton(_api.PoSSwitcher) From a90fb3fe4215e8445f964b3f801a790124e529a8 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 27 Sep 2024 22:11:45 +0800 Subject: [PATCH 28/36] Why did you not fail earlier? --- .../Nethermind.Core/ContainerBuilderExtensions.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs index 77e7a93db34..1c30a830b55 100644 --- a/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Core/ContainerBuilderExtensions.cs @@ -30,9 +30,11 @@ public static ContainerBuilder AddPropertiesFrom(this ContainerBuilder config foreach (PropertyInfo propertyInfo in properties) { - configuration.Register(_ => propertyInfo.GetValue(source)!) - .ExternallyOwned() // Don't disposeee this - .As(propertyInfo.PropertyType); + object? val = propertyInfo.GetValue(source); + if (val != null) + { + configuration.RegisterInstance(val).As(propertyInfo.PropertyType); + } } return configuration; From c14ae8b4189cbcde07d21afa267f6b645464a357 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Sat, 28 Sep 2024 09:13:06 +0800 Subject: [PATCH 29/36] Use module. Module nice. --- .../Steps/InitializeNetwork.cs | 2 +- .../Nethermind.Merge.Plugin/MergePlugin.cs | 5 +- .../Synchronization/MergeSynchronizer.cs | 37 +-- .../Nethermind.Optimism/OptimismPlugin.cs | 6 +- .../OldStyleFullSynchronizerTests.cs | 3 +- .../SyncThreadTests.cs | 2 +- .../SynchronizerTests.cs | 4 +- .../Synchronizer.cs | 233 +++++++++--------- 8 files changed, 155 insertions(+), 137 deletions(-) diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 59687c9ba9f..69c2d4b786e 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -136,7 +136,7 @@ private async Task Initialize(CancellationToken cancellationToken) ContainerBuilder builder = new ContainerBuilder(); _api.ConfigureContainerBuilderFromApiWithNetwork(builder) .AddSingleton(No.BeaconSync); - Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); + builder.RegisterModule(new SynchronizerModule(_syncConfig)); IContainer container = builder.Build(); _api.ApiWithNetworkServiceContainer = container; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 06d52c6d48c..2bf987ade8e 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -447,8 +447,9 @@ public Task InitSynchronization() .AddSingleton(_beaconPivot) .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); - MergeSynchronizer.ConfigureMergeComponent(builder); + + builder.RegisterModule(new SynchronizerModule(_syncConfig)); + builder.RegisterModule(new MergeSynchronizerModule()); IContainer container = builder.Build(); _api.ApiWithNetworkServiceContainer = container; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index 015b8f72ddb..d44fc49f3ae 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -26,21 +26,6 @@ public class MergeSynchronizer( private readonly CancellationTokenSource? _syncCancellation = new(); private readonly ILogger _logger = logManager.GetClassLogger(); - public static void ConfigureMergeComponent(ContainerBuilder serviceCollection) - { - serviceCollection - .AddSingleton() - - .AddSingleton() - .AddScoped() - .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() - - .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), - scopeConfig => scopeConfig - .AddScoped, BeaconHeadersSyncFeed>() - .AddScoped, BeaconHeadersSyncDownloader>()); - } - public event EventHandler? SyncEvent { add => baseSynchronizer.SyncEvent += value; @@ -90,3 +75,25 @@ public void Dispose() baseSynchronizer.Dispose(); } } + +public class MergeSynchronizerModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + builder + .RegisterType() + .As() + .As>() + .InstancePerLifetimeScope(); + + builder + .AddSingleton() + .AddSingleton() + .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() + + .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), + scopeConfig => scopeConfig + .AddScoped, BeaconHeadersSyncFeed>() + .AddScoped, BeaconHeadersSyncDownloader>()); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index 56afe08a256..90e05a4c4b6 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -159,8 +159,10 @@ public Task InitSynchronization() .AddSingleton(_api.PoSSwitcher) .AddSingleton(_mergeConfig) .AddSingleton(_invalidChainTracker); - Synchronizer.ConfigureContainerBuilder(builder, _syncConfig); - MergeSynchronizer.ConfigureMergeComponent(builder); + + builder.RegisterModule(new SynchronizerModule(_syncConfig)); + builder.RegisterModule(new MergeSynchronizerModule()); + IContainer container = builder.Build(); _api.ApiWithNetworkServiceContainer = container; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index 70e958b6c53..89bd50a3ca4 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -87,7 +87,8 @@ public async Task Setup() .AddSingleton(No.BeaconSync) .AddSingleton(LimboLogs.Instance); dbProvider.ConfigureServiceCollection(builder); - Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + + builder.RegisterModule(new SynchronizerModule(syncConfig)); IContainer container = builder.Build(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index f6fac1169d0..be5768ee9bc 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -382,7 +382,7 @@ private SyncTestContext CreateSyncManager(int index) .AddSingleton(No.BeaconSync) .AddSingleton(logManager); dbProvider.ConfigureServiceCollection(builder); - Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + builder.RegisterModule(new SynchronizerModule(syncConfig)); IContainer container = builder.Build(); Synchronizer synchronizer = container.Resolve(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 753d0dc9e5f..0e7a62e9b44 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -370,11 +370,11 @@ ISyncConfig GetSyncConfig() => .AddSingleton(beaconPivot) .AddSingleton(_logManager); - Nethermind.Synchronization.Synchronizer.ConfigureContainerBuilder(builder, syncConfig); + builder.RegisterModule(new SynchronizerModule(syncConfig)); if (IsMerge(synchronizerType)) { - MergeSynchronizer.ConfigureMergeComponent(builder); + builder.RegisterModule(new MergeSynchronizerModule()); } IContainer container = builder.Build(); diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index e1d81daabcc..eb3b286d548 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -13,6 +13,7 @@ using Nethermind.Logging; using Nethermind.Stats; using Nethermind.Stats.Model; +using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.DbTuner; using Nethermind.Synchronization.FastBlocks; @@ -55,119 +56,6 @@ public class Synchronizer( public ISyncModeSelector SyncModeSelector => syncModeSelector; - public static void ConfigureContainerBuilder(ContainerBuilder builder, ISyncConfig syncConfig) - { - builder - .AddSingleton() - - .AddSingleton() - .AddSingleton() - .AddSingleton() - .AddSingleton() - .AddSingleton() - .AddSingleton() - - // For blocks. There are two block scope, Fast and Full - .AddScoped>() - .AddScoped, BlockDownloader>() - .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() - .AddScoped>() - - // For headers. There are two header scope, Fast and Beacon - .AddScoped>() - .AddScoped, HeadersSyncDownloader>() - .AddScoped, FastBlocksPeerAllocationStrategyFactory>() - .AddScoped>() - - // SyncProgress resolver need one header sync batch feed, which is the fast header one. - .Register(ctx => ctx - .ResolveNamed>(nameof(HeadersSyncFeed)) - .Feed) - .Named>(nameof(HeadersSyncFeed)); - - ConfigureSnapComponent(builder, syncConfig); - ConfigureReceiptSyncComponent(builder, syncConfig); - ConfigureBodiesSyncComponent(builder, syncConfig); - ConfigureStateSyncComponent(builder, syncConfig); - - builder - .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), - scopeConfig => - { - if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) - { - scopeConfig.AddScoped, NoopSyncFeed>(); - } - else - { - scopeConfig.AddScoped, HeadersSyncFeed>(); - } - }) - .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), - scopeConfig => scopeConfig.AddScoped, FastSyncFeed>()) - .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), - scopeConfig => scopeConfig.AddScoped, FullSyncFeed>()); - } - - private static void ConfigureSnapComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) - { - serviceCollection - .AddSingleton() - .AddSingleton(); - - ConfigureSingletonSyncFeed(serviceCollection); - - if (!syncConfig.FastSync || !syncConfig.SnapSync) - { - serviceCollection.AddSingleton, NoopSyncFeed>(); - } - } - - private static void ConfigureReceiptSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) - { - ConfigureSingletonSyncFeed(serviceCollection); - - if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || - !syncConfig.DownloadBodiesInFastSync || - !syncConfig.DownloadReceiptsInFastSync) - { - serviceCollection.AddSingleton, NoopSyncFeed>(); - } - } - - private static void ConfigureBodiesSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) - { - ConfigureSingletonSyncFeed(serviceCollection); - - if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || - !syncConfig.DownloadBodiesInFastSync) - { - serviceCollection.AddSingleton, NoopSyncFeed>(); - } - - } - - private static void ConfigureStateSyncComponent(ContainerBuilder serviceCollection, ISyncConfig syncConfig) - { - serviceCollection - .AddSingleton(); - - ConfigureSingletonSyncFeed(serviceCollection); - - // Disable it by setting noop - if (!syncConfig.FastSync) serviceCollection.AddSingleton, NoopSyncFeed>(); - } - - private static void ConfigureSingletonSyncFeed(ContainerBuilder serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory - { - serviceCollection - .AddSingleton, TFeed>() - .AddSingleton>() - .AddSingleton, TDownloader>() - .AddSingleton, TAllocationStrategy>() - .AddSingleton>(); - } - public virtual void Start() { if (!syncConfig.SynchronizationEnabled) @@ -373,3 +261,122 @@ public void Dispose() } } } + +public class SynchronizerModule(ISyncConfig syncConfig) : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + + builder + .AddSingleton() + + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + + // For blocks. There are two block scope, Fast and Full + .AddScoped>() + .AddScoped, BlockDownloader>() + .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() + .AddScoped>() + + // For headers. There are two header scope, Fast and Beacon + .AddScoped>() + .AddScoped, HeadersSyncDownloader>() + .AddScoped, FastBlocksPeerAllocationStrategyFactory>() + .AddScoped>() + + // SyncProgress resolver need one header sync batch feed, which is the fast header one. + .Register(ctx => ctx + .ResolveNamed>(nameof(HeadersSyncFeed)) + .Feed) + .Named>(nameof(HeadersSyncFeed)); + + ConfigureSnapComponent(builder); + ConfigureReceiptSyncComponent(builder); + ConfigureBodiesSyncComponent(builder); + ConfigureStateSyncComponent(builder); + + builder + .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), + scopeConfig => + { + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + else + { + scopeConfig.AddScoped, HeadersSyncFeed>(); + } + }) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), + scopeConfig => scopeConfig.AddScoped, FastSyncFeed>()) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), + scopeConfig => scopeConfig.AddScoped, FullSyncFeed>()); + } + + private void ConfigureSnapComponent(ContainerBuilder serviceCollection) + { + serviceCollection + .AddSingleton() + .AddSingleton(); + + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.SnapSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + } + + private void ConfigureReceiptSyncComponent(ContainerBuilder serviceCollection) + { + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync || + !syncConfig.DownloadReceiptsInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + } + + private void ConfigureBodiesSyncComponent(ContainerBuilder serviceCollection) + { + ConfigureSingletonSyncFeed(serviceCollection); + + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync || + !syncConfig.DownloadBodiesInFastSync) + { + serviceCollection.AddSingleton, NoopSyncFeed>(); + } + + } + + private void ConfigureStateSyncComponent(ContainerBuilder serviceCollection) + { + serviceCollection + .AddSingleton(); + + ConfigureSingletonSyncFeed(serviceCollection); + + // Disable it by setting noop + if (!syncConfig.FastSync) serviceCollection.AddSingleton, NoopSyncFeed>(); + } + + private static void ConfigureSingletonSyncFeed(ContainerBuilder serviceCollection) where TFeed : class, ISyncFeed where TDownloader : class, ISyncDownloader where TAllocationStrategy : class, IPeerAllocationStrategyFactory + { + serviceCollection + .AddSingleton, TFeed>() + .AddSingleton>() + .AddSingleton, TDownloader>() + .AddSingleton, TAllocationStrategy>() + .AddSingleton>(); + } + +} From 0f73b6b917b5e2eab1b59fa62b28c163baa477a0 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Mon, 30 Sep 2024 09:14:50 +0800 Subject: [PATCH 30/36] Fix fast sync being loaded in tests --- .../Synchronization/MergeSynchronizer.cs | 13 ++-- ...{FeedComponent.cs => SyncFeedComponent.cs} | 7 +- .../Synchronizer.cs | 72 +++++++++++-------- 3 files changed, 58 insertions(+), 34 deletions(-) rename src/Nethermind/Nethermind.Synchronization/{FeedComponent.cs => SyncFeedComponent.cs} (62%) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs index d44fc49f3ae..ac3ad98313f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeSynchronizer.cs @@ -17,7 +17,7 @@ namespace Nethermind.Merge.Plugin.Synchronization; public class MergeSynchronizer( - [KeyFilter(nameof(BeaconHeadersSyncFeed))] FeedComponent beaconHeaderComponent, + [KeyFilter(nameof(BeaconHeadersSyncFeed))] SyncFeedComponent beaconHeaderComponent, ISyncConfig syncConfig, Synchronizer baseSynchronizer, ILogManager logManager) @@ -91,9 +91,12 @@ protected override void Load(ContainerBuilder builder) .AddSingleton() .AddScoped, MergeBlocksSyncPeerAllocationStrategyFactory>() - .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), - scopeConfig => scopeConfig - .AddScoped, BeaconHeadersSyncFeed>() - .AddScoped, BeaconHeadersSyncDownloader>()); + .RegisterNamedComponentInItsOwnLifetime>(nameof(BeaconHeadersSyncFeed), ConfigureBeaconHeader); + } + + private void ConfigureBeaconHeader(ContainerBuilder scopeConfig) + { + scopeConfig.AddScoped, BeaconHeadersSyncFeed>() + .AddScoped, BeaconHeadersSyncDownloader>(); } } diff --git a/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs b/src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs similarity index 62% rename from src/Nethermind/Nethermind.Synchronization/FeedComponent.cs rename to src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs index fb27fe8c7f1..93b4fe0df45 100644 --- a/src/Nethermind/Nethermind.Synchronization/FeedComponent.cs +++ b/src/Nethermind/Nethermind.Synchronization/SyncFeedComponent.cs @@ -7,7 +7,12 @@ using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; -public class FeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope) : IDisposable, IAsyncDisposable +namespace Nethermind.Synchronization; + +/// +/// Container to make it simpler to get a bunch of components within a same named scoped. +/// +public class SyncFeedComponent(Lazy> feed, Lazy> dispatcher, Lazy> blockDownloader, ILifetimeScope lifetimeScope) : IDisposable, IAsyncDisposable { public ISyncFeed Feed => feed.Value; public SyncDispatcher Dispatcher => dispatcher.Value; diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index eb3b286d548..826406ca9de 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -31,19 +31,19 @@ public class Synchronizer( ISyncConfig syncConfig, ILogManager logManager, INodeStatsManager nodeStatsManager, - [KeyFilter(nameof(FullSyncFeed))] FeedComponent fullSyncComponent, - [KeyFilter(nameof(FastSyncFeed))] FeedComponent fastSyncComponent, - FeedComponent stateSyncComponent, - FeedComponent snapSyncComponent, - [KeyFilter(nameof(HeadersSyncFeed))] FeedComponent fastHeaderComponent, - FeedComponent oldBodiesComponent, - FeedComponent oldReceiptsComponent, + [KeyFilter(nameof(FullSyncFeed))] SyncFeedComponent fullSyncComponent, + [KeyFilter(nameof(FastSyncFeed))] SyncFeedComponent fastSyncComponent, + SyncFeedComponent stateSyncComponent, + SyncFeedComponent snapSyncComponent, + [KeyFilter(nameof(HeadersSyncFeed))] SyncFeedComponent fastHeaderComponent, + SyncFeedComponent oldBodiesComponent, + SyncFeedComponent oldReceiptsComponent, #pragma warning disable CS9113 // Parameter is unread. But it need to be instantiated to function SyncDbTuner syncDbTuner, MallocTrimmer mallocTrimmer, #pragma warning restore CS9113 // Parameter is unread. IProcessExitSource exitSource) - : ISynchronizer + : ISynchronizer { private const int FeedsTerminationTimeout = 5_000; @@ -279,20 +279,20 @@ protected override void Load(ContainerBuilder builder) .AddSingleton() // For blocks. There are two block scope, Fast and Full - .AddScoped>() + .AddScoped>() .AddScoped, BlockDownloader>() .AddScoped, BlocksSyncPeerAllocationStrategyFactory>() .AddScoped>() // For headers. There are two header scope, Fast and Beacon - .AddScoped>() + .AddScoped>() .AddScoped, HeadersSyncDownloader>() .AddScoped, FastBlocksPeerAllocationStrategyFactory>() .AddScoped>() // SyncProgress resolver need one header sync batch feed, which is the fast header one. .Register(ctx => ctx - .ResolveNamed>(nameof(HeadersSyncFeed)) + .ResolveNamed>(nameof(HeadersSyncFeed)) .Feed) .Named>(nameof(HeadersSyncFeed)); @@ -302,22 +302,38 @@ protected override void Load(ContainerBuilder builder) ConfigureStateSyncComponent(builder); builder - .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), - scopeConfig => - { - if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) - { - scopeConfig.AddScoped, NoopSyncFeed>(); - } - else - { - scopeConfig.AddScoped, HeadersSyncFeed>(); - } - }) - .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), - scopeConfig => scopeConfig.AddScoped, FastSyncFeed>()) - .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), - scopeConfig => scopeConfig.AddScoped, FullSyncFeed>()); + .RegisterNamedComponentInItsOwnLifetime>(nameof(HeadersSyncFeed), ConfigureFastHeader) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FastSyncFeed), ConfigureFastSync) + .RegisterNamedComponentInItsOwnLifetime>(nameof(FullSyncFeed), ConfigureFullSync); + } + + private void ConfigureFullSync(ContainerBuilder scopeConfig) + { + scopeConfig.AddScoped, FullSyncFeed>(); + } + + private void ConfigureFastSync(ContainerBuilder scopeConfig) + { + if (syncConfig.FastSync) + { + scopeConfig.AddScoped, FastSyncFeed>(); + } + else + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + } + + private void ConfigureFastHeader(ContainerBuilder scopeConfig) + { + if (!syncConfig.FastSync || !syncConfig.DownloadHeadersInFastSync) + { + scopeConfig.AddScoped, NoopSyncFeed>(); + } + else + { + scopeConfig.AddScoped, HeadersSyncFeed>(); + } } private void ConfigureSnapComponent(ContainerBuilder serviceCollection) @@ -373,7 +389,7 @@ private static void ConfigureSingletonSyncFeed, TFeed>() - .AddSingleton>() + .AddSingleton>() .AddSingleton, TDownloader>() .AddSingleton, TAllocationStrategy>() .AddSingleton>(); From ee4cf91f89c6f9a58557a5b815e86f8282a359a1 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Mon, 30 Sep 2024 20:31:15 +0800 Subject: [PATCH 31/36] Fic build --- src/Nethermind/Nethermind.Network.Test/NodesLoaderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Network.Test/NodesLoaderTests.cs b/src/Nethermind/Nethermind.Network.Test/NodesLoaderTests.cs index 20d7cbec54f..fc3ac709530 100644 --- a/src/Nethermind/Nethermind.Network.Test/NodesLoaderTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/NodesLoaderTests.cs @@ -92,7 +92,7 @@ public void Can_load_only_static_nodes() _networkConfig.StaticPeers = enode1String; _networkConfig.Bootnodes = enode2String; _networkConfig.OnlyStaticPeers = true; - List nodes = _loader.LoadInitialList(); + List nodes = _loader.DiscoverNodes(default).ToBlockingEnumerable().ToList(); Assert.That(nodes.Count, Is.EqualTo(1)); foreach (Node node in nodes) { From 399fe76883cfd23245064be2ee9b67070dcf14a5 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 1 Oct 2024 18:31:15 +0800 Subject: [PATCH 32/36] Update src/Nethermind/Nethermind.Api/IApiWithNetwork.cs Co-authored-by: Lukasz Rozmej --- src/Nethermind/Nethermind.Api/IApiWithNetwork.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs index 93bb0769d2f..0b0e05e2a6d 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithNetwork.cs @@ -59,7 +59,7 @@ public interface IApiWithNetwork : IApiWithBlockchain public ContainerBuilder ConfigureContainerBuilderFromApiWithNetwork(ContainerBuilder builder) { return ConfigureContainerBuilderFromApiWithBlockchain(builder) - .AddPropertiesFrom(this); + .AddPropertiesFrom(this); } } } From 2083c73609a986e5f5b026e24bd1c6b3662ea806 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 1 Oct 2024 18:31:41 +0800 Subject: [PATCH 33/36] Update src/Nethermind/Nethermind.Db/IDbProvider.cs Co-authored-by: Lukasz Rozmej --- src/Nethermind/Nethermind.Db/IDbProvider.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 38d2a09a813..84acd8dc223 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -55,14 +55,7 @@ void ConfigureServiceCollection(ContainerBuilder sc) var db = GetDb(dbName); sc.AddKeyedSingleton(dbName, db); sc.AddKeyedSingleton(dbName, db); - if (db is ITunableDb tunableDb) - { - sc.AddKeyedSingleton(dbName, tunableDb); - } - else - { - sc.AddKeyedSingleton(dbName, new NoopTunableDb()); - } + sc.AddKeyedSingleton(dbName, db as ITunableDb ?? new NoopTunableDb()); } IColumnsDb receiptColumnDb = GetColumnDb(DbNames.Receipts); From 101b77958f867cb9001f0eadbd1ca810afbdb732 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Tue, 1 Oct 2024 18:32:10 +0800 Subject: [PATCH 34/36] Update src/Nethermind/Nethermind.Db/IDbProvider.cs Co-authored-by: Lukasz Rozmej --- src/Nethermind/Nethermind.Db/IDbProvider.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 84acd8dc223..6a0380b0446 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -60,10 +60,7 @@ void ConfigureServiceCollection(ContainerBuilder sc) IColumnsDb receiptColumnDb = GetColumnDb(DbNames.Receipts); sc.AddSingleton>(receiptColumnDb); - if (receiptColumnDb is ITunableDb tunableDb2) - sc.AddKeyedSingleton(DbNames.Receipts, tunableDb2); - else - sc.AddKeyedSingleton(DbNames.Receipts, new NoopTunableDb()); + sc.AddKeyedSingleton(DbNames.Receipts, receiptColumnDb as ITunableDb ?? new NoopTunableDb()); } } } From 54680c7ce6aee78016a316c2816dbc18680e4b03 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 3 Oct 2024 08:33:42 +0800 Subject: [PATCH 35/36] Address comment --- .../Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs index 05f07fd5015..f04f755c736 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/NoopSyncFeed.cs @@ -18,24 +18,22 @@ public class NoopSyncFeed : ISyncFeed public Task PrepareRequest(CancellationToken token = default) { - throw new InvalidOperationException("null sync feed should not be used"); + return Task.FromResult(default); } public SyncResponseHandlingResult HandleResponse(T response, PeerInfo? peer = null) { - throw new InvalidOperationException("null sync feed should not be used"); + return SyncResponseHandlingResult.NotAssigned; } public bool IsMultiFeed { get; } public AllocationContexts Contexts { get; } public void Activate() { - throw new InvalidOperationException("null sync feed should not be used"); } public void Finish() { - throw new InvalidOperationException("null sync feed should not be used"); } public Task FeedTask => Task.CompletedTask; From c0dd79718c9dede050ff31269ace86df446e6a94 Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Fri, 4 Oct 2024 00:02:30 +0800 Subject: [PATCH 36/36] Update src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs Co-authored-by: Alex --- src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 07dc3ae4e8e..be6c36dcd63 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -106,7 +106,6 @@ public ContainerBuilder ConfigureContainerBuilderFromApiWithBlockchain(Container return ConfigureContainerBuilderFromApiWithStores(builder) .AddPropertiesFrom(this) .AddSingleton(NodeStorageFactory.WrapKeyValueStore(DbProvider!.StateDb)); - } } }