chore: added option for verbosity and debug level

draft: improved operation state mechanism

draft: improved operation state mechanism

draft: improved operation state mechanism

implemented a new entity to store cached block state

.gitignore

@@ -29,7 +29,6 @@ lerna-debug.log*
 
 # IDE - VSCode
 .vscode/*
-!.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json

config/config.yaml

@@ -3,7 +3,8 @@ db:
   synchronize: false
 app:
   port: <port>
-  schedulerInterval: <interval> #interval in seconds
+  verbose: <boolean>
+  debug: <boolean>
   network: <network>
   requestRetry:
     delay: <number> # delay in Milliseconds

config/dev.config.yaml

@@ -3,7 +3,8 @@ db:
   synchronize: true
 app:
   port: 3000
-  schedulerInterval: 10
+  verbose: false
+  debug: true
   network: regtest
   requestRetry:
     delay: 3000
@@ -18,4 +19,3 @@ bitcoinCore:
   rpcPass: password
   rpcUser: admin
   rpcPort: 18443
-

config/e2e.config.yaml

@@ -3,7 +3,8 @@ db:
   synchronize: true
 app:
   port: 3000
-  schedulerInterval: 10
+  verbose: false
+  debug: true
   network: regtest
   requestRetry:
     delay: 500

src/block-data-providers/base-block-data-provider.abstract.ts

@@ -5,44 +5,20 @@ import {
     TransactionInput,
     TransactionOutput,
 } from '@/indexer/indexer.service';
-import { TransactionsService } from '@/transactions/transactions.service';
-import { SchedulerRegistry } from '@nestjs/schedule';
-import { CronJob } from 'cron';
 import { ConfigService } from '@nestjs/config';
+import { BlockStateService } from '@/block-state/block-state.service';
+import { BlockState } from '@/block-state/block-state.entity';
 
-export interface BaseOperationState {
-    indexedBlockHeight: number;
-    blockCache: Record<number, string>;
-}
-
-export abstract class BaseBlockDataProvider<
-    OperationState extends BaseOperationState,
-> {
+export abstract class BaseBlockDataProvider<OperationState> {
     protected abstract readonly logger: Logger;
     protected abstract readonly operationStateKey: string;
-    protected cacheSize = 6;
-    protected readonly CRON_JOB_NAME = 'providerSync';
 
     protected constructor(
         protected readonly configService: ConfigService,
         private readonly indexerService: IndexerService,
         private readonly operationStateService: OperationStateService,
-        private readonly transactionService: TransactionsService,
-        private readonly schedulerRegistry: SchedulerRegistry,
-    ) {
-        const schedulerIntervalInSeconds = this.configService.get<string>(
-            'app.schedulerInterval',
-        );
-
-        const job = new CronJob(
-            `*/${schedulerIntervalInSeconds} * * * * *`,
-            () => this.sync(),
-        );
-        this.schedulerRegistry.addCronJob(this.CRON_JOB_NAME, job);
-        job.start();
-    }
-
-    abstract sync(): void;
+        protected readonly blockStateService: BlockStateService,
+    ) {}
 
     async indexTransaction(
         txid: string,
@@ -68,69 +44,38 @@ export abstract class BaseBlockDataProvider<
         )?.state;
     }
 
-    async setState(partialState: Partial<OperationState>): Promise<void> {
-        const oldState = (await this.getState()) || ({} as OperationState);
-
-        if (partialState.blockCache) {
-            const updatedBlockCache = {
-                ...oldState.blockCache,
-                ...partialState.blockCache,
-            };
-
-            if (this.cacheSize < Object.keys(updatedBlockCache).length) {
-                delete updatedBlockCache[oldState.indexedBlockHeight - 5];
-            }
-
-            partialState.blockCache = updatedBlockCache;
-        }
-
-        const newState = {
-            ...oldState,
-            ...partialState,
-        };
-
+    async setState(
+        state: OperationState,
+        blockState: BlockState,
+    ): Promise<void> {
         await this.operationStateService.setOperationState(
             this.operationStateKey,
-            newState,
+            state,
         );
+
+        await this.blockStateService.addBlockState(blockState);
     }
 
     abstract getBlockHash(height: number): Promise<string>;
 
     async traceReorg(): Promise<number> {
-        const { indexedBlockHeight, blockCache } = await this.getState();
-        let counter = indexedBlockHeight;
+        let state = await this.blockStateService.getCurrentBlockState();
 
-        if (Object.keys(blockCache).length === 0) {
-            return indexedBlockHeight;
-        }
-
-        while (true) {
-            const storedBlockHash = blockCache[counter];
+        if (!state) return null;
 
-            if (storedBlockHash === undefined) {
-                throw new Error('Reorgs levels deep');
-            }
+        while (state) {
+            const fetchedBlockHash = await this.getBlockHash(state.blockHeight);
 
-            const fetchedBlockHash = await this.getBlockHash(counter);
+            if (state.blockHash === fetchedBlockHash) return state.blockHeight;
 
-            if (storedBlockHash === fetchedBlockHash) {
-                return counter;
-            }
-            console.log(
-                'reorg found at count: ',
-                counter,
-                ' and hash: ',
-                storedBlockHash,
-                ' ',
-                fetchedBlockHash,
-            );
+            await this.blockStateService.removeState(state);
 
-            await this.transactionService.deleteTransactionByBlockHash(
-                storedBlockHash,
+            this.logger.log(
+                `Reorg found at height: ${state.blockHeight}, Wrong hash: ${state.blockHash}, Correct hash: ${fetchedBlockHash}`,
             );
-
-            --counter;
+            state = await this.blockStateService.getCurrentBlockState();
         }
+
+        throw new Error('Cannot Reorgs, blockchain state exhausted');
     }
 }

src/block-data-providers/bitcoin-core/interfaces.ts

@@ -1,5 +1,4 @@
 import { TransactionInput, TransactionOutput } from '@/indexer/indexer.service';
-import { BaseOperationState } from '@/block-data-providers/base-block-data-provider.abstract';
 
 export interface Block {
     height: number;
@@ -40,10 +39,9 @@ export interface Output {
     };
 }
 
-export interface BitcoinCoreOperationState extends BaseOperationState {
-    currentBlockHeight: number;
+export type BitcoinCoreOperationState = {
     indexedBlockHeight: number;
-}
+};
 
 export type Transaction = {
     txid: string;

src/block-data-providers/bitcoin-core/provider.spec.ts

@@ -12,6 +12,7 @@ import {
     rawTransactions,
 } from '@/block-data-providers/bitcoin-core/provider-fixtures';
 import { Test, TestingModule } from '@nestjs/testing';
+import { BlockStateService } from '@/block-state/block-state.service';
 
 describe('Bitcoin Core Provider', () => {
     let provider: BitcoinCoreProvider;
@@ -46,6 +47,10 @@ describe('Bitcoin Core Provider', () => {
                         getOperationState: jest.fn(),
                     },
                 },
+                {
+                    provide: BlockStateService,
+                    useClass: jest.fn(),
+                },
             ],
         }).compile();
 

src/block-data-providers/bitcoin-core/provider.ts

@@ -7,7 +7,7 @@ import {
     SATS_PER_BTC,
     TAPROOT_ACTIVATION_HEIGHT,
 } from '@/common/constants';
-import { Cron, CronExpression, SchedulerRegistry } from '@nestjs/schedule';
+import { Cron, CronExpression } from '@nestjs/schedule';
 import {
     IndexerService,
     TransactionInput,
@@ -28,7 +28,7 @@ import {
 import { AxiosRequestConfig } from 'axios';
 import * as currency from 'currency.js';
 import { AxiosRetryConfig, makeRequest } from '@/common/request';
-import { TransactionsService } from '@/transactions/transactions.service';
+import { BlockStateService } from '@/block-state/block-state.service';
 
 @Injectable()
 export class BitcoinCoreProvider
@@ -45,15 +45,13 @@ export class BitcoinCoreProvider
         configService: ConfigService,
         indexerService: IndexerService,
         operationStateService: OperationStateService,
-        transactionService: TransactionsService,
-        schedulerRegistry: SchedulerRegistry,
+        blockStateService: BlockStateService,
     ) {
         super(
             configService,
             indexerService,
             operationStateService,
-            transactionService,
-            schedulerRegistry,
+            blockStateService,
         );
 
         const { protocol, rpcPort, rpcHost } =
@@ -66,35 +64,42 @@ export class BitcoinCoreProvider
     }
 
     async onApplicationBootstrap() {
-        const getState = await this.getState();
-        if (getState) {
+        const currentState = await this.getState();
+        if (currentState) {
             this.logger.log(
                 `Restoring state from previous run: ${JSON.stringify(
-                    getState,
+                    currentState,
                 )}`,
             );
         } else {
             this.logger.log('No previous state found. Starting from scratch.');
-            const state: BitcoinCoreOperationState = {
-                currentBlockHeight: 0,
-                blockCache: {},
-                indexedBlockHeight:
-                    this.configService.get<BitcoinNetwork>('app.network') ===
-                    BitcoinNetwork.MAINNET
-                        ? TAPROOT_ACTIVATION_HEIGHT - 1
-                        : 0,
-            };
-
-            await this.setState(state);
+
+            const blockHeight =
+                this.configService.get<BitcoinNetwork>('app.network') ===
+                BitcoinNetwork.MAINNET
+                    ? TAPROOT_ACTIVATION_HEIGHT - 1
+                    : 0;
+            const blockHash = await this.getBlockHash(blockHeight);
+
+            await this.setState(
+                {
+                    indexedBlockHeight: blockHeight,
+                },
+                {
+                    blockHash,
+                    blockHeight,
+                },
+            );
         }
     }
 
+    @Cron(CronExpression.EVERY_10_SECONDS)
     async sync() {
         if (this.isSyncing) return;
         this.isSyncing = true;
 
-        console.log("sync running");
         const state = await this.getState();
+
         if (!state) {
             throw new Error('State not found');
         }
@@ -113,7 +118,8 @@ export class BitcoinCoreProvider
             const networkInfo = await this.getNetworkInfo();
             const verbosityLevel = this.versionToVerbosity(networkInfo.version);
 
-            let height = (await this.traceReorg()) + 1;
+            let height =
+                ((await this.traceReorg()) ?? state.indexedBlockHeight) + 1;
 
             for (height; height <= tipHeight; height++) {
                 const [transactions, blockHash] = await this.processBlock(
@@ -133,9 +139,10 @@ export class BitcoinCoreProvider
                     );
                 }
 
-                await this.setState({
-                    indexedBlockHeight: height,
-                    blockCache: { [height]: blockHash },
+                state.indexedBlockHeight = height;
+                await this.setState(state, {
+                    blockHash: blockHash,
+                    blockHeight: height,
                 });
             }
         } finally {

src/block-data-providers/block-provider.module.ts

@@ -7,16 +7,15 @@ import { IndexerService } from '@/indexer/indexer.service';
 import { ProviderType } from '@/common/enum';
 import { BitcoinCoreProvider } from '@/block-data-providers/bitcoin-core/provider';
 import { EsploraProvider } from '@/block-data-providers/esplora/provider';
-import { TransactionsService } from '@/transactions/transactions.service';
-import { TransactionsModule } from '@/transactions/transactions.module';
-import { SchedulerRegistry } from '@nestjs/schedule';
+import { BlockStateService } from '@/block-state/block-state.service';
+import { BlockStateModule } from '@/block-state/block-state.module';
 
 @Module({
     imports: [
         OperationStateModule,
         IndexerModule,
         ConfigModule,
-        TransactionsModule,
+        BlockStateModule,
     ],
     controllers: [],
     providers: [
@@ -26,32 +25,28 @@ import { SchedulerRegistry } from '@nestjs/schedule';
                 ConfigService,
                 IndexerService,
                 OperationStateService,
-                TransactionsService,
-                SchedulerRegistry,
+                BlockStateService,
             ],
             useFactory: (
                 configService: ConfigService,
                 indexerService: IndexerService,
                 operationStateService: OperationStateService,
-                transactionService: TransactionsService,
-                schedulerRegistry: SchedulerRegistry,
+                blockStateService: BlockStateService,
             ) => {
                 switch (configService.get<ProviderType>('providerType')) {
                     case ProviderType.ESPLORA:
                         return new EsploraProvider(
                             configService,
                             indexerService,
                             operationStateService,
-                            transactionService,
-                            schedulerRegistry,
+                            blockStateService,
                         );
                     case ProviderType.BITCOIN_CORE_RPC:
                         return new BitcoinCoreProvider(
                             configService,
                             indexerService,
                             operationStateService,
-                            transactionService,
-                            schedulerRegistry,
+                            blockStateService,
                         );
                     default:
                         throw Error('unrecognised provider type in config');