Sync latest changes

package.json

@@ -35,8 +35,7 @@
     "typescript": "^4.9.5"
   },
   "resolutions": {
-    "node-fetch": "2.6.7",
-    "@subql/types": "2.2.1-1"
+    "node-fetch": "2.6.7"
   },
   "scripts": {
     "build": "yarn workspaces foreach -ptA run build",
@@ -58,6 +57,7 @@
   },
   "packageManager": "[email protected]",
   "dependencies": {
+    "@subql/common": "^3.0.1",
     "near-api-js": "^1.1.0"
   }
 }

packages/common-near/src/project/models.ts

@@ -1,7 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {ProcessorImpl} from '@subql/common';
+import {BlockFilterImpl, ProcessorImpl} from '@subql/common';
 import {BaseMapping, FileReference, Processor} from '@subql/types-core';
 import {
   CustomDataSourceAsset as NearCustomDataSourceAsset,
@@ -23,12 +23,9 @@ import {
   NearReceiptHandler,
   NearReceiptFilter,
 } from '@subql/types-near';
-import BN from 'bn.js';
 import {plainToClass, Transform, Type} from 'class-transformer';
 import {
-  ArrayMaxSize,
   IsArray,
-  IsBoolean,
   IsEnum,
   IsInt,
   IsOptional,
@@ -42,18 +39,7 @@ import {
   ValidateIf,
 } from 'class-validator';
 
-export class BlockFilter implements NearBlockFilter {
-  @IsOptional()
-  @IsArray()
-  @ArrayMaxSize(2)
-  specVersion?: [number, number];
-  @IsOptional()
-  @IsInt()
-  modulo?: number;
-  @IsOptional()
-  @IsString()
-  timestamp?: string;
-}
+export class BlockFilter extends BlockFilterImpl implements NearBlockFilter {}
 
 export class TransactionFilter extends BlockFilter implements NearTransactionFilter {
   @IsOptional()

packages/common-near/src/project/versioned/v1_0_0/model.ts

@@ -15,23 +15,12 @@ import {BaseMapping, NodeSpec, ParentProject, QuerySpec, RunnerSpecs} from '@sub
 import {
   CustomDatasourceTemplate,
   NearCustomDatasource,
-  NearNetworkFilter,
   NearProjectManifestV1_0_0,
   NearRuntimeDatasource,
   RuntimeDatasourceTemplate,
 } from '@subql/types-near';
 import {plainToClass, Transform, TransformFnParams, Type} from 'class-transformer';
-import {
-  Equals,
-  IsArray,
-  IsNotEmpty,
-  IsObject,
-  IsOptional,
-  IsString,
-  Validate,
-  ValidateNested,
-  validateSync,
-} from 'class-validator';
+import {Equals, IsArray, IsNotEmpty, IsObject, IsOptional, IsString, ValidateNested} from 'class-validator';
 import {CustomDataSourceBase, RuntimeDataSourceBase} from '../../models';
 
 const NEAR_NODE_NAME = `@subql/node-near`;

packages/node/package.json

@@ -24,10 +24,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "dev",
+    "@subql/common": "^3.0.1",
     "@subql/common-near": "workspace:*",
-    "@subql/node-core": "4.2.4-26",
-    "@subql/types-core": "0.0.0-2",
+    "@subql/node-core": "^5.0.1",
     "@subql/types-near": "workspace:*",
     "cron-converter": "^1.0.2",
     "eventemitter2": "^6.4.5",

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -26,7 +26,7 @@ import {
   dynamicDsHostFunctions,
   HostUnfinalizedBlocks,
 } from '@subql/node-core';
-import { Store } from '@subql/types';
+import { Store } from '@subql/types-core';
 import { NearDatasource } from '@subql/types-near';
 import {
   NearProjectDs,
@@ -48,6 +48,7 @@ async function createIndexerWorker(
   unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
   connectionPoolState: ConnectionPoolStateManager<NearApiConnection>,
   root: string,
+  startHeight: number,
 ): Promise<IndexerWorker> {
   const indexerWorker = Worker.create<
     IInitIndexerWorker,
@@ -70,7 +71,7 @@ async function createIndexerWorker(
     root,
   );
 
-  await indexerWorker.initWorker();
+  await indexerWorker.initWorker(startHeight);
 
   return indexerWorker;
 }
@@ -113,6 +114,7 @@ export class WorkerBlockDispatcherService
           unfinalizedBlocksSevice,
           connectionPoolState,
           project.root,
+          projectService.startHeight,
         ),
     );
   }

packages/node/src/indexer/indexer.manager.ts

@@ -38,7 +38,6 @@ import {
   DsProcessorService,
 } from './ds-processor.service';
 import { DynamicDsService } from './dynamic-ds.service';
-import { ProjectService } from './project.service';
 import { SandboxService } from './sandbox.service';
 import { BlockContent } from './types';
 import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
@@ -68,7 +67,6 @@ export class IndexerManager extends BaseIndexerManager<
     dsProcessorService: DsProcessorService,
     dynamicDsService: DynamicDsService,
     unfinalizedBlocksService: UnfinalizedBlocksService,
-    @Inject('IProjectService') private projectService: ProjectService,
   ) {
     super(
       apiService,
@@ -82,11 +80,6 @@ export class IndexerManager extends BaseIndexerManager<
     );
   }
 
-  async start(): Promise<void> {
-    await this.projectService.init();
-    logger.info('indexer manager started');
-  }
-
   @profiler()
   async indexBlock(
     block: BlockContent,

packages/node/src/indexer/project.service.ts

@@ -10,7 +10,6 @@ import {
   StoreService,
   NodeConfig,
   IProjectUpgradeService,
-  mainThreadOnly,
 } from '@subql/node-core';
 import { Sequelize } from '@subql/x-sequelize';
 import { NearProjectDs, SubqueryProject } from '../configure/SubqueryProject';
@@ -65,7 +64,6 @@ export class ProjectService extends BaseProjectService<
     return new Date(block.header.timestamp / 1_000_000);
   }
 
-  @mainThreadOnly()
   protected onProjectChange(project: SubqueryProject): void | Promise<void> {
     // TODO update this when implementing skipBlock feature for Eth
     // this.apiService.updateBlockFetching();

packages/node/src/indexer/sandbox.service.ts

@@ -10,8 +10,7 @@ import {
   NodeConfig,
   StoreService,
 } from '@subql/node-core';
-import { Store } from '@subql/types';
-import { BaseDataSource } from '@subql/types-core';
+import { Store, BaseDataSource } from '@subql/types-core';
 import { ApiService } from './api.service';
 
 /* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */

packages/node/src/indexer/worker/worker-fetch.module.ts

@@ -1,7 +1,6 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import { isMainThread } from 'worker_threads';
 import { Module } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
@@ -29,12 +28,8 @@ import { WorkerUnfinalizedBlocksService } from '../worker/worker.unfinalizedBloc
     ConnectionPoolService,
     {
       provide: ConnectionPoolStateManager,
-      useFactory: () => {
-        if (isMainThread) {
-          throw new Error('Expected to be worker thread');
-        }
-        return new WorkerConnectionPoolStateManager((global as any).host);
-      },
+      useFactory: () =>
+        new WorkerConnectionPoolStateManager((global as any).host),
     },
     ConnectionPoolService,
     {
@@ -65,12 +60,7 @@ import { WorkerUnfinalizedBlocksService } from '../worker/worker.unfinalizedBloc
     DsProcessorService,
     {
       provide: DynamicDsService,
-      useFactory: () => {
-        if (isMainThread) {
-          throw new Error('Expected to be worker thread');
-        }
-        return new WorkerDynamicDsService((global as any).host);
-      },
+      useFactory: () => new WorkerDynamicDsService((global as any).host),
     },
     {
       provide: 'IProjectService',
@@ -79,12 +69,8 @@ import { WorkerUnfinalizedBlocksService } from '../worker/worker.unfinalizedBloc
     WorkerService,
     {
       provide: UnfinalizedBlocksService,
-      useFactory: () => {
-        if (isMainThread) {
-          throw new Error('Expected to be worker thread');
-        }
-        return new WorkerUnfinalizedBlocksService((global as any).host);
-      },
+      useFactory: () =>
+        new WorkerUnfinalizedBlocksService((global as any).host),
     },
   ],
   exports: [],

packages/node/src/indexer/worker/worker.ts

@@ -25,23 +25,23 @@ import {
   IBaseIndexerWorker,
   initWorkerServices,
 } from '@subql/node-core';
-import { IndexerManager } from '../indexer.manager';
+import { ProjectService } from '../project.service';
 import { WorkerModule } from './worker.module';
 import { WorkerService } from './worker.service';
 
 const logger = getLogger(`worker #${threadId}`);
 
-async function initWorker(): Promise<void> {
+async function initWorker(startHeight: number): Promise<void> {
   try {
     const app = await NestFactory.create(WorkerModule, {
       logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging
     });
 
     await app.init();
 
-    const indexerManager = app.get(IndexerManager);
+    const projectService: ProjectService = app.get('IProjectService');
     // Initialise async services, we do this here rather than in factories so we can capture one off events
-    await indexerManager.start();
+    await projectService.init(startHeight);
 
     const workerService = app.get(WorkerService);
 

packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts

@@ -1,6 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import { isMainThread } from 'worker_threads';
 import { Injectable } from '@nestjs/common';
 import {
   IUnfinalizedBlocksService,
@@ -14,7 +15,11 @@ import { nearHeaderToHeader } from '../unfinalizedBlocks.service';
 export class WorkerUnfinalizedBlocksService
   implements IUnfinalizedBlocksService<BlockContent>
 {
-  constructor(private host: HostUnfinalizedBlocks) {}
+  constructor(private host: HostUnfinalizedBlocks) {
+    if (isMainThread) {
+      throw new Error('Expected to be worker thread');
+    }
+  }
 
   async processUnfinalizedBlocks(block: BlockContent): Promise<number | null> {
     return this.processUnfinalizedBlockHeader(

packages/node/src/subcommands/reindex.module.ts

@@ -7,6 +7,7 @@ import { SchedulerRegistry } from '@nestjs/schedule';
 import {
   DbModule,
   ForceCleanService,
+  PoiService,
   ReindexService,
   StoreCacheService,
   StoreService,
@@ -23,6 +24,7 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
     StoreService,
     ReindexService,
     ForceCleanService,
+    PoiService,
     UnfinalizedBlocksService,
     DynamicDsService,
     DsProcessorService,

packages/types/package.json

@@ -16,7 +16,7 @@
     "/dist"
   ],
   "dependencies": {
-    "@subql/types-core": "0.0.0-2",
+    "@subql/types-core": "^0.0.1",
     "bn.js": "5.2.1",
     "near-api-js": "^1.1.0"
   },