Merge pull request #27 from subquery/sync-20231031

Sync @subql dependencies

packages/common-stellar/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Update `@subql/common` and relevant changes to support endBlock feature (#27)
 
 ## [3.0.1] - 2023-10-13
 ### Changed

packages/common-stellar/package.json

@@ -13,7 +13,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.1.2",
+    "@subql/common": "^3.3.0",
     "@subql/types-stellar": "workspace:*",
     "js-yaml": "^4.1.0",
     "pino": "^6.13.3",

packages/common-stellar/src/project/models.ts

@@ -1,7 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {forbidNonWhitelisted, ProcessorImpl} from '@subql/common';
+import {BaseDataSource, forbidNonWhitelisted, ProcessorImpl} from '@subql/common';
 import {Processor, FileReference} from '@subql/types-core';
 import {
   StellarHandlerKind,
@@ -201,7 +201,10 @@ export class StellarProcessorOptions implements SubqlStellarProcessorOptions {
   address?: string;
 }
 
-export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>> implements SubqlRuntimeDatasource<M> {
+export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>>
+  extends BaseDataSource
+  implements SubqlRuntimeDatasource<M>
+{
   @IsEnum(StellarDatasourceKind, {
     groups: [StellarDatasourceKind.Runtime],
   })
@@ -210,9 +213,6 @@ export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>>
   @ValidateNested()
   mapping: M;
   @IsOptional()
-  @IsInt()
-  startBlock?: number;
-  @IsOptional()
   assets?: Map<string, FileReference>;
   @IsOptional()
   @ValidateNested()
@@ -226,6 +226,7 @@ export class FileReferenceImpl implements FileReference {
 }
 
 export class CustomDataSourceBase<K extends string, M extends SubqlMapping = SubqlMapping<SubqlCustomHandler>, O = any>
+  extends BaseDataSource
   implements SubqlCustomDatasource<K, M, O>
 {
   @IsString()

packages/node/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Update `@subql/node-core` with fixes and support for endBlock feature (#27)
 
 ## [3.0.2] - 2023-10-13
 ### Changed

packages/node/package.json

@@ -24,9 +24,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.1.2",
+    "@subql/common": "^3.3.0",
     "@subql/common-stellar": "workspace:*",
-    "@subql/node-core": "^6.0.2",
+    "@subql/node-core": "^6.2.0",
     "@subql/testing": "^2.0.0",
     "@subql/types": "^2.2.1-1",
     "@subql/types-stellar": "workspace:*",

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -7,28 +7,18 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   getLogger,
   NodeConfig,
-  Worker,
   SmartBatchService,
   StoreService,
   PoiService,
   PoiSyncService,
   StoreCacheService,
   IProjectService,
-  IDynamicDsService,
-  HostStore,
-  HostDynamicDS,
   WorkerBlockDispatcher,
-  IUnfinalizedBlocksService,
-  HostConnectionPoolState,
-  ConnectionPoolStateManager,
-  connectionPoolStateHostFunctions,
-  baseWorkerFunctions,
-  storeHostFunctions,
-  dynamicDsHostFunctions,
   IProjectUpgradeService,
-  HostUnfinalizedBlocks,
+  ConnectionPoolStateManager,
+  createIndexerWorker,
+  InMemoryCacheService,
 } from '@subql/node-core';
-import { Store } from '@subql/types-core';
 import {
   StellarProjectDs,
   SubqueryProject,
@@ -37,45 +27,14 @@ import { StellarApiConnection } from '../../stellar/api.connection';
 import { StellarBlockWrapped } from '../../stellar/block.stellar';
 import { DynamicDsService } from '../dynamic-ds.service';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';
+import { IIndexerWorker } from '../worker/worker';
 
 const logger = getLogger('WorkerBlockDispatcherService');
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
 };
 
-async function createIndexerWorker(
-  store: Store,
-  dynamicDsService: IDynamicDsService<StellarProjectDs>,
-  unfinalizedBlocksService: IUnfinalizedBlocksService<StellarBlockWrapped>,
-  connectionPoolState: ConnectionPoolStateManager<StellarApiConnection>,
-  root: string,
-  startHeight: number,
-): Promise<IndexerWorker> {
-  const indexerWorker = Worker.create<
-    IInitIndexerWorker,
-    HostDynamicDS<StellarProjectDs> & HostStore & HostUnfinalizedBlocks
-  >(
-    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
-    [...baseWorkerFunctions, 'initWorker'],
-    {
-      ...storeHostFunctions(store),
-      ...dynamicDsHostFunctions(dynamicDsService),
-      unfinalizedBlocksProcess:
-        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(
-          unfinalizedBlocksService,
-        ),
-      ...connectionPoolStateHostFunctions(connectionPoolState),
-    },
-    root,
-  );
-
-  await indexerWorker.initWorker(startHeight);
-
-  return indexerWorker;
-}
-
 @Injectable()
 export class WorkerBlockDispatcherService
   extends WorkerBlockDispatcher<StellarProjectDs, IndexerWorker>
@@ -89,6 +48,7 @@ export class WorkerBlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgradeService: IProjectUpgradeService,
     smartBatchService: SmartBatchService,
+    cacheService: InMemoryCacheService,
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
@@ -111,8 +71,16 @@ export class WorkerBlockDispatcherService
       project,
       dynamicDsService,
       () =>
-        createIndexerWorker(
+        createIndexerWorker<
+          IIndexerWorker,
+          StellarApiConnection,
+          StellarBlockWrapped,
+          StellarProjectDs
+        >(
+          path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
+          [],
           storeService.getStore(),
+          cacheService.getCache(),
           dynamicDsService,
           unfinalizedBlocksSevice,
           connectionPoolState,

packages/node/src/indexer/fetch.module.ts

@@ -17,6 +17,7 @@ import {
   NodeConfig,
   IProjectUpgradeService,
   ProjectUpgradeSevice,
+  InMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { StellarApiConnection } from '../stellar/api.connection';
@@ -36,6 +37,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
 
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
     StoreCacheService,
     {
@@ -82,6 +84,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         apiService: ApiService,
         indexerManager: IndexerManager,
         smartBatchService: SmartBatchService,
+        cacheService: InMemoryCacheService,
         storeService: StoreService,
         storeCacheService: StoreCacheService,
         poiService: PoiService,
@@ -98,6 +101,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               smartBatchService,
+              cacheService,
               storeService,
               storeCacheService,
               poiService,
@@ -130,6 +134,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         ApiService,
         IndexerManager,
         SmartBatchService,
+        InMemoryCacheService,
         StoreService,
         StoreCacheService,
         PoiService,

packages/node/src/indexer/worker/worker-fetch.module.ts

@@ -6,14 +6,13 @@ import { Module } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   ApiService,
-  StoreService,
-  PoiService,
   ConnectionPoolService,
-  StoreCacheService,
   WorkerDynamicDsService,
   ConnectionPoolStateManager,
   WorkerConnectionPoolStateManager,
   ProjectUpgradeSevice,
+  InMemoryCacheService,
+  WorkerInMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../../configure/SubqueryProject';
 import { StellarApiService } from '../../stellar';
@@ -89,6 +88,10 @@ import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.servi
         return new WorkerUnfinalizedBlocksService((global as any).host);
       },
     },
+    {
+      provide: InMemoryCacheService,
+      useFactory: () => new WorkerInMemoryCacheService((global as any).host),
+    },
   ],
 })
 export class WorkerFetchModule {}

packages/node/src/indexer/worker/worker.ts

@@ -54,10 +54,6 @@ async function initWorker(startHeight: number): Promise<void> {
 }
 export type IIndexerWorker = IBaseIndexerWorker;
 
-export type IInitIndexerWorker = IIndexerWorker & {
-  initWorker: typeof initWorker;
-};
-
 (global as any).host = createWorkerHost([], {
   initWorker,
 });

packages/node/src/subcommands/testing.module.ts

@@ -9,7 +9,9 @@ import {
   ConnectionPoolService,
   ConnectionPoolStateManager,
   DbModule,
+  InMemoryCacheService,
   PoiService,
+  PoiSyncService,
   ProjectUpgradeSevice,
   StoreService,
   TestRunner,
@@ -28,10 +30,12 @@ import { TestingService } from './testing.service';
 
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
     TestingService,
     EventEmitter2,
     PoiService,
+    PoiSyncService,
     SandboxService,
     DsProcessorService,
     DynamicDsService,