subquery · jiqiang90 · Apr 9, 2024 · Apr 9, 2024 · Apr 9, 2024 · Apr 9, 2024
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -85,7 +85,7 @@ module.exports = {
       2,
       'line',
       [
-        //Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+        //Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
         {pattern: ' Copyright \\d{4}(-\\d{4})? SubQuery Pte Ltd authors & contributors'},
         ' SPDX-License-Identifier: GPL-3.0',
       ],

diff --git a/packages/common-algorand/CHANGELOG.md b/packages/common-algorand/CHANGELOG.md
@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- version bump with `@subql/common`
+
 ## [3.2.2] - 2024-02-23
 ### Changed
 - version bump with `@subql/common`

diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.5.0",
     "@subql/types-algorand": "workspace:*",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",

diff --git a/packages/common-algorand/src/index.ts b/packages/common-algorand/src/index.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 export * from './project';
diff --git a/packages/common-algorand/src/project/index.ts b/packages/common-algorand/src/project/index.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 export * from './load';

diff --git a/packages/common-algorand/src/project/load.ts b/packages/common-algorand/src/project/load.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {AlgorandProjectManifestVersioned, VersionedProjectManifest} from './versioned';

diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {BaseDataSource, ProcessorImpl} from '@subql/common';
@@ -141,7 +141,7 @@
        case AlgorandHandlerKind.Transaction:
          return plainToClass(TransactionHandler, handler);
        default:
          throw new Error(`handler ${(handler as any).kind} not supported`);
      }
    });
  })
@@ -177,7 +177,7 @@
  file: string;
 }

 export class CustomDataSourceBase<K extends string, M extends CustomMapping, O = any>
  extends BaseDataSource
  implements AlgorandCustomDataSource<K, M>
 {

diff --git a/packages/common-algorand/src/project/types.ts b/packages/common-algorand/src/project/types.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {AlgorandDataSource} from '@subql/types-algorand';

diff --git a/packages/common-algorand/src/project/utils.ts b/packages/common-algorand/src/project/utils.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {

diff --git a/packages/common-algorand/src/project/validation/is-string-or-object.validation.ts b/packages/common-algorand/src/project/validation/is-string-or-object.validation.ts
@@ -1,10 +1,10 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {registerDecorator, ValidationOptions} from 'class-validator';

 export function IsStringOrObject(validationOptions?: ValidationOptions) {
  return function (object: Object, propertyName: string) {
    registerDecorator({
      name: 'IsStringOrObject',
      target: object.constructor,

diff --git a/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts b/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {AlgorandDataSource} from '@subql/types-algorand';

diff --git a/packages/common-algorand/src/project/versioned/index.ts b/packages/common-algorand/src/project/versioned/index.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 export * from './ProjectManifestVersioned';

diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/index.ts b/packages/common-algorand/src/project/versioned/v1_0_0/index.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 export * from './model';
diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {
@@ -23,7 +23,7 @@
 import {
  Equals,
  IsArray,
  IsNotEmpty,
  IsObject,
  IsOptional,
  IsString,
@@ -80,7 +80,7 @@

 export class AlgorandCustomDataSourceV1_0_0Impl<
    K extends string = string,
    M extends BaseMapping<any> = BaseMapping<any>
  >
  extends CustomDataSourceBase<K, M>
  implements AlgorandCustomDataSource<K, M>

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Updated with node-core.Now dictionary support multiple dictionary endpoints, indexer will fetch and switch dictionaries base on available blocks
+### Fixed
+- Updated with node-core ,also fixed:
+  - Fix modulo block didn't apply correctly with multiple dataSources
+  - Now when `workers` set to 0, it will use block dispatcher instead of throw and exit
+
 ## [3.9.1] - 2024-03-14
 ### Changed
 - Update `@subql/node-core` to 4.7.2 with graphql comments escaping fix

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -25,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.4.1",
+    "@subql/common": "^3.5.0",
     "@subql/common-algorand": "workspace:*",
-    "@subql/node-core": "^7.4.2",
+    "@subql/node-core": "^8.0.1",
     "@subql/types-algorand": "workspace:*",
     "algosdk": "^2.2.0",
     "axios": "^1.3.4",

diff --git a/packages/node/src/algorand/algorand.spec.ts b/packages/node/src/algorand/algorand.spec.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { INestApplication } from '@nestjs/common';
@@ -32,7 +32,7 @@
    root: './',
    schema: new GraphQLSchema({}),
    templates: [],
  } as any;
 }

 // eslint-disable-next-line jest/no-export
@@ -114,7 +114,7 @@
    };

    expect(
      filterTransaction(tx as any, {
        txType: 'acfg',
        sender: '7JMGBIDKQRR4MC3DNC73QU4QUNNN43VNY5RYPN2FRWEG6NXAHQMCPD4BIQ',
      }),
@@ -122,7 +122,7 @@
  });

  // This is failing since switching from algo explorer api. This is due to a node configuration limit
  it.skip('paginate large blocks', async () => {
    [app, apiService] = await prepareApiService(
      testnetEndpoint,
      testnetChainId,

diff --git a/packages/node/src/algorand/api.algorand.ts b/packages/node/src/algorand/api.algorand.ts
@@ -1,8 +1,8 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { TokenHeader } from '@subql/common-algorand';
-import { delay, getLogger } from '@subql/node-core';
+import { delay, getLogger, IBlock } from '@subql/node-core';
 import {
   AlgorandBlock,
   AlgorandTransaction,
@@ -11,7 +11,7 @@ import {
 import algosdk, { Indexer } from 'algosdk';
 import axios from 'axios';
 import { omit } from 'lodash';
-import { camelCaseObjectKey } from './utils.algorand';
+import { camelCaseObjectKey, formatBlockUtil } from './utils.algorand';
 
 const logger = getLogger('api.algorand');
 
@@ -170,7 +170,7 @@ export class AlgorandApi {
   getSafeApi(height: number): SafeAPIService {
     return new SafeAPIService(this, height, this.endpoint);
   }
-  async fetchBlocks(blockNums: number[]): Promise<AlgorandBlock[]> {
+  async fetchBlocks(blockNums: number[]): Promise<IBlock<AlgorandBlock>[]> {
     let blocks: AlgorandBlock[] = [];
 
     for (let i = 0; i < blockNums.length; i++) {
@@ -185,14 +185,14 @@ export class AlgorandApi {
 
     blocks = [...blocks, ...fetchedBlocks];
 
-    blocks = await Promise.all(
+    const formattedBlocks = await Promise.all(
       blocks.map(async (block) => {
         block.hash = await this.getBlockHash(block.round, blocks);
-        return block;
+        return formatBlockUtil(block);
       }),
     );
 
-    return blocks;
+    return formattedBlocks;
   }
 
   private blockInCache(number: number): AlgorandBlock {

diff --git a/packages/node/src/algorand/api.connection.ts b/packages/node/src/algorand/api.connection.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import {
@@ -7,17 +7,21 @@ import {
   IApiConnectionSpecific,
   NetworkMetadataPayload,
   getLogger,
+  IBlock,
 } from '@subql/node-core';
 import { BlockContent } from '../indexer/types';
 import { AlgorandApi, SafeAPIService } from './api.algorand';
 
 const logger = getLogger('AlgorandApiConnection');
 
-type FetchFunc = (api: AlgorandApi, batch: number[]) => Promise<BlockContent[]>;
+type FetchFunc = (
+  api: AlgorandApi,
+  batch: number[],
+) => Promise<IBlock<BlockContent>[]>;
 
 export class AlgorandApiConnection
   implements
-    IApiConnectionSpecific<AlgorandApi, SafeAPIService, BlockContent[]>
+    IApiConnectionSpecific<AlgorandApi, SafeAPIService, IBlock<BlockContent>[]>
 {
   readonly networkMeta: NetworkMetadataPayload;
 
@@ -56,7 +60,7 @@ export class AlgorandApiConnection
     logger.debug('apiDisconnect is not implemented');
   }
 
-  async fetchBlocks(heights: number[]): Promise<BlockContent[]> {
+  async fetchBlocks(heights: number[]): Promise<IBlock<BlockContent>[]> {
     const blocks = await this.fetchBlocksBatches(this.unsafeApi, heights);
     return blocks;
   }

diff --git a/packages/node/src/algorand/api.service.algorand.ts b/packages/node/src/algorand/api.service.algorand.ts
@@ -1,9 +1,14 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { Inject, Injectable } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
-import { ApiService, ConnectionPoolService, getLogger } from '@subql/node-core';
+import {
+  ApiService,
+  ConnectionPoolService,
+  getLogger,
+  IBlock,
+} from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { BlockContent } from '../indexer/types';
 import { AlgorandApi, SafeAPIService } from './api.algorand';
@@ -15,7 +20,7 @@ const logger = getLogger('api');
 export class AlgorandApiService extends ApiService<
   AlgorandApi,
   SafeAPIService,
-  BlockContent[]
+  IBlock<BlockContent>[]
 > {
   constructor(
     @Inject('ISubqueryProject') private project: SubqueryProject,
@@ -41,8 +46,7 @@ export class AlgorandApiService extends ApiService<
         AlgorandApiConnection.create(endpoint, this.fetchBlockBatches),
       //eslint-disable-next-line @typescript-eslint/require-await
       async (connection: AlgorandApiConnection) => {
-        const api = connection.unsafeApi;
-        return api.getGenesisHash();
+        return connection.unsafeApi.getGenesisHash();
       },
     );
 
@@ -56,7 +60,7 @@ export class AlgorandApiService extends ApiService<
   async fetchBlockBatches(
     api: AlgorandApi,
     blocks: number[],
-  ): Promise<BlockContent[]> {
+  ): Promise<IBlock<BlockContent>[]> {
     return api.fetchBlocks(blocks);
   }
 }
diff --git a/packages/node/src/algorand/index.ts b/packages/node/src/algorand/index.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 export * from './api.algorand';

diff --git a/packages/node/src/algorand/utils.algorand.spec.ts b/packages/node/src/algorand/utils.algorand.spec.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { INestApplication } from '@nestjs/common';

diff --git a/packages/node/src/algorand/utils.algorand.ts b/packages/node/src/algorand/utils.algorand.ts
@@ -1,6 +1,7 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import { Header, IBlock } from '@subql/node-core';
 import {
   AlgorandBlock,
   AlgorandBlockFilter,
@@ -9,6 +10,24 @@ import {
 } from '@subql/types-algorand';
 import { Indexer, TransactionType } from 'algosdk';
 import { camelCase, get } from 'lodash';
+import { BlockContent } from '../indexer/types';
+
+export function algorandBlockToHeader(block: BlockContent): Header {
+  return {
+    blockHeight: block.round,
+    blockHash: block.round.toString(),
+    parentHash: block.previousBlockHash,
+  };
+}
+
+export function formatBlockUtil<B extends AlgorandBlock = AlgorandBlock>(
+  block: B,
+): IBlock<B> {
+  return {
+    block,
+    getHeader: () => algorandBlockToHeader(block),
+  };
+}
 
 export function camelCaseObjectKey(object: object) {
   if (Array.isArray(object)) {

diff --git a/packages/node/src/app.module.ts b/packages/node/src/app.module.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { Module } from '@nestjs/common';

diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import assert from 'assert';

diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { DynamicModule, Global, Module } from '@nestjs/common';

diff --git a/packages/node/src/indexer/api.service.spec.ts b/packages/node/src/indexer/api.service.spec.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { INestApplication } from '@nestjs/common';
@@ -67,7 +67,9 @@ describe('ApiService', () => {
 
     const block = (await apiService.api.fetchBlocks([50000]))[0];
 
-    expect(block.hash).toEqual('Gss169f22yVUBJzbNT9qXtQukjh0tgecvapaQY5NIRg=');
+    expect(block.block.hash).toEqual(
+      'Gss169f22yVUBJzbNT9qXtQukjh0tgecvapaQY5NIRg=',
+    );
   });
 
   it('waits on pending block to fetch hash', async () => {
@@ -83,6 +85,6 @@ describe('ApiService', () => {
     expect(fetchLatestBlock).not.toThrow();
 
     const block = await fetchLatestBlock();
-    expect(block.hash).toBeDefined();
+    expect(block.block.hash).toBeDefined();
   });
 });
diff --git a/packages/node/src/indexer/blockDispatcher/algorand-block-dispatcher.ts b/packages/node/src/indexer/blockDispatcher/algorand-block-dispatcher.ts
@@ -1,8 +1,10 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { IBlockDispatcher } from '@subql/node-core';
+import { AlgorandBlock } from '@subql/types-algorand';
 
-export interface IAlgorandBlockDispatcher extends IBlockDispatcher {
+export interface IAlgorandBlockDispatcher
+  extends IBlockDispatcher<AlgorandBlock> {
   init(onDynamicDsCreated: (height: number) => Promise<void>): Promise<void>;
 }
diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts
@@ -1,4 +1,4 @@
-// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
 import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
@@ -13,6 +13,7 @@ import {
   BlockDispatcher,
   ProcessBlockResponse,
   IProjectUpgradeService,
+  IBlock,
 } from '@subql/node-core';
 import { AlgorandBlock } from '@subql/types-algorand';
 import { AlgorandApiService } from '../../algorand';
@@ -58,9 +59,7 @@ export class BlockDispatcherService
       poiSyncService,
       project,
       dynamicDsService,
-      async (blockNums: number[]): Promise<AlgorandBlock[]> => {
-        return this.apiService.fetchBlocks(blockNums);
-      },
+      apiService.fetchBlocks.bind(apiService),
     );
   }
 
@@ -69,11 +68,11 @@ export class BlockDispatcherService
   }
 
   protected async indexBlock(
-    block: AlgorandBlock,
+    block: IBlock<AlgorandBlock>,
   ): Promise<ProcessBlockResponse> {
     return this.indexerManager.indexBlock(
       block,
-      await this.projectService.getDataSources(this.getBlockHeight(block)),
+      await this.projectService.getDataSources(block.getHeader().blockHeight),
     );
   }
 }