From f9d109a1312a71af8e080bf07f62036d57d24498 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Fri, 1 Sep 2023 12:43:47 +1200 Subject: [PATCH 01/15] Sync with main SKD for project upgrades feature --- packages/common-algorand/package.json | 2 +- packages/common-algorand/src/project/types.ts | 1 - .../src/project/versioned/v1_0_0/model.ts | 16 +- .../src/project/versioned/v1_0_0/types.ts | 22 +- packages/node/package.json | 4 +- packages/node/src/algorand/algorand.spec.ts | 2 +- packages/node/src/algorand/api.connection.ts | 3 +- .../node/src/algorand/api.service.algorand.ts | 4 +- .../node/src/configure/SubqueryProject.ts | 203 ++--- .../node/src/configure/configure.module.ts | 145 +--- packages/node/src/indexer/api.service.spec.ts | 2 +- .../block-dispatcher.service.ts | 13 +- .../worker-block-dispatcher.service.ts | 49 +- .../node/src/indexer/dictionary.service.ts | 16 +- .../node/src/indexer/ds-processor.service.ts | 2 - .../node/src/indexer/dynamic-ds.service.ts | 11 +- packages/node/src/indexer/fetch.module.ts | 5 + packages/node/src/indexer/fetch.service.ts | 63 +- packages/node/src/indexer/indexer.manager.ts | 17 +- packages/node/src/indexer/project.service.ts | 22 +- .../node/src/indexer/worker/worker.service.ts | 99 +-- packages/node/src/indexer/worker/worker.ts | 140 +--- .../worker.unfinalizedBlocks.service.ts | 14 +- packages/node/src/init.ts | 3 +- packages/node/src/main.ts | 15 +- packages/node/src/subcommands/reindex.init.ts | 3 +- .../node/src/subcommands/reindex.module.ts | 2 +- .../node/src/subcommands/reindex.service.ts | 54 -- packages/node/src/subcommands/testing.init.ts | 6 +- .../node/src/subcommands/testing.module.ts | 2 + .../node/src/subcommands/testing.service.ts | 9 +- packages/node/src/utils/reindex.ts | 63 -- packages/node/src/yargs.ts | 5 +- yarn.lock | 763 +++++++++++++++++- 34 files changed, 1010 insertions(+), 770 deletions(-) delete mode 100644 packages/node/src/subcommands/reindex.service.ts delete mode 100644 packages/node/src/utils/reindex.ts diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index ce35909e..83ce8639 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -13,7 +13,7 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "^2.4.0", + "@subql/common": "dev", "@subql/types-algorand": "workspace:*", "class-transformer": "0.4.0", "class-validator": "^0.13.2", diff --git a/packages/common-algorand/src/project/types.ts b/packages/common-algorand/src/project/types.ts index a65e613a..81ee820d 100644 --- a/packages/common-algorand/src/project/types.ts +++ b/packages/common-algorand/src/project/types.ts @@ -26,5 +26,4 @@ export interface AlgorandProjectNetworkConfig extends ProjectNetworkConfig { genesisHash?: string; chainId?: string; apiKey?: string | TokenHeader; - bypassBlocks?: (number | string)[]; } diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts index 3a63a209..490751bc 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts @@ -4,6 +4,8 @@ import { BaseMapping, NodeSpec, + ParentProject, + ParentProjectModel, ProjectManifestBaseImpl, QuerySpec, RunnerQueryBaseModel, @@ -30,7 +32,7 @@ import {IsStringOrObject} from '../../validation/is-string-or-object.validation' import { CustomDataSourceTemplate, CustomDataSourceV1_0_0, - ProjectManifestV1_0_0, + AlgorandProjectManifestV1_0_0, RuntimeDataSourceTemplate, RuntimeDataSourceV1_0_0, } from './types'; @@ -175,11 +177,16 @@ export class DeploymentV1_0_0 { keepDiscriminatorProperty: true, }) templates?: (RuntimeDataSourceTemplate | CustomDataSourceTemplate)[]; + + @IsOptional() + @IsObject() + @Type(() => ParentProjectModel) + parent?: ParentProject; } export class ProjectManifestV1_0_0Impl extends ProjectManifestBaseImpl - implements ProjectManifestV1_0_0 + implements AlgorandProjectManifestV1_0_0 { @Equals('1.0.0') specVersion: string; @@ -229,6 +236,11 @@ export class ProjectManifestV1_0_0Impl private _deployment: DeploymentV1_0_0; + @IsOptional() + @IsObject() + @Type(() => ParentProjectModel) + parent?: ParentProject; + toDeployment(): string { return yaml.dump(this._deployment, { sortKeys: true, diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/types.ts b/packages/common-algorand/src/project/versioned/v1_0_0/types.ts index 31d3d586..fd8ae447 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/types.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/types.ts @@ -1,7 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {TemplateBase} from '@subql/common'; +import {ProjectManifestV1_0_0, TemplateBase} from '@subql/common'; import { AlgorandCustomDataSource, AlgorandDataSource, @@ -10,7 +10,6 @@ import { AlgorandMapping, AlgorandRuntimeDataSource, } from '@subql/types-algorand'; -import {IAlgorandProjectManifest} from '../../types'; export interface SubqlMappingV1_0_0 extends AlgorandMapping { file: string; @@ -22,21 +21,10 @@ export type CustomDataSourceV1_0_0 = AlgorandCustomDataSource; export interface RuntimeDataSourceTemplate extends Omit, TemplateBase {} export interface CustomDataSourceTemplate extends Omit, TemplateBase {} -export interface ProjectManifestV1_0_0 extends IAlgorandProjectManifest { - name: string; - version: string; - schema: { - file: string; - }; - - network: { - endpoint?: string | string[]; - dictionary?: string; - chainId: string; - }; - - dataSources: (RuntimeDataSourceV1_0_0 | CustomDataSourceV1_0_0)[]; -} +export type AlgorandProjectManifestV1_0_0 = ProjectManifestV1_0_0< + RuntimeDataSourceV1_0_0 | CustomDataSourceV1_0_0, + RuntimeDataSourceTemplate | CustomDataSourceTemplate +>; export function isRuntimeDataSourceV1_0_0(dataSource: AlgorandDataSource): dataSource is RuntimeDataSourceV1_0_0 { return dataSource.kind === AlgorandDataSourceKind.Runtime && !!(dataSource as RuntimeDataSourceV1_0_0).mapping.file; diff --git a/packages/node/package.json b/packages/node/package.json index 82624d55..c991e3d5 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -24,9 +24,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^2.4.0", + "@subql/common": "dev", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "^4.0.1", + "@subql/node-core": "4.2.4-2", "@subql/types-algorand": "workspace:*", "@willsoto/nestjs-prometheus": "^4.4.0", "algosdk": "^2.2.0", diff --git a/packages/node/src/algorand/algorand.spec.ts b/packages/node/src/algorand/algorand.spec.ts index 3e599008..a6ee38b8 100644 --- a/packages/node/src/algorand/algorand.spec.ts +++ b/packages/node/src/algorand/algorand.spec.ts @@ -32,7 +32,7 @@ function testSubqueryProject(endpoint: string, chainId): SubqueryProject { root: './', schema: new GraphQLSchema({}), templates: [], - }; + } as any; } export const prepareApiService = async ( diff --git a/packages/node/src/algorand/api.connection.ts b/packages/node/src/algorand/api.connection.ts index 2aefdf24..932da986 100644 --- a/packages/node/src/algorand/api.connection.ts +++ b/packages/node/src/algorand/api.connection.ts @@ -16,7 +16,8 @@ const logger = getLogger('AlgorandApiConnection'); type FetchFunc = (api: AlgorandApi, batch: number[]) => Promise; export class AlgorandApiConnection - implements IApiConnectionSpecific + implements + IApiConnectionSpecific { readonly networkMeta: NetworkMetadataPayload; diff --git a/packages/node/src/algorand/api.service.algorand.ts b/packages/node/src/algorand/api.service.algorand.ts index 3df0bfb9..ef51e732 100644 --- a/packages/node/src/algorand/api.service.algorand.ts +++ b/packages/node/src/algorand/api.service.algorand.ts @@ -8,10 +8,8 @@ import { ApiService, ConnectionPoolService, getLogger, - IndexerEvent, NetworkMetadataPayload, } from '@subql/node-core'; -import { AlgorandBlock } from '@subql/types-algorand'; import { SubqueryProject } from '../configure/SubqueryProject'; import { BlockContent } from '../indexer/types'; import { AlgorandApi, SafeAPIService } from './api.algorand'; @@ -23,7 +21,7 @@ const logger = getLogger('api'); export class AlgorandApiService extends ApiService< AlgorandApi, SafeAPIService, - BlockContent + BlockContent[] > { networkMeta: NetworkMetadataPayload; constructor( diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index dd147038..0e6a2736 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -1,8 +1,14 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import assert from 'assert'; import { Injectable } from '@nestjs/common'; -import { Reader, RunnerSpecs, validateSemver } from '@subql/common'; +import { + ParentProject, + Reader, + RunnerSpecs, + validateSemver, +} from '@subql/common'; import { AlgorandProjectNetworkConfig, parseAlgorandProjectManifest, @@ -12,17 +18,26 @@ import { isRuntimeDs, AlgorandHandlerKind, isCustomDs, + RuntimeDataSourceTemplate, + CustomDataSourceTemplate, } from '@subql/common-algorand'; -import { getProjectRoot, updateDataSourcesV1_0_0 } from '@subql/node-core'; -import { AlgorandBlock } from '@subql/types-algorand'; +import { + insertBlockFiltersCronSchedules, + loadProjectTemplates, + SubqlProjectDs, + updateDataSourcesV1_0_0, +} from '@subql/node-core'; import { buildSchemaFromString } from '@subql/utils'; import Cron from 'cron-converter'; import { GraphQLSchema } from 'graphql'; -import { AlgorandApi } from '../algorand'; -export type SubqlProjectDs = AlgorandDataSource & { - mapping: AlgorandDataSource['mapping'] & { entryScript: string }; -}; +const { version: packageVersion } = require('../../package.json'); + +export type AlgorandProjectDs = SubqlProjectDs; + +export type AlgorandProjectDsTemplate = + | SubqlProjectDs + | SubqlProjectDs; export type SubqlProjectBlockFilter = BlockFilter & { cronSchedule?: { @@ -31,9 +46,9 @@ export type SubqlProjectBlockFilter = BlockFilter & { }; }; -export type SubqlProjectDsTemplate = Omit & { - name: string; -}; +// export type SubqlProjectDsTemplate = Omit & { +// name: string; +// }; const NOT_SUPPORT = (name: string) => { throw new Error(`Manifest specVersion ${name}() is not supported`); @@ -44,20 +59,42 @@ type NetworkConfig = AlgorandProjectNetworkConfig & { chainId: string }; @Injectable() export class SubqueryProject { - id: string; - root: string; - network: NetworkConfig; - dataSources: SubqlProjectDs[]; - schema: GraphQLSchema; - templates: SubqlProjectDsTemplate[]; - runner?: RunnerSpecs; + #dataSources: AlgorandProjectDs[]; + + constructor( + readonly id: string, + readonly root: string, + readonly network: NetworkConfig, + dataSources: AlgorandProjectDs[], + readonly schema: GraphQLSchema, + readonly templates: AlgorandProjectDsTemplate[], + readonly runner?: RunnerSpecs, + readonly parent?: ParentProject, + ) { + this.#dataSources = dataSources; + } + + get dataSources(): AlgorandProjectDs[] { + return this.#dataSources; + } + + async applyCronTimestamps( + getTimestamp: (height: number) => Promise, + ): Promise { + this.#dataSources = await insertBlockFiltersCronSchedules( + this.dataSources, + getTimestamp, + isRuntimeDs, + AlgorandHandlerKind.Block, + ); + } static async create( path: string, rawManifest: unknown, reader: Reader, + root: string, networkOverrides?: Partial, - root?: string, ): Promise { // rawManifest and reader can be reused here. // It has been pre-fetched and used for rebase manifest runner options with args @@ -75,12 +112,12 @@ export class SubqueryProject { NOT_SUPPORT('<1.0.0'); } - return loadProjectFromManifest1_0_0( + return loadProjectFromManifestBase( manifest.asV1_0_0, reader, path, - networkOverrides, root, + networkOverrides, ); } } @@ -100,11 +137,9 @@ async function loadProjectFromManifestBase( projectManifest: SUPPORT_MANIFEST, reader: Reader, path: string, + root: string, networkOverrides?: Partial, - root?: string, ): Promise { - root = root ?? (await getProjectRoot(reader)); - if (typeof projectManifest.network.endpoint === 'string') { projectManifest.network.endpoint = [projectManifest.network.endpoint]; } @@ -130,118 +165,34 @@ async function loadProjectFromManifestBase( } const schema = buildSchemaFromString(schemaString); + const templates = await loadProjectTemplates( + projectManifest.templates, + root, + reader, + isCustomDs, + ); + const runner = projectManifest.runner; + assert( + validateSemver(packageVersion, runner.node.version), + new Error( + `Runner require node version ${runner.node.version}, current node ${packageVersion}`, + ), + ); + const dataSources = await updateDataSourcesV1_0_0( projectManifest.dataSources, reader, root, isCustomDs, ); - return { - id: reader.root ? reader.root : path, //TODO, need to method to get project_id + return new SubqueryProject( + reader.root ? reader.root : path, //TODO, need to method to get project_id root, network, dataSources, schema, - templates: [], - }; -} - -const { version: packageVersion } = require('../../package.json'); - -async function loadProjectFromManifest1_0_0( - projectManifest: ProjectManifestV1_0_0Impl, - reader: Reader, - path: string, - networkOverrides?: Partial, - root?: string, -): Promise { - const project = await loadProjectFromManifestBase( - projectManifest, - reader, - path, - networkOverrides, - root, - ); - project.templates = await loadProjectTemplates( - projectManifest, - project.root, - reader, + templates, + runner, + projectManifest.parent, ); - project.runner = projectManifest.runner; - if (!validateSemver(packageVersion, project.runner.node.version)) { - throw new Error( - `Runner require node version ${project.runner.node.version}, current node ${packageVersion}`, - ); - } - return project; -} - -async function loadProjectTemplates( - projectManifest: ProjectManifestV1_0_0Impl, - root: string, - reader: Reader, -): Promise { - if (!projectManifest.templates || !projectManifest.templates.length) { - return []; - } - const dsTemplates = await updateDataSourcesV1_0_0( - projectManifest.templates, - reader, - root, - isCustomDs, - ); - return dsTemplates.map((ds, index) => ({ - ...ds, - name: projectManifest.templates[index].name, - })); -} - -// eslint-disable-next-line @typescript-eslint/require-await -export async function generateTimestampReferenceForBlockFilters( - dataSources: SubqlProjectDs[], - api: AlgorandApi, -): Promise { - const cron = new Cron(); - - dataSources = await Promise.all( - dataSources.map(async (ds) => { - if (isRuntimeDs(ds)) { - const startBlock = ds.startBlock ?? 1; - let block: AlgorandBlock; - let timestampReference: Date; - ds.mapping.handlers = await Promise.all( - ds.mapping.handlers.map(async (handler) => { - if (handler.kind === AlgorandHandlerKind.Block) { - if (handler.filter?.timestamp) { - if (!block) { - block = await api.getBlockByHeight(startBlock); - - timestampReference = new Date(block.timestamp); - } - try { - cron.fromString(handler.filter.timestamp); - } catch (e) { - throw new Error( - `Invalid Cron string: ${handler.filter.timestamp}`, - ); - } - - const schedule = cron.schedule(timestampReference); - (handler.filter as SubqlProjectBlockFilter).cronSchedule = { - schedule: schedule, - get next() { - return Date.parse(this.schedule.next().format()); - }, - }; - } - } - return handler; - }), - ); - } - return ds; - }), - ); - - return dataSources; } diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts index ebe63946..c89287e8 100644 --- a/packages/node/src/configure/configure.module.ts +++ b/packages/node/src/configure/configure.module.ts @@ -1,157 +1,48 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import assert from 'assert'; import { DynamicModule, Global, Module } from '@nestjs/common'; -import { - handleCreateSubqueryProjectError, - Reader, - ReaderFactory, -} from '@subql/common'; -import { AlgorandProjectNetworkConfig } from '@subql/common-algorand'; -import { - IConfig, - NodeConfig, - getLogger, - setLevel, - rebaseArgsWithManifest, - defaultSubqueryName, - validDbSchemaName, -} from '@subql/node-core'; -import { camelCase, last, omitBy, isNil } from 'lodash'; +import { NodeConfig, registerApp } from '@subql/node-core'; import { yargsOptions } from '../yargs'; import { SubqueryProject } from './SubqueryProject'; -const logger = getLogger('configure'); - -const YargsNameMapping: Record = {}; - -type Args = (typeof yargsOptions.argv)['argv']; - -function yargsToIConfig(yargs: Args): Partial { - return Object.entries(yargs).reduce((acc, [key, value]) => { - if (['_', '$0'].includes(key)) return acc; - - if (key === 'network-registry') { - try { - value = JSON.parse(value as string); - } catch (e) { - throw new Error('Argument `network-registry` is not valid JSON'); - } - } - acc[YargsNameMapping[key] ?? camelCase(key)] = value; - return acc; - }, {} as any); -} - -function warnDeprecations() { - const { argv } = yargsOptions; - if (argv['subquery-name']) { - logger.warn( - 'Note that argument --subquery-name has been deprecated in favour of --db-schema', - ); - } - if (argv.local) { - logger.warn('Note that argument --local has been deprecated'); - } -} +const pjson = require('../../package.json'); @Global() @Module({}) export class ConfigureModule { static async getInstance(): Promise<{ - config: NodeConfig; - project: () => Promise; + nodeConfig: NodeConfig; + project: SubqueryProject; }> { const { argv } = yargsOptions; - let config: NodeConfig; - let rawManifest: unknown; - let reader: Reader; - - const isTest = argv._[0] === 'test'; - - // Override order : Sub-command/Args/Flags > Manifest Runner options > Default configs - // Therefore, we should rebase the manifest runner options with args first but not the config in the end - if (argv.config) { - // get manifest options - config = NodeConfig.fromFile(argv.config, yargsToIConfig(argv), isTest); - reader = await ReaderFactory.create(config.subquery, { - ipfs: config.ipfs, - }); - rawManifest = await reader.getProjectSchema(); - rebaseArgsWithManifest(argv, rawManifest); - // use rebased argv generate config to override current config - config = NodeConfig.rebaseWithArgs(config, yargsToIConfig(argv), isTest); - } else { - if (!argv.subquery) { - logger.error( - 'Subquery path is missing neither in cli options nor in config file', - ); - yargsOptions.showHelp(); - process.exit(1); - } - - warnDeprecations(); - assert(argv.subquery, 'subquery path is missing'); - reader = await ReaderFactory.create(argv.subquery, { - ipfs: argv.ipfs, - }); - rawManifest = await reader.getProjectSchema(); - rebaseArgsWithManifest(argv, rawManifest); - // Create new nodeConfig with rebased argv - config = new NodeConfig( - defaultSubqueryName(yargsToIConfig(argv)), - isTest, - ); - } - - if (!validDbSchemaName(config.dbSchema)) { - process.exit(1); - } - - if (config.debug) { - setLevel('debug'); - } - - const project = async () => { - const p = await SubqueryProject.create( - argv.subquery, - rawManifest, - reader, - omitBy( - { - endpoint: config.networkEndpoints, - dictionary: config.networkDictionary, - }, - isNil, - ), - config.root, - ).catch((err) => { - const pjson = require('../../package.json'); - handleCreateSubqueryProjectError(err, pjson, rawManifest, logger); - process.exit(1); - }); - return p; - }; - - return { config, project }; + return registerApp( + argv, + SubqueryProject.create.bind(SubqueryProject), + yargsOptions.showHelp.bind(yargsOptions), + pjson, + ); } static async register(): Promise { - const { config, project } = await ConfigureModule.getInstance(); + const { nodeConfig, project } = await ConfigureModule.getInstance(); return { module: ConfigureModule, providers: [ { provide: NodeConfig, - useValue: config, + useValue: nodeConfig, }, { provide: 'ISubqueryProject', - useFactory: project, + useValue: project, + }, + { + provide: 'IProjectUpgradeService', + useValue: project, }, ], - exports: [NodeConfig, 'ISubqueryProject'], + exports: [NodeConfig, 'ISubqueryProject', 'IProjectUpgradeService'], }; } } diff --git a/packages/node/src/indexer/api.service.spec.ts b/packages/node/src/indexer/api.service.spec.ts index b6166faa..a47b4df2 100644 --- a/packages/node/src/indexer/api.service.spec.ts +++ b/packages/node/src/indexer/api.service.spec.ts @@ -27,7 +27,7 @@ function testSubqueryProject(endpoint: string): SubqueryProject { root: './', schema: new GraphQLSchema({}), templates: [], - }; + } as any; } jest.setTimeout(90000); diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index 58a23d83..c4352cb7 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -12,11 +12,12 @@ import { PoiService, BlockDispatcher, ProcessBlockResponse, + IProjectUpgradeService, } from '@subql/node-core'; import { AlgorandBlock } from '@subql/types-algorand'; import { AlgorandApiService } from '../../algorand'; import { - SubqlProjectDs, + AlgorandProjectDs, SubqueryProject, } from '../../configure/SubqueryProject'; import { DynamicDsService } from '../dynamic-ds.service'; @@ -27,7 +28,7 @@ import { IndexerManager } from '../indexer.manager'; */ @Injectable() export class BlockDispatcherService - extends BlockDispatcher + extends BlockDispatcher implements OnApplicationShutdown { constructor( @@ -35,7 +36,10 @@ export class BlockDispatcherService nodeConfig: NodeConfig, private indexerManager: IndexerManager, eventEmitter: EventEmitter2, - @Inject('IProjectService') projectService: IProjectService, + @Inject('IProjectService') + projectService: IProjectService, + @Inject('IProjectUpgradeService') + projectUpgradeService: IProjectUpgradeService, smartBatchService: SmartBatchService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -47,6 +51,7 @@ export class BlockDispatcherService nodeConfig, eventEmitter, projectService, + projectUpgradeService, smartBatchService, storeService, storeCacheService, @@ -68,7 +73,7 @@ export class BlockDispatcherService ): Promise { return this.indexerManager.indexBlock( block, - await this.projectService.getAllDataSources(this.getBlockHeight(block)), + await this.projectService.getDataSources(this.getBlockHeight(block)), ); } } diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index e6faf99e..27931752 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -20,18 +20,22 @@ import { HostConnectionPoolState, ConnectionPoolStateManager, connectionPoolStateHostFunctions, + baseWorkerFunctions, + storeHostFunctions, + dynamicDsHostFunctions, + IProjectUpgradeService, + HostUnfinalizedBlocks, } from '@subql/node-core'; import { Store } from '@subql/types'; import { AlgorandApiConnection } from '../../algorand'; import { - SubqlProjectDs, + AlgorandProjectDs, SubqueryProject, } from '../../configure/SubqueryProject'; import { DynamicDsService } from '../dynamic-ds.service'; import { BlockContent } from '../types'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker'; -import { HostUnfinalizedBlocks } from '../worker/worker.unfinalizedBlocks.service'; type IndexerWorker = IIndexerWorker & { terminate: () => Promise; @@ -39,43 +43,24 @@ type IndexerWorker = IIndexerWorker & { async function createIndexerWorker( store: Store, - dynamicDsService: IDynamicDsService, + dynamicDsService: IDynamicDsService, unfinalizedBlocksService: IUnfinalizedBlocksService, connectionPoolState: ConnectionPoolStateManager, root: string, ): Promise { const indexerWorker = Worker.create< IInitIndexerWorker, - // HostDynamicDS & HostStore & HostUnfinalizedBlocks - HostDynamicDS & + // HostDynamicDS & HostStore & HostUnfinalizedBlocks + HostDynamicDS & HostStore & HostUnfinalizedBlocks & HostConnectionPoolState >( path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), - [ - 'initWorker', - 'processBlock', - 'fetchBlock', - 'numFetchedBlocks', - 'numFetchingBlocks', - 'getStatus', - 'getMemoryLeft', - 'waitForWorkerBatchSize', - ], + [...baseWorkerFunctions, 'initWorker'], { - storeGet: store.get.bind(store), - storeGetByField: store.getByField.bind(store), - storeGetOneByField: store.getOneByField.bind(store), - storeSet: store.set.bind(store), - storeBulkCreate: store.bulkCreate.bind(store), - storeBulkUpdate: store.bulkUpdate.bind(store), - storeRemove: store.remove.bind(store), - storeBulkRemove: store.bulkRemove.bind(store), - dynamicDsCreateDynamicDatasource: - dynamicDsService.createDynamicDatasource.bind(dynamicDsService), - dynamicDsGetDynamicDatasources: - dynamicDsService.getDynamicDatasources.bind(dynamicDsService), + ...storeHostFunctions(store), + ...dynamicDsHostFunctions(dynamicDsService), unfinalizedBlocksProcess: unfinalizedBlocksService.processUnfinalizedBlockHeader.bind( unfinalizedBlocksService, @@ -92,13 +77,16 @@ async function createIndexerWorker( @Injectable() export class WorkerBlockDispatcherService - extends WorkerBlockDispatcher + extends WorkerBlockDispatcher implements OnApplicationShutdown { constructor( nodeConfig: NodeConfig, eventEmitter: EventEmitter2, - @Inject('IProjectService') projectService: IProjectService, + @Inject('IProjectService') + projectService: IProjectService, + @Inject('IProjectUpgradeService') + projectUpgadeService: IProjectUpgradeService, smartBatchService: SmartBatchService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -112,6 +100,7 @@ export class WorkerBlockDispatcherService nodeConfig, eventEmitter, projectService, + projectUpgadeService, smartBatchService, storeService, storeCacheService, @@ -134,7 +123,7 @@ export class WorkerBlockDispatcherService height: number, ): Promise { // const start = new Date(); - await worker.fetchBlock(height); + await worker.fetchBlock(height, null); // const end = new Date(); // const waitTime = end.getTime() - start.getTime(); diff --git a/packages/node/src/indexer/dictionary.service.ts b/packages/node/src/indexer/dictionary.service.ts index 022f1aa8..20c0252d 100644 --- a/packages/node/src/indexer/dictionary.service.ts +++ b/packages/node/src/indexer/dictionary.service.ts @@ -1,7 +1,8 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, DictionaryService as CoreDictionaryService, @@ -9,14 +10,17 @@ import { import { SubqueryProject } from '../configure/SubqueryProject'; @Injectable() -export class DictionaryService - extends CoreDictionaryService - implements OnApplicationShutdown -{ +export class DictionaryService extends CoreDictionaryService { constructor( @Inject('ISubqueryProject') protected project: SubqueryProject, nodeConfig: NodeConfig, + eventEmitter: EventEmitter2, ) { - super(project.network.dictionary, project.network.chainId, nodeConfig); + super( + project.network.dictionary, + project.network.chainId, + nodeConfig, + eventEmitter, + ); } } diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts index 56d4fa17..403238a2 100644 --- a/packages/node/src/indexer/ds-processor.service.ts +++ b/packages/node/src/indexer/ds-processor.service.ts @@ -1,8 +1,6 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import fs from 'fs'; -import path from 'path'; import { Inject, Injectable } from '@nestjs/common'; import { isCustomDs, diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index 5c1ca23f..154617d4 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -9,11 +9,14 @@ import { } from '@subql/node-core'; import { TransactionType } from 'algosdk'; import { cloneDeep } from 'lodash'; -import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; +import { + AlgorandProjectDs, + SubqueryProject, +} from '../configure/SubqueryProject'; import { DsProcessorService } from './ds-processor.service'; @Injectable() -export class DynamicDsService extends BaseDynamicDsService { +export class DynamicDsService extends BaseDynamicDsService { constructor( private readonly dsProcessorService: DsProcessorService, @Inject('ISubqueryProject') private readonly project: SubqueryProject, @@ -23,7 +26,7 @@ export class DynamicDsService extends BaseDynamicDsService { protected async getDatasource( params: DatasourceParams, - ): Promise { + ): Promise { const template = cloneDeep( this.project.templates.find((t) => t.name === params.templateName), ); @@ -37,7 +40,7 @@ export class DynamicDsService extends BaseDynamicDsService { const dsObj = { ...template, startBlock: params.startBlock, - } as SubqlProjectDs; + } as AlgorandProjectDs; delete dsObj.name; try { if (isCustomDs(dsObj)) { diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index e3bcee55..1f493571 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -16,6 +16,7 @@ import { PgMmrCacheService, MmrQueryService, ConnectionPoolStateManager, + IProjectUpgradeService, } from '@subql/node-core'; import { AlgorandApiConnection, AlgorandApiService } from '../algorand'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -70,6 +71,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; nodeConfig: NodeConfig, eventEmitter: EventEmitter2, projectService: ProjectService, + projectUpgradeService: IProjectUpgradeService, apiService: AlgorandApiService, indexerManager: IndexerManager, smartBatchService: SmartBatchService, @@ -86,6 +88,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; nodeConfig, eventEmitter, projectService, + projectUpgradeService, smartBatchService, storeService, storeCacheService, @@ -101,6 +104,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; indexerManager, eventEmitter, projectService, + projectUpgradeService, smartBatchService, storeService, storeCacheService, @@ -112,6 +116,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; NodeConfig, EventEmitter2, 'IProjectService', + 'IProjectUpgradeService', AlgorandApiService, IndexerManager, SmartBatchService, diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 5f0ae572..11de1480 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -1,10 +1,9 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import { Interval, SchedulerRegistry } from '@nestjs/schedule'; - +import { SchedulerRegistry } from '@nestjs/schedule'; import { isCustomDs, isRuntimeDs, @@ -12,24 +11,22 @@ import { AlgorandHandler, AlgorandDataSource, AlgorandRuntimeHandlerFilter, - isRuntimeDataSourceV1_0_0, } from '@subql/common-algorand'; -import { NodeConfig, BaseFetchService } from '@subql/node-core'; +import { NodeConfig, BaseFetchService, getModulos } from '@subql/node-core'; import { DictionaryQueryCondition } from '@subql/types'; import { AlgorandBlockFilter, DictionaryQueryEntry, } from '@subql/types-algorand'; -import { MetaData } from '@subql/utils'; -import { Indexer } from 'algosdk'; -import { range, sortBy, uniqBy, without } from 'lodash'; +import { sortBy, uniqBy } from 'lodash'; import { AlgorandApi, AlgorandApiService, calcInterval } from '../algorand'; -import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; +import { SubqueryProject } from '../configure/SubqueryProject'; import { isBaseHandler, isCustomHandler } from '../utils/project'; import { IAlgorandBlockDispatcher } from './blockDispatcher'; import { DictionaryService } from './dictionary.service'; import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; +import { ProjectService } from './project.service'; const BLOCK_TIME_VARIANCE = 5000; //ms const DICTIONARY_MAX_QUERY_SIZE = 10000; @@ -40,30 +37,29 @@ const INTERVAL_PERCENT = 0.9; @Injectable() export class FetchService extends BaseFetchService< - AlgorandApiService, AlgorandDataSource, IAlgorandBlockDispatcher, DictionaryService > { constructor( - apiService: AlgorandApiService, + private apiService: AlgorandApiService, nodeConfig: NodeConfig, + @Inject('IProjectService') projectService: ProjectService, @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IBlockDispatcher') blockDispatcher: IAlgorandBlockDispatcher, dictionaryService: DictionaryService, - dsProcessorService: DsProcessorService, + private dsProcessorService: DsProcessorService, dynamicDsService: DynamicDsService, eventEmitter: EventEmitter2, schedulerRegistry: SchedulerRegistry, ) { super( - apiService, nodeConfig, - project, + projectService, + project.network, blockDispatcher, dictionaryService, - dsProcessorService, dynamicDsService, eventEmitter, schedulerRegistry, @@ -74,20 +70,12 @@ export class FetchService extends BaseFetchService< return this.apiService.unsafeApi; } - buildDictionaryQueryEntries(startBlock: number): DictionaryQueryEntry[] { + buildDictionaryQueryEntries( + dataSources: AlgorandDataSource[], + ): DictionaryQueryEntry[] { const queryEntries: DictionaryQueryEntry[] = []; - const dataSources = this.project.dataSources.filter((ds) => - isRuntimeDataSourceV1_0_0(ds), - ); - - // Only run the ds that is equal or less than startBlock - // sort array from lowest ds.startBlock to highest - const filteredDs = dataSources - .concat(this.templateDynamicDatasouces) - .filter((ds) => ds.startBlock <= startBlock) - .sort((a, b) => a.startBlock - b.startBlock); - for (const ds of filteredDs) { + for (const ds of dataSources) { const plugin = isCustomDs(ds) ? this.dsProcessorService.getDsProcessor(ds) : undefined; @@ -167,22 +155,11 @@ export class FetchService extends BaseFetchService< } getModulos(): number[] { - const modulos: number[] = []; - for (const ds of this.project.dataSources) { - if (isCustomDs(ds)) { - continue; - } - for (const handler of ds.mapping.handlers) { - if ( - handler.kind === AlgorandHandlerKind.Block && - handler.filter && - handler.filter.modulo - ) { - modulos.push(handler.filter.modulo); - } - } - } - return modulos; + return getModulos( + this.projectService.getAllDataSources(), + isCustomDs, + AlgorandHandlerKind.Block, + ); } private getBaseHandlerKind( diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index c5372dbf..3f147304 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -21,10 +21,8 @@ import { import { AlgorandBlock, AlgorandCustomDataSource, - AlgorandCustomHandler, AlgorandDataSource, AlgorandTransaction, - RuntimeHandlerInputMap, SafeAPI, } from '@subql/types-algorand'; import { @@ -34,8 +32,7 @@ import { filterBlock, filterTransaction, } from '../algorand'; -import { SubqlProjectDs } from '../configure/SubqueryProject'; -import { yargsOptions } from '../yargs'; +import { AlgorandProjectDs } from '../configure/SubqueryProject'; import { asSecondLayerHandlerProcessor_1_0_0, DsProcessorService, @@ -116,8 +113,8 @@ export class IndexerManager extends BaseIndexerManager< protected async indexBlockData( block: BlockContent, - dataSources: SubqlProjectDs[], - getVM: (d: SubqlProjectDs) => Promise, + dataSources: AlgorandProjectDs[], + getVM: (d: AlgorandProjectDs) => Promise, ): Promise { await this.indexBlockContent(block, dataSources, getVM); for (const tx of block.transactions) { @@ -127,8 +124,8 @@ export class IndexerManager extends BaseIndexerManager< private async indexBlockContent( block: AlgorandBlock, - dataSources: SubqlProjectDs[], - getVM: (d: SubqlProjectDs) => Promise, + dataSources: AlgorandProjectDs[], + getVM: (d: AlgorandProjectDs) => Promise, ): Promise { for (const ds of dataSources) { await this.indexData(AlgorandHandlerKind.Block, block, ds, getVM); @@ -137,8 +134,8 @@ export class IndexerManager extends BaseIndexerManager< private async indexBlockTransactionContent( txn: AlgorandTransaction, - dataSources: SubqlProjectDs[], - getVM: (d: SubqlProjectDs) => Promise, + dataSources: AlgorandProjectDs[], + getVM: (d: AlgorandProjectDs) => Promise, ): Promise { for (const ds of dataSources) { await this.indexData(AlgorandHandlerKind.Transaction, txn, ds, getVM); diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 34c26907..1dfa5ca9 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -10,13 +10,13 @@ import { BaseProjectService, StoreService, NodeConfig, + IProjectUpgradeService, } from '@subql/node-core'; import { Sequelize } from '@subql/x-sequelize'; import { AlgorandApiService } from '../algorand'; import { SubqueryProject, - generateTimestampReferenceForBlockFilters, - SubqlProjectDs, + AlgorandProjectDs, } from '../configure/SubqueryProject'; import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; @@ -28,7 +28,7 @@ const { version: packageVersion } = require('../../package.json'); @Injectable() export class ProjectService extends BaseProjectService< AlgorandApiService, - SubqlProjectDs + AlgorandProjectDs > { protected packageVersion = packageVersion; @@ -40,6 +40,8 @@ export class ProjectService extends BaseProjectService< mmrQueryService: MmrQueryService, sequelize: Sequelize, @Inject('ISubqueryProject') project: SubqueryProject, + @Inject('IProjectUpgradeService') + protected readonly projectUpgradeService: IProjectUpgradeService, storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, @@ -54,6 +56,7 @@ export class ProjectService extends BaseProjectService< mmrQueryService, sequelize, project, + projectUpgradeService, storeService, nodeConfig, dynamicDsService, @@ -62,13 +65,14 @@ export class ProjectService extends BaseProjectService< ); } - protected async generateTimestampReferenceForBlockFilters( - ds: SubqlProjectDs[], - ): Promise { - return generateTimestampReferenceForBlockFilters(ds, this.apiService.api); + protected async getBlockTimestamp(height: number): Promise { + const block = await this.apiService.api.getBlockByHeight(height); + + return new Date(block.timestamp); } - protected getStartBlockDatasources(): SubqlProjectDs[] { - return this.project.dataSources; + protected onProjectChange(project: SubqueryProject): void | Promise { + // TODO update this when implementing skipBlock feature for Eth + // this.apiService.updateBlockFetching(); } } diff --git a/packages/node/src/indexer/worker/worker.service.ts b/packages/node/src/indexer/worker/worker.service.ts index bc42a519..e402bd7c 100644 --- a/packages/node/src/indexer/worker/worker.service.ts +++ b/packages/node/src/indexer/worker/worker.service.ts @@ -1,17 +1,16 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { threadId } from 'node:worker_threads'; import { Inject, Injectable } from '@nestjs/common'; import { NodeConfig, - getLogger, - AutoQueue, - memoryLock, IProjectService, + BaseWorkerService, + IProjectUpgradeService, } from '@subql/node-core'; +import { AlgorandBlock, AlgorandDataSource } from '@subql/types-algorand'; import { AlgorandApiService } from '../../algorand'; -import { SubqlProjectDs } from '../../configure/SubqueryProject'; +import { AlgorandProjectDs } from '../../configure/SubqueryProject'; import { IndexerManager } from '../indexer.manager'; import { BlockContent } from '../types'; @@ -30,84 +29,38 @@ export type WorkerStatusResponse = { toFetchBlocks: number; }; -const logger = getLogger(`Worker Service #${threadId}`); - @Injectable() -export class WorkerService { - private fetchedBlocks: Record = {}; - private _isIndexing = false; - - private queue: AutoQueue; - +export class WorkerService extends BaseWorkerService< + BlockContent, + FetchBlockResponse, + AlgorandDataSource, + {} +> { constructor( private apiService: AlgorandApiService, private indexerManager: IndexerManager, @Inject('IProjectService') - private projectService: IProjectService, + projectService: IProjectService, + @Inject('IProjectUpgradeService') + projectUpgradeService: IProjectUpgradeService, nodeConfig: NodeConfig, ) { - this.queue = new AutoQueue(undefined, nodeConfig.batchSize); - } - - async fetchBlock(height: number): Promise { - try { - return await this.queue.put(async () => { - // If a dynamic ds is created we might be asked to fetch blocks again, use existing result - if (!this.fetchedBlocks[height]) { - if (memoryLock.isLocked()) { - const start = Date.now(); - await memoryLock.waitForUnlock(); - const end = Date.now(); - logger.debug(`memory lock wait time: ${end - start}ms`); - } - - const [block] = await this.apiService.fetchBlocks([height]); - this.fetchedBlocks[height] = block; - } - - const block = this.fetchedBlocks[height]; - // Return info to get the runtime version, this lets the worker thread know - return { - parentHash: block.previousBlockHash, - }; - }); - } catch (e) { - logger.error(e, `Failed to fetch block ${height}`); - } + super(projectService, projectUpgradeService, nodeConfig); } - async processBlock(height: number): Promise { - try { - this._isIndexing = true; - const block = this.fetchedBlocks[height]; - - if (!block) { - throw new Error(`Block ${height} has not been fetched`); - } - - delete this.fetchedBlocks[height]; - - return await this.indexerManager.indexBlock( - block, - await this.projectService.getAllDataSources(height), - ); - } catch (e) { - logger.error(e, `Failed to index block ${height}: ${e.stack}`); - throw e; - } finally { - this._isIndexing = false; - } + protected async fetchChainBlock(heights: number): Promise { + const [block] = await this.apiService.fetchBlocks([heights]); + return block; } - - get numFetchedBlocks(): number { - return Object.keys(this.fetchedBlocks).length; + protected toBlockResponse(block: AlgorandBlock): { parentHash: string } { + return { + parentHash: block.previousBlockHash, + }; } - - get numFetchingBlocks(): number { - return this.queue.size; - } - - get isIndexing(): boolean { - return this._isIndexing; + protected async processFetchedBlock( + block: AlgorandBlock, + dataSources: AlgorandDataSource[], + ): Promise { + return this.indexerManager.indexBlock(block, dataSources); } } diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index 8150876f..8cfc809d 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -16,51 +16,25 @@ initLogger( argv.logLevel as string | undefined, ); -import assert from 'assert'; import { threadId } from 'node:worker_threads'; -import { getHeapStatistics } from 'v8'; -import { INestApplication } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { - waitForBatchSize, - WorkerHost, getLogger, NestLogger, - hostStoreKeys, - HostStore, - hostDynamicDsKeys, - HostDynamicDS, - ProcessBlockResponse, - HostConnectionPoolState, - hostConnectionPoolStateKeys, + createWorkerHost, + IBaseIndexerWorker, + initWorkerServices, } from '@subql/node-core'; -import { AlgorandApiConnection } from '../../algorand'; -import { SubqlProjectDs } from '../../configure/SubqueryProject'; import { IndexerManager } from '../indexer.manager'; import { WorkerModule } from './worker.module'; -import { - FetchBlockResponse, - WorkerService, - WorkerStatusResponse, -} from './worker.service'; -import { - HostUnfinalizedBlocks, - hostUnfinalizedBlocksKeys, -} from './worker.unfinalizedBlocks.service'; -let app: INestApplication; -let workerService: WorkerService; +import { WorkerService } from './worker.service'; const logger = getLogger(`worker #${threadId}`); async function initWorker(): Promise { try { - if (app) { - logger.warn('Worker already initialised'); - return; - } - - app = await NestFactory.create(WorkerModule, { - logger: new NestLogger(), // TIP: If the worker is crashing comment out this line for better logging + const app = await NestFactory.create(WorkerModule, { + logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging }); await app.init(); @@ -69,7 +43,9 @@ async function initWorker(): Promise { // Initialise async services, we do this here rather than in factories so we can capture one off events await indexerManager.start(); - workerService = app.get(WorkerService); + const workerService = app.get(WorkerService); + + initWorkerServices(app, workerService); } catch (e) { console.log('Failed to start worker', e); logger.error(e, 'Failed to start worker'); @@ -77,102 +53,12 @@ async function initWorker(): Promise { } } -async function fetchBlock(height: number): Promise { - assert(workerService, 'Not initialised'); - - return workerService.fetchBlock(height); -} - -async function processBlock(height: number): Promise { - assert(workerService, 'Not initialised'); - - return workerService.processBlock(height); -} - -// eslint-disable-next-line @typescript-eslint/require-await -async function numFetchedBlocks(): Promise { - return workerService.numFetchedBlocks; -} - -// eslint-disable-next-line @typescript-eslint/require-await -async function numFetchingBlocks(): Promise { - return workerService.numFetchingBlocks; -} - -// eslint-disable-next-line @typescript-eslint/require-await -async function getStatus(): Promise { - return { - threadId, - fetchedBlocks: workerService.numFetchedBlocks, - toFetchBlocks: workerService.numFetchingBlocks, - isIndexing: workerService.isIndexing, - }; -} - -// eslint-disable-next-line @typescript-eslint/require-await -async function getMemoryLeft(): Promise { - const totalHeap = getHeapStatistics().heap_size_limit; - const heapUsed = process.memoryUsage().heapUsed; - - return totalHeap - heapUsed; -} - -async function waitForWorkerBatchSize(heapSizeInBytes: number): Promise { - await waitForBatchSize(heapSizeInBytes); -} - -// Register these functions to be exposed to worker host -(global as any).host = WorkerHost.create< - HostStore & - HostDynamicDS & - HostUnfinalizedBlocks & - HostConnectionPoolState, - IInitIndexerWorker ->( - [ - ...hostStoreKeys, - ...hostDynamicDsKeys, - ...hostUnfinalizedBlocksKeys, - ...hostConnectionPoolStateKeys, - ], - { - initWorker, - fetchBlock, - processBlock, - numFetchedBlocks, - numFetchingBlocks, - getStatus, - getMemoryLeft, - waitForWorkerBatchSize, - }, - logger, -); - -// Export types to be used on the parent -type InitWorker = typeof initWorker; -type FetchBlock = typeof fetchBlock; -type ProcessBlock = typeof processBlock; -type NumFetchedBlocks = typeof numFetchedBlocks; -type NumFetchingBlocks = typeof numFetchingBlocks; -type GetWorkerStatus = typeof getStatus; -type GetMemoryLeft = typeof getMemoryLeft; -type WaitForWorkerBatchSize = typeof waitForWorkerBatchSize; - -export type IIndexerWorker = { - processBlock: ProcessBlock; - fetchBlock: FetchBlock; - numFetchedBlocks: NumFetchedBlocks; - numFetchingBlocks: NumFetchingBlocks; - getStatus: GetWorkerStatus; - getMemoryLeft: GetMemoryLeft; - waitForWorkerBatchSize: WaitForWorkerBatchSize; -}; +export type IIndexerWorker = IBaseIndexerWorker; export type IInitIndexerWorker = IIndexerWorker & { - initWorker: InitWorker; + initWorker: typeof initWorker; }; -process.on('uncaughtException', (e) => { - logger.error(e, 'Uncaught Exception'); - throw e; +(global as any).host = createWorkerHost([], { + initWorker, }); diff --git a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts index fb525abb..aba72e42 100644 --- a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts @@ -2,18 +2,14 @@ // SPDX-License-Identifier: GPL-3.0 import { Injectable } from '@nestjs/common'; -import { IUnfinalizedBlocksService, Header } from '@subql/node-core'; +import { + IUnfinalizedBlocksService, + Header, + HostUnfinalizedBlocks, +} from '@subql/node-core'; import { BlockContent } from '../types'; import { algorandBlockToHeader } from '../unfinalizedBlocks.service'; -export type HostUnfinalizedBlocks = { - unfinalizedBlocksProcess: (header: Header) => Promise; -}; - -export const hostUnfinalizedBlocksKeys: (keyof HostUnfinalizedBlocks)[] = [ - 'unfinalizedBlocksProcess', -]; - @Injectable() export class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService diff --git a/packages/node/src/init.ts b/packages/node/src/init.ts index 190cae88..d087a132 100644 --- a/packages/node/src/init.ts +++ b/packages/node/src/init.ts @@ -4,7 +4,6 @@ import { NestFactory } from '@nestjs/core'; import { findAvailablePort } from '@subql/common'; import { getLogger, NestLogger } from '@subql/node-core'; -import { AlgorandApiService } from './algorand'; import { AppModule } from './app.module'; import { FetchService } from './indexer/fetch.service'; import { yargsOptions } from './yargs'; @@ -42,7 +41,7 @@ export async function bootstrap(): Promise { try { const app = await NestFactory.create(AppModule, { - logger: debug ? new NestLogger() : false, + logger: new NestLogger(debug), }); await app.init(); diff --git a/packages/node/src/main.ts b/packages/node/src/main.ts index 4bbbfb2d..4c1c50a3 100644 --- a/packages/node/src/main.ts +++ b/packages/node/src/main.ts @@ -13,16 +13,9 @@ initLogger( argv.logLevel as string | undefined, ); -// Lazy import, to allow logger to be initialised before bootstrap() -// As bootstrap runs services that requires logger -const { bootstrap } = require('./init'); -if ( - !( - argv._[0] === 'test' || - argv._[0] === 'mmr-migrate' || - argv._[0] === 'mmr-regen' || - argv._[0] === 'force-clean' - ) -) { +if (argv._[0] === undefined) { + // Lazy import, to allow logger to be initialised before bootstrap() + // As bootstrap runs services that requires logger + const { bootstrap } = require('./init'); void bootstrap(); } diff --git a/packages/node/src/subcommands/reindex.init.ts b/packages/node/src/subcommands/reindex.init.ts index 32310792..5d1e9e15 100644 --- a/packages/node/src/subcommands/reindex.init.ts +++ b/packages/node/src/subcommands/reindex.init.ts @@ -2,9 +2,8 @@ // SPDX-License-Identifier: GPL-3.0 import { NestFactory } from '@nestjs/core'; -import { getLogger } from '@subql/node-core'; +import { getLogger, ReindexService } from '@subql/node-core'; import { ReindexModule } from './reindex.module'; -import { ReindexService } from './reindex.service'; const logger = getLogger('CLI-Reindex'); export async function reindexInit(targetHeight: number): Promise { diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts index 9de149da..0e1dfdbd 100644 --- a/packages/node/src/subcommands/reindex.module.ts +++ b/packages/node/src/subcommands/reindex.module.ts @@ -9,6 +9,7 @@ import { ForceCleanService, MmrService, PgMmrCacheService, + ReindexService, StoreCacheService, StoreService, } from '@subql/node-core'; @@ -16,7 +17,6 @@ import { AlgorandApiService } from '../algorand'; import { ConfigureModule } from '../configure/configure.module'; import { DsProcessorService } from '../indexer/ds-processor.service'; import { DynamicDsService } from '../indexer/dynamic-ds.service'; -import { ReindexService } from './reindex.service'; @Module({ providers: [ diff --git a/packages/node/src/subcommands/reindex.service.ts b/packages/node/src/subcommands/reindex.service.ts deleted file mode 100644 index 7b472e46..00000000 --- a/packages/node/src/subcommands/reindex.service.ts +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Inject, Injectable } from '@nestjs/common'; -import { - getLogger, - MmrService, - NodeConfig, - StoreService, - ForceCleanService, - BaseReindexService, -} from '@subql/node-core'; -import { AlgorandDataSource } from '@subql/types-algorand'; -import { Sequelize } from '@subql/x-sequelize'; -import { SubqueryProject } from '../configure/SubqueryProject'; -import { DynamicDsService } from '../indexer/dynamic-ds.service'; -import { BlockContent } from '../indexer/types'; -import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; - -const logger = getLogger('Reindex'); - -@Injectable() -export class ReindexService extends BaseReindexService< - SubqueryProject, - AlgorandDataSource, - BlockContent -> { - constructor( - sequelize: Sequelize, - nodeConfig: NodeConfig, - storeService: StoreService, - mmrService: MmrService, - @Inject('ISubqueryProject') project: SubqueryProject, - forceCleanService: ForceCleanService, - unfinalizedBlocksService: UnfinalizedBlocksService, - dynamicDsService: DynamicDsService, - ) { - super( - sequelize, - nodeConfig, - storeService, - mmrService, - project, - forceCleanService, - unfinalizedBlocksService, - dynamicDsService, - ); - } - - // eslint-disable-next-line @typescript-eslint/require-await - protected async getStartBlockDatasources(): Promise { - return this.project.dataSources; - } -} diff --git a/packages/node/src/subcommands/testing.init.ts b/packages/node/src/subcommands/testing.init.ts index fa1053df..2471b069 100644 --- a/packages/node/src/subcommands/testing.init.ts +++ b/packages/node/src/subcommands/testing.init.ts @@ -8,11 +8,9 @@ import { TestingService } from './testing.service'; const logger = getLogger('Testing'); export async function testingInit(): Promise { try { - const { config, project } = await ConfigureModule.getInstance(); - const subqueryProject = await project(); + const { nodeConfig, project } = await ConfigureModule.getInstance(); - const testingService = new TestingService(config, subqueryProject); - await testingService.init(); + const testingService = new TestingService(nodeConfig, project); await testingService.run(); } catch (e) { logger.error(e, 'Testing failed'); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 2047fded..43795070 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -6,6 +6,7 @@ import { EventEmitter2, EventEmitterModule } from '@nestjs/event-emitter'; import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule'; import { ConnectionPoolService, + ConnectionPoolStateManager, DbModule, NodeConfig, PoiService, @@ -36,6 +37,7 @@ import { TestingService } from './testing.service'; DynamicDsService, ProjectService, UnfinalizedBlocksService, + ConnectionPoolStateManager, ConnectionPoolService, { provide: 'IProjectService', diff --git a/packages/node/src/subcommands/testing.service.ts b/packages/node/src/subcommands/testing.service.ts index ecc21252..de05756e 100644 --- a/packages/node/src/subcommands/testing.service.ts +++ b/packages/node/src/subcommands/testing.service.ts @@ -10,7 +10,10 @@ import { TestRunner, } from '@subql/node-core'; import { AlgorandApi, AlgorandApiService, SafeAPIService } from '../algorand'; -import { SubqlProjectDs, SubqueryProject } from '../configure/SubqueryProject'; +import { + AlgorandProjectDs, + SubqueryProject, +} from '../configure/SubqueryProject'; import { IndexerManager } from '../indexer/indexer.manager'; import { ProjectService } from '../indexer/project.service'; import { BlockContent } from '../indexer/types'; @@ -21,7 +24,7 @@ export class TestingService extends BaseTestingService< AlgorandApi, SafeAPIService, BlockContent, - SubqlProjectDs + AlgorandProjectDs > { constructor( nodeConfig: NodeConfig, @@ -31,7 +34,7 @@ export class TestingService extends BaseTestingService< } async getTestRunner(): Promise< - TestRunner + TestRunner > { const testContext = await NestFactory.createApplicationContext( TestingModule, diff --git a/packages/node/src/utils/reindex.ts b/packages/node/src/utils/reindex.ts deleted file mode 100644 index 7cbedfac..00000000 --- a/packages/node/src/utils/reindex.ts +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { getLogger, MmrService, StoreService } from '@subql/node-core'; -import { Sequelize } from '@subql/x-sequelize'; -import { DynamicDsService } from '../indexer/dynamic-ds.service'; -import { ForceCleanService } from '../subcommands/forceClean.service'; - -const logger = getLogger('Reindex'); - -export async function reindex( - startHeight: number, - blockOffset: number | undefined, - targetBlockHeight: number, - lastProcessedHeight: number, - storeService: StoreService, - dynamicDsService: DynamicDsService, - mmrService: MmrService, - sequelize: Sequelize, - forceCleanService?: ForceCleanService, -): Promise { - if (!lastProcessedHeight || lastProcessedHeight < targetBlockHeight) { - logger.warn( - `Skipping reindexing to block ${targetBlockHeight}: current indexing height ${lastProcessedHeight} is behind requested block`, - ); - return; - } - - // if startHeight is greater than the targetHeight, just force clean - if (targetBlockHeight < startHeight) { - logger.info( - `targetHeight: ${targetBlockHeight} is less than startHeight: ${startHeight}. Hence executing force-clean`, - ); - if (!forceCleanService) { - logger.error('ForceCleanService not provided, cannot force clean'); - process.exit(1); - } - await forceCleanService.forceClean(); - } else { - logger.info(`Reindexing to block: ${targetBlockHeight}`); - const transaction = await sequelize.transaction(); - try { - await Promise.all([ - storeService.rewind(targetBlockHeight, transaction), - dynamicDsService.resetDynamicDatasource(targetBlockHeight), - ]); - - if (blockOffset) { - await mmrService.deleteMmrNode(targetBlockHeight + 1, blockOffset); - } - - // Flush metadata changes from above Promise.all - await storeService.storeCache.metadata.flush(transaction); - - await transaction.commit(); - logger.info('Reindex Success'); - } catch (err) { - logger.error(err, 'Reindexing failed'); - await transaction.rollback(); - throw err; - } - } -} diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index 12170ff9..acb6183a 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -221,9 +221,9 @@ export const yargsOptions = yargs(hideBin(process.argv)) }, 'disable-historical': { demandOption: false, - default: false, describe: 'Disable storing historical state entities', type: 'boolean', + // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig }, ipfs: { demandOption: false, @@ -391,14 +391,15 @@ export const yargsOptions = yargs(hideBin(process.argv)) }, 'unfinalized-blocks': { demandOption: false, - default: false, describe: 'Enable to fetch and index unfinalized blocks', type: 'boolean', + // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig }, unsafe: { type: 'boolean', demandOption: false, describe: 'Allows usage of any built-in module within the sandbox', + // NOTE: don't set a default for this. It will break apply args from manifest. The default should be set in NodeConfig }, workers: { alias: 'w', diff --git a/yarn.lock b/yarn.lock index eaac75d4..7f0a1e71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1925,7 +1925,7 @@ __metadata: languageName: node linkType: hard -"@ipld/dag-cbor@npm:^6.0.5": +"@ipld/dag-cbor@npm:^6.0.3, @ipld/dag-cbor@npm:^6.0.5": version: 6.0.15 resolution: "@ipld/dag-cbor@npm:6.0.15" dependencies: @@ -1935,6 +1935,26 @@ __metadata: languageName: node linkType: hard +"@ipld/dag-cbor@npm:^7.0.0": + version: 7.0.3 + resolution: "@ipld/dag-cbor@npm:7.0.3" + dependencies: + cborg: ^1.6.0 + multiformats: ^9.5.4 + checksum: c0c59907ab6146a214c1ecb2341cc02904bc952255e15544554990690f7841380a87636d5937aaa23e9004b1c141e90238277d088ed6932b5b0e6d2e6ee1fe02 + languageName: node + linkType: hard + +"@ipld/dag-json@npm:^8.0.1": + version: 8.0.11 + resolution: "@ipld/dag-json@npm:8.0.11" + dependencies: + cborg: ^1.5.4 + multiformats: ^9.5.4 + checksum: 5ce25e4ed4004839a0dc18a51b09d0e2bda02a00bc15e8066809ddcedf5927ef8829a7dacaaf71ba0eb1c8699599130389af6d137da1d6f524394f5ddb0763f0 + languageName: node + linkType: hard + "@ipld/dag-pb@npm:^2.1.3": version: 2.1.18 resolution: "@ipld/dag-pb@npm:2.1.18" @@ -2848,6 +2868,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/is@npm:^0.14.0": + version: 0.14.0 + resolution: "@sindresorhus/is@npm:0.14.0" + checksum: 971e0441dd44ba3909b467219a5e242da0fc584048db5324cfb8048148fa8dcc9d44d71e3948972c4f6121d24e5da402ef191420d1266a95f713bb6d6e59c98a + languageName: node + linkType: hard + "@sinonjs/commons@npm:^1.7.0": version: 1.8.6 resolution: "@sinonjs/commons@npm:1.8.6" @@ -2866,9 +2893,9 @@ __metadata: languageName: node linkType: hard -"@subql/apollo-links@npm:^0.5.3": - version: 0.5.5 - resolution: "@subql/apollo-links@npm:0.5.5" +"@subql/apollo-links@npm:^0.5.8": + version: 0.5.8 + resolution: "@subql/apollo-links@npm:0.5.8" dependencies: "@metamask/eth-sig-util": 5.1.0 apollo-link-error: ^1.1.13 @@ -2879,7 +2906,7 @@ __metadata: peerDependencies: "@apollo/client": "*" graphql: "*" - checksum: 9075c7e52a08ce7e60e466f16364cf7cc66965544765079c0a6ad879c7edc818c5e92e0c3bdd4fd9191091335e02563d9676ff79546b712269ba08db77df8f8e + checksum: 34366e720bfe5c5dece380b93ec61d754f0bc804d07ff43e46aaca792f26f202b112a47cc2729a43e55181190b322aa1665618162f94add5cb341fb553f22116 languageName: node linkType: hard @@ -2887,7 +2914,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": ^2.4.0 + "@subql/common": dev "@subql/types-algorand": "workspace:*" "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 @@ -2906,19 +2933,37 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:2.4.0, @subql/common@npm:^2.4.0": - version: 2.4.0 - resolution: "@subql/common@npm:2.4.0" +"@subql/common@npm:2.6.1-2": + version: 2.6.1-2 + resolution: "@subql/common@npm:2.6.1-2" dependencies: axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 fs-extra: ^10.1.0 - ipfs-http-client: ^52.0.3 + ipfs-http-client: 56 js-yaml: ^4.1.0 reflect-metadata: ^0.1.13 semver: ^7.5.2 - checksum: 921cdc4740194fb21b6b204b000d8b95b5b11a0c696c9152503c5e70f31edf34106518372debbaf6f6892fe76c5de5275d4c37ce87331a361c1a1471575b9421 + update-notifier: 5.1.0 + checksum: fba2a5141977a2c66a2191c7b0abb9cf95114e940731a590a881e08caff91565fb708e013185dccd29950cf97cdf89ed90ff5489c2f137e5e0f8134e5945e95c + languageName: node + linkType: hard + +"@subql/common@npm:dev": + version: 2.6.1-3 + resolution: "@subql/common@npm:2.6.1-3" + dependencies: + axios: ^0.27.2 + class-transformer: ^0.5.1 + class-validator: ^0.14.0 + fs-extra: ^10.1.0 + ipfs-http-client: 56 + js-yaml: ^4.1.0 + reflect-metadata: ^0.1.13 + semver: ^7.5.2 + update-notifier: 5.1.0 + checksum: 397c21df879b4310b4aab0f4108b942a53c069d7186cff196ef0cd245ce72e357bd3f6f5891256c9850354f813aac349a10bf44a906f0b72d44619fb27cdc39a languageName: node linkType: hard @@ -2934,9 +2979,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^2.4.0 + "@subql/common": dev "@subql/common-algorand": "workspace:*" - "@subql/node-core": ^4.0.1 + "@subql/node-core": 4.2.4-2 "@subql/types-algorand": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -2966,18 +3011,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:^4.0.1": - version: 4.0.1 - resolution: "@subql/node-core@npm:4.0.1" +"@subql/node-core@npm:4.2.4-2": + version: 4.2.4-2 + resolution: "@subql/node-core@npm:4.2.4-2" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 - "@subql/apollo-links": ^0.5.3 - "@subql/common": 2.4.0 + "@subql/apollo-links": ^0.5.8 + "@subql/common": 2.6.1-2 "@subql/testing": 2.0.2 - "@subql/types": 2.1.3 + "@subql/types": 2.2.1-0 "@subql/utils": 2.4.3 "@subql/x-merkle-mountain-range": ^2.0.0-0.1.3 "@subql/x-sequelize": 6.32.0-0.0.2 @@ -2994,7 +3039,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: fc1f389b5f06dfde74acfcfa3f205c4be27742547c29dd74194c1fe96a0f63fd60546baa3c2f12c74d48d5958269cd22ced8b6bc45e9cd4db1feb677776b3327 + checksum: ebaf6aa24a0b267a4304a49a4f24539d4ff3d7f3f5eb94c7789b792f2cbb6af52d55ff40b731617f5bc872c8bf995882605a7ccefb24a13a9e6733d158dec802 languageName: node linkType: hard @@ -3018,7 +3063,16 @@ __metadata: languageName: unknown linkType: soft -"@subql/types@npm:2.1.3, @subql/types@npm:^2.1.3": +"@subql/types@npm:2.2.1-0": + version: 2.2.1-0 + resolution: "@subql/types@npm:2.2.1-0" + peerDependencies: + "@polkadot/api": ^10 + checksum: 48639f5d3dc4b1ad103df8769744c2cc8fd7db6c27e0a41939c2576ff943ba3606a4fe3d7b97b7cce383add7b471504b1614f3134faed4c6422d93b169de9198 + languageName: node + linkType: hard + +"@subql/types@npm:^2.1.3": version: 2.1.3 resolution: "@subql/types@npm:2.1.3" peerDependencies: @@ -3115,6 +3169,15 @@ __metadata: languageName: node linkType: hard +"@szmarczak/http-timer@npm:^1.1.2": + version: 1.1.2 + resolution: "@szmarczak/http-timer@npm:1.1.2" + dependencies: + defer-to-connect: ^1.0.1 + checksum: 4d9158061c5f397c57b4988cde33a163244e4f02df16364f103971957a32886beb104d6180902cbe8b38cb940e234d9f98a4e486200deca621923f62f50a06fe + languageName: node + linkType: hard + "@tootallnate/once@npm:1": version: 1.1.2 resolution: "@tootallnate/once@npm:1.1.2" @@ -3349,6 +3412,15 @@ __metadata: languageName: node linkType: hard +"@types/keyv@npm:^3.1.1": + version: 3.1.4 + resolution: "@types/keyv@npm:3.1.4" + dependencies: + "@types/node": "*" + checksum: e009a2bfb50e90ca9b7c6e8f648f8464067271fd99116f881073fa6fa76dc8d0133181dd65e6614d5fb1220d671d67b0124aef7d97dc02d7e342ab143a47779d + languageName: node + linkType: hard + "@types/lodash@npm:^4.14.178": version: 4.14.194 resolution: "@types/lodash@npm:4.14.194" @@ -3467,6 +3539,15 @@ __metadata: languageName: node linkType: hard +"@types/responselike@npm:^1.0.0": + version: 1.0.0 + resolution: "@types/responselike@npm:1.0.0" + dependencies: + "@types/node": "*" + checksum: e99fc7cc6265407987b30deda54c1c24bb1478803faf6037557a774b2f034c5b097ffd65847daa87e82a61a250d919f35c3588654b0fdaa816906650f596d1b0 + languageName: node + linkType: hard + "@types/semver@npm:^7, @types/semver@npm:^7.3.12": version: 7.5.0 resolution: "@types/semver@npm:7.5.0" @@ -3946,6 +4027,15 @@ __metadata: languageName: node linkType: hard +"ansi-align@npm:^3.0.0": + version: 3.0.1 + resolution: "ansi-align@npm:3.0.1" + dependencies: + string-width: ^4.1.0 + checksum: 6abfa08f2141d231c257162b15292467081fa49a208593e055c866aa0455b57f3a86b5a678c190c618faa79b4c59e254493099cb700dd9cf2293c6be2c8f5d8d + languageName: node + linkType: hard + "ansi-escapes@npm:^4.2.1, ansi-escapes@npm:^4.3.0": version: 4.3.2 resolution: "ansi-escapes@npm:4.3.2" @@ -4011,6 +4101,13 @@ __metadata: languageName: node linkType: hard +"any-signal@npm:^3.0.0": + version: 3.0.1 + resolution: "any-signal@npm:3.0.1" + checksum: 073eb14c365b7552f9f16fbf36cd76171e4a0fe156a8faa865fe1d5ac4ed2f5c5ab6e3faad0ac0d4c69511b5892971c5573baa8a1cbf85fe250d0c54ff0734ff + languageName: node + linkType: hard + "anymatch@npm:^3.0.3, anymatch@npm:~3.1.2": version: 3.1.3 resolution: "anymatch@npm:3.1.3" @@ -4496,6 +4593,22 @@ __metadata: languageName: node linkType: hard +"boxen@npm:^5.0.0": + version: 5.1.2 + resolution: "boxen@npm:5.1.2" + dependencies: + ansi-align: ^3.0.0 + camelcase: ^6.2.0 + chalk: ^4.1.0 + cli-boxes: ^2.2.1 + string-width: ^4.2.2 + type-fest: ^0.20.2 + widest-line: ^3.1.0 + wrap-ansi: ^7.0.0 + checksum: 82d03e42a72576ff235123f17b7c505372fe05c83f75f61e7d4fa4bcb393897ec95ce766fecb8f26b915f0f7a7227d66e5ec7cef43f5b2bd9d3aeed47ec55877 + languageName: node + linkType: hard + "brace-expansion@npm:^1.1.7": version: 1.1.11 resolution: "brace-expansion@npm:1.1.11" @@ -4538,7 +4651,7 @@ __metadata: languageName: node linkType: hard -"browser-readablestream-to-it@npm:^1.0.1, browser-readablestream-to-it@npm:^1.0.3": +"browser-readablestream-to-it@npm:^1.0.0, browser-readablestream-to-it@npm:^1.0.1, browser-readablestream-to-it@npm:^1.0.3": version: 1.0.3 resolution: "browser-readablestream-to-it@npm:1.0.3" checksum: 07895bbc54cdeea62c8e9b7e32d374ec5c340ed1d0bc0c6cd6f1e0561ad931b160a3988426c763672ddf38ac1f75e45b9d8ae267b43f387183edafcad625f30a @@ -4653,6 +4766,21 @@ __metadata: languageName: node linkType: hard +"cacheable-request@npm:^6.0.0": + version: 6.1.0 + resolution: "cacheable-request@npm:6.1.0" + dependencies: + clone-response: ^1.0.2 + get-stream: ^5.1.0 + http-cache-semantics: ^4.0.0 + keyv: ^3.0.0 + lowercase-keys: ^2.0.0 + normalize-url: ^4.1.0 + responselike: ^1.0.2 + checksum: b510b237b18d17e89942e9ee2d2a077cb38db03f12167fd100932dfa8fc963424bfae0bfa1598df4ae16c944a5484e43e03df8f32105b04395ee9495e9e4e9f1 + languageName: node + linkType: hard + "call-bind@npm:^1.0.0, call-bind@npm:^1.0.2": version: 1.0.2 resolution: "call-bind@npm:1.0.2" @@ -4700,6 +4828,15 @@ __metadata: languageName: node linkType: hard +"cborg@npm:^1.6.0": + version: 1.10.2 + resolution: "cborg@npm:1.10.2" + bin: + cborg: cli.js + checksum: 7743a8f125046ac27fb371c4ea18af54fbe853f7210f1ffacc6504a79566480c39d52ac4fbc1a5b5155e27b13c3b58955dc29db1bf20c4d651549d55fec2fa7f + languageName: node + linkType: hard + "chalk@npm:^2.0.0": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -4764,6 +4901,13 @@ __metadata: languageName: node linkType: hard +"ci-info@npm:^2.0.0": + version: 2.0.0 + resolution: "ci-info@npm:2.0.0" + checksum: 3b374666a85ea3ca43fa49aa3a048d21c9b475c96eb13c133505d2324e7ae5efd6a454f41efe46a152269e9b6a00c9edbe63ec7fa1921957165aae16625acd67 + languageName: node + linkType: hard + "ci-info@npm:^3.2.0": version: 3.8.0 resolution: "ci-info@npm:3.8.0" @@ -4820,6 +4964,13 @@ __metadata: languageName: node linkType: hard +"cli-boxes@npm:^2.2.1": + version: 2.2.1 + resolution: "cli-boxes@npm:2.2.1" + checksum: be79f8ec23a558b49e01311b39a1ea01243ecee30539c880cf14bf518a12e223ef40c57ead0cb44f509bffdffc5c129c746cd50d863ab879385370112af4f585 + languageName: node + linkType: hard + "cli-cursor@npm:^3.1.0": version: 3.1.0 resolution: "cli-cursor@npm:3.1.0" @@ -4867,6 +5018,15 @@ __metadata: languageName: node linkType: hard +"clone-response@npm:^1.0.2": + version: 1.0.3 + resolution: "clone-response@npm:1.0.3" + dependencies: + mimic-response: ^1.0.0 + checksum: 4e671cac39b11c60aa8ba0a450657194a5d6504df51bca3fac5b3bd0145c4f8e8464898f87c8406b83232e3bc5cca555f51c1f9c8ac023969ebfbf7f6bdabb2e + languageName: node + linkType: hard + "clone@npm:^1.0.2": version: 1.0.4 resolution: "clone@npm:1.0.4" @@ -4971,6 +5131,20 @@ __metadata: languageName: node linkType: hard +"configstore@npm:^5.0.1": + version: 5.0.1 + resolution: "configstore@npm:5.0.1" + dependencies: + dot-prop: ^5.2.0 + graceful-fs: ^4.1.2 + make-dir: ^3.0.0 + unique-string: ^2.0.0 + write-file-atomic: ^3.0.0 + xdg-basedir: ^4.0.0 + checksum: 60ef65d493b63f96e14b11ba7ec072fdbf3d40110a94fb7199d1c287761bdea5c5244e76b2596325f30c1b652213aa75de96ea20afd4a5f82065e61ea090988e + languageName: node + linkType: hard + "consola@npm:^2.15.0": version: 2.15.3 resolution: "consola@npm:2.15.3" @@ -5094,6 +5268,13 @@ __metadata: languageName: node linkType: hard +"crypto-random-string@npm:^2.0.0": + version: 2.0.0 + resolution: "crypto-random-string@npm:2.0.0" + checksum: 0283879f55e7c16fdceacc181f87a0a65c53bc16ffe1d58b9d19a6277adcd71900d02bb2c4843dd55e78c51e30e89b0fec618a7f170ebcc95b33182c28f05fd6 + languageName: node + linkType: hard + "cssom@npm:^0.4.4": version: 0.4.4 resolution: "cssom@npm:0.4.4" @@ -5117,6 +5298,16 @@ __metadata: languageName: node linkType: hard +"dag-jose@npm:^1.0.0": + version: 1.0.0 + resolution: "dag-jose@npm:1.0.0" + dependencies: + "@ipld/dag-cbor": ^6.0.3 + multiformats: ^9.0.2 + checksum: 2d95e938f08c383718dd6f07d92063f164bb85be831f624f122687ca3f4d365a76320c3f6f16cfbe013c581198504728abd226567775ef35fb40176e1cb23112 + languageName: node + linkType: hard + "data-urls@npm:^2.0.0": version: 2.0.0 resolution: "data-urls@npm:2.0.0" @@ -5172,6 +5363,15 @@ __metadata: languageName: node linkType: hard +"decompress-response@npm:^3.3.0": + version: 3.3.0 + resolution: "decompress-response@npm:3.3.0" + dependencies: + mimic-response: ^1.0.0 + checksum: 952552ac3bd7de2fc18015086b09468645c9638d98a551305e485230ada278c039c91116e946d07894b39ee53c0f0d5b6473f25a224029344354513b412d7380 + languageName: node + linkType: hard + "dedent@npm:^0.7.0": version: 0.7.0 resolution: "dedent@npm:0.7.0" @@ -5179,6 +5379,13 @@ __metadata: languageName: node linkType: hard +"deep-extend@npm:^0.6.0": + version: 0.6.0 + resolution: "deep-extend@npm:0.6.0" + checksum: 7be7e5a8d468d6b10e6a67c3de828f55001b6eb515d014f7aeb9066ce36bd5717161eb47d6a0f7bed8a9083935b465bc163ee2581c8b128d29bf61092fdf57a7 + languageName: node + linkType: hard + "deep-is@npm:^0.1.3, deep-is@npm:~0.1.3": version: 0.1.4 resolution: "deep-is@npm:0.1.4" @@ -5202,6 +5409,13 @@ __metadata: languageName: node linkType: hard +"defer-to-connect@npm:^1.0.1": + version: 1.1.3 + resolution: "defer-to-connect@npm:1.1.3" + checksum: 9491b301dcfa04956f989481ba7a43c2231044206269eb4ab64a52d6639ee15b1252262a789eb4239fb46ab63e44d4e408641bae8e0793d640aee55398cb3930 + languageName: node + linkType: hard + "deferred-leveldown@npm:~5.3.0": version: 5.3.0 resolution: "deferred-leveldown@npm:5.3.0" @@ -5338,6 +5552,15 @@ __metadata: languageName: node linkType: hard +"dot-prop@npm:^5.2.0": + version: 5.3.0 + resolution: "dot-prop@npm:5.3.0" + dependencies: + is-obj: ^2.0.0 + checksum: d5775790093c234ef4bfd5fbe40884ff7e6c87573e5339432870616331189f7f5d86575c5b5af2dcf0f61172990f4f734d07844b1f23482fff09e3c4bead05ea + languageName: node + linkType: hard + "dotenv@npm:^15.0.1": version: 15.0.1 resolution: "dotenv@npm:15.0.1" @@ -5352,6 +5575,13 @@ __metadata: languageName: node linkType: hard +"duplexer3@npm:^0.1.4": + version: 0.1.5 + resolution: "duplexer3@npm:0.1.5" + checksum: e677cb4c48f031ca728601d6a20bf6aed4c629d69ef9643cb89c67583d673c4ec9317cc6427501f38bd8c368d3a18f173987cc02bd99d8cf8fe3d94259a22a20 + languageName: node + linkType: hard + "eastasianwidth@npm:^0.2.0": version: 0.2.0 resolution: "eastasianwidth@npm:0.2.0" @@ -5586,6 +5816,13 @@ __metadata: languageName: node linkType: hard +"escape-goat@npm:^2.0.0": + version: 2.1.1 + resolution: "escape-goat@npm:2.1.1" + checksum: ce05c70c20dd7007b60d2d644b625da5412325fdb57acf671ba06cb2ab3cd6789e2087026921a05b665b0a03fadee2955e7fc0b9a67da15a6551a980b260eba7 + languageName: node + linkType: hard + "escape-html@npm:~1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" @@ -6377,7 +6614,16 @@ __metadata: languageName: node linkType: hard -"get-stream@npm:^5.0.0": +"get-stream@npm:^4.1.0": + version: 4.1.0 + resolution: "get-stream@npm:4.1.0" + dependencies: + pump: ^3.0.0 + checksum: 443e1914170c15bd52ff8ea6eff6dfc6d712b031303e36302d2778e3de2506af9ee964d6124010f7818736dcfde05c04ba7ca6cc26883106e084357a17ae7d73 + languageName: node + linkType: hard + +"get-stream@npm:^5.0.0, get-stream@npm:^5.1.0": version: 5.2.0 resolution: "get-stream@npm:5.2.0" dependencies: @@ -6448,6 +6694,15 @@ __metadata: languageName: node linkType: hard +"global-dirs@npm:^3.0.0": + version: 3.0.1 + resolution: "global-dirs@npm:3.0.1" + dependencies: + ini: 2.0.0 + checksum: 70147b80261601fd40ac02a104581432325c1c47329706acd773f3a6ce99bb36d1d996038c85ccacd482ad22258ec233c586b6a91535b1a116b89663d49d6438 + languageName: node + linkType: hard + "globals@npm:^11.1.0": version: 11.12.0 resolution: "globals@npm:11.12.0" @@ -6496,7 +6751,26 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": +"got@npm:^9.6.0": + version: 9.6.0 + resolution: "got@npm:9.6.0" + dependencies: + "@sindresorhus/is": ^0.14.0 + "@szmarczak/http-timer": ^1.1.2 + cacheable-request: ^6.0.0 + decompress-response: ^3.3.0 + duplexer3: ^0.1.4 + get-stream: ^4.1.0 + lowercase-keys: ^1.0.1 + mimic-response: ^1.0.1 + p-cancelable: ^1.0.0 + to-readable-stream: ^1.0.0 + url-parse-lax: ^3.0.0 + checksum: 941807bd9704bacf5eb401f0cc1212ffa1f67c6642f2d028fd75900471c221b1da2b8527f4553d2558f3faeda62ea1cf31665f8b002c6137f5de8732f07370b0 + languageName: node + linkType: hard + +"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 @@ -6588,6 +6862,13 @@ __metadata: languageName: node linkType: hard +"has-yarn@npm:^2.1.0": + version: 2.1.0 + resolution: "has-yarn@npm:2.1.0" + checksum: 5eb1d0bb8518103d7da24532bdbc7124ffc6d367b5d3c10840b508116f2f1bcbcf10fd3ba843ff6e2e991bdf9969fd862d42b2ed58aade88343326c950b7e7f7 + languageName: node + linkType: hard + "has@npm:^1.0.3": version: 1.0.3 resolution: "has@npm:1.0.3" @@ -6650,7 +6931,7 @@ __metadata: languageName: node linkType: hard -"http-cache-semantics@npm:^4.1.0": +"http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.0": version: 4.1.1 resolution: "http-cache-semantics@npm:4.1.1" checksum: 83ac0bc60b17a3a36f9953e7be55e5c8f41acc61b22583060e8dedc9dd5e3607c823a88d0926f9150e571f90946835c7fe150732801010845c72cd8bbff1a236 @@ -6790,6 +7071,13 @@ __metadata: languageName: node linkType: hard +"import-lazy@npm:^2.1.0": + version: 2.1.0 + resolution: "import-lazy@npm:2.1.0" + checksum: 05294f3b9dd4971d3a996f0d2f176410fb6745d491d6e73376429189f5c1c3d290548116b2960a7cf3e89c20cdf11431739d1d2d8c54b84061980795010e803a + languageName: node + linkType: hard + "import-local@npm:^3.0.2": version: 3.1.0 resolution: "import-local@npm:3.1.0" @@ -6847,6 +7135,20 @@ __metadata: languageName: node linkType: hard +"ini@npm:2.0.0": + version: 2.0.0 + resolution: "ini@npm:2.0.0" + checksum: e7aadc5fb2e4aefc666d74ee2160c073995a4061556b1b5b4241ecb19ad609243b9cceafe91bae49c219519394bbd31512516cb22a3b1ca6e66d869e0447e84e + languageName: node + linkType: hard + +"ini@npm:~1.3.0": + version: 1.3.8 + resolution: "ini@npm:1.3.8" + checksum: dfd98b0ca3a4fc1e323e38a6c8eb8936e31a97a918d3b377649ea15bdb15d481207a0dda1021efbd86b464cae29a0d33c1d7dcaf6c5672bee17fa849bc50a1b3 + languageName: node + linkType: hard + "interface-datastore@npm:^5.2.0": version: 5.2.0 resolution: "interface-datastore@npm:5.2.0" @@ -6864,6 +7166,17 @@ __metadata: languageName: node linkType: hard +"interface-datastore@npm:^6.0.2": + version: 6.1.1 + resolution: "interface-datastore@npm:6.1.1" + dependencies: + interface-store: ^2.0.2 + nanoid: ^3.0.2 + uint8arrays: ^3.0.0 + checksum: a0388adabf029be229bbfce326bbe64fd3353373512e7e6ed4283e06710bfa141db118e3536f8535a65016a0abeec631b888d42790b00637879d6ae56cf728cd + languageName: node + linkType: hard + "interface-store@npm:^1.0.2": version: 1.0.2 resolution: "interface-store@npm:1.0.2" @@ -6871,6 +7184,13 @@ __metadata: languageName: node linkType: hard +"interface-store@npm:^2.0.2": + version: 2.0.2 + resolution: "interface-store@npm:2.0.2" + checksum: 0e80adb1de9ff57687cfa1b08499702b72cacf33a7e0320ac7781989f3685d73f2a84996358f540250229afa19c7acebf03085087762f718035622ea6a1a5b8a + languageName: node + linkType: hard + "internal-slot@npm:^1.0.5": version: 1.0.5 resolution: "internal-slot@npm:1.0.5" @@ -6903,6 +7223,19 @@ __metadata: languageName: node linkType: hard +"ipfs-core-types@npm:^0.10.3": + version: 0.10.3 + resolution: "ipfs-core-types@npm:0.10.3" + dependencies: + "@ipld/dag-pb": ^2.1.3 + interface-datastore: ^6.0.2 + ipfs-unixfs: ^6.0.3 + multiaddr: ^10.0.0 + multiformats: ^9.5.1 + checksum: 2a36b0ac72b98011aed333b57253d87fd5888e9e102fa3dd8f9e2dcf1b5e8c8f5582fbe3957ec24a445cec207943ea4ce4cc839770a794b273fd8b66a7a0de9d + languageName: node + linkType: hard + "ipfs-core-types@npm:^0.7.3": version: 0.7.3 resolution: "ipfs-core-types@npm:0.7.3" @@ -6938,6 +7271,60 @@ __metadata: languageName: node linkType: hard +"ipfs-core-utils@npm:^0.14.3": + version: 0.14.3 + resolution: "ipfs-core-utils@npm:0.14.3" + dependencies: + any-signal: ^3.0.0 + blob-to-it: ^1.0.1 + browser-readablestream-to-it: ^1.0.1 + debug: ^4.1.1 + err-code: ^3.0.1 + ipfs-core-types: ^0.10.3 + ipfs-unixfs: ^6.0.3 + ipfs-utils: ^9.0.6 + it-all: ^1.0.4 + it-map: ^1.0.4 + it-peekable: ^1.0.2 + it-to-stream: ^1.0.0 + merge-options: ^3.0.4 + multiaddr: ^10.0.0 + multiaddr-to-uri: ^8.0.0 + multiformats: ^9.5.1 + nanoid: ^3.1.23 + parse-duration: ^1.0.0 + timeout-abort-controller: ^3.0.0 + uint8arrays: ^3.0.0 + checksum: ed1bceef0677e050a06db3cc0fdd659ffaac256d89fedfa01633747675c1519ba01985d5adf5db40d7d82631028c363564a2c53678b9a665c54c75d287065d08 + languageName: node + linkType: hard + +"ipfs-http-client@npm:56": + version: 56.0.3 + resolution: "ipfs-http-client@npm:56.0.3" + dependencies: + "@ipld/dag-cbor": ^7.0.0 + "@ipld/dag-json": ^8.0.1 + "@ipld/dag-pb": ^2.1.3 + any-signal: ^3.0.0 + dag-jose: ^1.0.0 + debug: ^4.1.1 + err-code: ^3.0.1 + ipfs-core-types: ^0.10.3 + ipfs-core-utils: ^0.14.3 + ipfs-utils: ^9.0.6 + it-first: ^1.0.6 + it-last: ^1.0.4 + merge-options: ^3.0.4 + multiaddr: ^10.0.0 + multiformats: ^9.5.1 + parse-duration: ^1.0.0 + stream-to-it: ^0.2.2 + uint8arrays: ^3.0.0 + checksum: d45fe955b368655ff37bbaeaa72eb968399cffb6ac1ae44bd7898d834259fbe8b2394b73626d2338cd753b57b4f99ebaa8e7dfbead887c73f36c0070185cd9d6 + languageName: node + linkType: hard + "ipfs-http-client@npm:^52.0.3": version: 52.0.5 resolution: "ipfs-http-client@npm:52.0.5" @@ -7001,6 +7388,30 @@ __metadata: languageName: node linkType: hard +"ipfs-utils@npm:^9.0.6": + version: 9.0.14 + resolution: "ipfs-utils@npm:9.0.14" + dependencies: + any-signal: ^3.0.0 + browser-readablestream-to-it: ^1.0.0 + buffer: ^6.0.1 + electron-fetch: ^1.7.2 + err-code: ^3.0.1 + is-electron: ^2.2.0 + iso-url: ^1.1.5 + it-all: ^1.0.4 + it-glob: ^1.0.1 + it-to-stream: ^1.0.0 + merge-options: ^3.0.4 + nanoid: ^3.1.20 + native-fetch: ^3.0.0 + node-fetch: ^2.6.8 + react-native-fetch-api: ^3.0.0 + stream-to-it: ^0.2.2 + checksum: 08108e03ea7b90e0fa11b76a4e24bd29d7e027c603494b53c1cc37b367fb559eaeea7b0f10b2e83ee419d50cdcb4d8105febdf185cab75c7e55afd4c8ed51aba + languageName: node + linkType: hard + "is-array-buffer@npm:^3.0.1, is-array-buffer@npm:^3.0.2": version: 3.0.2 resolution: "is-array-buffer@npm:3.0.2" @@ -7054,6 +7465,17 @@ __metadata: languageName: node linkType: hard +"is-ci@npm:^2.0.0": + version: 2.0.0 + resolution: "is-ci@npm:2.0.0" + dependencies: + ci-info: ^2.0.0 + bin: + is-ci: bin.js + checksum: 77b869057510f3efa439bbb36e9be429d53b3f51abd4776eeea79ab3b221337fe1753d1e50058a9e2c650d38246108beffb15ccfd443929d77748d8c0cc90144 + languageName: node + linkType: hard + "is-core-module@npm:^2.11.0, is-core-module@npm:^2.12.0": version: 2.12.0 resolution: "is-core-module@npm:2.12.0" @@ -7123,6 +7545,16 @@ __metadata: languageName: node linkType: hard +"is-installed-globally@npm:^0.4.0": + version: 0.4.0 + resolution: "is-installed-globally@npm:0.4.0" + dependencies: + global-dirs: ^3.0.0 + is-path-inside: ^3.0.2 + checksum: 3359840d5982d22e9b350034237b2cda2a12bac1b48a721912e1ab8e0631dd07d45a2797a120b7b87552759a65ba03e819f1bd63f2d7ab8657ec0b44ee0bf399 + languageName: node + linkType: hard + "is-interactive@npm:^1.0.0": version: 1.0.0 resolution: "is-interactive@npm:1.0.0" @@ -7153,6 +7585,13 @@ __metadata: languageName: node linkType: hard +"is-npm@npm:^5.0.0": + version: 5.0.0 + resolution: "is-npm@npm:5.0.0" + checksum: 9baff02b0c69a3d3c79b162cb2f9e67fb40ef6d172c16601b2e2471c21e9a4fa1fc9885a308d7bc6f3a3cd2a324c27fa0bf284c133c3349bb22571ab70d041cc + languageName: node + linkType: hard + "is-number-object@npm:^1.0.4": version: 1.0.7 resolution: "is-number-object@npm:1.0.7" @@ -7169,7 +7608,14 @@ __metadata: languageName: node linkType: hard -"is-path-inside@npm:^3.0.3": +"is-obj@npm:^2.0.0": + version: 2.0.0 + resolution: "is-obj@npm:2.0.0" + checksum: c9916ac8f4621962a42f5e80e7ffdb1d79a3fab7456ceaeea394cd9e0858d04f985a9ace45be44433bf605673c8be8810540fe4cc7f4266fc7526ced95af5a08 + languageName: node + linkType: hard + +"is-path-inside@npm:^3.0.2, is-path-inside@npm:^3.0.3": version: 3.0.3 resolution: "is-path-inside@npm:3.0.3" checksum: abd50f06186a052b349c15e55b182326f1936c89a78bf6c8f2b707412517c097ce04bc49a0ca221787bc44e1049f51f09a2ffb63d22899051988d3a618ba13e9 @@ -7277,6 +7723,13 @@ __metadata: languageName: node linkType: hard +"is-yarn-global@npm:^0.3.0": + version: 0.3.0 + resolution: "is-yarn-global@npm:0.3.0" + checksum: bca013d65fee2862024c9fbb3ba13720ffca2fe750095174c1c80922fdda16402b5c233f5ac9e265bc12ecb5446e7b7f519a32d9541788f01d4d44e24d2bf481 + languageName: node + linkType: hard + "isarray@npm:~1.0.0": version: 1.0.0 resolution: "isarray@npm:1.0.0" @@ -7378,6 +7831,16 @@ __metadata: languageName: node linkType: hard +"it-glob@npm:^1.0.1": + version: 1.0.2 + resolution: "it-glob@npm:1.0.2" + dependencies: + "@types/minimatch": ^3.0.4 + minimatch: ^3.0.4 + checksum: 629e7b66510006041df98882acfd73ac785836d51fc3ffa5c83c7099f931b3287a64c5a3772e7c1e46b63f1d511a9222f5b637c50f1c738222b46d104ff2e91c + languageName: node + linkType: hard + "it-glob@npm:~0.0.11": version: 0.0.14 resolution: "it-glob@npm:0.0.14" @@ -8051,6 +8514,13 @@ __metadata: languageName: node linkType: hard +"json-buffer@npm:3.0.0": + version: 3.0.0 + resolution: "json-buffer@npm:3.0.0" + checksum: 0cecacb8025370686a916069a2ff81f7d55167421b6aa7270ee74e244012650dd6bce22b0852202ea7ff8624fce50ff0ec1bdf95914ccb4553426e290d5a63fa + languageName: node + linkType: hard + "json-parse-even-better-errors@npm:^2.3.0": version: 2.3.1 resolution: "json-parse-even-better-errors@npm:2.3.1" @@ -8126,6 +8596,15 @@ __metadata: languageName: node linkType: hard +"keyv@npm:^3.0.0": + version: 3.1.0 + resolution: "keyv@npm:3.1.0" + dependencies: + json-buffer: 3.0.0 + checksum: bb7e8f3acffdbafbc2dd5b63f377fe6ec4c0e2c44fc82720449ef8ab54f4a7ce3802671ed94c0f475ae0a8549703353a2124561fcf3317010c141b32ca1ce903 + languageName: node + linkType: hard + "kleur@npm:^3.0.3": version: 3.0.3 resolution: "kleur@npm:3.0.3" @@ -8133,6 +8612,15 @@ __metadata: languageName: node linkType: hard +"latest-version@npm:^5.1.0": + version: 5.1.0 + resolution: "latest-version@npm:5.1.0" + dependencies: + package-json: ^6.3.0 + checksum: fbc72b071eb66c40f652441fd783a9cca62f08bf42433651937f078cd9ef94bf728ec7743992777826e4e89305aef24f234b515e6030503a2cbee7fc9bdc2c0f + languageName: node + linkType: hard + "level-codec@npm:^9.0.0": version: 9.0.2 resolution: "level-codec@npm:9.0.2" @@ -8415,6 +8903,20 @@ __metadata: languageName: node linkType: hard +"lowercase-keys@npm:^1.0.0, lowercase-keys@npm:^1.0.1": + version: 1.0.1 + resolution: "lowercase-keys@npm:1.0.1" + checksum: 4d045026595936e09953e3867722e309415ff2c80d7701d067546d75ef698dac218a4f53c6d1d0e7368b47e45fd7529df47e6cb56fbb90523ba599f898b3d147 + languageName: node + linkType: hard + +"lowercase-keys@npm:^2.0.0": + version: 2.0.0 + resolution: "lowercase-keys@npm:2.0.0" + checksum: 24d7ebd56ccdf15ff529ca9e08863f3c54b0b9d1edb97a3ae1af34940ae666c01a1e6d200707bce730a8ef76cb57cc10e65f245ecaaf7e6bc8639f2fb460ac23 + languageName: node + linkType: hard + "lru-cache@npm:8.0.4": version: 8.0.4 resolution: "lru-cache@npm:8.0.4" @@ -8621,6 +9123,13 @@ __metadata: languageName: node linkType: hard +"mimic-response@npm:^1.0.0, mimic-response@npm:^1.0.1": + version: 1.0.1 + resolution: "mimic-response@npm:1.0.1" + checksum: 034c78753b0e622bc03c983663b1cdf66d03861050e0c8606563d149bc2b02d63f62ce4d32be4ab50d0553ae0ffe647fc34d1f5281184c6e1e8cf4d85e8d9823 + languageName: node + linkType: hard + "minimalistic-assert@npm:^1.0.0, minimalistic-assert@npm:^1.0.1": version: 1.0.1 resolution: "minimalistic-assert@npm:1.0.1" @@ -8846,7 +9355,7 @@ __metadata: languageName: node linkType: hard -"multiformats@npm:^9.4.1, multiformats@npm:^9.4.2, multiformats@npm:^9.4.5, multiformats@npm:^9.5.4": +"multiformats@npm:^9.0.2, multiformats@npm:^9.4.1, multiformats@npm:^9.4.2, multiformats@npm:^9.4.5, multiformats@npm:^9.5.1, multiformats@npm:^9.5.4": version: 9.9.0 resolution: "multiformats@npm:9.9.0" checksum: d3e8c1be400c09a014f557ea02251a2710dbc9fca5aa32cc702ff29f636c5471e17979f30bdcb0a9cbb556f162a8591dc2e1219c24fc21394a56115b820bb84e @@ -8866,7 +9375,7 @@ __metadata: languageName: node linkType: hard -"nanoid@npm:^3.0.2, nanoid@npm:^3.1.12, nanoid@npm:^3.1.20": +"nanoid@npm:^3.0.2, nanoid@npm:^3.1.12, nanoid@npm:^3.1.20, nanoid@npm:^3.1.23": version: 3.3.6 resolution: "nanoid@npm:3.3.6" bin: @@ -9029,6 +9538,13 @@ __metadata: languageName: node linkType: hard +"normalize-url@npm:^4.1.0": + version: 4.5.1 + resolution: "normalize-url@npm:4.5.1" + checksum: 9a9dee01df02ad23e171171893e56e22d752f7cff86fb96aafeae074819b572ea655b60f8302e2d85dbb834dc885c972cc1c573892fea24df46b2765065dd05a + languageName: node + linkType: hard + "npm-run-path@npm:^4.0.0, npm-run-path@npm:^4.0.1": version: 4.0.1 resolution: "npm-run-path@npm:4.0.1" @@ -9183,6 +9699,13 @@ __metadata: languageName: node linkType: hard +"p-cancelable@npm:^1.0.0": + version: 1.1.0 + resolution: "p-cancelable@npm:1.1.0" + checksum: 2db3814fef6d9025787f30afaee4496a8857a28be3c5706432cbad76c688a6db1874308f48e364a42f5317f5e41e8e7b4f2ff5c8ff2256dbb6264bc361704ece + languageName: node + linkType: hard + "p-defer@npm:^3.0.0": version: 3.0.0 resolution: "p-defer@npm:3.0.0" @@ -9252,6 +9775,18 @@ __metadata: languageName: node linkType: hard +"package-json@npm:^6.3.0": + version: 6.5.0 + resolution: "package-json@npm:6.5.0" + dependencies: + got: ^9.6.0 + registry-auth-token: ^4.0.0 + registry-url: ^5.0.0 + semver: ^6.2.0 + checksum: cc9f890d3667d7610e6184decf543278b87f657d1ace0deb4a9c9155feca738ef88f660c82200763d3348010f4e42e9c7adc91e96ab0f86a770955995b5351e2 + languageName: node + linkType: hard + "packet-reader@npm:1.0.0": version: 1.0.0 resolution: "packet-reader@npm:1.0.0" @@ -9536,6 +10071,13 @@ __metadata: languageName: node linkType: hard +"prepend-http@npm:^2.0.0": + version: 2.0.0 + resolution: "prepend-http@npm:2.0.0" + checksum: 7694a9525405447662c1ffd352fcb41b6410c705b739b6f4e3a3e21cf5fdede8377890088e8934436b8b17ba55365a615f153960f30877bf0d0392f9e93503ea + languageName: node + linkType: hard + "prettier-linter-helpers@npm:^1.0.0": version: 1.0.0 resolution: "prettier-linter-helpers@npm:1.0.0" @@ -9723,6 +10265,15 @@ __metadata: languageName: node linkType: hard +"pupa@npm:^2.1.1": + version: 2.1.1 + resolution: "pupa@npm:2.1.1" + dependencies: + escape-goat: ^2.0.0 + checksum: 49529e50372ffdb0cccf0efa0f3b3cb0a2c77805d0d9cc2725bd2a0f6bb414631e61c93a38561b26be1259550b7bb6c2cb92315aa09c8bf93f3bdcb49f2b2fb7 + languageName: node + linkType: hard + "qs@npm:6.11.0": version: 6.11.0 resolution: "qs@npm:6.11.0" @@ -9784,6 +10335,20 @@ __metadata: languageName: node linkType: hard +"rc@npm:1.2.8, rc@npm:^1.2.8": + version: 1.2.8 + resolution: "rc@npm:1.2.8" + dependencies: + deep-extend: ^0.6.0 + ini: ~1.3.0 + minimist: ^1.2.0 + strip-json-comments: ~2.0.1 + bin: + rc: ./cli.js + checksum: 2e26e052f8be2abd64e6d1dabfbd7be03f80ec18ccbc49562d31f617d0015fbdbcf0f9eed30346ea6ab789e0fdfe4337f033f8016efdbee0df5354751842080e + languageName: node + linkType: hard + "react-is@npm:^16.13.1, react-is@npm:^16.7.0": version: 16.13.1 resolution: "react-is@npm:16.13.1" @@ -9807,6 +10372,15 @@ __metadata: languageName: node linkType: hard +"react-native-fetch-api@npm:^3.0.0": + version: 3.0.0 + resolution: "react-native-fetch-api@npm:3.0.0" + dependencies: + p-defer: ^3.0.0 + checksum: f10f435060551c470711ba0b3663e3d49c7701aae84ea645d66992d756b13e923fb5762b324d3583d85c1c0def4138b9cc3f686bab1c1bc10d3ad82dc7175c99 + languageName: node + linkType: hard + "readable-stream@npm:^2.2.2": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" @@ -9915,6 +10489,24 @@ __metadata: languageName: node linkType: hard +"registry-auth-token@npm:^4.0.0": + version: 4.2.2 + resolution: "registry-auth-token@npm:4.2.2" + dependencies: + rc: 1.2.8 + checksum: c5030198546ecfdcbcb0722cbc3e260c4f5f174d8d07bdfedd4620e79bfdf17a2db735aa230d600bd388fce6edd26c0a9ed2eb7e9b4641ec15213a28a806688b + languageName: node + linkType: hard + +"registry-url@npm:^5.0.0": + version: 5.1.0 + resolution: "registry-url@npm:5.1.0" + dependencies: + rc: ^1.2.8 + checksum: bcea86c84a0dbb66467b53187fadebfea79017cddfb4a45cf27530d7275e49082fe9f44301976eb0164c438e395684bcf3dae4819b36ff9d1640d8cc60c73df9 + languageName: node + linkType: hard + "regjsparser@npm:^0.9.1": version: 0.9.1 resolution: "regjsparser@npm:0.9.1" @@ -10010,6 +10602,15 @@ __metadata: languageName: node linkType: hard +"responselike@npm:^1.0.2": + version: 1.0.2 + resolution: "responselike@npm:1.0.2" + dependencies: + lowercase-keys: ^1.0.0 + checksum: 2e9e70f1dcca3da621a80ce71f2f9a9cad12c047145c6ece20df22f0743f051cf7c73505e109814915f23f9e34fb0d358e22827723ee3d56b623533cab8eafcd + languageName: node + linkType: hard + "restore-cursor@npm:^3.1.0": version: 3.1.0 resolution: "restore-cursor@npm:3.1.0" @@ -10027,6 +10628,13 @@ __metadata: languageName: node linkType: hard +"retimer@npm:^3.0.0": + version: 3.0.0 + resolution: "retimer@npm:3.0.0" + checksum: f88309196e9d4f2d4be0c76eafc27a9f102c74b40b391ce730785b052c345d7bd59c3e4411a4c422f89f19a42b97b28034639e2f06c63133a06ec8958e9e7516 + languageName: node + linkType: hard + "retry-as-promised@npm:^7.0.3": version: 7.0.4 resolution: "retry-as-promised@npm:7.0.4" @@ -10157,6 +10765,15 @@ __metadata: languageName: node linkType: hard +"semver-diff@npm:^3.1.1": + version: 3.1.1 + resolution: "semver-diff@npm:3.1.1" + dependencies: + semver: ^6.3.0 + checksum: 8bbe5a5d7add2d5e51b72314a9215cd294d71f41cdc2bf6bd59ee76411f3610b576172896f1d191d0d7294cb9f2f847438d2ee158adacc0c224dca79052812fe + languageName: node + linkType: hard + "semver@npm:7.x, semver@npm:^7.3.2, semver@npm:^7.3.4, semver@npm:^7.3.5, semver@npm:^7.3.7": version: 7.5.1 resolution: "semver@npm:7.5.1" @@ -10186,6 +10803,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^6.2.0": + version: 6.3.1 + resolution: "semver@npm:6.3.1" + bin: + semver: bin/semver.js + checksum: ae47d06de28836adb9d3e25f22a92943477371292d9b665fb023fae278d345d508ca1958232af086d85e0155aee22e313e100971898bbb8d5d89b8b1d4054ca2 + languageName: node + linkType: hard + "semver@npm:^7.5.2": version: 7.5.3 resolution: "semver@npm:7.5.3" @@ -10511,7 +11137,7 @@ __metadata: languageName: node linkType: hard -"string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": +"string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.0.0, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.2, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3" dependencies: @@ -10639,6 +11265,13 @@ __metadata: languageName: node linkType: hard +"strip-json-comments@npm:~2.0.1": + version: 2.0.1 + resolution: "strip-json-comments@npm:2.0.1" + checksum: 1074ccb63270d32ca28edfb0a281c96b94dc679077828135141f27d52a5a398ef5e78bcf22809d23cadc2b81dfbe345eb5fd8699b385c8b1128907dec4a7d1e1 + languageName: node + linkType: hard + "subql-mono@workspace:.": version: 0.0.0-use.local resolution: "subql-mono@workspace:." @@ -10820,6 +11453,15 @@ __metadata: languageName: node linkType: hard +"timeout-abort-controller@npm:^3.0.0": + version: 3.0.0 + resolution: "timeout-abort-controller@npm:3.0.0" + dependencies: + retimer: ^3.0.0 + checksum: c74387e6472a1e151d89b4af8c2a18ac72d566f3cd6ec9b203e3cda969fb5c54acba0a8d20cda8e9772efda52e27bea59c3423bab785b65f0287c9419184607e + languageName: node + linkType: hard + "tmpl@npm:1.0.5": version: 1.0.5 resolution: "tmpl@npm:1.0.5" @@ -10834,6 +11476,13 @@ __metadata: languageName: node linkType: hard +"to-readable-stream@npm:^1.0.0": + version: 1.0.0 + resolution: "to-readable-stream@npm:1.0.0" + checksum: 2bd7778490b6214a2c40276065dd88949f4cf7037ce3964c76838b8cb212893aeb9cceaaf4352a4c486e3336214c350270f3263e1ce7a0c38863a715a4d9aeb5 + languageName: node + linkType: hard + "to-regex-range@npm:^5.0.1": version: 5.0.1 resolution: "to-regex-range@npm:5.0.1" @@ -11263,6 +11912,15 @@ __metadata: languageName: node linkType: hard +"unique-string@npm:^2.0.0": + version: 2.0.0 + resolution: "unique-string@npm:2.0.0" + dependencies: + crypto-random-string: ^2.0.0 + checksum: ef68f639136bcfe040cf7e3cd7a8dff076a665288122855148a6f7134092e6ed33bf83a7f3a9185e46c98dddc445a0da6ac25612afa1a7c38b8b654d6c02498e + languageName: node + linkType: hard + "universal-user-agent@npm:^6.0.0": version: 6.0.0 resolution: "universal-user-agent@npm:6.0.0" @@ -11305,6 +11963,28 @@ __metadata: languageName: node linkType: hard +"update-notifier@npm:5.1.0": + version: 5.1.0 + resolution: "update-notifier@npm:5.1.0" + dependencies: + boxen: ^5.0.0 + chalk: ^4.1.0 + configstore: ^5.0.1 + has-yarn: ^2.1.0 + import-lazy: ^2.1.0 + is-ci: ^2.0.0 + is-installed-globally: ^0.4.0 + is-npm: ^5.0.0 + is-yarn-global: ^0.3.0 + latest-version: ^5.1.0 + pupa: ^2.1.1 + semver: ^7.3.4 + semver-diff: ^3.1.1 + xdg-basedir: ^4.0.0 + checksum: 461e5e5b002419296d3868ee2abe0f9ab3e1846d9db642936d0c46f838872ec56069eddfe662c45ce1af0a8d6d5026353728de2e0a95ab2e3546a22ea077caf1 + languageName: node + linkType: hard + "uri-js@npm:^4.2.2": version: 4.4.1 resolution: "uri-js@npm:4.4.1" @@ -11314,6 +11994,15 @@ __metadata: languageName: node linkType: hard +"url-parse-lax@npm:^3.0.0": + version: 3.0.0 + resolution: "url-parse-lax@npm:3.0.0" + dependencies: + prepend-http: ^2.0.0 + checksum: 1040e357750451173132228036aff1fd04abbd43eac1fb3e4fca7495a078bcb8d33cb765fe71ad7e473d9c94d98fd67adca63bd2716c815a2da066198dd37217 + languageName: node + linkType: hard + "url-parse@npm:^1.5.3": version: 1.5.10 resolution: "url-parse@npm:1.5.10" @@ -11555,6 +12244,15 @@ __metadata: languageName: node linkType: hard +"widest-line@npm:^3.1.0": + version: 3.1.0 + resolution: "widest-line@npm:3.1.0" + dependencies: + string-width: ^4.0.0 + checksum: 03db6c9d0af9329c37d74378ff1d91972b12553c7d72a6f4e8525fe61563fa7adb0b9d6e8d546b7e059688712ea874edd5ded475999abdeedf708de9849310e0 + languageName: node + linkType: hard + "wkx@npm:^0.5.0": version: 0.5.0 resolution: "wkx@npm:0.5.0" @@ -11642,6 +12340,13 @@ __metadata: languageName: node linkType: hard +"xdg-basedir@npm:^4.0.0": + version: 4.0.0 + resolution: "xdg-basedir@npm:4.0.0" + checksum: 0073d5b59a37224ed3a5ac0dd2ec1d36f09c49f0afd769008a6e9cd3cd666bd6317bd1c7ce2eab47e1de285a286bad11a9b038196413cd753b79770361855f3c + languageName: node + linkType: hard + "xml-name-validator@npm:^3.0.0": version: 3.0.0 resolution: "xml-name-validator@npm:3.0.0" From ede4b06d43c35fda320d5105534867c4428061ca Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Fri, 1 Sep 2023 12:56:24 +1200 Subject: [PATCH 02/15] Include POI changes --- packages/node/package.json | 2 +- packages/node/src/indexer/fetch.module.ts | 8 +- packages/node/src/indexer/indexer.module.ts | 8 +- packages/node/src/indexer/project.service.ts | 6 - .../src/subcommands/forceClean.service.ts | 5 - .../node/src/subcommands/mmrMigrate.init.ts | 36 -- .../node/src/subcommands/mmrMigrate.module.ts | 16 - .../src/subcommands/mmrRegenerate.init.ts | 30 -- .../src/subcommands/mmrRegenerate.module.ts | 18 - .../node/src/subcommands/reindex.module.ts | 4 - packages/node/src/yargs.ts | 134 -------- yarn.lock | 323 ++---------------- 12 files changed, 38 insertions(+), 552 deletions(-) delete mode 100644 packages/node/src/subcommands/mmrMigrate.init.ts delete mode 100644 packages/node/src/subcommands/mmrMigrate.module.ts delete mode 100644 packages/node/src/subcommands/mmrRegenerate.init.ts delete mode 100644 packages/node/src/subcommands/mmrRegenerate.module.ts diff --git a/packages/node/package.json b/packages/node/package.json index c991e3d5..b5051527 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -26,7 +26,7 @@ "@nestjs/schedule": "^3.0.1", "@subql/common": "dev", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "4.2.4-2", + "@subql/node-core": "dev", "@subql/types-algorand": "workspace:*", "@willsoto/nestjs-prometheus": "^4.4.0", "algosdk": "^2.2.0", diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 1f493571..22f0da5b 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -6,15 +6,12 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { PoiBenchmarkService, IndexingBenchmarkService, - MmrService, StoreService, PoiService, NodeConfig, SmartBatchService, ConnectionPoolService, StoreCacheService, - PgMmrCacheService, - MmrQueryService, ConnectionPoolStateManager, IProjectUpgradeService, } from '@subql/node-core'; @@ -138,14 +135,11 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; DsProcessorService, DynamicDsService, PoiService, - MmrService, - MmrQueryService, - PgMmrCacheService, { useClass: ProjectService, provide: 'IProjectService', }, ], - exports: [StoreService, MmrService, StoreCacheService, MmrQueryService], + exports: [StoreService, StoreCacheService], }) export class FetchModule {} diff --git a/packages/node/src/indexer/indexer.module.ts b/packages/node/src/indexer/indexer.module.ts index b60719b8..1323dfe4 100644 --- a/packages/node/src/indexer/indexer.module.ts +++ b/packages/node/src/indexer/indexer.module.ts @@ -7,12 +7,9 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { StoreService, PoiService, - MmrService, ConnectionPoolService, StoreCacheService, WorkerDynamicDsService, - PgMmrCacheService, - MmrQueryService, ConnectionPoolStateManager, WorkerConnectionPoolStateManager, } from '@subql/node-core'; @@ -71,15 +68,12 @@ import { WorkerService } from './worker/worker.service'; }, }, PoiService, - MmrService, - PgMmrCacheService, - MmrQueryService, { provide: 'IProjectService', useClass: ProjectService, }, WorkerService, ], - exports: [StoreService, MmrService, MmrQueryService], + exports: [StoreService], }) export class IndexerModule {} diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 1dfa5ca9..7e63fd0c 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -5,8 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { PoiService, - MmrService, - MmrQueryService, BaseProjectService, StoreService, NodeConfig, @@ -36,8 +34,6 @@ export class ProjectService extends BaseProjectService< dsProcessorService: DsProcessorService, apiService: AlgorandApiService, poiService: PoiService, - mmrService: MmrService, - mmrQueryService: MmrQueryService, sequelize: Sequelize, @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IProjectUpgradeService') @@ -52,8 +48,6 @@ export class ProjectService extends BaseProjectService< dsProcessorService, apiService, poiService, - mmrService, - mmrQueryService, sequelize, project, projectUpgradeService, diff --git a/packages/node/src/subcommands/forceClean.service.ts b/packages/node/src/subcommands/forceClean.service.ts index 57418ce5..b068895b 100644 --- a/packages/node/src/subcommands/forceClean.service.ts +++ b/packages/node/src/subcommands/forceClean.service.ts @@ -80,11 +80,6 @@ export class ForceCleanService { } logger.info('force cleaned schema and tables'); - - if (fs.existsSync(this.nodeConfig.mmrPath)) { - await fs.promises.unlink(this.nodeConfig.mmrPath); - logger.info('force cleaned file based mmr'); - } } catch (err) { logger.error(err, 'failed to force clean'); throw err; diff --git a/packages/node/src/subcommands/mmrMigrate.init.ts b/packages/node/src/subcommands/mmrMigrate.init.ts deleted file mode 100644 index 599bbe94..00000000 --- a/packages/node/src/subcommands/mmrMigrate.init.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { NestFactory } from '@nestjs/core'; -import { - getLogger, - MigrationDirection, - MMRMigrateService, -} from '@subql/node-core'; -import { MMRMigrateModule } from './mmrMigrate.module'; - -const logger = getLogger('mmr-migrate'); - -export async function mmrMigrateInit( - direction: MigrationDirection, -): Promise { - try { - logger.info('Starting MMR migration process...'); - - const app = await NestFactory.create(MMRMigrateModule); - await app.init(); - - logger.info('MMRMigrateModule initialized.'); - - const mmrMigrateService = app.get(MMRMigrateService); - logger.info(`Migrating MMR data in ${direction} direction...`); - await mmrMigrateService.migrate(direction); - - logger.info('MMR migration completed successfully.'); - } catch (e) { - logger.error(e, 'MMR migration failed to execute'); - process.exit(1); - } - - process.exit(0); -} diff --git a/packages/node/src/subcommands/mmrMigrate.module.ts b/packages/node/src/subcommands/mmrMigrate.module.ts deleted file mode 100644 index d009e05e..00000000 --- a/packages/node/src/subcommands/mmrMigrate.module.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Module } from '@nestjs/common'; -import { DbModule, MMRMigrateFeatureModule } from '@subql/node-core'; -import { ConfigureModule } from '../configure/configure.module'; - -@Module({ - imports: [ - DbModule.forRoot(), - ConfigureModule.register(), - MMRMigrateFeatureModule, - ], - controllers: [], -}) -export class MMRMigrateModule {} diff --git a/packages/node/src/subcommands/mmrRegenerate.init.ts b/packages/node/src/subcommands/mmrRegenerate.init.ts deleted file mode 100644 index 72f89b2b..00000000 --- a/packages/node/src/subcommands/mmrRegenerate.init.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { NestFactory } from '@nestjs/core'; -import { getLogger, MmrRegenerateService } from '@subql/node-core'; -import { MmrRegenerateModule } from './mmrRegenerate.module'; - -const logger = getLogger('MMR-Regeneration'); - -export async function mmrRegenerateInit( - probeMode = false, - resetOnly = false, - unsafe = false, - targetHeight?: number, -): Promise { - try { - const app = await NestFactory.create(MmrRegenerateModule); - await app.init(); - const mmrRegenerateService = app.get(MmrRegenerateService); - app.enableShutdownHooks(); - await mmrRegenerateService.init(); - if (!probeMode) { - await mmrRegenerateService.regenerate(targetHeight, resetOnly, unsafe); - } - } catch (e) { - logger.error(e, 'Re-generate MMR failed to execute'); - process.exit(1); - } - process.exit(0); -} diff --git a/packages/node/src/subcommands/mmrRegenerate.module.ts b/packages/node/src/subcommands/mmrRegenerate.module.ts deleted file mode 100644 index b8f49693..00000000 --- a/packages/node/src/subcommands/mmrRegenerate.module.ts +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { Module } from '@nestjs/common'; -import { EventEmitterModule } from '@nestjs/event-emitter'; -import { DbModule, MmrRegenerateFeatureModule } from '@subql/node-core'; -import { ConfigureModule } from '../configure/configure.module'; - -@Module({ - imports: [ - DbModule.forRoot(), - ConfigureModule.register(), - MmrRegenerateFeatureModule, - EventEmitterModule.forRoot(), - ], - controllers: [], -}) -export class MmrRegenerateModule {} diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts index 0e1dfdbd..227540eb 100644 --- a/packages/node/src/subcommands/reindex.module.ts +++ b/packages/node/src/subcommands/reindex.module.ts @@ -7,8 +7,6 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { DbModule, ForceCleanService, - MmrService, - PgMmrCacheService, ReindexService, StoreCacheService, StoreService, @@ -23,8 +21,6 @@ import { DynamicDsService } from '../indexer/dynamic-ds.service'; StoreCacheService, StoreService, ReindexService, - MmrService, - PgMmrCacheService, ForceCleanService, DynamicDsService, DsProcessorService, diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index acb6183a..c953915b 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -63,127 +63,6 @@ export const yargsOptions = yargs(hideBin(process.argv)) return reindexInit(argv.targetHeight); }, }) - .command({ - command: 'mmr-regen', - describe: - 'Re-generate mmr between Filebased/Postgres mmr and Proof of index', - builder: (yargs) => - yargs.options({ - probe: { - type: 'boolean', - description: - 'Fetch latest mmr height information from file based/postgres DB and Poi table', - demandOption: false, - default: false, - }, - targetHeight: { - type: 'number', - description: 'Re-genrate mmr value from this block height', - demandOption: false, - }, - resetOnly: { - type: 'boolean', - description: - 'Only reset the mmr value in both POI and file based/postgres DB to target height', - demandOption: false, - default: false, - }, - unsafe: { - type: 'boolean', - description: 'Allow sync mmr from Poi table to file or a postgres DB', - demandOption: false, - default: false, - }, - 'mmr-store-type': { - demandOption: false, - describe: - 'When regenerate MMR store in either a file or a postgres DB', - type: 'string', - choices: ['file', 'postgres'], - default: 'postgres', - }, - 'mmr-path': { - alias: 'm', - demandOption: false, - describe: - 'File based only : local path of the merkle mountain range (.mmr) file', - type: 'string', - }, - 'db-schema': { - demandOption: false, - describe: 'Db schema name of the project', - type: 'string', - }, - subquery: { - alias: 'f', - demandOption: true, - default: process.cwd(), - describe: 'Local path or IPFS cid of the subquery project', - type: 'string', - }, - }), - handler: (argv) => { - initLogger( - argv.debug as boolean, - argv.outputFmt as 'json' | 'colored', - argv.logLevel as string | undefined, - ); - - // lazy import to make sure logger is instantiated before all other services - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { mmrRegenerateInit } = require('./subcommands/mmrRegenerate.init'); - return mmrRegenerateInit( - argv.probe, - argv.resetOnly, - argv.unsafe, - argv.targetHeight, - ); - }, - }) - .command({ - command: 'mmr-migrate', - describe: 'Migrate MMR data from storage file to postgres DB', - builder: (yargs) => - yargs.options({ - direction: { - type: 'string', - description: 'set direction of migration (file -> DB or DB -> file)', - demandOption: false, - choices: ['dbToFile', 'fileToDb'], - default: 'dbToFile', - }, - 'mmr-path': { - alias: 'm', - demandOption: false, - describe: 'Local path of the merkle mountain range (.mmr) file', - type: 'string', - }, - 'db-schema': { - demandOption: false, - describe: 'Db schema name of the project', - type: 'string', - }, - subquery: { - alias: 'f', - demandOption: true, - default: process.cwd(), - describe: 'Local path or IPFS cid of the subquery project', - type: 'string', - }, - }), - handler: (argv) => { - initLogger( - argv.debug as boolean, - argv.outputFmt as 'json' | 'colored', - argv.logLevel as string | undefined, - ); - - // lazy import to make sure logger is instantiated before all other services - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { mmrMigrateInit } = require('./subcommands/mmrMigrate.init'); - return mmrMigrateInit(argv.direction); - }, - }) .options({ 'batch-size': { demandOption: false, @@ -242,12 +121,6 @@ export const yargsOptions = yargs(hideBin(process.argv)) type: 'string', choices: ['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent'], }, - 'mmr-path': { - alias: 'm', - demandOption: false, - describe: 'Local path of the merkle mountain range (.mmr) file', - type: 'string', - }, 'multi-chain': { demandOption: false, default: false, @@ -290,13 +163,6 @@ export const yargsOptions = yargs(hideBin(process.argv)) type: 'boolean', default: false, }, - 'mmr-store-type': { - demandOption: false, - describe: 'Store MMR in either a file or a postgres DB', - type: 'string', - choices: ['file', 'postgres'], - default: 'postgres', - }, 'query-limit': { demandOption: false, describe: diff --git a/yarn.lock b/yarn.lock index 7f0a1e71..168ac75a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2893,20 +2893,22 @@ __metadata: languageName: node linkType: hard -"@subql/apollo-links@npm:^0.5.8": - version: 0.5.8 - resolution: "@subql/apollo-links@npm:0.5.8" +"@subql/apollo-links@npm:^1.0.2": + version: 1.0.2 + resolution: "@subql/apollo-links@npm:1.0.2" dependencies: "@metamask/eth-sig-util": 5.1.0 apollo-link-error: ^1.1.13 axios: ^0.27.2 buffer: ^6.0.3 + cross-fetch: ^4.0.0 ethers: ^5.6.8 jwt-decode: ^3.1.2 + lru-cache: ^10.0.1 peerDependencies: "@apollo/client": "*" graphql: "*" - checksum: 34366e720bfe5c5dece380b93ec61d754f0bc804d07ff43e46aaca792f26f202b112a47cc2729a43e55181190b322aa1665618162f94add5cb341fb553f22116 + checksum: b7e3511d78d136bdda4e1ae1722c22522a10dacbe7b0908fe7065233cd1ed2e61a6e3a599b9a37b356bedf48833212eb037593324bc409def6e562c094605376 languageName: node linkType: hard @@ -2933,24 +2935,7 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:2.6.1-2": - version: 2.6.1-2 - resolution: "@subql/common@npm:2.6.1-2" - dependencies: - axios: ^0.27.2 - class-transformer: ^0.5.1 - class-validator: ^0.14.0 - fs-extra: ^10.1.0 - ipfs-http-client: 56 - js-yaml: ^4.1.0 - reflect-metadata: ^0.1.13 - semver: ^7.5.2 - update-notifier: 5.1.0 - checksum: fba2a5141977a2c66a2191c7b0abb9cf95114e940731a590a881e08caff91565fb708e013185dccd29950cf97cdf89ed90ff5489c2f137e5e0f8134e5945e95c - languageName: node - linkType: hard - -"@subql/common@npm:dev": +"@subql/common@npm:2.6.1-3, @subql/common@npm:dev": version: 2.6.1-3 resolution: "@subql/common@npm:2.6.1-3" dependencies: @@ -2981,7 +2966,7 @@ __metadata: "@nestjs/testing": ^9.4.0 "@subql/common": dev "@subql/common-algorand": "workspace:*" - "@subql/node-core": 4.2.4-2 + "@subql/node-core": dev "@subql/types-algorand": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -3011,20 +2996,19 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:4.2.4-2": - version: 4.2.4-2 - resolution: "@subql/node-core@npm:4.2.4-2" +"@subql/node-core@npm:dev": + version: 4.2.4-4 + resolution: "@subql/node-core@npm:4.2.4-4" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 - "@subql/apollo-links": ^0.5.8 - "@subql/common": 2.6.1-2 + "@subql/apollo-links": ^1.0.2 + "@subql/common": 2.6.1-3 "@subql/testing": 2.0.2 "@subql/types": 2.2.1-0 "@subql/utils": 2.4.3 - "@subql/x-merkle-mountain-range": ^2.0.0-0.1.3 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.1.1 async-lock: ^1.4.0 @@ -3039,7 +3023,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: ebaf6aa24a0b267a4304a49a4f24539d4ff3d7f3f5eb94c7789b792f2cbb6af52d55ff40b731617f5bc872c8bf995882605a7ccefb24a13a9e6733d158dec802 + checksum: d6d59c19984d9ee4f3a91dc878223b25840205c4a4a3c4217e6f7da7d005a48b9546bda4e799c9b6dcc6ee90659efeceb99e374d1c518919588abdc26bcb4636 languageName: node linkType: hard @@ -3104,21 +3088,6 @@ __metadata: languageName: node linkType: hard -"@subql/x-merkle-mountain-range@npm:^2.0.0-0.1.3": - version: 2.0.0-0.1.3 - resolution: "@subql/x-merkle-mountain-range@npm:2.0.0-0.1.3" - dependencies: - bignumber.js: ^8.1.1 - buffer: ^5.4.3 - js-sha3: ^0.8.0 - level: ^6.0.0 - rlp: ^2.2.3 - semaphore-async-await: ^1.5.1 - sha.js: ^2.4.11 - checksum: fcd17a07b77c79730fe2847f4b2fefa376beae59b942b6eb71ffefa683f4e1988e00a2ed974e422560ccb914f72e98067c6b8c6923128371b6e2611dde866265 - languageName: node - linkType: hard - "@subql/x-sequelize@npm:6.32.0-0.0.2": version: 6.32.0-0.0.2 resolution: "@subql/x-sequelize@npm:6.32.0-0.0.2" @@ -3833,32 +3802,6 @@ __metadata: languageName: node linkType: hard -"abstract-leveldown@npm:^6.2.1": - version: 6.3.0 - resolution: "abstract-leveldown@npm:6.3.0" - dependencies: - buffer: ^5.5.0 - immediate: ^3.2.3 - level-concat-iterator: ~2.0.0 - level-supports: ~1.0.0 - xtend: ~4.0.0 - checksum: 121a8509d8c6a540e656c2a69e5b8d853d4df71072011afefc868b98076991bb00120550e90643de9dc18889c675f62413409eeb4c8c204663124c7d215e4ec3 - languageName: node - linkType: hard - -"abstract-leveldown@npm:~6.2.1, abstract-leveldown@npm:~6.2.3": - version: 6.2.3 - resolution: "abstract-leveldown@npm:6.2.3" - dependencies: - buffer: ^5.5.0 - immediate: ^3.2.3 - level-concat-iterator: ~2.0.0 - level-supports: ~1.0.0 - xtend: ~4.0.0 - checksum: 00202b2eb7955dd7bc04f3e44d225e60160cedb8f96fe6ae0e6dca9c356d57071f001ece8ae1d53f48095c4c036d92b3440f2bc7666730610ddea030f9fbde4a - languageName: node - linkType: hard - "accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -4491,13 +4434,6 @@ __metadata: languageName: node linkType: hard -"bignumber.js@npm:^8.1.1": - version: 8.1.1 - resolution: "bignumber.js@npm:8.1.1" - checksum: 71544348507cd16c2c152699a8e5146964d9cf4f46108075bcd16a964dffb3963256259fc6ffc9753b4d33dfd0cf863a6b564103a3a20e10002f12a1902e8075 - languageName: node - linkType: hard - "bignumber.js@npm:^9.0.0": version: 9.1.1 resolution: "bignumber.js@npm:9.1.1" @@ -4546,7 +4482,7 @@ __metadata: languageName: node linkType: hard -"bn.js@npm:^5.2.0, bn.js@npm:^5.2.1": +"bn.js@npm:^5.2.1": version: 5.2.1 resolution: "bn.js@npm:5.2.1" checksum: 3dd8c8d38055fedfa95c1d5fc3c99f8dd547b36287b37768db0abab3c239711f88ff58d18d155dd8ad902b0b0cee973747b7ae20ea12a09473272b0201c9edd3 @@ -4704,7 +4640,7 @@ __metadata: languageName: node linkType: hard -"buffer@npm:^5.4.3, buffer@npm:^5.5.0, buffer@npm:^5.6.0": +"buffer@npm:^5.5.0": version: 5.7.1 resolution: "buffer@npm:5.7.1" dependencies: @@ -5257,6 +5193,15 @@ __metadata: languageName: node linkType: hard +"cross-fetch@npm:^4.0.0": + version: 4.0.0 + resolution: "cross-fetch@npm:4.0.0" + dependencies: + node-fetch: ^2.6.12 + checksum: ecca4f37ffa0e8283e7a8a590926b66713a7ef7892757aa36c2d20ffa27b0ac5c60dcf453119c809abe5923fc0bae3702a4d896bfb406ef1077b0d0018213e24 + languageName: node + linkType: hard + "cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.2, cross-spawn@npm:^7.0.3": version: 7.0.3 resolution: "cross-spawn@npm:7.0.3" @@ -5416,16 +5361,6 @@ __metadata: languageName: node linkType: hard -"deferred-leveldown@npm:~5.3.0": - version: 5.3.0 - resolution: "deferred-leveldown@npm:5.3.0" - dependencies: - abstract-leveldown: ~6.2.1 - inherits: ^2.0.3 - checksum: 5631e153528bb9de1aa60d59a5065d1a519374c5e4c1d486f2190dba4008dcf5c2ee8dd7f2f81396fc4d5a6bb6e7d0055e3dfe68afe00da02adaa3bf329addf7 - languageName: node - linkType: hard - "define-properties@npm:^1.1.3, define-properties@npm:^1.1.4, define-properties@npm:^1.2.0": version: 1.2.0 resolution: "define-properties@npm:1.2.0" @@ -5655,18 +5590,6 @@ __metadata: languageName: node linkType: hard -"encoding-down@npm:^6.3.0": - version: 6.3.0 - resolution: "encoding-down@npm:6.3.0" - dependencies: - abstract-leveldown: ^6.2.1 - inherits: ^2.0.3 - level-codec: ^9.0.0 - level-errors: ^2.0.0 - checksum: 74043e6d9061a470614ff61d708c849259ab32932a428fd5ddfb0878719804f56a52f59b31cccd95fddc2e636c0fd22dc3e02481fb98d5bf1bdbbbc44ca09bdc - languageName: node - linkType: hard - "encoding@npm:^0.1.13": version: 0.1.13 resolution: "encoding@npm:0.1.13" @@ -5716,17 +5639,6 @@ __metadata: languageName: node linkType: hard -"errno@npm:~0.1.1": - version: 0.1.8 - resolution: "errno@npm:0.1.8" - dependencies: - prr: ~1.0.1 - bin: - errno: cli.js - checksum: 1271f7b9fbb3bcbec76ffde932485d1e3561856d21d847ec613a9722ee924cdd4e523a62dc71a44174d91e898fe21fdc8d5b50823f4b5e0ce8c35c8271e6ef4a - languageName: node - linkType: hard - "error-ex@npm:^1.3.1": version: 1.3.2 resolution: "error-ex@npm:1.3.2" @@ -7054,13 +6966,6 @@ __metadata: languageName: node linkType: hard -"immediate@npm:^3.2.3": - version: 3.3.0 - resolution: "immediate@npm:3.3.0" - checksum: 634b4305101e2452eba6c07d485bf3e415995e533c94b9c3ffbc37026fa1be34def6e4f2276b0dc2162a3f91628564a4bfb26280278b89d3ee54624e854d2f5f - languageName: node - linkType: hard - "import-fresh@npm:^3.0.0, import-fresh@npm:^3.2.1": version: 3.3.0 resolution: "import-fresh@npm:3.3.0" @@ -7128,7 +7033,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -8621,109 +8526,6 @@ __metadata: languageName: node linkType: hard -"level-codec@npm:^9.0.0": - version: 9.0.2 - resolution: "level-codec@npm:9.0.2" - dependencies: - buffer: ^5.6.0 - checksum: 289003d51b8afcdd24c4d318606abf2bae81975e4b527d7349abfdbacc8fef26711f2f24e2d20da0e1dce0bb216a856c9433ccb9ca25fa78a96aed9f51e506ed - languageName: node - linkType: hard - -"level-concat-iterator@npm:~2.0.0": - version: 2.0.1 - resolution: "level-concat-iterator@npm:2.0.1" - checksum: 562583ef1292215f8e749c402510cb61c4d6fccf4541082b3d21dfa5ecde9fcccfe52bdcb5cfff9d2384e7ce5891f44df9439a6ddb39b0ffe31015600b4a828a - languageName: node - linkType: hard - -"level-errors@npm:^2.0.0, level-errors@npm:~2.0.0": - version: 2.0.1 - resolution: "level-errors@npm:2.0.1" - dependencies: - errno: ~0.1.1 - checksum: aca5d7670e2a40609db8d7743fce289bb5202c0bc13e4a78f81f36a6642e9abc0110f48087d3d3c2c04f023d70d4ee6f2db0e20c63d29b3fda323a67bfff6526 - languageName: node - linkType: hard - -"level-iterator-stream@npm:~4.0.0": - version: 4.0.2 - resolution: "level-iterator-stream@npm:4.0.2" - dependencies: - inherits: ^2.0.4 - readable-stream: ^3.4.0 - xtend: ^4.0.2 - checksum: 239e2c7e62bffb485ed696bcd3b98de7a2bc455d13be4fce175ae3544fe9cda81c2ed93d3e88b61380ae6d28cce02511862d77b86fb2ba5b5cf00471f3c1eccc - languageName: node - linkType: hard - -"level-js@npm:^5.0.0": - version: 5.0.2 - resolution: "level-js@npm:5.0.2" - dependencies: - abstract-leveldown: ~6.2.3 - buffer: ^5.5.0 - inherits: ^2.0.3 - ltgt: ^2.1.2 - checksum: 3c7f75979bb8c042e95a58245b8fe1230bb0f56a11ee418e08156e3eadda371efae6eb7b9bf10bf1e08e0b1b2a25d80c026858ca99ffd49109d6541e3d9d3b37 - languageName: node - linkType: hard - -"level-packager@npm:^5.1.0": - version: 5.1.1 - resolution: "level-packager@npm:5.1.1" - dependencies: - encoding-down: ^6.3.0 - levelup: ^4.3.2 - checksum: befe2aa54f2010a6ecf7ddce392c8dee225e1839205080a2704d75e560e28b01191b345494696196777b70d376e3eaae4c9e7c330cc70d3000839f5b18dd78f2 - languageName: node - linkType: hard - -"level-supports@npm:~1.0.0": - version: 1.0.1 - resolution: "level-supports@npm:1.0.1" - dependencies: - xtend: ^4.0.2 - checksum: 5d6bdb88cf00c3d9adcde970db06a548c72c5a94bf42c72f998b58341a105bfe2ea30d313ce1e84396b98cc9ddbc0a9bd94574955a86e929f73c986e10fc0df0 - languageName: node - linkType: hard - -"level@npm:^6.0.0": - version: 6.0.1 - resolution: "level@npm:6.0.1" - dependencies: - level-js: ^5.0.0 - level-packager: ^5.1.0 - leveldown: ^5.4.0 - checksum: bd4981f94162469a82a6c98d267d814d9d4a7beed4fc3d18fbe3b156f71cf4c6d35b424d14c46d401dbf0cd91425e842950a7cd17ddf7bf57acdab5af4c278da - languageName: node - linkType: hard - -"leveldown@npm:^5.4.0": - version: 5.6.0 - resolution: "leveldown@npm:5.6.0" - dependencies: - abstract-leveldown: ~6.2.1 - napi-macros: ~2.0.0 - node-gyp: latest - node-gyp-build: ~4.1.0 - checksum: 06d4683170d7fc661acd65457e531b42ad66480e9339d3154ba6d0de38ff0503d7d017c1c6eba12732b5488ecd2915c70c8dc3a7d67f4a836f3de34b8a993949 - languageName: node - linkType: hard - -"levelup@npm:^4.3.2": - version: 4.4.0 - resolution: "levelup@npm:4.4.0" - dependencies: - deferred-leveldown: ~5.3.0 - level-errors: ~2.0.0 - level-iterator-stream: ~4.0.0 - level-supports: ~1.0.0 - xtend: ~4.0.0 - checksum: 5a09e34c78cd7c23f9f6cb73563f1ebe8121ffc5f9f5f232242529d4fbdd40e8d1ffb337d2defa0b842334e0dbd4028fbfe7a072eebfe2c4d07174f0aa4aabca - languageName: node - linkType: hard - "leven@npm:^3.1.0": version: 3.1.0 resolution: "leven@npm:3.1.0" @@ -8924,6 +8726,13 @@ __metadata: languageName: node linkType: hard +"lru-cache@npm:^10.0.1": + version: 10.0.1 + resolution: "lru-cache@npm:10.0.1" + checksum: 06f8d0e1ceabd76bb6f644a26dbb0b4c471b79c7b514c13c6856113879b3bf369eb7b497dad4ff2b7e2636db202412394865b33c332100876d838ad1372f0181 + languageName: node + linkType: hard + "lru-cache@npm:^5.1.1": version: 5.1.1 resolution: "lru-cache@npm:5.1.1" @@ -8949,13 +8758,6 @@ __metadata: languageName: node linkType: hard -"ltgt@npm:^2.1.2": - version: 2.2.1 - resolution: "ltgt@npm:2.2.1" - checksum: 7e3874296f7538bc8087b428ac4208008d7b76916354b34a08818ca7c83958c1df10ec427eeeaad895f6b81e41e24745b18d30f89abcc21d228b94f6961d50a2 - languageName: node - linkType: hard - "luxon@npm:^3.2.1": version: 3.3.0 resolution: "luxon@npm:3.3.0" @@ -9384,13 +9186,6 @@ __metadata: languageName: node linkType: hard -"napi-macros@npm:~2.0.0": - version: 2.0.0 - resolution: "napi-macros@npm:2.0.0" - checksum: 30384819386977c1f82034757014163fa60ab3c5a538094f778d38788bebb52534966279956f796a92ea771c7f8ae072b975df65de910d051ffbdc927f62320c - languageName: node - linkType: hard - "native-abort-controller@npm:^1.0.3": version: 1.0.4 resolution: "native-abort-controller@npm:1.0.4" @@ -9444,17 +9239,6 @@ __metadata: languageName: node linkType: hard -"node-gyp-build@npm:~4.1.0": - version: 4.1.1 - resolution: "node-gyp-build@npm:4.1.1" - bin: - node-gyp-build: ./bin.js - node-gyp-build-optional: ./optional.js - node-gyp-build-test: ./build-test.js - checksum: 959d42221cc44b92700003efae741652bc4e379e4cf375830ddde03ba43c89f99694bf0883078ed0d4e03ffe2f85decab0572e04068d3900b8538d165dbc17df - languageName: node - linkType: hard - "node-gyp@npm:latest": version: 9.3.1 resolution: "node-gyp@npm:9.3.1" @@ -10227,13 +10011,6 @@ __metadata: languageName: node linkType: hard -"prr@npm:~1.0.1": - version: 1.0.1 - resolution: "prr@npm:1.0.1" - checksum: 3bca2db0479fd38f8c4c9439139b0c42dcaadcc2fbb7bb8e0e6afaa1383457f1d19aea9e5f961d5b080f1cfc05bfa1fe9e45c97a1d3fd6d421950a73d3108381 - languageName: node - linkType: hard - "psl@npm:^1.1.33": version: 1.9.0 resolution: "psl@npm:1.9.0" @@ -10674,17 +10451,6 @@ __metadata: languageName: node linkType: hard -"rlp@npm:^2.2.3": - version: 2.2.7 - resolution: "rlp@npm:2.2.7" - dependencies: - bn.js: ^5.2.0 - bin: - rlp: bin/rlp - checksum: 3db4dfe5c793f40ac7e0be689a1f75d05e6f2ca0c66189aeb62adab8c436b857ab4420a419251ee60370d41d957a55698fc5e23ab1e1b41715f33217bc4bb558 - languageName: node - linkType: hard - "rotating-file-stream@npm:^3.0.2": version: 3.1.0 resolution: "rotating-file-stream@npm:3.1.0" @@ -10710,7 +10476,7 @@ __metadata: languageName: node linkType: hard -"safe-buffer@npm:5.2.1, safe-buffer@npm:^5.0.1, safe-buffer@npm:~5.2.0": +"safe-buffer@npm:5.2.1, safe-buffer@npm:~5.2.0": version: 5.2.1 resolution: "safe-buffer@npm:5.2.1" checksum: b99c4b41fdd67a6aaf280fcd05e9ffb0813654894223afb78a31f14a19ad220bba8aba1cb14eddce1fcfb037155fe6de4e861784eb434f7d11ed58d1e70dd491 @@ -10758,13 +10524,6 @@ __metadata: languageName: node linkType: hard -"semaphore-async-await@npm:^1.5.1": - version: 1.5.1 - resolution: "semaphore-async-await@npm:1.5.1" - checksum: 2dedf7c59ba5f2da860fed95a81017189de6257cbe06c9de0ff2e610a3ae427e9bde1ab7685a62b03ebc28982dee437110492215d75fd6dc8257ce7a38e66b74 - languageName: node - linkType: hard - "semver-diff@npm:^3.1.1": version: 3.1.1 resolution: "semver-diff@npm:3.1.1" @@ -10886,18 +10645,6 @@ __metadata: languageName: node linkType: hard -"sha.js@npm:^2.4.11": - version: 2.4.11 - resolution: "sha.js@npm:2.4.11" - dependencies: - inherits: ^2.0.1 - safe-buffer: ^5.0.1 - bin: - sha.js: ./bin.js - checksum: ebd3f59d4b799000699097dadb831c8e3da3eb579144fd7eb7a19484cbcbb7aca3c68ba2bb362242eb09e33217de3b4ea56e4678184c334323eca24a58e3ad07 - languageName: node - linkType: hard - "shebang-command@npm:^2.0.0": version: 2.0.0 resolution: "shebang-command@npm:2.0.0" @@ -12361,7 +12108,7 @@ __metadata: languageName: node linkType: hard -"xtend@npm:^4.0.0, xtend@npm:^4.0.2, xtend@npm:~4.0.0": +"xtend@npm:^4.0.0": version: 4.0.2 resolution: "xtend@npm:4.0.2" checksum: ac5dfa738b21f6e7f0dd6e65e1b3155036d68104e67e5d5d1bde74892e327d7e5636a076f625599dc394330a731861e87343ff184b0047fef1360a7ec0a5a36a From 1c3fc5b050174473f8d09f27ae02637a6fcc7cac Mon Sep 17 00:00:00 2001 From: Naveen V Date: Tue, 19 Sep 2023 05:33:31 +0000 Subject: [PATCH 03/15] update node-core and fix testing module --- package.json | 3 +- packages/node/package.json | 3 +- .../node/src/algorand/api.service.algorand.ts | 94 +++---------------- .../node/src/subcommands/testing.module.ts | 35 +------ .../node/src/subcommands/testing.service.ts | 14 ++- packages/types/src/project.ts | 2 +- yarn.lock | 83 ++++++++-------- 7 files changed, 73 insertions(+), 161 deletions(-) diff --git a/package.json b/package.json index 22d73f59..8550a7da 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "typescript": "^4.9.5" }, "resolutions": { - "node-fetch": "2.6.7" + "node-fetch": "2.6.7", + "@subql/types": "2.2.1-1" }, "scripts": { "build": "yarn workspaces foreach -ptA run build", diff --git a/packages/node/package.json b/packages/node/package.json index b5051527..c5ef6f3a 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -26,9 +26,8 @@ "@nestjs/schedule": "^3.0.1", "@subql/common": "dev", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "dev", + "@subql/node-core": "4.2.4-16", "@subql/types-algorand": "workspace:*", - "@willsoto/nestjs-prometheus": "^4.4.0", "algosdk": "^2.2.0", "axios": "^1.3.4", "cron-converter": "^1.0.2", diff --git a/packages/node/src/algorand/api.service.algorand.ts b/packages/node/src/algorand/api.service.algorand.ts index ef51e732..cac555f9 100644 --- a/packages/node/src/algorand/api.service.algorand.ts +++ b/packages/node/src/algorand/api.service.algorand.ts @@ -23,66 +23,16 @@ export class AlgorandApiService extends ApiService< SafeAPIService, BlockContent[] > { - networkMeta: NetworkMetadataPayload; constructor( @Inject('ISubqueryProject') private project: SubqueryProject, connectionPoolService: ConnectionPoolService, - private eventEmitter: EventEmitter2, + eventEmitter: EventEmitter2, ) { - super(connectionPoolService); - } - - async connectWithRetry( - endpoint: string, - index: number, - maxRetries: number, - retryInterval: number, - ): Promise { - let retries = 0; - - const tryConnect = async () => { - try { - const connection = await AlgorandApiConnection.create( - endpoint, - this.fetchBlockBatches, - ); - const api = connection.unsafeApi; - - if (!this.networkMeta) { - this.networkMeta = connection.networkMeta; - } - - if (this.project.network.chainId !== api.getGenesisHash()) { - throw this.metadataMismatchError( - 'ChainId', - this.project.network.chainId, - api.getGenesisHash(), - ); - } - - await this.connectionPoolService.addToConnections(connection, endpoint); - } catch (error) { - if (retries < maxRetries) { - retries++; - logger.warn( - `Failed to start up endpoint ${endpoint} (retry ${retries}/${maxRetries}): ${error.message}`, - ); - setTimeout(() => { - tryConnect(); - }, retryInterval); - } else { - logger.error( - `Failed to start up endpoint ${endpoint} after ${maxRetries} retries: ${error.message}`, - ); - } - } - }; - - await tryConnect(); + super(connectionPoolService, eventEmitter); } async init(): Promise { - let network: ProjectNetworkV1_0_0; + let network; try { network = this.project.network; @@ -91,22 +41,16 @@ export class AlgorandApiService extends ApiService< process.exit(1); } - const endpoints = Array.isArray(network.endpoint) - ? network.endpoint - : [network.endpoint]; - - const maxRetries = 3; - const retryInterval = 30000; // 30 seconds - const connectionPromises: Promise[] = []; - - endpoints.forEach((endpoint, i) => { - connectionPromises.push( - this.connectWithRetry(endpoint, i, maxRetries, retryInterval), - ); - }); - - // Wait for at least one successful connection before proceeding - await Promise.race(connectionPromises); + await this.createConnections( + network, + (endpoint) => + AlgorandApiConnection.create(endpoint, this.fetchBlockBatches), + //eslint-disable-next-line @typescript-eslint/require-await + async (connection: AlgorandApiConnection) => { + const api = connection.unsafeApi; + return api.getGenesisHash(); + }, + ); return this; } @@ -115,18 +59,6 @@ export class AlgorandApiService extends ApiService< return this.unsafeApi; } - private metadataMismatchError( - metadata: string, - expected: string, - actual: string, - ): Error { - return Error( - `Value of ${metadata} does not match across all endpoints. Please check that your endpoints are for the same network.\n - Expected: ${expected} - Actual: ${actual}`, - ); - } - async fetchBlockBatches( api: AlgorandApi, blocks: number[], diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 43795070..a04e7906 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -8,34 +8,28 @@ import { ConnectionPoolService, ConnectionPoolStateManager, DbModule, - NodeConfig, PoiService, StoreService, TestRunner, + StoreCacheService, } from '@subql/node-core'; -import { AlgorandApiConnection, AlgorandApiService } from '../algorand'; +import { AlgorandApiService } from '../algorand'; import { ConfigureModule } from '../configure/configure.module'; -import { SubqueryProject } from '../configure/SubqueryProject'; import { DsProcessorService } from '../indexer/ds-processor.service'; import { DynamicDsService } from '../indexer/dynamic-ds.service'; -import { FetchModule } from '../indexer/fetch.module'; import { IndexerManager } from '../indexer/indexer.manager'; import { ProjectService } from '../indexer/project.service'; import { SandboxService } from '../indexer/sandbox.service'; import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; -import { MetaModule } from '../meta/meta.module'; -import { TestingService } from './testing.service'; - @Module({ providers: [ StoreService, - TestingService, + StoreCacheService, EventEmitter2, PoiService, SandboxService, DsProcessorService, DynamicDsService, - ProjectService, UnfinalizedBlocksService, ConnectionPoolStateManager, ConnectionPoolService, @@ -43,37 +37,18 @@ import { TestingService } from './testing.service'; provide: 'IProjectService', useClass: ProjectService, }, - { - provide: AlgorandApiService, - useFactory: async ( - project: SubqueryProject, - connectionPoolService: ConnectionPoolService, - eventEmitter: EventEmitter2, - ) => { - const apiService = new AlgorandApiService( - project, - connectionPoolService, - eventEmitter, - ); - await apiService.init(); - return apiService; - }, - inject: ['ISubqueryProject', ConnectionPoolService, EventEmitter2], - }, - IndexerManager, + AlgorandApiService, SchedulerRegistry, TestRunner, { provide: 'IApi', - useClass: AlgorandApiService, + useExisting: AlgorandApiService, }, { provide: 'IIndexerManager', useClass: IndexerManager, }, ], - - imports: [MetaModule, FetchModule], controllers: [], exports: [TestRunner], }) diff --git a/packages/node/src/subcommands/testing.service.ts b/packages/node/src/subcommands/testing.service.ts index de05756e..18222a21 100644 --- a/packages/node/src/subcommands/testing.service.ts +++ b/packages/node/src/subcommands/testing.service.ts @@ -34,7 +34,15 @@ export class TestingService extends BaseTestingService< } async getTestRunner(): Promise< - TestRunner + [ + close: () => Promise, + runner: TestRunner< + AlgorandApi, + SafeAPIService, + BlockContent, + AlgorandProjectDs + >, + ] > { const testContext = await NestFactory.createApplicationContext( TestingModule, @@ -45,14 +53,14 @@ export class TestingService extends BaseTestingService< await testContext.init(); - const projectService: ProjectService = testContext.get(ProjectService); + const projectService: ProjectService = testContext.get('IProjectService'); const apiService = testContext.get(AlgorandApiService); // Initialise async services, we do this here rather than in factories, so we can capture one off events await apiService.init(); await projectService.init(); - return testContext.get(TestRunner); + return [testContext.close.bind(testContext), testContext.get(TestRunner)]; } async indexBlock( diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index ea7e876f..8bf660ea 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -162,7 +162,7 @@ export interface AlgorandDataSourceProcessor< export interface DictionaryQueryCondition { field: string; - value: string | string[]; + value: string | boolean | string[]; matcher?: string; // defaults to "equalTo", use "contains" for JSON } diff --git a/yarn.lock b/yarn.lock index 168ac75a..64c1a3e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2935,7 +2935,24 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:2.6.1-3, @subql/common@npm:dev": +"@subql/common@npm:2.7.1-0": + version: 2.7.1-0 + resolution: "@subql/common@npm:2.7.1-0" + dependencies: + axios: ^0.27.2 + class-transformer: ^0.5.1 + class-validator: ^0.14.0 + fs-extra: ^10.1.0 + ipfs-http-client: 56 + js-yaml: ^4.1.0 + reflect-metadata: ^0.1.13 + semver: ^7.5.2 + update-notifier: 5.1.0 + checksum: d4bf3b39d817c966a0b30944b21c47653a303039e65df341c3464c6f7a3f913e27ea1f0ce2e14a86f35c0fb172bb1e92b3191aed104603ecbc832fd754acd2cd + languageName: node + linkType: hard + +"@subql/common@npm:dev": version: 2.6.1-3 resolution: "@subql/common@npm:2.6.1-3" dependencies: @@ -2966,7 +2983,7 @@ __metadata: "@nestjs/testing": ^9.4.0 "@subql/common": dev "@subql/common-algorand": "workspace:*" - "@subql/node-core": dev + "@subql/node-core": 4.2.4-16 "@subql/types-algorand": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -2974,7 +2991,6 @@ __metadata: "@types/pino": ^6.3.12 "@types/tar": ^6.1.1 "@types/yargs": ^16.0.4 - "@willsoto/nestjs-prometheus": ^4.4.0 algosdk: ^2.2.0 axios: ^1.3.4 cron-converter: ^1.0.2 @@ -2996,21 +3012,21 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:dev": - version: 4.2.4-4 - resolution: "@subql/node-core@npm:4.2.4-4" +"@subql/node-core@npm:4.2.4-16": + version: 4.2.4-16 + resolution: "@subql/node-core@npm:4.2.4-16" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 2.6.1-3 + "@subql/common": 2.7.1-0 "@subql/testing": 2.0.2 - "@subql/types": 2.2.1-0 - "@subql/utils": 2.4.3 + "@subql/types": 2.2.1-1 + "@subql/utils": 2.4.4-1 "@subql/x-sequelize": 6.32.0-0.0.2 - "@willsoto/nestjs-prometheus": ^5.1.1 + "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 async-mutex: ^0.4.0 cross-fetch: ^3.1.6 @@ -3023,7 +3039,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: d6d59c19984d9ee4f3a91dc878223b25840205c4a4a3c4217e6f7da7d005a48b9546bda4e799c9b6dcc6ee90659efeceb99e374d1c518919588abdc26bcb4636 + checksum: a837fef368287bdefff825d1e2f434b41e706b6f1453e40adf77f23b4bde54edaa86ff1e208645cefd9ef8a8a885ab6871f57f8c863a210b0dee60f35b2c2ddc languageName: node linkType: hard @@ -3047,27 +3063,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/types@npm:2.2.1-0": - version: 2.2.1-0 - resolution: "@subql/types@npm:2.2.1-0" +"@subql/types@npm:2.2.1-1": + version: 2.2.1-1 + resolution: "@subql/types@npm:2.2.1-1" peerDependencies: "@polkadot/api": ^10 - checksum: 48639f5d3dc4b1ad103df8769744c2cc8fd7db6c27e0a41939c2576ff943ba3606a4fe3d7b97b7cce383add7b471504b1614f3134faed4c6422d93b169de9198 + checksum: ba11a63169557bbddd61b9da0f609548e40915cf399dddd8a27a9ad970aac16ffc6d23d09f082d2b1bce5465ea1a7bca471213ffe69e8ace1475a08ebf74f695 languageName: node linkType: hard -"@subql/types@npm:^2.1.3": - version: 2.1.3 - resolution: "@subql/types@npm:2.1.3" - peerDependencies: - "@polkadot/api": ^10 - checksum: f9d198fe21b6d7a53a38330e903c264052145a9a6e04aeffea0618931e1a8c8d461e90dbfb9c61d6e3233caba6c9b6f0717ac6a182e0340047093133fc09ec45 - languageName: node - linkType: hard - -"@subql/utils@npm:2.4.3": - version: 2.4.3 - resolution: "@subql/utils@npm:2.4.3" +"@subql/utils@npm:2.4.4-1": + version: 2.4.4-1 + resolution: "@subql/utils@npm:2.4.4-1" dependencies: "@polkadot/util": ^12.2.1 "@polkadot/util-crypto": ^12.2.1 @@ -3084,7 +3091,7 @@ __metadata: rotating-file-stream: ^3.0.2 semver: ^7.5.2 tar: ^6.1.11 - checksum: fa6aa95bb0b158fedfec67a8609dbb66a047e68e55412b2498fcfeb2d85e8daff7f2df5d7852a70da3ec814ab773f31b28f5c3f7f72c8a224b6f04b9ef1a8e46 + checksum: f80fac522d9739a493b8afe8f69029996b1e597241a60c8e2641a6192da1a03300c76738f9bcdaedc973f208ed60c6e9608483e60dd8711b0d0843d42b647b42 languageName: node linkType: hard @@ -3705,23 +3712,13 @@ __metadata: languageName: node linkType: hard -"@willsoto/nestjs-prometheus@npm:^4.4.0": - version: 4.7.0 - resolution: "@willsoto/nestjs-prometheus@npm:4.7.0" - peerDependencies: - "@nestjs/common": ^7.0.0 || ^8.0.0 || ^9.0.0 - prom-client: ^13.0.0 || ^14.0.0 - checksum: 06020f2cb36bf6f98b1f7ca3494103cc77bea227ab766bb66e0294040c6a1399c3ff2a1781b7d163a5590cd9ccf0903fc08fc3bd9d3112d63c7400c4d6d55f23 - languageName: node - linkType: hard - -"@willsoto/nestjs-prometheus@npm:^5.1.1": - version: 5.1.1 - resolution: "@willsoto/nestjs-prometheus@npm:5.1.1" +"@willsoto/nestjs-prometheus@npm:^5.4.0": + version: 5.5.0 + resolution: "@willsoto/nestjs-prometheus@npm:5.5.0" peerDependencies: - "@nestjs/common": ^7.0.0 || ^8.0.0 || ^9.0.0 + "@nestjs/common": ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 prom-client: ^13.0.0 || ^14.0.0 - checksum: fa60f20442605c4243bc21b829be80a53ac2a0d1b7c5ab5c816048970ae57486cbba38c9d9707c989608501b27aa9a50acb5046dca76d19900a5719e068f57ae + checksum: a4bd0517f23ac80edd7eda68eebe138847ab045a7a53b9ab8e206c043aa7fff37e2d7c6d2209ec71b4d39ac7fddec8059ab3c827de579a2121838fe3e50aa90b languageName: node linkType: hard From 870471e0686eb7c40c76266baf6f8555b460e49b Mon Sep 17 00:00:00 2001 From: Naveen V Date: Tue, 19 Sep 2023 05:34:11 +0000 Subject: [PATCH 04/15] cleanup code --- packages/node/src/algorand/api.service.algorand.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/node/src/algorand/api.service.algorand.ts b/packages/node/src/algorand/api.service.algorand.ts index cac555f9..26b8662c 100644 --- a/packages/node/src/algorand/api.service.algorand.ts +++ b/packages/node/src/algorand/api.service.algorand.ts @@ -3,13 +3,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import { ProjectNetworkV1_0_0 } from '@subql/common-algorand'; -import { - ApiService, - ConnectionPoolService, - getLogger, - NetworkMetadataPayload, -} from '@subql/node-core'; +import { ApiService, ConnectionPoolService, getLogger } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { BlockContent } from '../indexer/types'; import { AlgorandApi, SafeAPIService } from './api.algorand'; From c0bfc44f2d93173424548fb816ada596233fb234 Mon Sep 17 00:00:00 2001 From: Naveen V Date: Wed, 27 Sep 2023 08:40:02 +0000 Subject: [PATCH 05/15] update node-core and common --- packages/common-algorand/package.json | 5 +- packages/common-algorand/src/project/load.ts | 7 -- .../common-algorand/src/project/models.ts | 8 +- packages/common-algorand/src/project/types.ts | 8 +- packages/common-algorand/src/project/utils.ts | 12 +-- .../versioned/ProjectManifestVersioned.ts | 4 +- .../src/project/versioned/v1_0_0/index.ts | 1 - .../src/project/versioned/v1_0_0/model.ts | 101 +++++------------ .../src/project/versioned/v1_0_0/types.ts | 31 ------ packages/node/package.json | 5 +- .../node/src/configure/SubqueryProject.ts | 30 +++--- .../worker-block-dispatcher.service.ts | 2 +- .../node/src/indexer/ds-processor.service.ts | 6 +- .../node/src/indexer/dynamic-ds.service.ts | 4 +- packages/node/src/indexer/fetch.service.ts | 6 +- packages/node/src/indexer/sandbox.service.ts | 2 +- packages/types/package.json | 1 + packages/types/src/global.ts | 7 +- packages/types/src/interfaces.ts | 24 ----- packages/types/src/project.ts | 66 +++++------- yarn.lock | 102 ++++++++++-------- 21 files changed, 160 insertions(+), 272 deletions(-) delete mode 100644 packages/common-algorand/src/project/versioned/v1_0_0/types.ts diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index 83ce8639..f4585eb5 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -13,9 +13,10 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "dev", + "@subql/common": "2.7.1-3", "@subql/types-algorand": "workspace:*", - "class-transformer": "0.4.0", + "@subql/types-core": "0.0.0-2", + "class-transformer": "^0.5.1", "class-validator": "^0.13.2", "fs-extra": "^10.1.0", "ipfs-http-client": "^52.0.3", diff --git a/packages/common-algorand/src/project/load.ts b/packages/common-algorand/src/project/load.ts index 047792cf..7e84e6bd 100644 --- a/packages/common-algorand/src/project/load.ts +++ b/packages/common-algorand/src/project/load.ts @@ -9,10 +9,3 @@ export function parseAlgorandProjectManifest(raw: unknown): AlgorandProjectManif projectManifest.validate(); return projectManifest; } - -export function loadSubstrateProjectManifest(file: string): AlgorandProjectManifestVersioned { - const doc = loadFromJsonOrYaml(getManifestPath(file)); - const projectManifest = new AlgorandProjectManifestVersioned(doc as VersionedProjectManifest); - projectManifest.validate(); - return projectManifest; -} diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts index 800c8c21..9fdc2fc1 100644 --- a/packages/common-algorand/src/project/models.ts +++ b/packages/common-algorand/src/project/models.ts @@ -1,7 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {BaseMapping, FileReference} from '@subql/common'; +import {BlockFilterImpl, ProcessorImpl} from '@subql/common'; import { CustomDataSourceAsset as AlgorandCustomDataSourceAsset, AlgorandBlockFilter, @@ -16,6 +16,8 @@ import { AlgorandTransactionHandler, AlgorandTransactionFilter, } from '@subql/types-algorand'; +import {BaseMapping, FileReference, Processor} from '@subql/types-core'; + import {TransactionType} from 'algosdk'; import {plainToClass, Transform, Type} from 'class-transformer'; import { @@ -198,7 +200,7 @@ export class CustomDataSourceBase; - @Type(() => FileReferenceImpl) + @Type(() => ProcessorImpl) @IsObject() - processor: FileReference; + processor: Processor; } diff --git a/packages/common-algorand/src/project/types.ts b/packages/common-algorand/src/project/types.ts index 81ee820d..7295aac2 100644 --- a/packages/common-algorand/src/project/types.ts +++ b/packages/common-algorand/src/project/types.ts @@ -1,8 +1,8 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {IProjectManifest, ProjectNetworkConfig} from '@subql/common'; import {AlgorandDataSource} from '@subql/types-algorand'; +import {IProjectManifest} from '@subql/types-core'; // All of these used to be redefined in this file, re-exporting for simplicity export { @@ -21,9 +21,3 @@ export { export type IAlgorandProjectManifest = IProjectManifest; export type TokenHeader = Record; - -export interface AlgorandProjectNetworkConfig extends ProjectNetworkConfig { - genesisHash?: string; - chainId?: string; - apiKey?: string | TokenHeader; -} diff --git a/packages/common-algorand/src/project/utils.ts b/packages/common-algorand/src/project/utils.ts index fcec8ad3..b182f523 100644 --- a/packages/common-algorand/src/project/utils.ts +++ b/packages/common-algorand/src/project/utils.ts @@ -10,15 +10,15 @@ import { AlgorandRuntimeDataSource, } from '@subql/types-algorand'; -export function isBlockHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { +export function isBlockHandlerProcessor, E>( + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === AlgorandHandlerKind.Block; } -export function isTransactionHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { +export function isTransactionHandlerProcessor, E>( + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === AlgorandHandlerKind.Transaction; } diff --git a/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts b/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts index 2c7af856..8b76168f 100644 --- a/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts +++ b/packages/common-algorand/src/project/versioned/ProjectManifestVersioned.ts @@ -13,7 +13,7 @@ const ALGORAND_SUPPORTED_VERSIONS = { type Versions = keyof typeof ALGORAND_SUPPORTED_VERSIONS; -type ProjectManifestImpls = InstanceType<(typeof ALGORAND_SUPPORTED_VERSIONS)[Versions]>; +export type ProjectManifestImpls = InstanceType<(typeof ALGORAND_SUPPORTED_VERSIONS)[Versions]>; export function manifestIsV1_0_0(manifest: IAlgorandProjectManifest): manifest is ProjectManifestV1_0_0Impl { return manifest.specVersion === '1.0.0'; @@ -43,7 +43,7 @@ export class AlgorandProjectManifestVersioned implements IAlgorandProjectManifes } toDeployment(): string | undefined { - return this._impl.toDeployment(); + return this._impl.deployment.toYaml(); } validate(): void { diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/index.ts b/packages/common-algorand/src/project/versioned/v1_0_0/index.ts index d9954cc2..26eab2f7 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/index.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/index.ts @@ -2,4 +2,3 @@ // SPDX-License-Identifier: GPL-3.0 export * from './model'; -export * from './types'; diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts index 490751bc..c38cf1a1 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts @@ -2,18 +2,25 @@ // SPDX-License-Identifier: GPL-3.0 import { - BaseMapping, - NodeSpec, - ParentProject, + BaseDeploymentV1_0_0, + FileType, ParentProjectModel, ProjectManifestBaseImpl, - QuerySpec, + RunnerNodeImpl, RunnerQueryBaseModel, - RunnerSpecs, + validateObject, + CommonProjectNetworkV1_0_0, SemverVersionValidator, } from '@subql/common'; -import {AlgorandCustomDataSource} from '@subql/types-algorand'; -import {plainToClass, Transform, Type} from 'class-transformer'; +import { + AlgorandCustomDataSource, + AlgorandProjectManifestV1_0_0, + AlgorandRuntimeDataSource, + CustomDatasourceTemplate, + RuntimeDatasourceTemplate, +} from '@subql/types-algorand'; +import {BaseMapping, NodeSpec, ParentProject, QuerySpec, RunnerSpecs} from '@subql/types-core'; +import {plainToInstance, Transform, Type} from 'class-transformer'; import { Equals, IsArray, @@ -29,14 +36,6 @@ import yaml from 'js-yaml'; import {CustomDataSourceBase, RuntimeDataSourceBase} from '../../models'; import {TokenHeader} from '../../types'; import {IsStringOrObject} from '../../validation/is-string-or-object.validation'; -import { - CustomDataSourceTemplate, - CustomDataSourceV1_0_0, - AlgorandProjectManifestV1_0_0, - RuntimeDataSourceTemplate, - RuntimeDataSourceV1_0_0, -} from './types'; - const ALGORAND_NODE_NAME = `@subql/node-algorand`; export class AlgorandRunnerNodeImpl implements NodeSpec { @@ -60,11 +59,6 @@ export class AlgorandRunnerSpecsImpl implements RunnerSpecs { query: QuerySpec; } -export class FileType { - @IsString() - file: string; -} - export class ProjectNetworkDeploymentV1_0_0 { @IsString() chainId: string; @@ -73,34 +67,13 @@ export class ProjectNetworkDeploymentV1_0_0 { bypassBlocks?: (number | string)[]; } -export class ProjectNetworkV1_0_0 extends ProjectNetworkDeploymentV1_0_0 { - @IsString({each: true}) - @IsOptional() - endpoint?: string | string[]; - - @IsString() - @IsOptional() - dictionary?: string; - - @IsString() - @IsOptional() - genesisHash?: string; - +export class ProjectNetworkV1_0_0 extends CommonProjectNetworkV1_0_0 { @IsStringOrObject() @IsOptional() apiKey?: string | TokenHeader; } -function validateObject(object: any, errorMessage = 'failed to validate object.'): void { - const errors = validateSync(object, {whitelist: true, forbidNonWhitelisted: true}); - if (errors?.length) { - // TODO: print error details - const errorMsgs = errors.map((e) => e.toString()).join('\n'); - throw new Error(`${errorMessage}\n${errorMsgs}`); - } -} - -export class AlgorandRuntimeDataSourceV1_0_0Impl extends RuntimeDataSourceBase implements RuntimeDataSourceV1_0_0 { +export class AlgorandRuntimeDataSourceV1_0_0Impl extends RuntimeDataSourceBase { validate(): void { return validateObject(this, 'failed to validate runtime datasource.'); } @@ -120,7 +93,7 @@ export class AlgorandCustomDataSourceV1_0_0Impl< export class RuntimeDataSourceTemplateImpl extends AlgorandRuntimeDataSourceV1_0_0Impl - implements RuntimeDataSourceTemplate + implements RuntimeDatasourceTemplate { @IsString() name: string; @@ -128,34 +101,27 @@ export class RuntimeDataSourceTemplateImpl export class CustomDataSourceTemplateImpl extends AlgorandCustomDataSourceV1_0_0Impl - implements CustomDataSourceTemplate + implements CustomDatasourceTemplate { @IsString() name: string; } -export class DeploymentV1_0_0 { +export class DeploymentV1_0_0 extends BaseDeploymentV1_0_0 { @Transform((params) => { if (params.value.genesisHash && !params.value.chainId) { params.value.chainId = params.value.genesisHash; } - return plainToClass(ProjectNetworkDeploymentV1_0_0, params.value); + return plainToInstance(ProjectNetworkDeploymentV1_0_0, params.value); }) @ValidateNested() @Type(() => ProjectNetworkDeploymentV1_0_0) network: ProjectNetworkDeploymentV1_0_0; - @Equals('1.0.0') - @IsString() - specVersion: string; @IsObject() @ValidateNested() @Type(() => AlgorandRunnerSpecsImpl) runner: RunnerSpecs; - @ValidateNested() - @Type(() => FileType) - schema: FileType; - @IsArray() @ValidateNested() @Type(() => AlgorandCustomDataSourceV1_0_0Impl, { @@ -165,7 +131,7 @@ export class DeploymentV1_0_0 { }, keepDiscriminatorProperty: true, }) - dataSources: (RuntimeDataSourceV1_0_0 | CustomDataSourceV1_0_0)[]; + dataSources: (AlgorandCustomDataSource | AlgorandRuntimeDataSource)[]; @IsOptional() @IsArray() @ValidateNested() @@ -176,18 +142,17 @@ export class DeploymentV1_0_0 { }, keepDiscriminatorProperty: true, }) - templates?: (RuntimeDataSourceTemplate | CustomDataSourceTemplate)[]; - - @IsOptional() - @IsObject() - @Type(() => ParentProjectModel) - parent?: ParentProject; + templates?: (RuntimeDatasourceTemplate | CustomDatasourceTemplate)[]; } export class ProjectManifestV1_0_0Impl extends ProjectManifestBaseImpl implements AlgorandProjectManifestV1_0_0 { + constructor() { + super(DeploymentV1_0_0); + } + @Equals('1.0.0') specVersion: string; @@ -215,7 +180,7 @@ export class ProjectManifestV1_0_0Impl }, keepDiscriminatorProperty: true, }) - dataSources: (RuntimeDataSourceV1_0_0 | CustomDataSourceV1_0_0)[]; + dataSources: (AlgorandRuntimeDataSource | AlgorandCustomDataSource)[]; @IsOptional() @IsArray() @@ -227,15 +192,13 @@ export class ProjectManifestV1_0_0Impl }, keepDiscriminatorProperty: true, }) - templates?: (RuntimeDataSourceTemplate | CustomDataSourceTemplate)[]; + templates?: (RuntimeDatasourceTemplate | CustomDatasourceTemplate)[]; @IsObject() @ValidateNested() @Type(() => AlgorandRunnerSpecsImpl) runner: RunnerSpecs; - private _deployment: DeploymentV1_0_0; - @IsOptional() @IsObject() @Type(() => ParentProjectModel) @@ -248,14 +211,6 @@ export class ProjectManifestV1_0_0Impl }); } - get deployment(): DeploymentV1_0_0 { - if (!this._deployment) { - this._deployment = plainToClass(DeploymentV1_0_0, this); - validateSync(this._deployment, {whitelist: true}); - } - return this._deployment; - } - validate(): void { const errors = validateSync(this.deployment, {whitelist: true, forbidNonWhitelisted: true}); if (errors?.length) { diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/types.ts b/packages/common-algorand/src/project/versioned/v1_0_0/types.ts deleted file mode 100644 index fd8ae447..00000000 --- a/packages/common-algorand/src/project/versioned/v1_0_0/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import {ProjectManifestV1_0_0, TemplateBase} from '@subql/common'; -import { - AlgorandCustomDataSource, - AlgorandDataSource, - AlgorandDataSourceKind, - AlgorandHandler, - AlgorandMapping, - AlgorandRuntimeDataSource, -} from '@subql/types-algorand'; - -export interface SubqlMappingV1_0_0 extends AlgorandMapping { - file: string; -} - -export type RuntimeDataSourceV1_0_0 = AlgorandRuntimeDataSource; -export type CustomDataSourceV1_0_0 = AlgorandCustomDataSource; - -export interface RuntimeDataSourceTemplate extends Omit, TemplateBase {} -export interface CustomDataSourceTemplate extends Omit, TemplateBase {} - -export type AlgorandProjectManifestV1_0_0 = ProjectManifestV1_0_0< - RuntimeDataSourceV1_0_0 | CustomDataSourceV1_0_0, - RuntimeDataSourceTemplate | CustomDataSourceTemplate ->; - -export function isRuntimeDataSourceV1_0_0(dataSource: AlgorandDataSource): dataSource is RuntimeDataSourceV1_0_0 { - return dataSource.kind === AlgorandDataSourceKind.Runtime && !!(dataSource as RuntimeDataSourceV1_0_0).mapping.file; -} diff --git a/packages/node/package.json b/packages/node/package.json index c5ef6f3a..54a48aed 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -24,10 +24,11 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "dev", + "@subql/common": "2.7.1-3", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "4.2.4-16", + "@subql/node-core": "4.2.4-26", "@subql/types-algorand": "workspace:*", + "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0", "axios": "^1.3.4", "cron-converter": "^1.0.2", diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 0e6a2736..3159643c 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -3,14 +3,8 @@ import assert from 'assert'; import { Injectable } from '@nestjs/common'; +import { validateSemver } from '@subql/common'; import { - ParentProject, - Reader, - RunnerSpecs, - validateSemver, -} from '@subql/common'; -import { - AlgorandProjectNetworkConfig, parseAlgorandProjectManifest, AlgorandDataSource, ProjectManifestV1_0_0Impl, @@ -18,8 +12,6 @@ import { isRuntimeDs, AlgorandHandlerKind, isCustomDs, - RuntimeDataSourceTemplate, - CustomDataSourceTemplate, } from '@subql/common-algorand'; import { insertBlockFiltersCronSchedules, @@ -27,6 +19,16 @@ import { SubqlProjectDs, updateDataSourcesV1_0_0, } from '@subql/node-core'; +import { + CustomDatasourceTemplate, + RuntimeDatasourceTemplate, +} from '@subql/types-algorand'; +import { + IProjectNetworkConfig, + ParentProject, + Reader, + RunnerSpecs, +} from '@subql/types-core'; import { buildSchemaFromString } from '@subql/utils'; import Cron from 'cron-converter'; import { GraphQLSchema } from 'graphql'; @@ -36,8 +38,8 @@ const { version: packageVersion } = require('../../package.json'); export type AlgorandProjectDs = SubqlProjectDs; export type AlgorandProjectDsTemplate = - | SubqlProjectDs - | SubqlProjectDs; + | SubqlProjectDs + | SubqlProjectDs; export type SubqlProjectBlockFilter = BlockFilter & { cronSchedule?: { @@ -55,7 +57,7 @@ const NOT_SUPPORT = (name: string) => { }; // This is the runtime type after we have mapped genesisHash to chainId and endpoint/dict have been provided when dealing with deployments -type NetworkConfig = AlgorandProjectNetworkConfig & { chainId: string }; +type NetworkConfig = IProjectNetworkConfig & { chainId: string }; @Injectable() export class SubqueryProject { @@ -94,7 +96,7 @@ export class SubqueryProject { rawManifest: unknown, reader: Reader, root: string, - networkOverrides?: Partial, + networkOverrides?: Partial, ): Promise { // rawManifest and reader can be reused here. // It has been pre-fetched and used for rebase manifest runner options with args @@ -138,7 +140,7 @@ async function loadProjectFromManifestBase( reader: Reader, path: string, root: string, - networkOverrides?: Partial, + networkOverrides?: Partial, ): Promise { if (typeof projectManifest.network.endpoint === 'string') { projectManifest.network.endpoint = [projectManifest.network.endpoint]; diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 27931752..ddb02d37 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -26,7 +26,7 @@ import { IProjectUpgradeService, HostUnfinalizedBlocks, } from '@subql/node-core'; -import { Store } from '@subql/types'; +import { Store } from '@subql/types-core'; import { AlgorandApiConnection } from '../../algorand'; import { AlgorandProjectDs, diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts index 403238a2..fc4dcafa 100644 --- a/packages/node/src/indexer/ds-processor.service.ts +++ b/packages/node/src/indexer/ds-processor.service.ts @@ -24,7 +24,7 @@ export interface DsPluginSandboxOption { export function isSecondLayerHandlerProcessor_0_0_0< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource, >( @@ -38,7 +38,7 @@ export function isSecondLayerHandlerProcessor_0_0_0< export function isSecondLayerHandlerProcessor_1_0_0< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource, >( @@ -51,7 +51,7 @@ export function isSecondLayerHandlerProcessor_1_0_0< export function asSecondLayerHandlerProcessor_1_0_0< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource, >( diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index 154617d4..f9841706 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -27,7 +27,7 @@ export class DynamicDsService extends BaseDynamicDsService { protected async getDatasource( params: DatasourceParams, ): Promise { - const template = cloneDeep( + const { name, ...template } = cloneDeep( this.project.templates.find((t) => t.name === params.templateName), ); @@ -41,7 +41,7 @@ export class DynamicDsService extends BaseDynamicDsService { ...template, startBlock: params.startBlock, } as AlgorandProjectDs; - delete dsObj.name; + try { if (isCustomDs(dsObj)) { dsObj.processor.options = { diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 11de1480..d1f0969b 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -13,11 +13,11 @@ import { AlgorandRuntimeHandlerFilter, } from '@subql/common-algorand'; import { NodeConfig, BaseFetchService, getModulos } from '@subql/node-core'; -import { DictionaryQueryCondition } from '@subql/types'; +import { AlgorandBlockFilter } from '@subql/types-algorand'; import { - AlgorandBlockFilter, + DictionaryQueryCondition, DictionaryQueryEntry, -} from '@subql/types-algorand'; +} from '@subql/types-core'; import { sortBy, uniqBy } from 'lodash'; import { AlgorandApi, AlgorandApiService, calcInterval } from '../algorand'; import { SubqueryProject } from '../configure/SubqueryProject'; diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index f9b92c1b..9678aa9c 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -3,7 +3,6 @@ import { isMainThread } from 'worker_threads'; import { Inject, Injectable } from '@nestjs/common'; -import { BaseDataSource } from '@subql/common'; import { hostStoreToStore, IndexerSandbox, @@ -12,6 +11,7 @@ import { StoreService, } from '@subql/node-core'; import { Store } from '@subql/types'; +import { BaseDataSource } from '@subql/types-core'; import { AlgorandApiService } from '../algorand'; /* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */ diff --git a/packages/types/package.json b/packages/types/package.json index cb31e956..523054a7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -16,6 +16,7 @@ "/dist" ], "peerDependencies": { + "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0" }, "devDependencies": { diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts index 96326c28..59d600b1 100644 --- a/packages/types/src/global.ts +++ b/packages/types/src/global.ts @@ -1,13 +1,8 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import Pino from 'pino'; -import {Store, DynamicDatasourceCreator, SafeAPI} from './interfaces'; +import {SafeAPI} from './interfaces'; declare global { const api: SafeAPI; - const logger: Pino.Logger; - const store: Store; - const chainId: string; - const createDynamicDatasource: DynamicDatasourceCreator; } diff --git a/packages/types/src/interfaces.ts b/packages/types/src/interfaces.ts index 415aaf34..588f2dc2 100644 --- a/packages/types/src/interfaces.ts +++ b/packages/types/src/interfaces.ts @@ -3,30 +3,6 @@ import type {TransactionType, Indexer} from 'algosdk'; -export interface Entity { - id: string; - _name?: string; - save?: () => Promise; -} - -export type FunctionPropertyNames = { - [K in keyof T]: T[K] extends Function ? K : never; -}[keyof T]; - -export interface Store { - get(entity: string, id: string): Promise; - getByField(entity: string, field: string, value: any, options?: {offset?: number; limit?: number}): Promise; - getOneByField(entity: string, field: string, value: any): Promise; - set(entity: string, id: string, data: Entity): Promise; - bulkCreate(entity: string, data: Entity[]): Promise; - //if fields in provided, only specify fields will be updated - bulkUpdate(entity: string, data: Entity[], fields?: string[]): Promise; - remove(entity: string, id: string): Promise; - bulkRemove(entity: string, ids: string[]): Promise; -} - -export type DynamicDatasourceCreator = (name: string, args: Record) => Promise; - export enum EOnCompletion { noop = 'noop', optin = 'optin', diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 8bf660ea..7b15437a 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -1,9 +1,23 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import { + BaseTemplateDataSource, + IProjectNetworkConfig, + CommonSubqueryProject, + DictionaryQueryEntry, + FileReference, + Processor, + ProjectManifestV1_0_0, +} from '@subql/types-core'; import type {Indexer} from 'algosdk'; import {AlgorandBlock, AlgorandTransaction} from './interfaces'; +export type RuntimeDatasourceTemplate = BaseTemplateDataSource; +export type CustomDatasourceTemplate = BaseTemplateDataSource; + +export type AlgorandProjectManifestV1_0_0 = ProjectManifestV1_0_0; + export enum AlgorandDataSourceKind { Runtime = 'algorand/Runtime', } @@ -23,21 +37,6 @@ type RuntimeFilterMap = { [AlgorandHandlerKind.Transaction]: AlgorandTransactionFilter; }; -export interface ProjectManifest { - specVersion: string; - description: string; - repository: string; - - schema: string; - - network: { - endpoint: string; - }; - - dataSources: AlgorandDataSource[]; - bypassBlocks?: number[]; -} - // [startSpecVersion?, endSpecVersion?] closed range export type SpecVersionRange = [number, number]; @@ -97,14 +96,8 @@ export interface AlgorandRuntimeDataSource< export type AlgorandDataSource = AlgorandRuntimeDataSource | AlgorandCustomDataSource; -export interface FileReference { - file: string; -} - export type CustomDataSourceAsset = FileReference; -export type Processor = FileReference & {options?: O}; - export interface AlgorandCustomDataSource< K extends string = string, M extends AlgorandMapping = AlgorandMapping, @@ -115,8 +108,6 @@ export interface AlgorandCustomDataSource< processor: Processor; } -//export type SubstrateBuiltinDataSource = IAlgorandDataSource; - export interface HandlerInputTransformer_0_0_0< T extends AlgorandHandlerKind, E, @@ -127,7 +118,7 @@ export interface HandlerInputTransformer_0_0_0< export interface HandlerInputTransformer_1_0_0< T extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > { @@ -142,7 +133,7 @@ export interface HandlerInputTransformer_1_0_0< type SecondLayerHandlerProcessorArray< K extends string, - T, + T extends Record, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > = SecondLayerHandlerProcessor; @@ -160,20 +151,9 @@ export interface AlgorandDataSourceProcessor< handlerProcessors: P; } -export interface DictionaryQueryCondition { - field: string; - value: string | boolean | string[]; - matcher?: string; // defaults to "equalTo", use "contains" for JSON -} - -export interface DictionaryQueryEntry { - entity: string; - conditions: DictionaryQueryCondition[]; -} - interface SecondLayerHandlerProcessorBase< K extends AlgorandHandlerKind, - F, + F extends Record, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > { baseHandlerKind: K; @@ -185,7 +165,7 @@ interface SecondLayerHandlerProcessorBase< // only allow one custom handler for each baseHandler kind export interface SecondLayerHandlerProcessor_0_0_0< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > extends SecondLayerHandlerProcessorBase { @@ -196,7 +176,7 @@ export interface SecondLayerHandlerProcessor_0_0_0< export interface SecondLayerHandlerProcessor_1_0_0< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > extends SecondLayerHandlerProcessorBase { @@ -207,7 +187,13 @@ export interface SecondLayerHandlerProcessor_1_0_0< export type SecondLayerHandlerProcessor< K extends AlgorandHandlerKind, - F, + F extends Record, E, DS extends AlgorandCustomDataSource = AlgorandCustomDataSource > = SecondLayerHandlerProcessor_0_0_0 | SecondLayerHandlerProcessor_1_0_0; + +export type AlgorandProject = CommonSubqueryProject< + IProjectNetworkConfig, + AlgorandRuntimeDataSource | DS, + BaseTemplateDataSource | BaseTemplateDataSource +>; diff --git a/yarn.lock b/yarn.lock index 64c1a3e6..78fd6ec5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,13 +2916,14 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": dev + "@subql/common": 2.7.1-3 "@subql/types-algorand": "workspace:*" + "@subql/types-core": 0.0.0-2 "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 "@types/pino": ^6.3.12 "@types/semver": ^7 - class-transformer: 0.4.0 + class-transformer: ^0.5.1 class-validator: ^0.13.2 fs-extra: ^10.1.0 ipfs-http-client: ^52.0.3 @@ -2935,27 +2936,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:2.7.1-0": - version: 2.7.1-0 - resolution: "@subql/common@npm:2.7.1-0" - dependencies: - axios: ^0.27.2 - class-transformer: ^0.5.1 - class-validator: ^0.14.0 - fs-extra: ^10.1.0 - ipfs-http-client: 56 - js-yaml: ^4.1.0 - reflect-metadata: ^0.1.13 - semver: ^7.5.2 - update-notifier: 5.1.0 - checksum: d4bf3b39d817c966a0b30944b21c47653a303039e65df341c3464c6f7a3f913e27ea1f0ce2e14a86f35c0fb172bb1e92b3191aed104603ecbc832fd754acd2cd - languageName: node - linkType: hard - -"@subql/common@npm:dev": - version: 2.6.1-3 - resolution: "@subql/common@npm:2.6.1-3" +"@subql/common@npm:2.7.1-3": + version: 2.7.1-3 + resolution: "@subql/common@npm:2.7.1-3" dependencies: + "@subql/types-core": 0.0.0-1 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2965,7 +2950,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 397c21df879b4310b4aab0f4108b942a53c069d7186cff196ef0cd245ce72e357bd3f6f5891256c9850354f813aac349a10bf44a906f0b72d44619fb27cdc39a + checksum: f05dcc0089f4e10abb438f5beb6dfc1156d2b1fb8b2053ebfbf9417e3f393df1ce5e14a50fcc57e429002704dfc2246b380649b9a3d0845bbb924165e6eb1861 languageName: node linkType: hard @@ -2981,10 +2966,11 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": dev + "@subql/common": 2.7.1-3 "@subql/common-algorand": "workspace:*" - "@subql/node-core": 4.2.4-16 + "@subql/node-core": 4.2.4-26 "@subql/types-algorand": "workspace:*" + "@subql/types-core": 0.0.0-2 "@types/express": ^4.17.13 "@types/jest": ^27.4.0 "@types/lodash": ^4.14.178 @@ -3012,18 +2998,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:4.2.4-16": - version: 4.2.4-16 - resolution: "@subql/node-core@npm:4.2.4-16" +"@subql/node-core@npm:4.2.4-26": + version: 4.2.4-26 + resolution: "@subql/node-core@npm:4.2.4-26" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 2.7.1-0 - "@subql/testing": 2.0.2 - "@subql/types": 2.2.1-1 + "@subql/common": 2.7.1-3 + "@subql/testing": 2.0.3-0 + "@subql/types": 2.2.1-3 "@subql/utils": 2.4.4-1 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 @@ -3039,16 +3025,16 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: a837fef368287bdefff825d1e2f434b41e706b6f1453e40adf77f23b4bde54edaa86ff1e208645cefd9ef8a8a885ab6871f57f8c863a210b0dee60f35b2c2ddc + checksum: f9e0c95c5a6584621942b3edbf106af28ac54f647086066fa73b818df49dd8bfe4e30559a93a0eea04596672b7558ece963d4464334f6987796cf2acc928c3a5 languageName: node linkType: hard -"@subql/testing@npm:2.0.2": - version: 2.0.2 - resolution: "@subql/testing@npm:2.0.2" +"@subql/testing@npm:2.0.3-0": + version: 2.0.3-0 + resolution: "@subql/testing@npm:2.0.3-0" dependencies: - "@subql/types": ^2.1.3 - checksum: c564a80d226509c5343b2ffb8f10f6836a16ace8767bd8805db68ae0b01f000ec5e66b889049d1ff3fc401b5e343d447a5407f14515dc572e5695e15c862a37a + "@subql/types-core": ^0.0.0-1 + checksum: ded64670e266309115f1f8003dbf0cef1219ceff7fa30c2f1389588d1e659685b74e44e70d3f3389654a0e2ba2d04dddb239b19253d35af494cc8a93548955b5 languageName: node linkType: hard @@ -3059,10 +3045,38 @@ __metadata: "@types/app-module-path": ^2.2.0 algosdk: ^2.2.0 peerDependencies: + "@subql/types-core": 0.0.0-2 algosdk: ^2.2.0 languageName: unknown linkType: soft +"@subql/types-core@npm:0.0.0-1": + version: 0.0.0-1 + resolution: "@subql/types-core@npm:0.0.0-1" + dependencies: + package-json-type: ^1.0.3 + checksum: fd39376cd64622e34711dfc86c660e1cbc4fac5c72e7ee410d19b0753601024a7455b0b144fe4a1c9903f788686eb2f1c3cb50a7c9a1eb39933102b729958a42 + languageName: node + linkType: hard + +"@subql/types-core@npm:0.0.0-2": + version: 0.0.0-2 + resolution: "@subql/types-core@npm:0.0.0-2" + dependencies: + package-json-type: ^1.0.3 + checksum: 3cd6a90391a8d206d0140bc84fff3b019a54488b711ce4f3d8c59f5b326dba9fc8b76d233add90b6a8dc953b09a7d3057f5132ec04a4b46e18c1aac49e8c685f + languageName: node + linkType: hard + +"@subql/types-core@npm:^0.0.0-1": + version: 0.0.0-0-ts-manifest + resolution: "@subql/types-core@npm:0.0.0-0-ts-manifest" + dependencies: + package-json-type: ^1.0.3 + checksum: e37e0df66471d763200baa00cf2bea526cdf0a5ee81b496473675e5404fe9553d56a6f7ebb6a6a22916981f6a2a2e2d303c538d8d3120fcd81267cc4703a88d2 + languageName: node + linkType: hard + "@subql/types@npm:2.2.1-1": version: 2.2.1-1 resolution: "@subql/types@npm:2.2.1-1" @@ -4855,13 +4869,6 @@ __metadata: languageName: node linkType: hard -"class-transformer@npm:0.4.0": - version: 0.4.0 - resolution: "class-transformer@npm:0.4.0" - checksum: 4057346b512705b1a5a68025b025a5be1ff847d7e0e76e82bbb937f832cf6235c5f66e874487d4bd35d67401411e8a3279c9d36bdd8c800336d5507ee6e94af0 - languageName: node - linkType: hard - "class-transformer@npm:^0.5.1": version: 0.5.1 resolution: "class-transformer@npm:0.5.1" @@ -9556,6 +9563,13 @@ __metadata: languageName: node linkType: hard +"package-json-type@npm:^1.0.3": + version: 1.0.3 + resolution: "package-json-type@npm:1.0.3" + checksum: ea93a2ee1cbc480feee94974158605463b8ce8f899e8a09dc008ac83c060fd10d8f38bfdd5f28222196dba1f57de4e65b46b057dc889a704967714da393c2ad9 + languageName: node + linkType: hard + "package-json@npm:^6.3.0": version: 6.5.0 resolution: "package-json@npm:6.5.0" From 89dbf3a2830170ad7a5f8e8da936fabaf7556947 Mon Sep 17 00:00:00 2001 From: Naveen V Date: Wed, 27 Sep 2023 08:47:29 +0000 Subject: [PATCH 06/15] remove unnecessary service injection for worker --- packages/node/nodemon.json | 2 +- .../node/src/configure/configure.module.ts | 11 +++++++- packages/node/src/indexer/project.service.ts | 9 ++++-- packages/node/src/indexer/sandbox.service.ts | 1 + .../src/indexer/unfinalizedBlocks.service.ts | 5 ++++ .../worker-fetch.module.ts} | 28 ++++++++----------- .../node/src/indexer/worker/worker.module.ts | 4 +-- 7 files changed, 36 insertions(+), 24 deletions(-) rename packages/node/src/indexer/{indexer.module.ts => worker/worker-fetch.module.ts} (70%) diff --git a/packages/node/nodemon.json b/packages/node/nodemon.json index 0fcae7bd..92c89123 100644 --- a/packages/node/nodemon.json +++ b/packages/node/nodemon.json @@ -2,5 +2,5 @@ "watch": ["src"], "ext": "ts", "ignore": ["src/**/*.spec.ts"], - "exec": "node -r dotenv/config -r tsconfig-paths/register -r ts-node/register src/main.ts" + "exec": "node --trace-exit --trace-sigint --trace-warnings --trace-uncaught --trace-deprecation -r dotenv/config -r tsconfig-paths/register -r ts-node/register src/main.ts" } diff --git a/packages/node/src/configure/configure.module.ts b/packages/node/src/configure/configure.module.ts index c89287e8..8361b7ad 100644 --- a/packages/node/src/configure/configure.module.ts +++ b/packages/node/src/configure/configure.module.ts @@ -41,8 +41,17 @@ export class ConfigureModule { provide: 'IProjectUpgradeService', useValue: project, }, + { + provide: 'Null', + useValue: null, + }, + ], + exports: [ + NodeConfig, + 'ISubqueryProject', + 'IProjectUpgradeService', + 'Null', ], - exports: [NodeConfig, 'ISubqueryProject', 'IProjectUpgradeService'], }; } } diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 7e63fd0c..610ae4ea 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import { isMainThread } from 'worker_threads'; import { Inject, Injectable } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { @@ -9,6 +10,7 @@ import { StoreService, NodeConfig, IProjectUpgradeService, + mainThreadOnly, } from '@subql/node-core'; import { Sequelize } from '@subql/x-sequelize'; import { AlgorandApiService } from '../algorand'; @@ -33,12 +35,12 @@ export class ProjectService extends BaseProjectService< constructor( dsProcessorService: DsProcessorService, apiService: AlgorandApiService, - poiService: PoiService, - sequelize: Sequelize, + @Inject(isMainThread ? PoiService : 'Null') poiService: PoiService, + @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize, @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IProjectUpgradeService') protected readonly projectUpgradeService: IProjectUpgradeService, - storeService: StoreService, + @Inject(isMainThread ? StoreService : 'Null') storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, eventEmitter: EventEmitter2, @@ -65,6 +67,7 @@ export class ProjectService extends BaseProjectService< return new Date(block.timestamp); } + @mainThreadOnly() protected onProjectChange(project: SubqueryProject): void | Promise { // TODO update this when implementing skipBlock feature for Eth // this.apiService.updateBlockFetching(); diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index 9678aa9c..be68337f 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -21,6 +21,7 @@ export class SandboxService { constructor( private readonly apiService: AlgorandApiService, + @Inject(isMainThread ? StoreService : 'Null') private readonly storeService: StoreService, private readonly nodeConfig: NodeConfig, @Inject('ISubqueryProject') private readonly project: ISubqueryProject, diff --git a/packages/node/src/indexer/unfinalizedBlocks.service.ts b/packages/node/src/indexer/unfinalizedBlocks.service.ts index 7b32ffb9..1cb5926c 100644 --- a/packages/node/src/indexer/unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/unfinalizedBlocks.service.ts @@ -7,6 +7,7 @@ import { Header, NodeConfig, StoreCacheService, + mainThreadOnly, } from '@subql/node-core'; import { AlgorandApiService } from '../algorand'; import { BlockContent } from './types'; @@ -29,20 +30,24 @@ export class UnfinalizedBlocksService extends BaseUnfinalizedBlocksService { const checkHealth = await this.apiService.api.api.makeHealthCheck().do(); const latestHeight = checkHealth.round; return this.getHeaderForHeight(latestHeight); } + @mainThreadOnly() protected async getHeaderForHash(hash: string): Promise
{ return this.getHeaderForHeight(parseInt(hash, 10)); } + @mainThreadOnly() protected async getHeaderForHeight(height: number): Promise
{ return algorandBlockToHeader( await this.apiService.api.getBlockByHeight(height), diff --git a/packages/node/src/indexer/indexer.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts similarity index 70% rename from packages/node/src/indexer/indexer.module.ts rename to packages/node/src/indexer/worker/worker-fetch.module.ts index 1323dfe4..c590ea12 100644 --- a/packages/node/src/indexer/indexer.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -5,29 +5,24 @@ import { isMainThread } from 'worker_threads'; import { Module } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { - StoreService, - PoiService, ConnectionPoolService, - StoreCacheService, WorkerDynamicDsService, ConnectionPoolStateManager, WorkerConnectionPoolStateManager, } from '@subql/node-core'; -import { AlgorandApiService, AlgorandApiConnection } from '../algorand'; -import { SubqueryProject } from '../configure/SubqueryProject'; -import { DsProcessorService } from './ds-processor.service'; -import { DynamicDsService } from './dynamic-ds.service'; -import { IndexerManager } from './indexer.manager'; -import { ProjectService } from './project.service'; -import { SandboxService } from './sandbox.service'; -import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; -import { WorkerService } from './worker/worker.service'; +import { AlgorandApiService, AlgorandApiConnection } from '../../algorand'; +import { SubqueryProject } from '../../configure/SubqueryProject'; +import { DsProcessorService } from '../ds-processor.service'; +import { DynamicDsService } from '../dynamic-ds.service'; +import { IndexerManager } from '../indexer.manager'; +import { ProjectService } from '../project.service'; +import { SandboxService } from '../sandbox.service'; +import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; +import { WorkerService } from './worker.service'; @Module({ providers: [ IndexerManager, - StoreCacheService, - StoreService, { provide: ConnectionPoolStateManager, useFactory: () => { @@ -67,13 +62,12 @@ import { WorkerService } from './worker/worker.service'; return new WorkerDynamicDsService((global as any).host); }, }, - PoiService, { provide: 'IProjectService', useClass: ProjectService, }, WorkerService, ], - exports: [StoreService], + exports: [], }) -export class IndexerModule {} +export class WorkerFetchModule {} diff --git a/packages/node/src/indexer/worker/worker.module.ts b/packages/node/src/indexer/worker/worker.module.ts index 9dc9f181..1c12324a 100644 --- a/packages/node/src/indexer/worker/worker.module.ts +++ b/packages/node/src/indexer/worker/worker.module.ts @@ -6,7 +6,7 @@ import { EventEmitterModule } from '@nestjs/event-emitter'; import { ScheduleModule } from '@nestjs/schedule'; import { DbModule } from '@subql/node-core'; import { ConfigureModule } from '../../configure/configure.module'; -import { IndexerModule } from '../indexer.module'; +import { WorkerFetchModule } from './worker-fetch.module'; @Module({ imports: [ @@ -14,7 +14,7 @@ import { IndexerModule } from '../indexer.module'; EventEmitterModule.forRoot(), ConfigureModule.register(), ScheduleModule.forRoot(), - IndexerModule, + WorkerFetchModule, ], controllers: [], }) From b292c144d4463993a4fc7e2d0c5678eface530da Mon Sep 17 00:00:00 2001 From: Naveen V Date: Wed, 27 Sep 2023 08:53:46 +0000 Subject: [PATCH 07/15] update jest --- .eslintrc.js | 1 + package.json | 1 + yarn.lock | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/.eslintrc.js b/.eslintrc.js index 686b1d46..3385fe9c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -29,6 +29,7 @@ module.exports = { 'eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:@typescript-eslint/recommended-requiring-type-checking', + 'plugin:jest/recommended', ], rules: { // rules turned off in upstream project (also required when recommended-requiring-type-checking is extended) diff --git a/package.json b/package.json index 8550a7da..15026395 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "eslint-config-prettier": "^8.3.0", "eslint-plugin-header": "^3.1.1", "eslint-plugin-import": "^2.25.4", + "eslint-plugin-jest": "^27.2.3", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-sort-destructure-keys": "^1.4.0", "husky": "^7.0.4", diff --git a/yarn.lock b/yarn.lock index 78fd6ec5..2b64201c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3656,6 +3656,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/scope-manager@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/scope-manager@npm:5.62.0" + dependencies: + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/visitor-keys": 5.62.0 + checksum: 6062d6b797fe1ce4d275bb0d17204c827494af59b5eaf09d8a78cdd39dadddb31074dded4297aaf5d0f839016d601032857698b0e4516c86a41207de606e9573 + languageName: node + linkType: hard + "@typescript-eslint/type-utils@npm:5.59.5": version: 5.59.5 resolution: "@typescript-eslint/type-utils@npm:5.59.5" @@ -3680,6 +3690,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/types@npm:5.62.0" + checksum: 48c87117383d1864766486f24de34086155532b070f6264e09d0e6139449270f8a9559cfef3c56d16e3bcfb52d83d42105d61b36743626399c7c2b5e0ac3b670 + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:5.59.5": version: 5.59.5 resolution: "@typescript-eslint/typescript-estree@npm:5.59.5" @@ -3698,6 +3715,24 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/typescript-estree@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/typescript-estree@npm:5.62.0" + dependencies: + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/visitor-keys": 5.62.0 + debug: ^4.3.4 + globby: ^11.1.0 + is-glob: ^4.0.3 + semver: ^7.3.7 + tsutils: ^3.21.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 3624520abb5807ed8f57b1197e61c7b1ed770c56dfcaca66372d584ff50175225798bccb701f7ef129d62c5989070e1ee3a0aa2d84e56d9524dcf011a2bb1a52 + languageName: node + linkType: hard + "@typescript-eslint/utils@npm:5.59.5": version: 5.59.5 resolution: "@typescript-eslint/utils@npm:5.59.5" @@ -3716,6 +3751,24 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/utils@npm:^5.10.0": + version: 5.62.0 + resolution: "@typescript-eslint/utils@npm:5.62.0" + dependencies: + "@eslint-community/eslint-utils": ^4.2.0 + "@types/json-schema": ^7.0.9 + "@types/semver": ^7.3.12 + "@typescript-eslint/scope-manager": 5.62.0 + "@typescript-eslint/types": 5.62.0 + "@typescript-eslint/typescript-estree": 5.62.0 + eslint-scope: ^5.1.1 + semver: ^7.3.7 + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || ^8.0.0 + checksum: ee9398c8c5db6d1da09463ca7bf36ed134361e20131ea354b2da16a5fdb6df9ba70c62a388d19f6eebb421af1786dbbd79ba95ddd6ab287324fc171c3e28d931 + languageName: node + linkType: hard + "@typescript-eslint/visitor-keys@npm:5.59.5": version: 5.59.5 resolution: "@typescript-eslint/visitor-keys@npm:5.59.5" @@ -3726,6 +3779,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/visitor-keys@npm:5.62.0": + version: 5.62.0 + resolution: "@typescript-eslint/visitor-keys@npm:5.62.0" + dependencies: + "@typescript-eslint/types": 5.62.0 + eslint-visitor-keys: ^3.3.0 + checksum: 976b05d103fe8335bef5c93ad3f76d781e3ce50329c0243ee0f00c0fcfb186c81df50e64bfdd34970148113f8ade90887f53e3c4938183afba830b4ba8e30a35 + languageName: node + linkType: hard + "@willsoto/nestjs-prometheus@npm:^5.4.0": version: 5.5.0 resolution: "@willsoto/nestjs-prometheus@npm:5.5.0" @@ -5854,6 +5917,24 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-jest@npm:^27.2.3": + version: 27.4.0 + resolution: "eslint-plugin-jest@npm:27.4.0" + dependencies: + "@typescript-eslint/utils": ^5.10.0 + peerDependencies: + "@typescript-eslint/eslint-plugin": ^5.0.0 || ^6.0.0 + eslint: ^7.0.0 || ^8.0.0 + jest: "*" + peerDependenciesMeta: + "@typescript-eslint/eslint-plugin": + optional: true + jest: + optional: true + checksum: c33593dba87e750123555c2de32fb174d6f2c92342571492f8dbde01bf61a8ac229dff31bd08fea16c3ca2c4843fc2fec985459c351319c019016767ed1cd78e + languageName: node + linkType: hard + "eslint-plugin-prettier@npm:^4.0.0": version: 4.2.1 resolution: "eslint-plugin-prettier@npm:4.2.1" @@ -11047,6 +11128,7 @@ __metadata: eslint-config-prettier: ^8.3.0 eslint-plugin-header: ^3.1.1 eslint-plugin-import: ^2.25.4 + eslint-plugin-jest: ^27.2.3 eslint-plugin-prettier: ^4.0.0 eslint-plugin-sort-destructure-keys: ^1.4.0 husky: ^7.0.4 From 6a6b5596f6d5b997f2d8c277cee8a0ff3cef572d Mon Sep 17 00:00:00 2001 From: Naveen V Date: Wed, 27 Sep 2023 09:03:35 +0000 Subject: [PATCH 08/15] fix worker injection --- .../node/src/indexer/worker/worker-fetch.module.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index c590ea12..faf7349c 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -19,6 +19,7 @@ import { ProjectService } from '../project.service'; import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { WorkerService } from './worker.service'; +import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service'; @Module({ providers: [ @@ -33,7 +34,6 @@ import { WorkerService } from './worker.service'; }, }, ConnectionPoolService, - UnfinalizedBlocksService, { provide: AlgorandApiService, useFactory: async ( @@ -66,6 +66,15 @@ import { WorkerService } from './worker.service'; provide: 'IProjectService', useClass: ProjectService, }, + { + provide: UnfinalizedBlocksService, + useFactory: () => { + if (isMainThread) { + throw new Error('Expected to be worker thread'); + } + return new WorkerUnfinalizedBlocksService((global as any).host); + }, + }, WorkerService, ], exports: [], From 7a6fc532b721685d88879a25d6f3c2cea26b95fa Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 2 Oct 2023 10:44:15 +1300 Subject: [PATCH 09/15] Update latest changes --- package.json | 3 +- packages/common-algorand/package.json | 3 +- packages/common-algorand/src/project/load.ts | 1 - .../common-algorand/src/project/models.ts | 2 +- .../src/project/versioned/v1_0_0/model.ts | 1 - packages/node/package.json | 4 +- .../node/src/configure/SubqueryProject.ts | 4 - .../worker-block-dispatcher.service.ts | 4 +- packages/node/src/indexer/indexer.manager.ts | 7 - packages/node/src/indexer/project.service.ts | 2 - packages/node/src/indexer/sandbox.service.ts | 3 +- .../src/indexer/worker/worker-fetch.module.ts | 24 +-- packages/node/src/indexer/worker/worker.ts | 8 +- .../worker.unfinalizedBlocks.service.ts | 7 +- .../node/src/subcommands/reindex.module.ts | 2 + packages/types/package.json | 4 +- packages/types/src/project.ts | 164 ++++++++++++++++-- yarn.lock | 55 +++--- 18 files changed, 204 insertions(+), 94 deletions(-) diff --git a/package.json b/package.json index 15026395..ffc216cb 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,7 @@ "typescript": "^4.9.5" }, "resolutions": { - "node-fetch": "2.6.7", - "@subql/types": "2.2.1-1" + "node-fetch": "2.6.7" }, "scripts": { "build": "yarn workspaces foreach -ptA run build", diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index f4585eb5..4f5f2f87 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -13,9 +13,8 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "2.7.1-3", + "@subql/common": "^3.0.1", "@subql/types-algorand": "workspace:*", - "@subql/types-core": "0.0.0-2", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", "fs-extra": "^10.1.0", diff --git a/packages/common-algorand/src/project/load.ts b/packages/common-algorand/src/project/load.ts index 7e84e6bd..6b04f5b3 100644 --- a/packages/common-algorand/src/project/load.ts +++ b/packages/common-algorand/src/project/load.ts @@ -1,7 +1,6 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {getManifestPath, loadFromJsonOrYaml} from '@subql/common'; import {AlgorandProjectManifestVersioned, VersionedProjectManifest} from './versioned'; export function parseAlgorandProjectManifest(raw: unknown): AlgorandProjectManifestVersioned { diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts index 9fdc2fc1..c196ff6e 100644 --- a/packages/common-algorand/src/project/models.ts +++ b/packages/common-algorand/src/project/models.ts @@ -1,7 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {BlockFilterImpl, ProcessorImpl} from '@subql/common'; +import {ProcessorImpl} from '@subql/common'; import { CustomDataSourceAsset as AlgorandCustomDataSourceAsset, AlgorandBlockFilter, diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts index c38cf1a1..a33f1ab5 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts @@ -6,7 +6,6 @@ import { FileType, ParentProjectModel, ProjectManifestBaseImpl, - RunnerNodeImpl, RunnerQueryBaseModel, validateObject, CommonProjectNetworkV1_0_0, diff --git a/packages/node/package.json b/packages/node/package.json index 54a48aed..9dca3440 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -24,9 +24,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "2.7.1-3", + "@subql/common": "^3.0.1", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "4.2.4-26", + "@subql/node-core": "^5.0.1", "@subql/types-algorand": "workspace:*", "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0", diff --git a/packages/node/src/configure/SubqueryProject.ts b/packages/node/src/configure/SubqueryProject.ts index 3159643c..2594c8a7 100644 --- a/packages/node/src/configure/SubqueryProject.ts +++ b/packages/node/src/configure/SubqueryProject.ts @@ -48,10 +48,6 @@ export type SubqlProjectBlockFilter = BlockFilter & { }; }; -// export type SubqlProjectDsTemplate = Omit & { -// name: string; -// }; - const NOT_SUPPORT = (name: string) => { throw new Error(`Manifest specVersion ${name}() is not supported`); }; diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index ddb02d37..d47dee08 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -47,6 +47,7 @@ async function createIndexerWorker( unfinalizedBlocksService: IUnfinalizedBlocksService, connectionPoolState: ConnectionPoolStateManager, root: string, + startHeight: number, ): Promise { const indexerWorker = Worker.create< IInitIndexerWorker, @@ -70,7 +71,7 @@ async function createIndexerWorker( root, ); - await indexerWorker.initWorker(); + await indexerWorker.initWorker(startHeight); return indexerWorker; } @@ -114,6 +115,7 @@ export class WorkerBlockDispatcherService unfinalizedBlocksService, connectionPoolState, project.root, + projectService.startHeight, ), ); } diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index 3f147304..1d0c2426 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -38,7 +38,6 @@ import { DsProcessorService, } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; -import { ProjectService } from './project.service'; import { SandboxService } from './sandbox.service'; import { BlockContent } from './types'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @@ -63,7 +62,6 @@ export class IndexerManager extends BaseIndexerManager< sandboxService: SandboxService, dsProcessorService: DsProcessorService, dynamicDsService: DynamicDsService, - @Inject('IProjectService') private projectService: ProjectService, unfinalizedBlocksService: UnfinalizedBlocksService, ) { super( @@ -92,11 +90,6 @@ export class IndexerManager extends BaseIndexerManager< ); } - async start(): Promise { - await this.projectService.init(); - logger.info('indexer manager started'); - } - getBlockHeight(block: BlockContent): number { return block.round; } diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 610ae4ea..b62b7fb9 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -10,7 +10,6 @@ import { StoreService, NodeConfig, IProjectUpgradeService, - mainThreadOnly, } from '@subql/node-core'; import { Sequelize } from '@subql/x-sequelize'; import { AlgorandApiService } from '../algorand'; @@ -67,7 +66,6 @@ export class ProjectService extends BaseProjectService< return new Date(block.timestamp); } - @mainThreadOnly() protected onProjectChange(project: SubqueryProject): void | Promise { // TODO update this when implementing skipBlock feature for Eth // this.apiService.updateBlockFetching(); diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index be68337f..74c6dcb2 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -10,8 +10,7 @@ import { NodeConfig, StoreService, } from '@subql/node-core'; -import { Store } from '@subql/types'; -import { BaseDataSource } from '@subql/types-core'; +import { Store, BaseDataSource } from '@subql/types-core'; import { AlgorandApiService } from '../algorand'; /* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */ diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index faf7349c..be697da2 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -1,7 +1,6 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { isMainThread } from 'worker_threads'; import { Module } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { @@ -26,12 +25,8 @@ import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.servi IndexerManager, { provide: ConnectionPoolStateManager, - useFactory: () => { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - return new WorkerConnectionPoolStateManager((global as any).host); - }, + useFactory: () => + new WorkerConnectionPoolStateManager((global as any).host), }, ConnectionPoolService, { @@ -55,12 +50,7 @@ import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.servi DsProcessorService, { provide: DynamicDsService, - useFactory: () => { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - return new WorkerDynamicDsService((global as any).host); - }, + useFactory: () => new WorkerDynamicDsService((global as any).host), }, { provide: 'IProjectService', @@ -68,12 +58,8 @@ import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.servi }, { provide: UnfinalizedBlocksService, - useFactory: () => { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - return new WorkerUnfinalizedBlocksService((global as any).host); - }, + useFactory: () => + new WorkerUnfinalizedBlocksService((global as any).host), }, WorkerService, ], diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts index 8cfc809d..39a2e874 100644 --- a/packages/node/src/indexer/worker/worker.ts +++ b/packages/node/src/indexer/worker/worker.ts @@ -25,13 +25,13 @@ import { IBaseIndexerWorker, initWorkerServices, } from '@subql/node-core'; -import { IndexerManager } from '../indexer.manager'; +import { ProjectService } from '../project.service'; import { WorkerModule } from './worker.module'; import { WorkerService } from './worker.service'; const logger = getLogger(`worker #${threadId}`); -async function initWorker(): Promise { +async function initWorker(startHeight: number): Promise { try { const app = await NestFactory.create(WorkerModule, { logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging @@ -39,9 +39,9 @@ async function initWorker(): Promise { await app.init(); - const indexerManager = app.get(IndexerManager); + const projectService: ProjectService = app.get('IProjectService'); // Initialise async services, we do this here rather than in factories so we can capture one off events - await indexerManager.start(); + await projectService.init(startHeight); const workerService = app.get(WorkerService); diff --git a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts index aba72e42..340bcc3e 100644 --- a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts @@ -1,6 +1,7 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import { isMainThread } from 'worker_threads'; import { Injectable } from '@nestjs/common'; import { IUnfinalizedBlocksService, @@ -14,7 +15,11 @@ import { algorandBlockToHeader } from '../unfinalizedBlocks.service'; export class WorkerUnfinalizedBlocksService implements IUnfinalizedBlocksService { - constructor(private host: HostUnfinalizedBlocks) {} + constructor(private host: HostUnfinalizedBlocks) { + if (isMainThread) { + throw new Error('Expected to be worker thread'); + } + } async processUnfinalizedBlocks(block: BlockContent): Promise { return this.processUnfinalizedBlockHeader(algorandBlockToHeader(block)); diff --git a/packages/node/src/subcommands/reindex.module.ts b/packages/node/src/subcommands/reindex.module.ts index 227540eb..0194aaf1 100644 --- a/packages/node/src/subcommands/reindex.module.ts +++ b/packages/node/src/subcommands/reindex.module.ts @@ -7,6 +7,7 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { DbModule, ForceCleanService, + PoiService, ReindexService, StoreCacheService, StoreService, @@ -23,6 +24,7 @@ import { DynamicDsService } from '../indexer/dynamic-ds.service'; ReindexService, ForceCleanService, DynamicDsService, + PoiService, DsProcessorService, { // Used to work with DI for unfinalizedBlocksService but not used with reindex diff --git a/packages/types/package.json b/packages/types/package.json index 523054a7..dbeb0d37 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -16,11 +16,13 @@ "/dist" ], "peerDependencies": { - "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0" }, "devDependencies": { "@types/app-module-path": "^2.2.0", "algosdk": "^2.2.0" + }, + "dependencies": { + "@subql/types-core": "^0.0.1" } } diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 7b15437a..48253cdf 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -9,8 +9,9 @@ import { FileReference, Processor, ProjectManifestV1_0_0, + BlockFilter, } from '@subql/types-core'; -import type {Indexer} from 'algosdk'; +import type {Indexer, TransactionType} from 'algosdk'; import {AlgorandBlock, AlgorandTransaction} from './interfaces'; export type RuntimeDatasourceTemplate = BaseTemplateDataSource; @@ -18,12 +19,29 @@ export type CustomDatasourceTemplate = BaseTemplateDataSource; +/** + * Kind of Algorand datasource. + * @enum {string} + */ export enum AlgorandDataSourceKind { + /** + * The runtime kind of Algorand datasource. + */ Runtime = 'algorand/Runtime', } +/** + * Enum representing the kind of Algorand handler. + * @enum {string} + */ export enum AlgorandHandlerKind { + /** + * Handler for Algorand blocks. + */ Block = 'algorand/BlockHandler', + /** + * Handler for Algorand transactions. + */ Transaction = 'algorand/TransactionHandler', } @@ -33,58 +51,172 @@ export type RuntimeHandlerInputMap = { }; type RuntimeFilterMap = { - [AlgorandHandlerKind.Block]: {}; + [AlgorandHandlerKind.Block]: AlgorandBlockFilter; [AlgorandHandlerKind.Transaction]: AlgorandTransactionFilter; }; -// [startSpecVersion?, endSpecVersion?] closed range -export type SpecVersionRange = [number, number]; - -interface AlgorandBaseHandlerFilter { - specVersion?: SpecVersionRange; -} - -export interface AlgorandBlockFilter extends AlgorandBaseHandlerFilter { - modulo?: number; - timestamp?: string; -} +export type AlgorandBlockFilter = BlockFilter; +/** + * Represents a filter for Algorand transactions. + * Depending on the txType the other filters may or may not be relevant + * */ export interface AlgorandTransactionFilter { - txType?: string; + /** + * The type of transaction + * @example + * txType: 'axfer', + * @example + * txType: 'appl', + * */ + txType?: TransactionType | string; + /** + * Filter by the sender of a transction + * @example + * sender: '5GMADASEGJ324HR4GI2XZ2BNSN77ND45LLF4F4XDTAYX3YM6TX5YEU4FEA' + * */ sender?: string; + /** + * Filter by the receiver of a transaction + * @example + * receiver: 'V6CK3HRC4JBRBDIBB4JWOBMYNUYIP7SYHRPVHH5ZMJQME337C57IBIZVFI', + * */ receiver?: string; nonParticipant?: boolean; + /** + * Filter by the assetId of a transaction + * */ assetId?: number; newFreezeStatus?: boolean; address?: string; + /** + * Filter by the application id + * @example + * applicationId: 971368268, + * */ applicationId?: number; + /** + * Filter by the args of an application call + * @example + * applicationArgs: ['udVC+w=='], + * @example + * // Use null to skip filtering certain arguments + * applicationArgs: ['udVC+w==', null, null, null, 'AQ=='] + * */ applicationArgs?: string[]; } +/** + * Represents a handler for Algorand blocks. + * @type {AlgorandCustomHandler} + */ export type AlgorandBlockHandler = AlgorandCustomHandler; +/** + * Represents a handler for Algorand transactions. + * @type {AlgorandCustomHandler} + */ export type AlgorandTransactionHandler = AlgorandCustomHandler< AlgorandHandlerKind.Transaction, AlgorandTransactionFilter >; +/** + * Represents a generic custom handler for Algorand. + * @interface + * @template K - The kind of the handler (default: string). + * @template F - The filter type for the handler (default: Record). + */ export interface AlgorandCustomHandler> { - handler: string; + /** + * The kind of handler. For `algorand/Runtime` datasources this is either `Block` or `Transaction` kinds. + * The value of this will determine the filter options as well as the data provided to your handler function + * @type {AlgorandHandlerKind.Block | AlgorandHandlerKind.Transaction | string } + * @example + * kind: AlgorandHandlerKind.Block // Defined with an enum, this is used for runtime datasources + */ kind: K; + /** + * The name of your handler function. This must be defined and exported from your code. + * @type {string} + * @example + * handler: 'handleBlock' + */ + handler: string; + /** + * The filter for the handler. The handler kind will determine the possible filters (optional). + * + * @type {F} + * @example + * filter: { + txType: 'axfer', + assetId: 27165954, + sender: 'ZW3ISEHZUHPO7OZGMKLKIIMKVICOUDRCERI454I3DB2BH52HGLSO67W754', + } + */ filter?: F; } +/** + * Represents a runtime handler for Algorand, which can be a block handler or transaction handler. + * @type {AlgorandBlockHandler | AlgorandTransactionHandler} + */ export type AlgorandRuntimeHandler = AlgorandBlockHandler | AlgorandTransactionHandler; +/** + * Represents a handler for Algorand, which can be a runtime handler or a custom handler with unknown filter type. + * @type {AlgorandRuntimeHandler | AlgorandCustomHandler} + */ export type AlgorandHandler = AlgorandRuntimeHandler | AlgorandCustomHandler; +/** + * Represents a filter for Algorand runtime handlers, which can be a block filter or transaction filter. + * @type {AlgorandBlockFilter | AlgorandTransactionFilter} + */ export type AlgorandRuntimeHandlerFilter = AlgorandBlockFilter | AlgorandTransactionFilter; +/** + * Represents a mapping for Algorand handlers, extending FileReference. + * @interface + * @extends {FileReference} + */ export interface AlgorandMapping extends FileReference { + /** + * An array of handlers associated with the mapping. + * @type {T[]} + * @example + * handlers: [{ + kind: AlgorandRuntimeHandler.Transaction, + handler: 'handleTransfer', + filter: { + txType: 'axfer', + assetId: 27165954, + sender: 'ZW3ISEHZUHPO7OZGMKLKIIMKVICOUDRCERI454I3DB2BH52HGLSO67W754', + } + }] + */ handlers: T[]; } +/** + * Represents a Algorand datasource interface with generic parameters. + * @interface + * @template M - The mapping type for the datasource. + */ interface IAlgorandDataSource { - name?: string; + /** + * The kind of the datasource. + * @type {string} + */ kind: string; + /** + * The starting block number for the datasource. If not specified, 1 will be used (optional). + * @type {number} + * @default 1 + */ startBlock?: number; + /** + * The mapping associated with the datasource. + * This contains the handlers. + * @type {M} + */ mapping: M; } diff --git a/yarn.lock b/yarn.lock index 2b64201c..838a17a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,9 +2916,8 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": 2.7.1-3 + "@subql/common": ^3.0.1 "@subql/types-algorand": "workspace:*" - "@subql/types-core": 0.0.0-2 "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 "@types/pino": ^6.3.12 @@ -2936,11 +2935,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:2.7.1-3": - version: 2.7.1-3 - resolution: "@subql/common@npm:2.7.1-3" +"@subql/common@npm:3.0.1, @subql/common@npm:^3.0.1": + version: 3.0.1 + resolution: "@subql/common@npm:3.0.1" dependencies: - "@subql/types-core": 0.0.0-1 + "@subql/types-core": 0.0.1 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2950,7 +2949,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: f05dcc0089f4e10abb438f5beb6dfc1156d2b1fb8b2053ebfbf9417e3f393df1ce5e14a50fcc57e429002704dfc2246b380649b9a3d0845bbb924165e6eb1861 + checksum: 05cd5cdb0e22e7458bbe5c56adc04f5d9d57ae86be7b3efdccd0673a0630290f321d2bb742f9503ab001ccd048125553b75bbe38e82adc6af774f019e41c1e30 languageName: node linkType: hard @@ -2966,9 +2965,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": 2.7.1-3 + "@subql/common": ^3.0.1 "@subql/common-algorand": "workspace:*" - "@subql/node-core": 4.2.4-26 + "@subql/node-core": ^5.0.1 "@subql/types-algorand": "workspace:*" "@subql/types-core": 0.0.0-2 "@types/express": ^4.17.13 @@ -2998,18 +2997,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:4.2.4-26": - version: 4.2.4-26 - resolution: "@subql/node-core@npm:4.2.4-26" +"@subql/node-core@npm:^5.0.1": + version: 5.0.1 + resolution: "@subql/node-core@npm:5.0.1" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 2.7.1-3 + "@subql/common": 3.0.1 "@subql/testing": 2.0.3-0 - "@subql/types": 2.2.1-3 + "@subql/types": 3.0.1 "@subql/utils": 2.4.4-1 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 @@ -3025,7 +3024,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: f9e0c95c5a6584621942b3edbf106af28ac54f647086066fa73b818df49dd8bfe4e30559a93a0eea04596672b7558ece963d4464334f6987796cf2acc928c3a5 + checksum: d6249dd269cdbe6d2fbf55a240e082ba444de44cd754558fbab2e0f5887155ded19c66d6212eda3a78bbf341ce28ca92960f7dbc3c7294741b82516995334424 languageName: node linkType: hard @@ -3042,29 +3041,29 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/types-algorand@workspace:packages/types" dependencies: + "@subql/types-core": ^0.0.1 "@types/app-module-path": ^2.2.0 algosdk: ^2.2.0 peerDependencies: - "@subql/types-core": 0.0.0-2 algosdk: ^2.2.0 languageName: unknown linkType: soft -"@subql/types-core@npm:0.0.0-1": - version: 0.0.0-1 - resolution: "@subql/types-core@npm:0.0.0-1" +"@subql/types-core@npm:0.0.0-2": + version: 0.0.0-2 + resolution: "@subql/types-core@npm:0.0.0-2" dependencies: package-json-type: ^1.0.3 - checksum: fd39376cd64622e34711dfc86c660e1cbc4fac5c72e7ee410d19b0753601024a7455b0b144fe4a1c9903f788686eb2f1c3cb50a7c9a1eb39933102b729958a42 + checksum: 3cd6a90391a8d206d0140bc84fff3b019a54488b711ce4f3d8c59f5b326dba9fc8b76d233add90b6a8dc953b09a7d3057f5132ec04a4b46e18c1aac49e8c685f languageName: node linkType: hard -"@subql/types-core@npm:0.0.0-2": - version: 0.0.0-2 - resolution: "@subql/types-core@npm:0.0.0-2" +"@subql/types-core@npm:0.0.1, @subql/types-core@npm:^0.0.1": + version: 0.0.1 + resolution: "@subql/types-core@npm:0.0.1" dependencies: package-json-type: ^1.0.3 - checksum: 3cd6a90391a8d206d0140bc84fff3b019a54488b711ce4f3d8c59f5b326dba9fc8b76d233add90b6a8dc953b09a7d3057f5132ec04a4b46e18c1aac49e8c685f + checksum: 2324d331f94562c0fecd6af92b22340ac361e6ad14f28bb05b686c42ebfa3847c890d397c87e33c4a6d47b82adf537b7336fdd392881c7727a2e585ad20937bb languageName: node linkType: hard @@ -3077,12 +3076,12 @@ __metadata: languageName: node linkType: hard -"@subql/types@npm:2.2.1-1": - version: 2.2.1-1 - resolution: "@subql/types@npm:2.2.1-1" +"@subql/types@npm:3.0.1": + version: 3.0.1 + resolution: "@subql/types@npm:3.0.1" peerDependencies: "@polkadot/api": ^10 - checksum: ba11a63169557bbddd61b9da0f609548e40915cf399dddd8a27a9ad970aac16ffc6d23d09f082d2b1bce5465ea1a7bca471213ffe69e8ace1475a08ebf74f695 + checksum: 08bf82ecff0e6170b8962b685688aada7192019f9d7e0c309a395d8fc477d564a676e29d4f84232ae7fd8fb1feb9f0f7ca21e82ff83e4bcc11de94262fc368da languageName: node linkType: hard From b75efa6283cdb47595d850ddb5d5802588e900ef Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 3 Oct 2023 12:08:30 +1300 Subject: [PATCH 10/15] Update deps, add test for dictionary validation --- packages/common-algorand/package.json | 2 +- .../common-algorand/src/project/models.ts | 6 +-- .../src/project/versioned/v1_0_0/model.ts | 2 +- packages/node/package.json | 4 +- .../src/indexer/dictionary.service.spec.ts | 33 ++++++++++++ .../node/src/indexer/dictionary.service.ts | 6 +++ packages/node/src/indexer/fetch.service.ts | 4 ++ packages/types/package.json | 2 +- packages/types/src/project.ts | 50 ++++++------------- yarn.lock | 46 ++++++++--------- 10 files changed, 88 insertions(+), 67 deletions(-) create mode 100644 packages/node/src/indexer/dictionary.service.spec.ts diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index 4f5f2f87..64a819d2 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -13,7 +13,7 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "^3.0.1", + "@subql/common": "^3.1.0", "@subql/types-algorand": "workspace:*", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", diff --git a/packages/common-algorand/src/project/models.ts b/packages/common-algorand/src/project/models.ts index c196ff6e..acacf7af 100644 --- a/packages/common-algorand/src/project/models.ts +++ b/packages/common-algorand/src/project/models.ts @@ -132,7 +132,7 @@ export class CustomHandler implements AlgorandCustomHandler { filter?: Record; } -export class RuntimeMapping implements BaseMapping { +export class RuntimeMapping implements BaseMapping { @Transform((params) => { const handlers: AlgorandRuntimeHandler[] = params.value; return handlers.map((handler) => { @@ -154,7 +154,7 @@ export class RuntimeMapping implements BaseMapping, AlgorandCustomHandler> { +export class CustomMapping implements BaseMapping { @IsArray() @Type(() => CustomHandler) @ValidateNested() @@ -183,7 +183,7 @@ export class FileReferenceImpl implements FileReference { } export class CustomDataSourceBase - implements AlgorandCustomDataSource + implements AlgorandCustomDataSource { @IsString() kind: K; diff --git a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts index a33f1ab5..13899456 100644 --- a/packages/common-algorand/src/project/versioned/v1_0_0/model.ts +++ b/packages/common-algorand/src/project/versioned/v1_0_0/model.ts @@ -80,7 +80,7 @@ export class AlgorandRuntimeDataSourceV1_0_0Impl extends RuntimeDataSourceBase { export class AlgorandCustomDataSourceV1_0_0Impl< K extends string = string, - M extends BaseMapping = BaseMapping, any> + M extends BaseMapping = BaseMapping > extends CustomDataSourceBase implements AlgorandCustomDataSource diff --git a/packages/node/package.json b/packages/node/package.json index 9dca3440..7214c9b7 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -24,9 +24,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.0.1", + "@subql/common": "^3.1.0", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "^5.0.1", + "@subql/node-core": "^5.0.2", "@subql/types-algorand": "workspace:*", "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0", diff --git a/packages/node/src/indexer/dictionary.service.spec.ts b/packages/node/src/indexer/dictionary.service.spec.ts new file mode 100644 index 00000000..1f5a1f96 --- /dev/null +++ b/packages/node/src/indexer/dictionary.service.spec.ts @@ -0,0 +1,33 @@ +// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { NodeConfig } from '@subql/node-core'; +import { DictionaryService } from './dictionary.service'; + +describe('dictionary service', () => { + let dictionaryService: DictionaryService; + + beforeEach(() => { + dictionaryService = new DictionaryService( + { + network: { + chainId: 'wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=', + dictionary: + 'https://api.subquery.network/sq/subquery/Algorand-Dictionary', + }, + } as any, + { dictionaryTimeout: 10000 } as NodeConfig, + new EventEmitter2(), + ); + }); + + it('successfully validates metatada', async () => { + /* Genesis hash is unused with cosmos, chainId is used from project instead */ + await expect( + dictionaryService.initValidation( + 'wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=', + ), + ).resolves.toBeTruthy(); + }); +}); diff --git a/packages/node/src/indexer/dictionary.service.ts b/packages/node/src/indexer/dictionary.service.ts index 20c0252d..755c4731 100644 --- a/packages/node/src/indexer/dictionary.service.ts +++ b/packages/node/src/indexer/dictionary.service.ts @@ -7,6 +7,7 @@ import { NodeConfig, DictionaryService as CoreDictionaryService, } from '@subql/node-core'; +import { MetaData } from '@subql/utils'; import { SubqueryProject } from '../configure/SubqueryProject'; @Injectable() @@ -23,4 +24,9 @@ export class DictionaryService extends CoreDictionaryService { eventEmitter, ); } + + protected validateChainMeta(metaData: MetaData): boolean { + // Chain id is used as genesis hash + return this.project.network.chainId === metaData.genesisHash; + } } diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index d1f0969b..65f46011 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -131,6 +131,10 @@ export class FetchService extends BaseFetchService< ); } + protected getGenesisHash(): string { + return this.apiService.networkMeta.genesisHash; + } + protected async getFinalizedHeight(): Promise { const checkHealth = await this.api.api.makeHealthCheck().do(); return checkHealth.round; diff --git a/packages/types/package.json b/packages/types/package.json index dbeb0d37..9a64b3aa 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -23,6 +23,6 @@ "algosdk": "^2.2.0" }, "dependencies": { - "@subql/types-core": "^0.0.1" + "@subql/types-core": "^0.1.0" } } diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 48253cdf..354f376c 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -10,6 +10,10 @@ import { Processor, ProjectManifestV1_0_0, BlockFilter, + BaseHandler, + BaseMapping, + BaseDataSource, + BaseCustomDataSource, } from '@subql/types-core'; import type {Indexer, TransactionType} from 'algosdk'; import {AlgorandBlock, AlgorandTransaction} from './interfaces'; @@ -126,7 +130,8 @@ export type AlgorandTransactionHandler = AlgorandCustomHandler< * @template K - The kind of the handler (default: string). * @template F - The filter type for the handler (default: Record). */ -export interface AlgorandCustomHandler> { +export interface AlgorandCustomHandler> + extends BaseHandler { /** * The kind of handler. For `algorand/Runtime` datasources this is either `Block` or `Transaction` kinds. * The value of this will determine the filter options as well as the data provided to your handler function @@ -136,15 +141,6 @@ export interface AlgorandCustomHandler extends FileReference { +export interface AlgorandMapping extends BaseMapping { /** - * An array of handlers associated with the mapping. * @type {T[]} * @example * handlers: [{ @@ -200,25 +195,7 @@ export interface AlgorandMapping ex * @interface * @template M - The mapping type for the datasource. */ -interface IAlgorandDataSource { - /** - * The kind of the datasource. - * @type {string} - */ - kind: string; - /** - * The starting block number for the datasource. If not specified, 1 will be used (optional). - * @type {number} - * @default 1 - */ - startBlock?: number; - /** - * The mapping associated with the datasource. - * This contains the handlers. - * @type {M} - */ - mapping: M; -} +type IAlgorandDataSource = BaseDataSource; export interface AlgorandRuntimeDataSource< M extends AlgorandMapping = AlgorandMapping @@ -232,12 +209,13 @@ export type CustomDataSourceAsset = FileReference; export interface AlgorandCustomDataSource< K extends string = string, - M extends AlgorandMapping = AlgorandMapping, - O = any -> extends IAlgorandDataSource { + M extends AlgorandMapping = AlgorandMapping +> extends BaseCustomDataSource { + /** + * The kind of the custom datasource. This should follow the pattern `algorand/*`. + * @type {K} + */ kind: K; - assets: Map; - processor: Processor; } export interface HandlerInputTransformer_0_0_0< diff --git a/yarn.lock b/yarn.lock index 838a17a9..4ff79cdc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,7 +2916,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": ^3.0.1 + "@subql/common": ^3.1.0 "@subql/types-algorand": "workspace:*" "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 @@ -2935,11 +2935,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.0.1, @subql/common@npm:^3.0.1": - version: 3.0.1 - resolution: "@subql/common@npm:3.0.1" +"@subql/common@npm:3.1.0, @subql/common@npm:^3.1.0": + version: 3.1.0 + resolution: "@subql/common@npm:3.1.0" dependencies: - "@subql/types-core": 0.0.1 + "@subql/types-core": 0.1.0 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2949,7 +2949,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 05cd5cdb0e22e7458bbe5c56adc04f5d9d57ae86be7b3efdccd0673a0630290f321d2bb742f9503ab001ccd048125553b75bbe38e82adc6af774f019e41c1e30 + checksum: 43c7207acbc943cf1196c6ea0ecc6187b5a0a08a1f358861da5c2fa2db1b1a94ad6232850ef85e0c1a49274fd96d5c39204d860aa18d0e655fcad0b402df0220 languageName: node linkType: hard @@ -2965,9 +2965,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.0.1 + "@subql/common": ^3.1.0 "@subql/common-algorand": "workspace:*" - "@subql/node-core": ^5.0.1 + "@subql/node-core": ^5.0.2 "@subql/types-algorand": "workspace:*" "@subql/types-core": 0.0.0-2 "@types/express": ^4.17.13 @@ -2997,18 +2997,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:^5.0.1": - version: 5.0.1 - resolution: "@subql/node-core@npm:5.0.1" +"@subql/node-core@npm:^5.0.2": + version: 5.0.2 + resolution: "@subql/node-core@npm:5.0.2" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 3.0.1 + "@subql/common": 3.1.0 "@subql/testing": 2.0.3-0 - "@subql/types": 3.0.1 + "@subql/types": 3.1.0 "@subql/utils": 2.4.4-1 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 @@ -3024,7 +3024,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: d6249dd269cdbe6d2fbf55a240e082ba444de44cd754558fbab2e0f5887155ded19c66d6212eda3a78bbf341ce28ca92960f7dbc3c7294741b82516995334424 + checksum: e746752c239b4cdd9426ce664cee4330ca38e8bf8882646637404cecdb5f3e1ec846255ab5b3e19127de0c7c0d27ae22983400699b71b2a25bac473b1cb057a0 languageName: node linkType: hard @@ -3041,7 +3041,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/types-algorand@workspace:packages/types" dependencies: - "@subql/types-core": ^0.0.1 + "@subql/types-core": ^0.1.0 "@types/app-module-path": ^2.2.0 algosdk: ^2.2.0 peerDependencies: @@ -3058,12 +3058,12 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:0.0.1, @subql/types-core@npm:^0.0.1": - version: 0.0.1 - resolution: "@subql/types-core@npm:0.0.1" +"@subql/types-core@npm:0.1.0, @subql/types-core@npm:^0.1.0": + version: 0.1.0 + resolution: "@subql/types-core@npm:0.1.0" dependencies: package-json-type: ^1.0.3 - checksum: 2324d331f94562c0fecd6af92b22340ac361e6ad14f28bb05b686c42ebfa3847c890d397c87e33c4a6d47b82adf537b7336fdd392881c7727a2e585ad20937bb + checksum: fd86c2cf5148a4ad3af5c3a4fa7f0180f7a4d90fd3f6d96bc0e48eab6d73767220c859c751b21bbe08565d31eac5e8d0cee0148dee7ee7e0e273ff46182d3131 languageName: node linkType: hard @@ -3076,12 +3076,12 @@ __metadata: languageName: node linkType: hard -"@subql/types@npm:3.0.1": - version: 3.0.1 - resolution: "@subql/types@npm:3.0.1" +"@subql/types@npm:3.1.0": + version: 3.1.0 + resolution: "@subql/types@npm:3.1.0" peerDependencies: "@polkadot/api": ^10 - checksum: 08bf82ecff0e6170b8962b685688aada7192019f9d7e0c309a395d8fc477d564a676e29d4f84232ae7fd8fb1feb9f0f7ca21e82ff83e4bcc11de94262fc368da + checksum: 31cd2870513e69b51c3acdd5016d2ed8fb8ce405366f912b26422b9169b236a5c5c6bb3b6a7592f7209f842d963083a08ebe3c299e81620e5752d4cc3522d552 languageName: node linkType: hard From 7af5505f54adc456482aadd4984780d2ce2ee86c Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 3 Oct 2023 14:30:42 +1300 Subject: [PATCH 11/15] Fix main entry point, publish docker to subquery-network org --- .github/workflows/node-docker.yml | 50 +++++++++++++++++++++++++++++++ packages/node/src/main.ts | 3 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node-docker.yml b/.github/workflows/node-docker.yml index 482ac3f0..a05210db 100644 --- a/.github/workflows/node-docker.yml +++ b/.github/workflows/node-docker.yml @@ -58,3 +58,53 @@ jobs: - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} + node-build-push-docker-subquery: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 100 + token: ${{ secrets.REPO_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: subquerynetwork + password: ${{ secrets.SQ_DOCKERHUB_TOKEN }} + + ## node + - name: Get updated node version + id: get-node-version + run: | + sh .github/workflows/scripts/nodeVersion.sh + + - name: Build and push + if: github.event.inputs.isLatest == 'false' + uses: docker/build-push-action@v2 + with: + push: true + platforms: arm64,amd64 + file: ./packages/node/Dockerfile + tags: subquerynetwork/subql-node-algorand:v${{ steps.get-node-version.outputs.NODE_VERSION }}, + build-args: RELEASE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }} + + - name: Build and push + if: github.event.inputs.isLatest == 'true' + uses: docker/build-push-action@v2 + with: + push: true + platforms: arm64,amd64 + file: ./packages/node/Dockerfile + tags: subquerynetwork/subql-node-algorand:v${{ steps.get-node-version.outputs.NODE_VERSION }},subquerynetwork/subql-node-algorand:latest + build-args: RELEASE_VERSION=${{ steps.get-node-version.outputs.NODE_VERSION }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + diff --git a/packages/node/src/main.ts b/packages/node/src/main.ts index 4c1c50a3..8d4742d8 100644 --- a/packages/node/src/main.ts +++ b/packages/node/src/main.ts @@ -13,7 +13,8 @@ initLogger( argv.logLevel as string | undefined, ); -if (argv._[0] === undefined) { +// Check for no subcommand +if (!argv._[0]) { // Lazy import, to allow logger to be initialised before bootstrap() // As bootstrap runs services that requires logger const { bootstrap } = require('./init'); From da4856e60018591ade0f7236d54e0d156b5f040e Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 4 Oct 2023 10:45:53 +1300 Subject: [PATCH 12/15] Update deps --- packages/common-algorand/package.json | 2 +- packages/node/package.json | 4 +- packages/node/src/algorand/algorand.spec.ts | 3 +- yarn.lock | 53 ++++++++++++--------- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/packages/common-algorand/package.json b/packages/common-algorand/package.json index 64a819d2..6c7d914d 100644 --- a/packages/common-algorand/package.json +++ b/packages/common-algorand/package.json @@ -13,7 +13,7 @@ "main": "dist/index.js", "license": "GPL-3.0", "dependencies": { - "@subql/common": "^3.1.0", + "@subql/common": "^3.1.1", "@subql/types-algorand": "workspace:*", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", diff --git a/packages/node/package.json b/packages/node/package.json index 7214c9b7..7dbe17e0 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -24,9 +24,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.1.0", + "@subql/common": "^3.1.1", "@subql/common-algorand": "workspace:*", - "@subql/node-core": "^5.0.2", + "@subql/node-core": "^5.0.3", "@subql/types-algorand": "workspace:*", "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0", diff --git a/packages/node/src/algorand/algorand.spec.ts b/packages/node/src/algorand/algorand.spec.ts index a6ee38b8..b113535d 100644 --- a/packages/node/src/algorand/algorand.spec.ts +++ b/packages/node/src/algorand/algorand.spec.ts @@ -24,7 +24,7 @@ function testSubqueryProject(endpoint: string, chainId): SubqueryProject { return { network: { chainId, - endpoint, + endpoint: [endpoint], dictionary: `https://api.subquery.network/sq/subquery/Algorand-Dictionary`, }, dataSources: [], @@ -35,6 +35,7 @@ function testSubqueryProject(endpoint: string, chainId): SubqueryProject { } as any; } +// eslint-disable-next-line jest/no-export export const prepareApiService = async ( endpoint: string = mainnetEndpoint, chainId: string = mainnetChainId, diff --git a/yarn.lock b/yarn.lock index 4ff79cdc..b333ea72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2916,7 +2916,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/common-algorand@workspace:packages/common-algorand" dependencies: - "@subql/common": ^3.1.0 + "@subql/common": ^3.1.1 "@subql/types-algorand": "workspace:*" "@types/bn.js": 4.11.6 "@types/js-yaml": ^4.0.4 @@ -2935,11 +2935,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.1.0, @subql/common@npm:^3.1.0": - version: 3.1.0 - resolution: "@subql/common@npm:3.1.0" +"@subql/common@npm:3.1.1, @subql/common@npm:^3.1.1": + version: 3.1.1 + resolution: "@subql/common@npm:3.1.1" dependencies: - "@subql/types-core": 0.1.0 + "@subql/types-core": 0.1.1 axios: ^0.27.2 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2949,7 +2949,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 43c7207acbc943cf1196c6ea0ecc6187b5a0a08a1f358861da5c2fa2db1b1a94ad6232850ef85e0c1a49274fd96d5c39204d860aa18d0e655fcad0b402df0220 + checksum: 1d1e3070f0a2f2368a7ba24f9b98012ca88362d51e7e53ed146026aa8c2844e8d58335496a19341facf4422aa2cac89b05e87860b0a4f73e542ca91f8240cdd0 languageName: node linkType: hard @@ -2965,9 +2965,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.1.0 + "@subql/common": ^3.1.1 "@subql/common-algorand": "workspace:*" - "@subql/node-core": ^5.0.2 + "@subql/node-core": ^5.0.3 "@subql/types-algorand": "workspace:*" "@subql/types-core": 0.0.0-2 "@types/express": ^4.17.13 @@ -2997,18 +2997,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:^5.0.2": - version: 5.0.2 - resolution: "@subql/node-core@npm:5.0.2" +"@subql/node-core@npm:^5.0.3": + version: 5.0.3 + resolution: "@subql/node-core@npm:5.0.3" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 "@subql/apollo-links": ^1.0.2 - "@subql/common": 3.1.0 + "@subql/common": 3.1.1 "@subql/testing": 2.0.3-0 - "@subql/types": 3.1.0 + "@subql/types": 3.1.1 "@subql/utils": 2.4.4-1 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 @@ -3024,7 +3024,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: e746752c239b4cdd9426ce664cee4330ca38e8bf8882646637404cecdb5f3e1ec846255ab5b3e19127de0c7c0d27ae22983400699b71b2a25bac473b1cb057a0 + checksum: 8c620eb056436da207f079fd38fe34a599022417ca174187ee26e6b65e9442ea917ea1118920edbad5eb6dce88820124c1c5492333164e9b4c8edfba488d0154 languageName: node linkType: hard @@ -3058,12 +3058,12 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:0.1.0, @subql/types-core@npm:^0.1.0": - version: 0.1.0 - resolution: "@subql/types-core@npm:0.1.0" +"@subql/types-core@npm:0.1.1": + version: 0.1.1 + resolution: "@subql/types-core@npm:0.1.1" dependencies: package-json-type: ^1.0.3 - checksum: fd86c2cf5148a4ad3af5c3a4fa7f0180f7a4d90fd3f6d96bc0e48eab6d73767220c859c751b21bbe08565d31eac5e8d0cee0148dee7ee7e0e273ff46182d3131 + checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa languageName: node linkType: hard @@ -3076,12 +3076,21 @@ __metadata: languageName: node linkType: hard -"@subql/types@npm:3.1.0": - version: 3.1.0 - resolution: "@subql/types@npm:3.1.0" +"@subql/types-core@npm:^0.1.0": + version: 0.1.0 + resolution: "@subql/types-core@npm:0.1.0" + dependencies: + package-json-type: ^1.0.3 + checksum: fd86c2cf5148a4ad3af5c3a4fa7f0180f7a4d90fd3f6d96bc0e48eab6d73767220c859c751b21bbe08565d31eac5e8d0cee0148dee7ee7e0e273ff46182d3131 + languageName: node + linkType: hard + +"@subql/types@npm:3.1.1": + version: 3.1.1 + resolution: "@subql/types@npm:3.1.1" peerDependencies: "@polkadot/api": ^10 - checksum: 31cd2870513e69b51c3acdd5016d2ed8fb8ce405366f912b26422b9169b236a5c5c6bb3b6a7592f7209f842d963083a08ebe3c299e81620e5752d4cc3522d552 + checksum: 0b1765dfea7e180366cb846b6e8c398bb30be762d79823508c0e4a2953be6785316858f0efe67469cbb693f0bf3db0a81c625b7265cdcc095bd449df75e60d63 languageName: node linkType: hard From 888525010709672be28d5f1e78371be306182544 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 4 Oct 2023 10:53:11 +1300 Subject: [PATCH 13/15] Fix test --- packages/node/src/indexer/api.service.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node/src/indexer/api.service.spec.ts b/packages/node/src/indexer/api.service.spec.ts index a47b4df2..1b26c904 100644 --- a/packages/node/src/indexer/api.service.spec.ts +++ b/packages/node/src/indexer/api.service.spec.ts @@ -18,7 +18,7 @@ const ENDPOINT = 'https://mainnet-idx.algonode.cloud/'; function testSubqueryProject(endpoint: string): SubqueryProject { return { network: { - endpoint, + endpoint: [endpoint], dictionary: `https://api.subquery.network/sq/subquery/Algorand-Dictionary`, chainId: 'wGHE2Pwdvd7S12BL5FaOP20EGYesN73ktiC1qzkkit8=', }, From f227f00674ea36a04919d73d6e82af29bfb3eedd Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 4 Oct 2023 14:35:01 +1300 Subject: [PATCH 14/15] Update deps --- packages/types/package.json | 2 +- yarn.lock | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/types/package.json b/packages/types/package.json index 9a64b3aa..a7740297 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -23,6 +23,6 @@ "algosdk": "^2.2.0" }, "dependencies": { - "@subql/types-core": "^0.1.0" + "@subql/types-core": "^0.1.1" } } diff --git a/yarn.lock b/yarn.lock index b333ea72..890026a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3041,7 +3041,7 @@ __metadata: version: 0.0.0-use.local resolution: "@subql/types-algorand@workspace:packages/types" dependencies: - "@subql/types-core": ^0.1.0 + "@subql/types-core": ^0.1.1 "@types/app-module-path": ^2.2.0 algosdk: ^2.2.0 peerDependencies: @@ -3058,7 +3058,7 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:0.1.1": +"@subql/types-core@npm:0.1.1, @subql/types-core@npm:^0.1.1": version: 0.1.1 resolution: "@subql/types-core@npm:0.1.1" dependencies: @@ -3076,15 +3076,6 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:^0.1.0": - version: 0.1.0 - resolution: "@subql/types-core@npm:0.1.0" - dependencies: - package-json-type: ^1.0.3 - checksum: fd86c2cf5148a4ad3af5c3a4fa7f0180f7a4d90fd3f6d96bc0e48eab6d73767220c859c751b21bbe08565d31eac5e8d0cee0148dee7ee7e0e273ff46182d3131 - languageName: node - linkType: hard - "@subql/types@npm:3.1.1": version: 3.1.1 resolution: "@subql/types@npm:3.1.1" From 36685cc7cd7971165fde3705474fb6c361560e6e Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 4 Oct 2023 15:05:50 +1300 Subject: [PATCH 15/15] Remove types-core from node --- packages/node/package.json | 1 - yarn.lock | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/packages/node/package.json b/packages/node/package.json index 7dbe17e0..7119236d 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -28,7 +28,6 @@ "@subql/common-algorand": "workspace:*", "@subql/node-core": "^5.0.3", "@subql/types-algorand": "workspace:*", - "@subql/types-core": "0.0.0-2", "algosdk": "^2.2.0", "axios": "^1.3.4", "cron-converter": "^1.0.2", diff --git a/yarn.lock b/yarn.lock index 890026a2..89a5e982 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2969,7 +2969,6 @@ __metadata: "@subql/common-algorand": "workspace:*" "@subql/node-core": ^5.0.3 "@subql/types-algorand": "workspace:*" - "@subql/types-core": 0.0.0-2 "@types/express": ^4.17.13 "@types/jest": ^27.4.0 "@types/lodash": ^4.14.178 @@ -3049,15 +3048,6 @@ __metadata: languageName: unknown linkType: soft -"@subql/types-core@npm:0.0.0-2": - version: 0.0.0-2 - resolution: "@subql/types-core@npm:0.0.0-2" - dependencies: - package-json-type: ^1.0.3 - checksum: 3cd6a90391a8d206d0140bc84fff3b019a54488b711ce4f3d8c59f5b326dba9fc8b76d233add90b6a8dc953b09a7d3057f5132ec04a4b46e18c1aac49e8c685f - languageName: node - linkType: hard - "@subql/types-core@npm:0.1.1, @subql/types-core@npm:^0.1.1": version: 0.1.1 resolution: "@subql/types-core@npm:0.1.1"