[release] 20231026 Sync @subql dependencies (#71)

* Update @subql deps

* Apply changes from node-core, update changelogs

* [release] 20231026

* Fix test

---------

Co-authored-by: stwiname <[email protected]>

.yarn/versions/a42fee2f.yml

@@ -1,2 +1,3 @@
 undecided:
-  - "@subql/common-near"
+  - "@subql/node-near"
+  - "@subql/types-near"

packages/common-near/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.0.1] - 2023-10-26
+### Changed
+- Update @subql/common
+
 ## [3.0.0] - 2023-10-04
 ### Added
 - Parent field to manifest for project upgrades
@@ -50,7 +54,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add transaction receipts handlers and filters (#13)
 
 ## [1.0.0] - 2023-01-26
-[Unreleased]: https://github.com/subquery/subql-near/compare/common-near/3.0.0...HEAD
+[Unreleased]: https://github.com/subquery/subql-near/compare/common-near/3.0.1...HEAD
+[3.0.1]: https://github.com/subquery/subql-near/compare/common-near/3.0.0...common-near/3.0.1
 [3.0.0]: https://github.com/subquery/subql-near/compare/commonm-near/2.4.1...commonm-near/3.0.0
 [2.4.1]: https://github.com/subquery/subql-near/compare/common-near/2.4.0...common-near/2.4.1
 [2.4.0]: https://github.com/subquery/subql-near/compare/common-near/2.3.0...common-near/2.4.0

packages/common-near/package.json

@@ -1,19 +1,20 @@
 {
   "name": "@subql/common-near",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "",
   "scripts": {
     "build": "rm -rf dist && tsc -b",
     "test": "echo \"Error: no test specified\" && exit 1",
-    "format": "prettier --write \"src/**/*.ts\""
+    "format": "prettier --write \"src/**/*.ts\"",
+    "changelog:release": "echo \"Updating changelog $npm_package_version\" && npx chan release $npm_package_version --git-url \"https://github.com/subquery/subql-near\" --release-prefix=\"common-near/\""
   },
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",
   "author": "Ian He",
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.1.1",
+    "@subql/common": "^3.2.0",
     "@subql/types-near": "workspace:*",
     "js-yaml": "^4.1.0",
     "reflect-metadata": "^0.1.13"

packages/node/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.1.0] - 2023-10-26
+### Changed
+- Update node-core and make according changes
+
 ## [3.0.0] - 2023-10-04
 ### Changed
 - Update `@subql/node-core` and sync with main SDK.
@@ -132,7 +136,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add `toJson` method to function args (#9)
 
 ## 1.18.0 - 2023-01-26
-[Unreleased]: https://github.com/subquery/subql-near/compare/node-near/3.0.0...HEAD
+[Unreleased]: https://github.com/subquery/subql-near/compare/node-near/3.1.0...HEAD
+[3.1.0]: https://github.com/subquery/subql-near/compare/node-near/3.0.0...node-near/3.1.0
 [3.0.0]: https://github.com/subquery/subql-near/compare/node-near/2.10.2...node-near/3.0.0
 [2.10.2]: https://github.com/subquery/subql-near/compare/node-near/2.10.1...node-near/2.10.2
 [2.10.1]: https://github.com/subquery/subql-near/compare/node-near/2.10.0...node-near/2.10.1

packages/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-near",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "",
   "author": "Ian He",
   "license": "GPL-3.0",
@@ -10,7 +10,8 @@
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "nest start",
     "start:dev": "nodemon",
-    "start:prod": "node dist/main"
+    "start:prod": "node dist/main",
+    "changelog:release": "echo \"Updating changelog $npm_package_version\" && npx chan release $npm_package_version --git-url \"https://github.com/subquery/subql-near\" --release-prefix=\"node-near/\""
   },
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",
@@ -24,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.1.1",
+    "@subql/common": "^3.2.0",
     "@subql/common-near": "workspace:*",
-    "@subql/node-core": "^5.0.3",
+    "@subql/node-core": "^6.1.1",
     "@subql/types-near": "workspace:*",
     "cron-converter": "^1.0.2",
     "eventemitter2": "^6.4.5",

packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts

@@ -13,6 +13,7 @@ import {
   BlockDispatcher,
   ProcessBlockResponse,
   IProjectUpgradeService,
+  PoiSyncService,
 } from '@subql/node-core';
 import {
   NearProjectDs,
@@ -43,6 +44,7 @@ export class BlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
   ) {
@@ -55,6 +57,7 @@ export class BlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       async (blockNums: number[]): Promise<BlockContent[]> => {

packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts

@@ -25,6 +25,7 @@ import {
   storeHostFunctions,
   dynamicDsHostFunctions,
   HostUnfinalizedBlocks,
+  PoiSyncService,
 } from '@subql/node-core';
 import { Store } from '@subql/types-core';
 import { NearDatasource } from '@subql/types-near';
@@ -91,6 +92,7 @@ export class WorkerBlockDispatcherService
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
+    poiSyncService: PoiSyncService,
     @Inject('ISubqueryProject') project: SubqueryProject,
     dynamicDsService: DynamicDsService,
     unfinalizedBlocksSevice: UnfinalizedBlocksService,
@@ -105,6 +107,7 @@ export class WorkerBlockDispatcherService
       storeService,
       storeCacheService,
       poiService,
+      poiSyncService,
       project,
       dynamicDsService,
       () =>

packages/node/src/indexer/fetch.module.ts

@@ -14,6 +14,7 @@ import {
   StoreCacheService,
   ConnectionPoolStateManager,
   IProjectUpgradeService,
+  PoiSyncService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { ApiService } from './api.service';
@@ -59,6 +60,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         storeService: StoreService,
         storeCacheService: StoreCacheService,
         poiService: PoiService,
+        poiSyncService: PoiSyncService,
         project: SubqueryProject,
         dynamicDsService: DynamicDsService,
         unfinalizedBlocks: UnfinalizedBlocksService,
@@ -74,6 +76,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
               unfinalizedBlocks,
@@ -90,6 +93,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               storeService,
               storeCacheService,
               poiService,
+              poiSyncService,
               project,
               dynamicDsService,
             ),
@@ -104,6 +108,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         StoreService,
         StoreCacheService,
         PoiService,
+        PoiSyncService,
         'ISubqueryProject',
         DynamicDsService,
         UnfinalizedBlocksService,
@@ -119,6 +124,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
     DsProcessorService,
     DynamicDsService,
     PoiService,
+    PoiSyncService,
     {
       useClass: ProjectService,
       provide: 'IProjectService',

packages/node/src/indexer/project.service.spec.ts

@@ -32,6 +32,7 @@ describe('ProjectService', () => {
       null,
       null,
       null,
+      {} as any,
       null,
       null,
       null,

packages/node/src/indexer/project.service.ts

@@ -10,6 +10,7 @@ import {
   StoreService,
   NodeConfig,
   IProjectUpgradeService,
+  PoiSyncService,
 } from '@subql/node-core';
 import { Sequelize } from '@subql/x-sequelize';
 import { NearProjectDs, SubqueryProject } from '../configure/SubqueryProject';
@@ -33,6 +34,8 @@ export class ProjectService extends BaseProjectService<
     dsProcessorService: DsProcessorService,
     apiService: ApiService,
     @Inject(isMainThread ? PoiService : 'Null') poiService: PoiService,
+    @Inject(isMainThread ? PoiSyncService : 'Null')
+    poiSyncService: PoiSyncService,
     @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize,
     @Inject('ISubqueryProject') project: SubqueryProject,
     @Inject('IProjectUpgradeService')
@@ -47,6 +50,7 @@ export class ProjectService extends BaseProjectService<
       dsProcessorService,
       apiService,
       poiService,
+      poiSyncService,
       sequelize,
       project,
       projectUpgradeService,

packages/node/src/indexer/worker/worker.ts

@@ -34,7 +34,7 @@ const logger = getLogger(`worker #${threadId}`);
 async function initWorker(startHeight: number): Promise<void> {
   try {
     const app = await NestFactory.create(WorkerModule, {
-      logger: new NestLogger(argv.debug), // TIP: If the worker is crashing comment out this line for better logging
+      logger: new NestLogger(!!argv.debug), // TIP: If the worker is crashing comment out this line for better logging
     });
 
     await app.init();

packages/node/src/init.ts

@@ -26,7 +26,7 @@ export async function bootstrap(): Promise<void> {
 
   try {
     const app = await NestFactory.create(AppModule, {
-      logger: new NestLogger(debug),
+      logger: new NestLogger(!!debug),
     });
     await app.init();
 

packages/node/src/subcommands/testing.service.ts

@@ -44,7 +44,7 @@ export class TestingService extends BaseTestingService<
     const testContext = await NestFactory.createApplicationContext(
       TestingModule,
       {
-        logger: new NestLogger(),
+        logger: new NestLogger(!!this.nodeConfig.debug),
       },
     );
 

packages/node/src/yargs.ts

@@ -13,7 +13,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -30,7 +30,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     builder: {},
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -53,7 +53,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
       }),
     handler: (argv) => {
       initLogger(
-        argv.debug as boolean,
+        argv.debug as string,
         argv.outputFmt as 'json' | 'colored',
         argv.logLevel as string | undefined,
       );
@@ -63,81 +63,6 @@ export const yargsOptions = yargs(hideBin(process.argv))
       return reindexInit(argv.targetHeight);
     },
   })
-  .command({
-    command: 'mmr-regen',
-    describe:
-      'Re-generate mmr between Filebased/Postgres mmr and Proof of index',
-    builder: (yargs) =>
-      yargs.options({
-        probe: {
-          type: 'boolean',
-          description:
-            'Fetch latest mmr height information from file based/postgres DB and Poi table',
-          demandOption: false,
-          default: false,
-        },
-        resetOnly: {
-          type: 'boolean',
-          description:
-            'Only reset the mmr value in both POI and file based/postgres DB to target height',
-          demandOption: false,
-          default: false,
-        },
-        targetHeight: {
-          type: 'number',
-          description: 'Re-genrate mmr value from this block height',
-          demandOption: false,
-        },
-        unsafe: {
-          type: 'boolean',
-          description: 'Allow sync mmr from Poi table to file or a postgres DB',
-          demandOption: false,
-          default: false,
-        },
-      }),
-    handler: (argv) => {
-      initLogger(
-        argv.debug as boolean,
-        argv.outputFmt as 'json' | 'colored',
-        argv.logLevel as string | undefined,
-      );
-
-      // lazy import to make sure logger is instantiated before all other services
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      const { mmrRegenerateInit } = require('./subcommands/mmrRegenerate.init');
-      return mmrRegenerateInit(
-        argv.probe,
-        argv.resetOnly,
-        argv.unsafe,
-        argv.targetHeight,
-      );
-    },
-  })
-  .command({
-    command: 'mmr-migrate',
-    describe: 'Migrate MMR data from storage file to postgres DB',
-    builder: (yargs) =>
-      yargs.options({
-        direction: {
-          type: 'string',
-          description: 'set direction of migration (file -> DB or DB -> file)',
-          demandOption: false,
-          choices: ['dbToFile', 'fileToDb'],
-          default: 'dbToFile',
-        },
-      }),
-    handler: (argv) => {
-      initLogger(
-        argv.debug as boolean,
-        argv.outputFmt as 'json' | 'colored',
-        argv.logLevel as string | undefined,
-      );
-      // lazy import to make sure logger is instantiated before all other services
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      const { mmrMigrateInit } = require('./subcommands/mmrMigrate.init');
-      return mmrMigrateInit(argv.direction);
-    },
-  })
   // Note we must have default command $0 at last to avoid override
   .command({
     command: '$0', //default command
@@ -311,10 +236,8 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe:
-        'Show debug information to console output. will forcefully set log level to debug',
-      type: 'boolean',
-      default: false,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      type: 'string',
     },
     ipfs: {
       demandOption: false,

packages/types/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.0.1] - 2023-10-26
+### Changed
+- Update @subql/types-core
+
 ## [3.0.0] - 2023-10-04
 ### Changed
 - Use `@subql/types-core`
@@ -43,7 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Args type for funciton calls (#9)
 
 ## [1.0.0] - 2023-01-26
-[Unreleased]: https://github.com/subquery/subql-near/compare/types-near/3.0.0...HEAD
+[Unreleased]: https://github.com/subquery/subql-near/compare/types-near/3.0.1...HEAD
+[3.0.1]: https://github.com/subquery/subql-near/compare/types-near/3.0.0...types-near/3.0.1
 [3.0.0]: https://github.com/subquery/subql-near/compare/types-near/2.1.3...types-near/3.0.0
 [2.1.3]: https://github.com/subquery/subql-near/compare/types-near/v2.1.2...types-near/v2.1.3
 [2.1.2]: https://github.com/subquery/subql-near/compare/types-near/v2.1.1...types-near/v2.1.2