Milestone 3 support

packages/common-stellar/package.json

@@ -17,7 +17,8 @@
     "@subql/types-stellar": "workspace:*",
     "js-yaml": "^4.1.0",
     "pino": "^6.13.3",
-    "reflect-metadata": "^0.1.13"
+    "reflect-metadata": "^0.1.13",
+    "stellar-sdk": "^10.4.1"
   },
   "peerDependencies": {
     "class-transformer": "*",

packages/common-stellar/src/project/models.ts

@@ -5,7 +5,7 @@
 import {
   StellarHandlerKind,
   StellarDatasourceKind,
-  StellarEventFilter,
+  SorobanEventFilter,
   SubqlCustomHandler,
   SubqlMapping,
   SubqlHandler,
@@ -14,13 +14,112 @@
   SubqlCustomDatasource,
   FileReference,
   CustomDataSourceAsset,
+  StellarBlockFilter,
+  StellarTransactionFilter,
+  StellarOperationFilter,
+  StellarEffectFilter,
+  SubqlBlockHandler,
+  SubqlTransactionHandler,
+  SubqlOperationHandler,
+  SubqlEffectHandler,
   SubqlEventHandler,
+  SubqlSorobanTransactionHandler,
 } from '@subql/types-stellar';
 import {plainToClass, Transform, Type} from 'class-transformer';
 import {IsArray, IsEnum, IsInt, IsOptional, IsString, IsObject, ValidateNested} from 'class-validator';
+import {Horizon} from 'stellar-sdk';
 import {SubqlStellarProcessorOptions} from './types';
 
-export class EventFilter implements StellarEventFilter {
+export class BlockFilter implements StellarBlockFilter {
+  @IsOptional()
+  @IsInt()
+  modulo?: number;
+  @IsOptional()
+  @IsString()
+  timestamp?: string;
+}
+
+export class TransactionFilter implements StellarTransactionFilter {
+  @IsOptional()
+  @IsString()
+  account?: string;
+}
+
+export class OperationFilter implements StellarOperationFilter {
+  @IsOptional()
+  type: Horizon.OperationResponseType;
+
+  @IsOptional()
+  @IsString()
+  source_account?: string;
+}
+
+export class EffectFilter implements StellarEffectFilter {
+  @IsOptional()
+  @IsString()
+  type?: string;
+
+  @IsOptional()
+  @IsString()
+  account?: string;
+}
+
+export class BlockHandler implements SubqlBlockHandler {
+  @IsObject()
+  @IsOptional()
+  @Type(() => BlockFilter)
+  filter?: BlockFilter;
+  @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.Block]})
+  kind: StellarHandlerKind.Block;
+  @IsString()
+  handler: string;
+}
+
+export class TransactionHandler implements SubqlTransactionHandler {
+  @IsObject()
+  @IsOptional()
+  @Type(() => TransactionFilter)
+  filter?: TransactionFilter;
+  @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.Transaction]})
+  kind: StellarHandlerKind.Transaction;
+  @IsString()
+  handler: string;
+}
+
+export class SorobanTransactionHandler implements SubqlSorobanTransactionHandler {
+  @IsObject()
+  @IsOptional()
+  @Type(() => TransactionFilter)
+  filter?: TransactionFilter;
+  @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.SorobanTransaction]})
+  kind: StellarHandlerKind.SorobanTransaction;
+  @IsString()
+  handler: string;
+}
+
+export class OperationHandler implements SubqlOperationHandler {
+  @IsObject()
+  @IsOptional()
+  @Type(() => OperationFilter)
+  filter?: OperationFilter;
+  @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.Operation]})
+  kind: StellarHandlerKind.Operation;
+  @IsString()
+  handler: string;
+}
+
+export class EffectHandler implements SubqlEffectHandler {
+  @IsObject()
+  @IsOptional()
+  @Type(() => EffectFilter)
+  filter?: EffectFilter;
+  @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.Effects]})
+  kind: StellarHandlerKind.Effects;
+  @IsString()
+  handler: string;
+}
+
+export class EventFilter implements SorobanEventFilter {
   @IsOptional()
   @IsString()
   contractId?: string;
@@ -34,7 +133,7 @@
   @IsOptional()
   @ValidateNested()
   @Type(() => EventFilter)
-  filter?: StellarEventFilter;
+  filter?: SorobanEventFilter;
   @IsEnum(StellarHandlerKind, {groups: [StellarHandlerKind.Event]})
   kind: StellarHandlerKind.Event;
   @IsString()
@@ -56,10 +155,20 @@
     const handlers: SubqlHandler[] = params.value;
     return handlers.map((handler) => {
       switch (handler.kind) {
+        case StellarHandlerKind.Block:
+          return plainToClass(BlockHandler, handler);
+        case StellarHandlerKind.Transaction:
+          return plainToClass(TransactionHandler, handler);
+        case StellarHandlerKind.SorobanTransaction:
+          return plainToClass(SorobanTransactionHandler, handler);
+        case StellarHandlerKind.Operation:
+          return plainToClass(OperationHandler, handler);
+        case StellarHandlerKind.Effects:
+          return plainToClass(EffectHandler, handler);
         case StellarHandlerKind.Event:
           return plainToClass(EventHandler, handler);
         default:
           throw new Error(`handler ${(handler as any).kind} not supported`);
       }
     });
   })

packages/common-stellar/src/project/project.spec.ts

@@ -23,13 +23,43 @@ describe('project.yaml', () => {
     expect(() => loadStellarProjectManifest(path.join(projectsDir, 'project_1.0.0.yaml'))).not.toThrow();
   });
 
-  it('get v1.0.0 deployment mapping filter', () => {
+  it('get v1.0.0 deployment mapping filter - transaction', () => {
     const manifestVersioned = loadStellarProjectManifest(path.join(projectsDir, 'project_1.0.0.yaml'));
 
     const deployment = manifestVersioned.asV1_0_0.deployment;
     const filter = deployment.dataSources[0].mapping.handlers[0].filter;
     const deploymentString = manifestVersioned.toDeployment();
     expect(filter).not.toBeNull();
+    expect(deploymentString).toContain('account');
+  });
+
+  it('get v1.0.0 deployment mapping filter - operation', () => {
+    const manifestVersioned = loadStellarProjectManifest(path.join(projectsDir, 'project_1.0.0.yaml'));
+
+    const deployment = manifestVersioned.asV1_0_0.deployment;
+    const filter = deployment.dataSources[0].mapping.handlers[1].filter;
+    const deploymentString = manifestVersioned.toDeployment();
+    expect(filter).not.toBeNull();
+    expect(deploymentString).toContain('account_merge');
+  });
+
+  it('get v1.0.0 deployment mapping filter - effect', () => {
+    const manifestVersioned = loadStellarProjectManifest(path.join(projectsDir, 'project_1.0.0.yaml'));
+
+    const deployment = manifestVersioned.asV1_0_0.deployment;
+    const filter = deployment.dataSources[0].mapping.handlers[2].filter;
+    const deploymentString = manifestVersioned.toDeployment();
+    expect(filter).not.toBeNull();
+    expect(deploymentString).toContain('account_credited');
+  });
+
+  it('get v1.0.0 deployment mapping filter - events', () => {
+    const manifestVersioned = loadStellarProjectManifest(path.join(projectsDir, 'project_1.0.0.yaml'));
+
+    const deployment = manifestVersioned.asV1_0_0.deployment;
+    const filter = deployment.dataSources[0].mapping.handlers[3].filter;
+    const deploymentString = manifestVersioned.toDeployment();
+    expect(filter).not.toBeNull();
     expect(deploymentString).toContain('COUNTER');
   });
 

packages/common-stellar/src/project/types.ts

@@ -13,7 +13,10 @@ export {
   StellarHandlerKind,
   SubqlDatasource as SubqlStellarDataSource,
   SubqlCustomDatasource as SubqlStellarCustomDataSource,
-  StellarEventFilter,
+  StellarBlockFilter,
+  StellarTransactionFilter,
+  StellarOperationFilter,
+  StellarEffectFilter,
   SubqlDatasourceProcessor,
   SubqlHandlerFilter,
   StellarDatasourceKind,
@@ -25,4 +28,5 @@ export type IStellarProjectManifest = IProjectManifest<SubqlDatasource>;
 export interface StellarProjectNetworkConfig extends ProjectNetworkConfig {
   genesisHash?: string;
   chainId?: string;
+  soroban: string;
 }

packages/common-stellar/src/project/utils.ts

@@ -10,6 +10,36 @@ import {
   SubqlRuntimeDatasource,
 } from '@subql/types-stellar';
 
+export function isBlockHandlerProcessor<B>(
+  hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
+): hp is SecondLayerHandlerProcessor<StellarHandlerKind.Block, unknown, B> {
+  return hp.baseHandlerKind === StellarHandlerKind.Block;
+}
+
+export function isTransactionHandlerProcessor<T>(
+  hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
+): hp is SecondLayerHandlerProcessor<StellarHandlerKind.Transaction, unknown, T> {
+  return hp.baseHandlerKind === StellarHandlerKind.Transaction;
+}
+
+export function isSorobanTransactionHandlerProcessor<T>(
+  hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
+): hp is SecondLayerHandlerProcessor<StellarHandlerKind.SorobanTransaction, unknown, T> {
+  return hp.baseHandlerKind === StellarHandlerKind.SorobanTransaction;
+}
+
+export function isOperationHandlerProcessor<O>(
+  hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
+): hp is SecondLayerHandlerProcessor<StellarHandlerKind.Operation, unknown, O> {
+  return hp.baseHandlerKind === StellarHandlerKind.Operation;
+}
+
+export function isEffectHandlerProcessor<E>(
+  hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
+): hp is SecondLayerHandlerProcessor<StellarHandlerKind.Effects, unknown, E> {
+  return hp.baseHandlerKind === StellarHandlerKind.Effects;
+}
+
 export function isEventHandlerProcessor<E>(
   hp: SecondLayerHandlerProcessor<StellarHandlerKind, unknown, unknown>
 ): hp is SecondLayerHandlerProcessor<StellarHandlerKind.Event, unknown, E> {

packages/common-stellar/src/project/versioned/v1_0_0/model.ts

@@ -54,7 +54,7 @@
 
 export class StellarCustomDataSourceImpl<
     K extends string = string,
     M extends BaseMapping<any, any> = BaseMapping<Record<string, unknown>, any>
   >
   extends CustomDataSourceBase<K, M>
   implements SubqlCustomDatasource<K, M>
@@ -105,6 +105,9 @@
   endpoint?: string | string[];
   @IsString()
   @IsOptional()
+  soroban?: string;
+  @IsString()
+  @IsOptional()
   dictionary?: string;
 }
 

packages/common-stellar/test/project_1.0.0.yaml

@@ -25,8 +25,22 @@ dataSources:
     mapping:
       file: './dist/index.js'
       handlers:
+        - handler: handleTransaction
+          kind: stellar/TransactionHandler
+          filter:
+            account: 'GAKNXHJ5PCZYFIBNBWB4RCQHH6GDEO7Z334N74BOQUQCHKOURQEPMXCH'
+        - handler: handleOperation
+          kind: stellar/OperationHandler
+          filter:
+            source_account: 'GAKNXHJ5PCZYFIBNBWB4RCQHH6GDEO7Z334N74BOQUQCHKOURQEPMXCH'
+            type: 'account_merge'
+        - handler: handleEffect
+          kind: stellar/EffectHandler
+          filter:
+            type: 'account_credited'
+            account: 'GAKNXHJ5PCZYFIBNBWB4RCQHH6GDEO7Z334N74BOQUQCHKOURQEPMXCH'
         - handler: handleEvent
-          kind: stellar/EventHandler
+          kind: soroban/EventHandler
           filter:
             contractId: '34799b3411940249e73d21def8eb2c2b56b285ab8d1ca76d6780ec10a0e8f02e'
             topics:

packages/node/package.json

@@ -41,6 +41,7 @@
     "rimraf": "^3.0.2",
     "rxjs": "^7.5.2",
     "soroban-client": "^0.9.1",
+    "stellar-sdk": "^10.4.1",
     "yargs": "^16.2.0"
   },
   "peerDependencies": {

packages/node/src/configure/SubqueryProject.ts

@@ -9,6 +9,8 @@
   SubqlStellarDataSource,
   ProjectManifestV1_0_0Impl,
   isCustomDs,
+  StellarHandlerKind,
+  isRuntimeDs,
 } from '@subql/common-stellar';
 import { getProjectRoot, updateDataSourcesV1_0_0 } from '@subql/node-core';
 import { buildSchemaFromString } from '@subql/utils';
@@ -71,7 +73,7 @@
   }
 }
 
 function processChainId(network: any): NetworkConfig {
   if (network.chainId && network.genesisHash) {
     throw new Error('Please only provide one of chainId and genesisHash');
   } else if (network.genesisHash && !network.chainId) {
@@ -103,7 +105,7 @@
 
   if (!network.endpoint) {
     throw new Error(
-      `Network endpoint must be provided for network. chainId="${network.chainId}"`,
+      `Stellar network endpoint must be provided for network. chainId="${network.chainId}"`,
     );
   }
 
@@ -126,6 +128,18 @@
 
   const templates = await loadProjectTemplates(projectManifest, root, reader);
 
+  if (
+    dsHasSorobanEventHandler([
+      ...dataSources,
+      ...(templates as SubqlProjectDs[]),
+    ]) &&
+    !network.soroban
+  ) {
+    throw new Error(
+      `Soroban network endpoint must be provided for network. chainId="${network.chainId}"`,
+    );
+  }
+
   return {
     id: reader.root ? reader.root : path, //TODO, need to method to get project_id
     root,
@@ -136,7 +150,7 @@
   };
 }
 
 const { version: packageVersion } = require('../../package.json');
 
 async function loadProjectFromManifest1_0_0(
   projectManifest: ProjectManifestV1_0_0Impl,
@@ -182,6 +196,21 @@
   }));
 }
 
+export function dsHasSorobanEventHandler(
+  dataSources: SubqlProjectDs[],
+): boolean {
+  return (
+    dataSources.findIndex(function (ds) {
+      return (
+        isRuntimeDs(ds) &&
+        ds.mapping.handlers.findIndex(function (handler) {
+          return handler.kind === StellarHandlerKind.Event;
+        }) !== -1
+      );
+    }) !== -1
+  );
+}
+
 /*
 // eslint-disable-next-line @typescript-eslint/require-await
 export async function generateTimestampReferenceForBlockFilters(