feat: add EVM indexer

package.json

@@ -19,6 +19,11 @@
   },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@ethersproject/abi": "^5.7.0",
+    "@ethersproject/address": "^5.7.0",
+    "@ethersproject/keccak256": "^5.7.0",
+    "@ethersproject/providers": "^5.7.2",
+    "@ethersproject/strings": "^5.7.0",
     "@graphql-tools/schema": "^8.5.1",
     "bluebird": "^3.7.2",
     "connection-string": "^4.3.5",

src/checkpoint.ts

@@ -6,7 +6,8 @@ import { Pool as PgPool } from 'pg';
 import getGraphQL, { CheckpointsGraphQLObject, MetadataGraphQLObject } from './graphql';
 import { GqlEntityController } from './graphql/controller';
 import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpoints';
-import { BaseProvider, StarknetProvider, BlockNotFoundError } from './providers';
+import { BlockNotFoundError } from './providers';
+import { EvmIndexer } from './providers/evm/indexer';
 import { createLogger, Logger, LogLevel } from './utils/logger';
 import { getConfigChecksum, getContractsFromConfig } from './utils/checkpoint';
 import { extendSchema } from './utils/graphql';
@@ -15,41 +16,38 @@ import { AsyncMySqlPool, createMySqlPool } from './mysql';
 import { createPgPool } from './pg';
 import { checkpointConfigSchema } from './schemas';
 import { register } from './register';
-import {
-  ContractSourceConfig,
-  CheckpointConfig,
-  CheckpointOptions,
-  CheckpointWriters,
-  TemplateSource
-} from './types';
-
-const BLOCK_PRELOAD = 1000;
+import { ContractSourceConfig, CheckpointConfig, CheckpointOptions, TemplateSource } from './types';
+
+const BLOCK_PRELOAD_START_RANGE = 1000;
+const BLOCK_RELOAD_MIN_RANGE = 10;
+const BLOCK_PRELOAD_STEP = 100;
+const BLOCK_PRELOAD_TARGET = 10;
 const BLOCK_PRELOAD_OFFSET = 50;
 const DEFAULT_FETCH_INTERVAL = 2000;
 
 export default class Checkpoint {
   public config: CheckpointConfig;
-  public writer: CheckpointWriters;
   public opts?: CheckpointOptions;
   public schema: string;
 
   private readonly entityController: GqlEntityController;
   private readonly log: Logger;
-  private readonly networkProvider: BaseProvider;
+  private readonly indexer: EvmIndexer;
 
   private dbConnection: string;
   private knex: Knex;
   private mysqlPool?: AsyncMySqlPool;
   private pgPool?: PgPool;
   private checkpointsStore?: CheckpointsStore;
   private activeTemplates: TemplateSource[] = [];
+  private preloadStep: number = BLOCK_PRELOAD_START_RANGE;
   private preloadedBlocks: number[] = [];
   private preloadEndBlock = 0;
   private cpBlocksCache: number[] | null;
 
   constructor(
     config: CheckpointConfig,
-    writer: CheckpointWriters,
+    indexer: EvmIndexer,
     schema: string,
     opts?: CheckpointOptions
   ) {
@@ -59,12 +57,9 @@ export default class Checkpoint {
     }
 
     this.config = config;
-    this.writer = writer;
     this.opts = opts;
     this.schema = extendSchema(schema);
 
-    this.validateConfig();
-
     this.entityController = new GqlEntityController(this.schema, config);
 
     this.log = createLogger({
@@ -79,8 +74,14 @@ export default class Checkpoint {
         : {})
     });
 
-    const NetworkProvider = opts?.NetworkProvider || StarknetProvider;
-    this.networkProvider = new NetworkProvider({ instance: this, log: this.log, abis: opts?.abis });
+    this.indexer = indexer;
+    this.indexer.init({
+      instance: this,
+      log: this.log,
+      abis: opts?.abis
+    });
+
+    this.validateConfig();
 
     this.cpBlocksCache = [];
 
@@ -139,7 +140,7 @@ export default class Checkpoint {
   }
 
   public get sourceContracts() {
-    return this.networkProvider.formatAddresses(getContractsFromConfig(this.config));
+    return this.indexer.getProvider().formatAddresses(getContractsFromConfig(this.config));
   }
 
   public getCurrentSources(blockNumber: number) {
@@ -159,7 +160,7 @@ export default class Checkpoint {
     this.log.debug('starting');
 
     await this.validateStore();
-    await this.networkProvider.init();
+    await this.indexer.getProvider().init();
 
     const templateSources = await this.store.getTemplateSources();
     await Promise.all(
@@ -177,7 +178,7 @@ export default class Checkpoint {
 
     const blockNum = await this.getStartBlockNum();
     this.preloadEndBlock =
-      (await this.networkProvider.getLatestBlockNumber()) - BLOCK_PRELOAD_OFFSET;
+      (await this.indexer.getProvider().getLatestBlockNumber()) - BLOCK_PRELOAD_OFFSET;
 
     return await this.next(blockNum);
   }
@@ -332,8 +333,14 @@ export default class Checkpoint {
     let currentBlock = blockNum;
 
     while (currentBlock <= this.preloadEndBlock) {
-      const endBlock = Math.min(currentBlock + BLOCK_PRELOAD, this.preloadEndBlock);
-      const checkpoints = await this.networkProvider.getCheckpointsRange(currentBlock, endBlock);
+      const endBlock = Math.min(currentBlock + this.preloadStep, this.preloadEndBlock);
+      const checkpoints = await this.indexer
+        .getProvider()
+        .getCheckpointsRange(currentBlock, endBlock);
+
+      const increase =
+        checkpoints.length > BLOCK_PRELOAD_TARGET ? -BLOCK_PRELOAD_STEP : +BLOCK_PRELOAD_STEP;
+      this.preloadStep = Math.max(BLOCK_RELOAD_MIN_RANGE, this.preloadStep + increase);
 
       if (checkpoints.length > 0) {
         this.preloadedBlocks = checkpoints.map(cp => cp.blockNumber).sort();
@@ -363,7 +370,7 @@ export default class Checkpoint {
 
     try {
       const initialSources = this.getCurrentSources(blockNum);
-      const nextBlockNumber = await this.networkProvider.processBlock(blockNum);
+      const nextBlockNumber = await this.indexer.getProvider().processBlock(blockNum);
       const sources = this.getCurrentSources(nextBlockNumber);
 
       if (initialSources.length !== sources.length) {
@@ -375,7 +382,7 @@ export default class Checkpoint {
       if (err instanceof BlockNotFoundError) {
         if (this.config.optimistic_indexing) {
           try {
-            await this.networkProvider.processPool(blockNum);
+            await this.indexer.getProvider().processPool(blockNum);
           } catch (err) {
             this.log.error({ blockNumber: blockNum, err }, 'error occurred during pool processing');
           }
@@ -480,7 +487,9 @@ export default class Checkpoint {
     ];
 
     const missingAbis = usedAbis.filter(abi => !this.opts?.abis?.[abi]);
-    const missingWriters = usedWriters.filter(writer => !this.writer[writer.fn]);
+    const missingWriters = usedWriters.filter(
+      writer => !this.indexer.getHandlers().includes(writer.fn)
+    );
 
     if (missingAbis.length > 0) {
       throw new Error(
@@ -498,7 +507,7 @@ export default class Checkpoint {
   }
 
   private async validateStore() {
-    const networkIdentifier = await this.networkProvider.getNetworkIdentifier();
+    const networkIdentifier = await this.indexer.getProvider().getNetworkIdentifier();
     const configChecksum = getConfigChecksum(this.config);
 
     const storedNetworkIdentifier = await this.store.getMetadata(MetadataId.NetworkIdentifier);

src/index.ts

@@ -3,6 +3,7 @@ export { LogLevel } from './utils/logger';
 export { AsyncMySqlPool } from './mysql';
 export { createGetLoader } from './graphql';
 export { Model } from './orm';
+export * from './providers';
 export * from './types';
 
 export default Checkpoint;

src/providers/base.ts

@@ -3,10 +3,9 @@ import Checkpoint from '../checkpoint';
 import { CheckpointRecord } from '../stores/checkpoints';
 import { Logger } from '../utils/logger';
 import { AsyncMySqlPool } from '../mysql';
-import { CheckpointConfig, CheckpointWriters, ContractSourceConfig } from '../types';
+import { CheckpointConfig, ContractSourceConfig } from '../types';
 
-type Instance = {
-  writer: CheckpointWriters;
+export type Instance = {
   config: CheckpointConfig;
   getCurrentSources(blockNumber: number): ContractSourceConfig[];
   setLastIndexedBlock(blockNum: number);
@@ -76,7 +75,28 @@ export class BaseProvider {
 
   async getCheckpointsRange(fromBlock: number, toBlock: number): Promise<CheckpointRecord[]> {
     throw new Error(
-      `getEventsRange method was not defined when fetching events from ${fromBlock} to ${toBlock}`
+      `getCheckpointsRange method was not defined when fetching events from ${fromBlock} to ${toBlock}`
     );
   }
 }
+
+export class BaseIndexer {
+  protected provider?: BaseProvider;
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  init({ instance, log, abis }: { instance: Instance; log: Logger; abis?: Record<string, any> }) {
+    throw new Error('init method was not defined');
+  }
+
+  public getProvider() {
+    if (!this.provider) {
+      throw new Error('Provider not initialized');
+    }
+
+    return this.provider;
+  }
+
+  public getHandlers(): string[] {
+    throw new Error('getHandlers method was not defined');
+  }
+}

src/providers/evm/index.ts

@@ -0,0 +1,3 @@
+export { EvmProvider } from './provider';
+export { EvmIndexer } from './indexer';
+export * from './types';

src/providers/evm/indexer.ts

@@ -0,0 +1,21 @@
+import { Logger } from '../../utils/logger';
+import { Instance, BaseIndexer } from '../base';
+import { EvmProvider } from './provider';
+import { Writer } from './types';
+
+export class EvmIndexer extends BaseIndexer {
+  private writers: Record<string, Writer>;
+
+  constructor(writers: Record<string, Writer>) {
+    super();
+    this.writers = writers;
+  }
+
+  init({ instance, log, abis }: { instance: Instance; log: Logger; abis?: Record<string, any> }) {
+    this.provider = new EvmProvider({ instance, log, abis, writers: this.writers });
+  }
+
+  public getHandlers(): string[] {
+    return Object.keys(this.writers);
+  }
+}