Merge pull request #14 from AvailSpace/master

Data indexer Avail mainnet

.env → .env.example

@@ -1,11 +1,11 @@
-DB_NAME=squid_avail
-DB_PORT=23798
+DB_NAME=squid_avail_turing
+DB_PORT=23799
 GQL_PORT=3000
 NETWORK=testnet
-RPC_ENDPOINT=wss://rpc-testnet.avail.tools/ws
+RPC_ENDPOINT=""
 ARCHIVE_LOOKUP_NAME_TESTNET=data-avail
 SUPPORT_HOT_BLOCKS=true
 USE_ONLY_RPC=false # If true, archive is not used and only NODE_RPC_WS will be used
 START_BLOCK=0
-PROCESSOR_BATCH_SIZE=100
+PROCESSOR_BATCH_SIZE=80
 SS58_CODEC=42

docker-compose.yml

@@ -1,10 +1,17 @@
-version: "3.7"
+version: "3.8"
+
+networks:
+  avail_network:
+    driver: bridge
 
 services:
   db:
     image: postgres:15
+    container_name: postgres
+    networks:
+      - avail_network
     environment:
-      POSTGRES_DB: squid_avail
+      POSTGRES_DB: squid_avail_turing
       POSTGRES_PASSWORD: postgres
     shm_size: 1gb
     ports:
@@ -13,25 +20,26 @@ services:
      - avail-data:/var/lib/postgresql/data
     restart: unless-stopped
 
-  # app:
-  #   build:
-  #     context: .
-  #     dockerfile: Dockerfile
-  #   working_dir: /app
-  #   volumes:
-  #     - .:/app
-  #   ports:
-  #     - "3000:3000"
-  #   command: ["sqd", "process"]
-  #   env_file:
-  #     - .env
-  #   depends_on: 
-  #     - db
-  #   environment:
-  #     DB_HOST: db
-  #     DB_PORT: 5432
-
-  #   restart: unless-stopped
-
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    working_dir: /app
+    container_name: avail-squid
+    networks:
+      - avail_network
+    volumes:
+      - .:/app
+    ports:
+      - "3001:3000"
+    command: ["sqd", "process"]
+    env_file:
+      - .env
+    depends_on: 
+      - db
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+    restart: unless-stopped
 volumes:
   avail-data:

package.json

@@ -14,6 +14,7 @@
     "@subsquid/substrate-processor": "^8.1.1",
     "@subsquid/typeorm-migration": "^1.3.0",
     "@subsquid/typeorm-store": "^1.2.6",
+    "buffer": "^6.0.3",
     "dotenv": "^16.4.4",
     "graphql_typeorm": "^2.2.0",
     "pg": "8.11.3",

schema.graphql

@@ -249,4 +249,29 @@ type Amount {
   amount: BigInt
   symbol: String!
   decimal: Int!
-}
+}
+
+type Remark @entity {
+  id: ID!
+  action: String
+  extrinsicHash: String
+  timestamp: DateTime! @index
+  blockNumber: Int! @index
+  sender: Account
+  fee: Amount
+  dataRaw: String
+  dataValue: String
+}
+
+type DataAvailability @entity {
+  id: ID!
+  action: String
+  extrinsicHash: String
+  timestamp: DateTime! @index
+  blockNumber: Int! @index
+  sender: Account
+  fee: Amount
+  dataRaw: String
+  dataValue: String
+  isJson: Boolean
+}

src/constants/handler.ts

@@ -11,6 +11,8 @@ import { NpoolUnbondHandler } from "../process/npoolUnbondHandler";
 import { NpoolBondExtraHandler } from "../process/npoolBondExtraHandler";
 import { NpoolWithdrawUnbondHandler } from "../process/npoolWithdrawHandler";
 import { NpoolPaidOutHandler } from "../process/npoolPaidOutHandler";
+import { RemarkHandler } from "../process/remarkHandler";
+import { DataAvailabilityHandler } from "../process/dataAvailabilityHandler";
 import { IHandler } from "../interfaces/interfaces";
 
 export const HandlerMap: Record<string, IHandler> = {
@@ -26,7 +28,10 @@ export const HandlerMap: Record<string, IHandler> = {
     "NominationPools.Unbonded": new NpoolUnbondHandler(),
     "Npool.BondExtra": new NpoolBondExtraHandler(),
     "NominationPools.Withdrawn": new NpoolWithdrawUnbondHandler(),
-    "NominationPools.PaidOut": new NpoolPaidOutHandler()
+    "NominationPools.PaidOut": new NpoolPaidOutHandler(),
+    "System.remark": new RemarkHandler(),
+    "System.remark_with_event": new RemarkHandler(),
+    "DataAvailability.submit_data": new DataAvailabilityHandler()
 };
 
 export const CallArr = [
@@ -42,4 +47,7 @@ export const CallArr = [
     "NominationPools.unbond",
     "NominationPools.bond_extra",
     "NominationPools.withdraw_unbonded",
+    "System.remark",
+    "System.remark_with_event",
+    "DataAvailability.submit_data"
 ]

src/model/generated/dataAvailability.model.ts

@@ -0,0 +1,44 @@
+import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, ManyToOne as ManyToOne_} from "typeorm"
+import * as marshal from "./marshal"
+import {Account} from "./account.model"
+import {Amount} from "./_amount"
+
+@Entity_()
+export class DataAvailability {
+    constructor(props?: Partial<DataAvailability>) {
+        Object.assign(this, props)
+    }
+
+    @PrimaryColumn_()
+    id!: string
+
+    @Column_("text", {nullable: true})
+    action!: string | undefined | null
+
+    @Column_("text", {nullable: true})
+    extrinsicHash!: string | undefined | null
+
+    @Index_()
+    @Column_("timestamp with time zone", {nullable: false})
+    timestamp!: Date
+
+    @Index_()
+    @Column_("int4", {nullable: false})
+    blockNumber!: number
+
+    @Index_()
+    @ManyToOne_(() => Account, {nullable: true})
+    sender!: Account | undefined | null
+
+    @Column_("jsonb", {transformer: {to: obj => obj == null ? undefined : obj.toJSON(), from: obj => obj == null ? undefined : new Amount(undefined, obj)}, nullable: true})
+    fee!: Amount | undefined | null
+
+    @Column_("text", {nullable: true})
+    dataRaw!: string | undefined | null
+
+    @Column_("text", {nullable: true})
+    dataValue!: string | undefined | null
+
+    @Column_("bool", {nullable: true})
+    isJson!: boolean | undefined | null
+}

src/model/generated/index.ts

@@ -13,3 +13,5 @@ export * from "./nominationPoolBondExtra.model"
 export * from "./nominationPoolUnbond.model"
 export * from "./nominationPoolPaidOut.model"
 export * from "./nominationPoolWithdrawUnbonded.model"
+export * from "./remark.model"
+export * from "./dataAvailability.model"

src/model/generated/remark.model.ts

@@ -0,0 +1,41 @@
+import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, ManyToOne as ManyToOne_} from "typeorm"
+import * as marshal from "./marshal"
+import {Account} from "./account.model"
+import {Amount} from "./_amount"
+
+@Entity_()
+export class Remark {
+    constructor(props?: Partial<Remark>) {
+        Object.assign(this, props)
+    }
+
+    @PrimaryColumn_()
+    id!: string
+
+    @Column_("text", {nullable: true})
+    action!: string | undefined | null
+
+    @Column_("text", {nullable: true})
+    extrinsicHash!: string | undefined | null
+
+    @Index_()
+    @Column_("timestamp with time zone", {nullable: false})
+    timestamp!: Date
+
+    @Index_()
+    @Column_("int4", {nullable: false})
+    blockNumber!: number
+
+    @Index_()
+    @ManyToOne_(() => Account, {nullable: true})
+    sender!: Account | undefined | null
+
+    @Column_("jsonb", {transformer: {to: obj => obj == null ? undefined : obj.toJSON(), from: obj => obj == null ? undefined : new Amount(undefined, obj)}, nullable: true})
+    fee!: Amount | undefined | null
+
+    @Column_("text", {nullable: true})
+    dataRaw!: string | undefined | null
+
+    @Column_("text", {nullable: true})
+    dataValue!: string | undefined | null
+}

src/process/dataAvailabilityHandler.ts

@@ -0,0 +1,69 @@
+import { Call } from "@subsquid/substrate-processor";
+import { DataAvailability, Amount } from "../model";
+import { Fields, ctx } from "../processor";
+import { DataRawAddress,  } from "../util/interfaces";
+import { AccountQuerier } from "./accountHandler";
+import { IHandler } from "../interfaces/interfaces";
+import { Buffer } from 'buffer';
+
+export class DataAvailabilityHandler implements IHandler {
+    availabilityData: Map<string, DataAvailability> = new Map();
+    async process(call: Call<Fields>, accountInstance: AccountQuerier){
+        let addressHex = (call.extrinsic!.signature!.address as DataRawAddress).value;
+
+        let fee = {
+            amount: call.extrinsic!.fee || BigInt(0),
+            symbol: "AVL",
+            decimal: 18
+        }
+        const idExist = this.availabilityData.get(call.id);
+        const dataValue = decodeData(call.args.data);
+        if(!idExist){
+         this.availabilityData.set(call.id, new DataAvailability({
+            id: call.id,
+            action: call.name,
+            extrinsicHash: call.extrinsic!.hash,
+            timestamp: new Date(call.block.timestamp!),
+            blockNumber: call.extrinsic!.block.height,
+            sender: accountInstance.getAccountId(addressHex),
+            fee: new Amount(fee),
+            dataRaw: call.args.data,
+            dataValue: dataValue,
+            isJson: isJson(dataValue) ? true : false
+         }));
+        }
+    }
+
+    async save(){
+        await ctx.store.save([...this.availabilityData.values()]);
+    }
+}
+
+function isJson(str:string):boolean {
+    try {
+        const parsed = JSON.parse(str);
+        return typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed);
+    } catch (error) {
+        return false;
+    }
+}
+
+function decodeData(str:string):string {
+    let strInput = str.slice(2, str.length);
+    const bytesData:Buffer = Buffer.from(strInput, 'hex');
+    let res = bytesData.toString().replace(/<Buffer |>/g, '');
+    if (isValidHexString(res)) return res;
+    else if (isJson(res)) return res;
+    else if (isValidString(res)) return res;
+    else return str;
+}
+
+function isValidHexString(str:string):boolean {
+    const hexRegex = /^0x[0-9A-Fa-f]+$/;
+    return hexRegex.test(str);
+}
+
+function isValidString(str: string): boolean {
+    const regex = /^[\x20-\x7E]*$/;
+    return regex.test(str);
+}

src/process/remarkHandler.ts

@@ -0,0 +1,39 @@
+import { Call } from "@subsquid/substrate-processor";
+import { Remark, Amount } from "../model";
+import { Fields, ctx } from "../processor";
+import { DataRawAddress,  } from "../util/interfaces";
+import { AccountQuerier } from "./accountHandler";
+import { IHandler } from "../interfaces/interfaces";
+
+export class RemarkHandler implements IHandler {
+    remarkData: Map<string, Remark> = new Map();
+    async process(call: Call<Fields>, accountInstance: AccountQuerier){
+        let addressHex = (call.extrinsic!.signature!.address as DataRawAddress).value;
+
+        let fee = {
+            amount: call.extrinsic!.fee || BigInt(0),
+            symbol: "AVL",
+            decimal: 18
+        }
+
+        const idExist = this.remarkData.get(call.id);
+
+        if(!idExist){
+         this.remarkData.set(call.id, new Remark({
+            id: call.id,
+            action: call.name,
+            extrinsicHash: call.extrinsic!.hash,
+            timestamp: new Date(call.block.timestamp!),
+            blockNumber: call.extrinsic!.block.height,
+            sender: accountInstance.getAccountId(addressHex),
+            fee:new Amount(fee),
+            dataRaw: call.args.remark,
+            dataValue: call.args.remark,
+         }));
+        }
+    }
+
+    async save(){
+            await ctx.store.save([...this.remarkData.values()]);
+    }
+}