From 4b929df0a298687ffc2e5632ba6cd722be6c7e51 Mon Sep 17 00:00:00 2001 From: robin Date: Wed, 21 Aug 2024 13:25:12 +0800 Subject: [PATCH] protocol tvl --- .env.example | 1 + src/entities/blockTokenPrice.entity.ts | 15 ++ src/entities/dau.entity.ts | 6 +- src/migrations/1723508522029-migrations.ts | 2 +- src/statistics/statistic.controller.ts | 54 +++++ src/statistics/statistic.module.ts | 10 +- src/statistics/statistic.service.ts | 262 ++++++++++++++++++++- 7 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 src/entities/blockTokenPrice.entity.ts diff --git a/.env.example b/.env.example index 6b526b4..7d9c4a2 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,7 @@ REFER_DATABASE_CONNECTION_POOL_SIZE=100 EXPLORER_API_URL=https://explorer-api.zklink.io +POINTS_API_URL=https://app-api.zklink.io/points # PUFF PUFF_POINTS_TOKEN_ADDRESS=0x1B49eCf1A8323Db4abf48b2F5EFaA33F7DdAB3Fc diff --git a/src/entities/blockTokenPrice.entity.ts b/src/entities/blockTokenPrice.entity.ts new file mode 100644 index 0000000..66e4556 --- /dev/null +++ b/src/entities/blockTokenPrice.entity.ts @@ -0,0 +1,15 @@ +import { Entity, Column, PrimaryColumn } from "typeorm"; +import { BaseEntity } from "./base.entity"; +import { bigIntNumberTransformer } from "../transformers/bigIntNumber.transformer"; + +@Entity({ name: "blockTokenPrice" }) +export class BlockTokenPrice extends BaseEntity { + @PrimaryColumn({ type: "bigint", transformer: bigIntNumberTransformer }) + public readonly blockNumber: number; + + @PrimaryColumn({ type: "varchar" }) + public readonly priceId: string; + + @Column({ type: "double precision" }) + public readonly usdPrice: number; +} diff --git a/src/entities/dau.entity.ts b/src/entities/dau.entity.ts index 7987c8f..a63ccab 100644 --- a/src/entities/dau.entity.ts +++ b/src/entities/dau.entity.ts @@ -2,7 +2,7 @@ import { Entity, Column, PrimaryGeneratedColumn, Unique } from "typeorm"; import { BaseEntity } from "./base.entity"; @Entity({ name: "protocolDau" }) -@Unique(["date", "name"]) +@Unique(["date", "name", "type"]) export class ProtocolDau extends BaseEntity { @PrimaryGeneratedColumn({ type: "int" }) public readonly id: number; @@ -15,4 +15,8 @@ export class ProtocolDau extends BaseEntity { @Column({ type: "date" }) public readonly date: String; + + // 1: dau, 2: cumulative dau, 3: tvl + @Column({ type: "smallint", default: 1 }) + public readonly type: number; } diff --git a/src/migrations/1723508522029-migrations.ts b/src/migrations/1723508522029-migrations.ts index 7089ce7..79a4f72 100644 --- a/src/migrations/1723508522029-migrations.ts +++ b/src/migrations/1723508522029-migrations.ts @@ -5,7 +5,7 @@ export class Migrations1723508522029 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise { await queryRunner.query( - `CREATE TABLE "protocolDau" ("createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "id" SERIAL NOT NULL, "name" character varying NOT NULL, "amount" integer NOT NULL, "date" date NOT NULL, CONSTRAINT "UQ_1aaab071aea8a9562cb547fd5b5" UNIQUE ("date", "name"), CONSTRAINT "PK_20d32ec3e9e341c3fcaa9f3d267" PRIMARY KEY ("id"))`, + `CREATE TABLE "protocolDau" ("createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "id" SERIAL NOT NULL, "name" character varying NOT NULL, "amount" integer NOT NULL, "date" date NOT NULL, "type" smallint NOT NULL DEFAULT '1', CONSTRAINT "UQ_1aaab071aea8a9562cb547fd5b5" UNIQUE ("date", "name", "type"), CONSTRAINT "PK_20d32ec3e9e341c3fcaa9f3d267" PRIMARY KEY ("id"))`, ); } diff --git a/src/statistics/statistic.controller.ts b/src/statistics/statistic.controller.ts index b51843a..127a2ba 100644 --- a/src/statistics/statistic.controller.ts +++ b/src/statistics/statistic.controller.ts @@ -7,7 +7,9 @@ import { ApiTags, } from "@nestjs/swagger"; import { InjectRepository } from "@nestjs/typeorm"; +import { categoryBaseConfig } from "src/config/projectCategory.config"; import { ProtocolDau } from "src/entities/dau.entity"; +import { Project } from "src/entities/project.entity"; import { Between, Repository } from "typeorm"; @ApiTags("statistic") @@ -19,6 +21,8 @@ export class StatisticController { constructor( @InjectRepository(ProtocolDau) private readonly protocolDauRepository: Repository, + @InjectRepository(Project) + private readonly projectRepository: Repository, private readonly statisticService: StatisticService, ) {} @@ -39,6 +43,7 @@ export class StatisticController { where: { name, date: Between(startDate, endDate), + type: 1, }, order: { date: "desc", @@ -51,6 +56,55 @@ export class StatisticController { }; } + @Get("/protocol/cumulativeDau") + @ApiQuery({ name: "name", type: String, required: false }) + @ApiOperation({ summary: "get protocol cumulative dau" }) + public async getProtocolCumulativeDau( + @Query("page") page: number = 1, + @Query("size") size: number = 30, + @Query("name") name?: string, + ) { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - page * size); + const endDate = new Date(); + endDate.setDate(endDate.getDate() - 1 - (page - 1) * size); + + const data = await this.protocolDauRepository.find({ + where: { + name, + date: Between(startDate, endDate), + type: 2, + }, + order: { + date: "desc", + }, + }); + return { + errno: 0, + errmsg: "no error", + data, + }; + } + + @Get("/protocol/sector") + public async getSector() { + return { + errno: 0, + errmsg: "no error", + data: categoryBaseConfig, + }; + } + + @Get("/protocol/list") + public async getProjectList() { + const all = await this.projectRepository.find(); + return { + errno: 0, + errmsg: "no error", + data: all, + }; + } + @Get("/protocol/test") public async test() { await this.statisticService.statisticHistoryProtocolDau(); diff --git a/src/statistics/statistic.module.ts b/src/statistics/statistic.module.ts index 743a924..a6e9645 100644 --- a/src/statistics/statistic.module.ts +++ b/src/statistics/statistic.module.ts @@ -6,11 +6,19 @@ import { StatisticController } from "./statistic.controller"; import { ProtocolDau } from "src/entities/dau.entity"; import { BalanceOfLp } from "src/entities/balanceOfLp.entity"; import { Project } from "src/entities/project.entity"; +import { ExplorerService } from "src/common/service/explorer.service"; +import { BlockTokenPrice } from "src/entities/blockTokenPrice.entity"; @Module({ imports: [ - TypeOrmModule.forFeature([ProtocolDau, BalanceOfLp, Project]), + TypeOrmModule.forFeature([ + ProtocolDau, + BalanceOfLp, + Project, + BlockTokenPrice, + ]), ScheduleModule.forRoot(), + ExplorerService, ], providers: [StatisticService], controllers: [StatisticController], diff --git a/src/statistics/statistic.service.ts b/src/statistics/statistic.service.ts index 990bfe8..c98ce1a 100644 --- a/src/statistics/statistic.service.ts +++ b/src/statistics/statistic.service.ts @@ -2,17 +2,23 @@ import { BalanceOfLpRepository } from "./../repositories/balanceOfLp.repository" import { ConfigService } from "@nestjs/config"; import { Injectable, Logger } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; -import { In, Repository } from "typeorm"; +import { In, LessThanOrEqual, Repository } from "typeorm"; import { hexTransformer } from "src/transformers/hex.transformer"; import { Cron } from "@nestjs/schedule"; import { BalanceOfLp } from "src/entities/balanceOfLp.entity"; import { Project } from "src/entities/project.entity"; import { ProtocolDau } from "src/entities/dau.entity"; +import { ExplorerService } from "src/common/service/explorer.service"; +import { BlockTokenPrice } from "src/entities/blockTokenPrice.entity"; +import { Token } from "src/type/token"; +import BigNumber from "bignumber.js"; +import { ethers } from "ethers"; @Injectable() export class StatisticService { private readonly logger = new Logger(StatisticService.name); private novaBlocksGraph: string; + private readonly pointsApi: string; constructor( @InjectRepository(BalanceOfLp) @@ -21,11 +27,16 @@ export class StatisticService { private readonly projectRepository: Repository, @InjectRepository(ProtocolDau) private readonly protocolDauRepository: Repository, + @InjectRepository(BlockTokenPrice) + private readonly blockTokenPriceRepository: Repository, private readonly configService: ConfigService, + private readonly explorerService: ExplorerService, ) { this.novaBlocksGraph = configService.getOrThrow( "NOVA_POINT_BLOCK_GRAPH_API", ); + + this.pointsApi = configService.getOrThrow("POINTS_API_URL"); } // Historical data, run only once @@ -33,7 +44,7 @@ export class StatisticService { public async statisticHistoryProtocolDau() { const result: { min: string; max: string }[] = await this.balanceOfLpRepository.query( - `select date(min("createdAt")) as min, date(max("createdAt")) as max from "balancesOfLp"`, + `select date(min("createdAt")) as min, date(max("createdAt")) as max from "blockAddressPointOfLp"`, ); const minDate = new Date(result[0].min); @@ -47,14 +58,16 @@ export class StatisticService { } for (let start = minDate; start <= maxDate; ) { + this.logger.log(`start statistics date: ${start.toISOString()}`); await this.statisticProtocolDauByDay(start); + await this.statisticCumulativeDauByDay(start); start.setDate(start.getDate() + 1); } this.logger.log("finished history statistics protocol dau"); } - @Cron("0 15 0 * * *") + @Cron("0 30 0 * * *") public async statisticProtocolDau() { const yesterday = new Date(); yesterday.setDate(yesterday.getDate() - 1); @@ -62,6 +75,68 @@ export class StatisticService { this.logger.log("finished statistics protocol dau"); } + public async statisticCumulativeDauByDay(day: Date) { + // This parameter is not important + const begin = + new Date( + day.getFullYear(), + day.getMonth(), + day.getDate(), + 0, + 0, + 0, + ).getTime() / 1000; + const end = + new Date( + day.getFullYear(), + day.getMonth(), + day.getDate(), + 23, + 59, + 59, + ).getTime() / 1000; + + const [_minBlock, maxBlock] = await this.getBlockRangeByTime(begin, end); + + if (!maxBlock) { + this.logger.error("Statistical getBlockRangeByTime failure"); + return; + } + + const allProject = await this.projectRepository.find(); + allProject; + + const groupByName: { [name: string]: Buffer[] } = allProject.reduce( + (group, project) => { + const { name } = project; + group[name] = group[name] ?? []; + group[name].push(hexTransformer.to(project.pairAddress)); + return group; + }, + {}, + ); + + for (const name in groupByName) { + const pairAddressList = groupByName[name]; + const count: { count: number }[] = await this.balanceOfLpRepository.query( + `select count(distinct address) from "blockAddressPointOfLp" where "pairAddress"=any($1) and "blockNumber" <= $2 and type in ('txNum', 'txVol')`, + [pairAddressList, maxBlock], + ); + + await this.protocolDauRepository + .createQueryBuilder() + .insert() + .values({ + name, + amount: count[0].count, + date: day.toISOString().split("T")[0], + type: 2, + }) + .orIgnore() + .execute(); + } + } + public async statisticProtocolDauByDay(day: Date) { const begin = new Date( @@ -105,7 +180,7 @@ export class StatisticService { for (const name in groupByName) { const pairAddressList = groupByName[name]; const count: { count: number }[] = await this.balanceOfLpRepository.query( - `select count(distinct address) from "balancesOfLp" where "pairAddress"=any($1) and "blockNumber" between $2 and $3`, + `select count(distinct address) from "blockAddressPointOfLp" where "pairAddress"=any($1) and "blockNumber" between $2 and $3 and type in ('txNum', 'txVol')`, [pairAddressList, minBlock, maxBlock], ); @@ -116,8 +191,156 @@ export class StatisticService { name, amount: count[0].count, date: day.toISOString().split("T")[0], + type: 1, }) - .orIgnore() + .orUpdate(["amount"]) + .execute(); + } + } + + // Historical data, run only once + @Cron("0 15 0 * * *") + public async statisticHistoryTvl() { + const result: { min: string; max: string }[] = + await this.balanceOfLpRepository.query( + `select date(min("createdAt")) as min, date(max("createdAt")) as max from "balanceOfLp"`, + ); + + const minDate = new Date(result[0].min); + let maxDate = new Date(result[0].max); + // Not counted today + if ( + maxDate.toISOString().split("T")[0] === + new Date().toISOString().split("T")[0] + ) { + maxDate.setDate(maxDate.getDate() - 1); + } + + const tokenMap = await this.getSupportTokens(); + + for (let start = minDate; start <= maxDate; ) { + this.logger.log(`start statistics tvl date: ${start.toISOString()}`); + await this.statisticTvlByDay(start, tokenMap); + start.setDate(start.getDate() + 1); + } + + this.logger.log("finished history statistics protocol tvl"); + } + + @Cron("0 30 0 * * *") + public async statisticTvl() { + const yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + + const tokenMap = await this.getSupportTokens(); + await this.statisticTvlByDay(yesterday, tokenMap); + this.logger.log("finished statistics tvl"); + } + + public async statisticTvlByDay( + day: Date, + tokenMap: Map< + string, + { + address: { + l2Address: string; + }; + symbol: string; + decimals: number; + cpPriceId: string; + } + >, + ) { + const begin = + new Date( + day.getFullYear(), + day.getMonth(), + day.getDate(), + 0, + 0, + 0, + ).getTime() / 1000; + const end = + new Date( + day.getFullYear(), + day.getMonth(), + day.getDate(), + 23, + 59, + 59, + ).getTime() / 1000; + + const [minBlock, maxBlock] = await this.getBlockRangeByTime(begin, end); + + if (!minBlock || !maxBlock) { + this.logger.error("Statistical getBlockRangeByTime failure"); + return; + } + + const allProject = await this.projectRepository.find(); + allProject; + + const groupByName: { [name: string]: Buffer[] } = allProject.reduce( + (group, project) => { + const { name } = project; + group[name] = group[name] ?? []; + group[name].push(hexTransformer.to(project.pairAddress)); + return group; + }, + {}, + ); + + let maxBlockNumberCurday; + for (const name in groupByName) { + const pairAddressList = groupByName[name]; + + if (!maxBlockNumberCurday) { + const maxBlockRes: { max: number }[] = + await this.balanceOfLpRepository.query( + `select max("blockNumber") from "balanceOfLp" where "pairAddress" = any($1) and "blockNumber" between $2 and $3`, + [pairAddressList, minBlock, maxBlock], + ); + maxBlockNumberCurday = maxBlockRes[0].max; + } + + const result: { tokenAddress: Buffer; balance: number }[] = + await this.balanceOfLpRepository.query( + `select "tokenAddress", sum(balance) as balance from "balanceOfLp" where "pairAddress" = any($1) and "blockNumber" = $2 group by "tokenAddress"`, + [pairAddressList, maxBlockNumberCurday], + ); + let tvl = BigNumber(0); + for (const tokenBalance of result) { + const l2Address = hexTransformer.from( + tokenBalance.tokenAddress, + ) as string; + const token = tokenMap.get(l2Address); + const latestPrice = await this.blockTokenPriceRepository.findOne({ + where: { + priceId: token.cpPriceId, + blockNumber: LessThanOrEqual(maxBlockNumberCurday), + }, + order: { + blockNumber: "desc", + }, + }); + + tvl.plus( + BigNumber(latestPrice.usdPrice).multipliedBy( + BigNumber(ethers.formatUnits(tokenBalance.balance, token.decimals)), + ), + ); + } + + await this.protocolDauRepository + .createQueryBuilder() + .insert() + .values({ + name, + amount: tvl.toNumber(), + date: day.toISOString().split("T")[0], + type: 3, + }) + .orUpdate(["amount"]) .execute(); } } @@ -166,4 +389,33 @@ export class StatisticService { } } } + + public async getSupportTokens() { + let maxRetry = 3; + while (maxRetry-- > 0) { + try { + const response = await fetch( + `${this.pointsApi}/tokens/getSupportTokens`, + ); + const data: { + address: { + l2Address: string; + }; + symbol: string; + decimals: number; + cpPriceId: string; + }[] = await response.json(); + + const tokenMap = new Map( + data.map((token) => [token.address.l2Address, token]), + ); + return tokenMap; + } catch (err) { + this.logger.error( + `Fetch getSupportTokens query data faild, remain retry count: ${maxRetry}`, + err.stack, + ); + } + } + } }