add entity valdiations, work out some issues

package.json

@@ -5,6 +5,7 @@
     "codegen": "echo '# GENERATED, DO NOT MODIFY\n' > schema.graphql && cat schema/*.graphql >> schema.graphql && sqd codegen && git add src/model/generated/*",
     "migration:generate": "sqd down && sqd up && sqd migration:generate && git add db/migrations/*",
     "generate": "npm-run-all codegen migration:generate",
+    "generate:validations": "ts-node scripts/generate-validations.ts",
     "build": "rm -rf lib && tsc",
     "prettier-check": "prettier --check src",
     "prettier-fix": "prettier --write src",

scripts/generate-validations.ts

@@ -2,6 +2,7 @@ import fs from 'fs'
 
 import { addresses } from './../src/utils/addresses'
 import { baseAddresses } from './../src/utils/addresses-base'
+import { retry } from './../src/utils/retry'
 
 const LIMIT = 1000
 
@@ -23,7 +24,7 @@ const executeQuery = async (query: string) => {
     throw err
   }
 }
-const takePortion = (arr: any[], takeEvery: number) => {
+const takeEvery = (arr: any[], takeEvery: number) => {
   if (takeEvery <= 0) {
     throw new Error('takeEvery must be greater than 0')
   }
@@ -78,7 +79,7 @@ const oTokenHistories = (prefix: string, address: string) => {
   return gql(`
     ${prefix}_oTokenHistories: oTokenHistories(
       limit: ${LIMIT},
-      orderBy: timestamp_ASC,
+      orderBy: id_ASC,
       where: { otoken_eq: "${address}" }
     ) {
       id
@@ -87,6 +88,7 @@ const oTokenHistories = (prefix: string, address: string) => {
       chainId
       balance
       otoken
+      address { address }
       type
       txHash
       value
@@ -185,7 +187,7 @@ const erc20Balances = (prefix: string, address: string) => {
   return gql(`
     ${prefix}_erc20Balances: erc20Balances(
       limit: ${LIMIT},
-      orderBy: id_ASC,
+      orderBy: [blockNumber_ASC, account_ASC],
       where: { address_eq: "${address}" }
     ) {
       id
@@ -215,13 +217,13 @@ const main = async () => {
   for (let i = 0; i < queries.length; i++) {
     const query = queries[i]
     console.log(`Executing: \`${query.replace(/(\n|\s)+/g, ' ').slice(0, 80)}\`...`)
-    const result = await executeQuery(query)
+    const result = await retry(() => executeQuery(query), 5)
     if (!result.data) {
       console.log(result)
       throw new Error('Query failed')
     }
     for (const key of Object.keys(result.data)) {
-      entities[key] = takePortion(result.data[key], 25)
+      entities[key] = takeEvery(result.data[key], 25)
     }
   }
 

src/base/validate.ts

@@ -1,4 +1,4 @@
-import { ERC20Balance, OToken, OTokenAPY, OTokenDailyStat, OTokenRebase } from '@model'
+import { ERC20Balance, OToken, OTokenAPY, OTokenDailyStat, OTokenHistory, OTokenRebase } from '@model'
 import { Context } from '@processor'
 import { env } from '@utils/env'
 import { entities } from '@validation/entities'
@@ -13,7 +13,7 @@ export const process = async (ctx: Context) => {
   for (const block of ctx.blocks) {
     await validateExpectations(ctx, block, OToken, firstBlock, entities.superoethb_oTokens)
     await validateExpectations(ctx, block, OTokenAPY, firstBlock, entities.superoethb_oTokenApies)
-    // await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.superoethb_oTokenHistories)
+    await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.superoethb_oTokenHistories)
     await validateExpectations(ctx, block, OTokenRebase, firstBlock, entities.superoethb_oTokenRebases)
     await validateExpectations(ctx, block, OTokenDailyStat, firstBlock, entities.superoethb_oTokenDailyStats)
     await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.superoethb_erc20Balances)

src/mainnet/validators/validate-mainnet.ts

@@ -12,6 +12,7 @@ export const process = async (ctx: Context) => {
   if (env.BLOCK_FROM) return
   for (const block of ctx.blocks) {
     await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.ogn_erc20Balances)
+    await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.oeth_erc20Balances)
     firstBlock = false
   }
 }

src/oeth/validators/validate-oeth/validate-oeth.ts

@@ -1,13 +1,13 @@
 import { sortBy } from 'lodash'
 
 import {
-  ERC20Balance,
   OETHDailyStat,
   OETHMorphoAave,
   OETHVault,
   OToken,
   OTokenAPY,
   OTokenDailyStat,
+  OTokenHistory,
   OTokenRebase,
   StrategyBalance,
 } from '@model'
@@ -25,10 +25,9 @@ export const process = async (ctx: Context) => {
   for (const block of ctx.blocks) {
     await validateExpectations(ctx, block, OToken, firstBlock, entities.oeth_oTokens)
     await validateExpectations(ctx, block, OTokenAPY, firstBlock, entities.oeth_oTokenApies)
-    // await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.oeth_oTokenHistories)
+    await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.oeth_oTokenHistories)
     await validateExpectations(ctx, block, OTokenRebase, firstBlock, entities.oeth_oTokenRebases)
     await validateExpectations(ctx, block, OTokenDailyStat, firstBlock, entities.oeth_oTokenDailyStats)
-    await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.oeth_erc20Balances)
     await validateExpectations(ctx, block, OETHVault, firstBlock, expectations.oethVaults)
     await validateExpectations(ctx, block, OETHMorphoAave, firstBlock, expectations.oethMorphoAave)
     await validateExpectations(ctx, block, StrategyBalance, firstBlock, expectations.strategyBalances)

src/ousd/validators/validate-ousd/validate-ousd.ts

@@ -1,4 +1,4 @@
-import { ERC20Balance, OToken, OTokenAPY, OTokenDailyStat, OTokenRebase } from '@model'
+import { ERC20Balance, OToken, OTokenAPY, OTokenDailyStat, OTokenHistory, OTokenRebase } from '@model'
 import { Context } from '@processor'
 import { env } from '@utils/env'
 import { entities } from '@validation/entities'
@@ -13,7 +13,7 @@ export const process = async (ctx: Context) => {
   for (const block of ctx.blocks) {
     await validateExpectations(ctx, block, OToken, firstBlock, entities.ousd_oTokens)
     await validateExpectations(ctx, block, OTokenAPY, firstBlock, entities.ousd_oTokenApies)
-    // await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.ousd_oTokenHistories)
+    await validateExpectations(ctx, block, OTokenHistory, firstBlock, entities.ousd_oTokenHistories)
     await validateExpectations(ctx, block, OTokenRebase, firstBlock, entities.ousd_oTokenRebases)
     await validateExpectations(ctx, block, OTokenDailyStat, firstBlock, entities.ousd_oTokenDailyStats)
     await validateExpectations(ctx, block, ERC20Balance, firstBlock, entities.ousd_erc20Balances)

src/templates/otoken/utils.ts

@@ -1,7 +1,7 @@
 import dayjs from 'dayjs'
 import utc from 'dayjs/plugin/utc'
 import { findLast, last } from 'lodash'
-import { LessThan, MoreThanOrEqual } from 'typeorm'
+import { In, LessThan, MoreThanOrEqual, Not } from 'typeorm'
 
 import * as otoken from '@abi/otoken'
 import { OTokenAPY, OTokenAddress, OTokenRebase, RebasingOption } from '@model'
@@ -208,14 +208,18 @@ export async function createRebaseAPY(
     days: 30,
   }
 
-  const last30daysAPYs = await ctx.store.find(OTokenAPY, {
-    where: {
-      chainId: ctx.chain.id,
-      otoken: otokenAddress,
-      date: MoreThanOrEqual(last30daysDateId.value),
-    },
-    order: { id: 'asc' },
-  })
+  let last30daysAPYs: OTokenAPY[] = apies.filter((apy) => apy.date >= last30daysDateId.value)
+  last30daysAPYs = last30daysAPYs.concat(
+    await ctx.store.find(OTokenAPY, {
+      where: {
+        chainId: ctx.chain.id,
+        otoken: otokenAddress,
+        date: MoreThanOrEqual(last30daysDateId.value),
+        id: Not(In(last30daysAPYs.map((apy) => apy.id))),
+      },
+      order: { id: 'asc' },
+    }),
+  )
 
   const blockDateId = generateId(block.header.timestamp)
   for (const i of [last7daysDateId, last14daysDateId, last30daysDateId]) {
@@ -224,7 +228,7 @@ export async function createRebaseAPY(
       return dateId >= i.value && dateId < blockDateId
     })
     // console.log(i.days, pastAPYs.length)
-    apy![i.key] = pastAPYs.reduce((acc, cur) => acc + cur.apy, apy!.apy) / (pastAPYs.length + 1)
+    apy![i.key] = pastAPYs.reduce((acc, cur) => acc + cur.apy, apy!.apy) / i.days
   }
 
   return rebase

src/utils/retry.ts

@@ -0,0 +1,14 @@
+export const retry = async <T>(fn: () => Promise<T>, retries: number) => {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await fn()
+    } catch (e) {
+      console.error(e)
+      if (i === retries - 1) {
+        throw e
+      } else {
+        console.log(`Retrying... ${i + 1} of ${retries}`)
+      }
+    }
+  }
+}