new revision

package.json

@@ -6,9 +6,10 @@
   "scripts": {
     "test": "jest",
     "start:dev": "nodemon --exec ts-node src/index.ts",
+    "start:dev:inspect": "nodemon --exec node --inspect -r ts-node/register src/index.ts",
     "prepare": "husky install",
     "build": "tsc",
-    "start": "node dist/src/index.js"
+    "start": "node --max_old_space_size=16384 dist/src/index.js"
   },
   "lint-staged": {
     "**/*": "prettier --write --ignore-unknown"
@@ -24,7 +25,7 @@
     "dotenv": "^16.0.3",
     "googleapis": "^118.0.0",
     "he": "^1.2.0",
-    "redis": "^4.6.6"
+    "redis": "^4.6.11"
   },
   "devDependencies": {
     "@types/decimal.js": "^7.4.0",
@@ -33,7 +34,7 @@
     "husky": "^8.0.3",
     "jest": "^29.5.0",
     "lint-staged": "^13.2.2",
-    "nodemon": "^2.0.22",
+    "nodemon": "^3.0.2",
     "prettier": "2.8.8",
     "redis-mock": "^0.56.3",
     "ts-jest": "^29.1.0",

src/dog20Indexing/indexer/getAndProcessInscriptionTransfersPerBlock/index.ts

@@ -3,15 +3,19 @@ import {
   findPreviousInscriptionTransfers,
 } from "../../../shared/database/queries/doge20";
 import { getInscriptionTransfersPerBlock } from "../../../shared/database/queries/ordinals/inscriptionTransfer";
-import { getInscriptionContent } from "../../../shared/database/queries/ordinals/inscriptions";
+import {
+  getInscriptionContent,
+  getInscriptionContentsBulk,
+} from "../../../shared/database/queries/ordinals/inscriptions";
 import { findStatusByName } from "../../../shared/database/queries/status";
 import { ignoreAndSaveInscriptionTransfer } from "../../utils/inscriptionTransfer";
 import { transformToDoge20InscriptionTransfer } from "../../utils/inscriptionTransfer/transformToDoge20InscriptionTransfer";
 import { processDoge20Inscription } from "./processDoge20Inscription";
 import { doge20TransferTypes } from "../types";
 import { deleteTransactionsForBlock } from "../../../shared/database/queries/ordinals/transactions";
+import { getRedisClient } from "../../../shared/redis";
 
-/* 
+/*
 IMPORTANT REQUIREMENTS:
 1. we index the inscription transfers in from oldest to newest within a block
 */
@@ -38,21 +42,47 @@ export const getAndProcessInscriptionTransfersPerBlock = async (
     return;
   }
 
+  const redisClient = await getRedisClient();
+
   // get all inscription transfers for the block. Oldest / smallest transactionIndex first
   const inscriptionTransfers = await getInscriptionTransfersPerBlock(
-    blocknumber
+    blocknumber,
+    redisClient
+  );
+
+  const inscriptionContents = await getInscriptionContentsBulk(
+    inscriptionTransfers.map((el) => el.inscription),
+    redisClient
   );
 
-  for (let inscriptionTransfer of inscriptionTransfers) {
-    const alreadyProcessed = await checkDuplicateInscriptionTransfer({
+  const checkPromises = inscriptionTransfers.map((inscriptionTransfer) =>
+    checkDuplicateInscriptionTransfer({
       tx_id: inscriptionTransfer.tx_id,
       inscription: inscriptionTransfer.inscription,
-    });
+    }).then((alreadyProcessed) => ({
+      tx_id: inscriptionTransfer.tx_id,
+      alreadyProcessed,
+    }))
+  );
+
+  const results = await Promise.all(checkPromises);
+  const processedCache = new Map(
+    results.map((result) => [result.tx_id, result.alreadyProcessed])
+  );
+
+  // for (let inscriptionTransfer of inscriptionTransfers) {
+  for (let i = 0; i < inscriptionTransfers.length; i++) {
+    const inscriptionTransfer = inscriptionTransfers[i];
+
+    const alreadyProcessed = processedCache.get(inscriptionTransfer.tx_id);
+
     if (alreadyProcessed) continue;
 
-    const content = await getInscriptionContent(
-      inscriptionTransfer.inscription
-    );
+    // const content = await getInscriptionContent(
+    //   inscriptionTransfer.inscription
+    // );
+
+    const content = inscriptionContents[i];
 
     if (!content) {
       // if we don't have content the content didn't fulfill the format requirements so we ignore it
@@ -71,7 +101,8 @@ export const getAndProcessInscriptionTransfersPerBlock = async (
       // save the inscription transfer as non-doge-20 and continue.
       await ignoreAndSaveInscriptionTransfer(
         inscriptionTransfer,
-        `invalid dog-20 inscription transfer type: ${inscriptionTransferType}`
+        `invalid dog-20 inscription transfer type: ${inscriptionTransferType}`,
+        redisClient
       );
       continue;
     }
@@ -83,7 +114,7 @@ export const getAndProcessInscriptionTransfersPerBlock = async (
       numOfPreviousInscriptionTransfers: previousInscriptionTransfers.length,
     });
 
-    await processDoge20Inscription(doge20InscriptionTransfer);
+    await processDoge20Inscription(doge20InscriptionTransfer, redisClient);
   }
 
   // we delete the txs for the block since we don't need them anymore

src/dog20Indexing/indexer/getAndProcessInscriptionTransfersPerBlock/processDoge20Inscription.ts

@@ -22,9 +22,12 @@ import {
 } from "../../utils/inscriptionTransfer";
 import { IDog20InscriptionTransfer, Doge20TransferType } from "../types";
 import { Decimal } from "../../../shared/utils/other/Decimal";
+import { RedisClientType } from "redis";
+import { checkTokenExistsCached } from "../../../shared/database/queries/doge20/doge20Token";
 
 export const processDoge20Inscription = async (
-  dog20InscriptionTransfer: IDog20InscriptionTransfer
+  dog20InscriptionTransfer: IDog20InscriptionTransfer,
+  redisClient: RedisClientType
 ) => {
   // if max, lim or amt exist we parse them as Decimal and check that they are not negative
   if (
@@ -77,8 +80,10 @@ export const processDoge20Inscription = async (
     ) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "max cannot be more than 20 characters"
+        "max cannot be more than 20 characters",
+        redisClient
       );
+
       return;
     }
 
@@ -89,8 +94,10 @@ export const processDoge20Inscription = async (
     ) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "max cannot be negative or null"
+        "max cannot be negative or null",
+        redisClient
       );
+
       return;
     }
 
@@ -101,13 +108,15 @@ export const processDoge20Inscription = async (
     ) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "lim cannot be negative or null"
+        "lim cannot be negative or null",
+        redisClient
       );
+
       return;
     }
 
     // check if already deployed
-    const isAlreadyDeployed = await checkTokenExists({
+    const isAlreadyDeployed = await checkTokenExistsCached({
       tick: dog20InscriptionTransfer.dog20Data.tick.toLowerCase(),
     });
 
@@ -130,15 +139,20 @@ export const processDoge20Inscription = async (
           p: dog20InscriptionTransfer.dog20Data.p,
         }
       );
-      saveInscriptionTransfer(dog20InscriptionTransfer.inscriptionTransfer, {
-        availableBalanceChange: new Decimal(0),
-        transferableBalanceChange: new Decimal(0),
-      });
+      await saveInscriptionTransfer(
+        dog20InscriptionTransfer.inscriptionTransfer,
+        {
+          availableBalanceChange: new Decimal(0),
+          transferableBalanceChange: new Decimal(0),
+        },
+        redisClient
+      );
       return;
     } else {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "tick already deployed"
+        "tick already deployed",
+        redisClient
       );
       return;
     }
@@ -153,15 +167,16 @@ export const processDoge20Inscription = async (
     }
 
     // check if token is deployed
-    const isTokenDeployed = await checkTokenExists({
+    const isTokenDeployed = await checkTokenExistsCached({
       tick: dog20InscriptionTransfer.dog20Data.tick,
     });
 
     if (!isTokenDeployed) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
         "tick not deployed " +
-          dog20InscriptionTransfer.dog20Data.tick.toLowerCase()
+          dog20InscriptionTransfer.dog20Data.tick.toLowerCase(),
+        redisClient
       );
       return;
     }
@@ -177,7 +192,8 @@ export const processDoge20Inscription = async (
     if (!isBelowLimit) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "above limit"
+        "above limit",
+        redisClient
       );
       return;
     }
@@ -187,7 +203,8 @@ export const processDoge20Inscription = async (
     if (!isSupplyLeft) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "above max supply"
+        "above max supply",
+        redisClient
       );
       return;
     }
@@ -207,7 +224,7 @@ export const processDoge20Inscription = async (
 
     if (isBalanceAlreadyExisting) {
       // update available balance
-      increaseAvailableBalance({
+      await increaseAvailableBalance({
         tick: dog20InscriptionTransfer.dog20Data.tick.toLowerCase(),
         address: dog20InscriptionTransfer.inscriptionTransfer.receiver,
         amountToBeAdded: new Decimal(amountToBeMinted),
@@ -227,13 +244,16 @@ export const processDoge20Inscription = async (
       tick: dog20InscriptionTransfer.dog20Data.tick,
       supplyIncrease: amountToBeMinted.toString(),
     });
+
     await saveInscriptionTransfer(
       dog20InscriptionTransfer.inscriptionTransfer,
       {
         availableBalanceChange: amountToBeMinted,
         transferableBalanceChange: new Decimal(0),
-      }
+      },
+      redisClient
     );
+
     return;
   }
   // OP TRANFSER | INSCRIBE
@@ -246,15 +266,16 @@ export const processDoge20Inscription = async (
     }
 
     // check if token is deployed
-    const isTokenDeployed = await checkTokenExists({
+    const isTokenDeployed = await checkTokenExistsCached({
       tick: dog20InscriptionTransfer.dog20Data.tick,
     });
 
     if (!isTokenDeployed) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
         "tick not deployed " +
-          dog20InscriptionTransfer.dog20Data.tick.toLowerCase()
+          dog20InscriptionTransfer.dog20Data.tick.toLowerCase(),
+        redisClient
       );
       return;
     }
@@ -271,7 +292,8 @@ export const processDoge20Inscription = async (
     if (!enoughAvailableBalance) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "not enough available balance"
+        "not enough available balance",
+        redisClient
       );
       return;
     }
@@ -291,7 +313,8 @@ export const processDoge20Inscription = async (
         transferableBalanceChange: new Decimal(
           dog20InscriptionTransfer.dog20Data.amt
         ),
-      }
+      },
+      redisClient
     );
     return;
   }
@@ -305,15 +328,16 @@ export const processDoge20Inscription = async (
     }
 
     // check if token is deployed
-    const isTokenDeployed = await checkTokenExists({
+    const isTokenDeployed = await checkTokenExistsCached({
       tick: dog20InscriptionTransfer.dog20Data.tick,
     });
 
     if (!isTokenDeployed) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
         "tick not deployed " +
-          dog20InscriptionTransfer.dog20Data.tick.toLowerCase()
+          dog20InscriptionTransfer.dog20Data.tick.toLowerCase(),
+        redisClient
       );
       return;
     }
@@ -335,7 +359,8 @@ export const processDoge20Inscription = async (
     if (!hasEnoughTransferableBalance) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "not enough transferable balance"
+        "not enough transferable balance",
+        redisClient
       );
       return;
     }
@@ -349,7 +374,8 @@ export const processDoge20Inscription = async (
     if (prevValidInscriptionTransfer.length === 0) {
       await ignoreAndSaveInscriptionTransfer(
         dog20InscriptionTransfer.inscriptionTransfer,
-        "There is one valid previous inscription transfer, but it is not valid"
+        "There is one valid previous inscription transfer, but it is not valid",
+        redisClient
       );
       return;
     } else if (prevValidInscriptionTransfer.length > 1) {
@@ -393,7 +419,8 @@ export const processDoge20Inscription = async (
         transferableBalanceChange: new Decimal(
           dog20InscriptionTransfer.dog20Data.amt
         ),
-      }
+      },
+      redisClient
     );
     return;
   } else {

src/dog20Indexing/utils/inscriptionTransfer/ignoreAndSaveInscriptionTransfer.ts

@@ -1,16 +1,21 @@
 import { IInscriptionTransfer } from "../../../shared/database/models/ordinals/InscriptionTransfers";
 import { createIgnoredInscriptionTransfer } from "../../../shared/database/queries/doge20";
+import { RedisClientType } from "redis";
 
 export const ignoreAndSaveInscriptionTransfer = async (
   inscriptionTransfer: IInscriptionTransfer,
-  reasonForIgnore: string
+  reasonForIgnore: string,
+  redisClient: RedisClientType
 ) => {
-  await createIgnoredInscriptionTransfer({
-    block_height: inscriptionTransfer.block_height,
-    receiver: inscriptionTransfer.receiver!,
-    tx_id: inscriptionTransfer.tx_id,
-    inscription: inscriptionTransfer.inscription,
-    transactionIndex: inscriptionTransfer.transactionIndex,
-    reasonForIgnore,
-  });
+  await createIgnoredInscriptionTransfer(
+    {
+      block_height: inscriptionTransfer.block_height,
+      receiver: inscriptionTransfer.receiver!,
+      tx_id: inscriptionTransfer.tx_id,
+      inscription: inscriptionTransfer.inscription,
+      transactionIndex: inscriptionTransfer.transactionIndex,
+      reasonForIgnore,
+    },
+    redisClient
+  );
 };