Use XP claims glossary (#574)

Resolves #555

Should be tested with https://github.com/tahowallet/contracts/pull/427

To be able to work with very large XP claim files we need to split them
into multiple files.

This code assumes that 
- we have a `XP_HOSTING_BASE_URL` env variable that will be used as a
base to fetch xp drop files and leaderboard files
- in the `assets/xp-data.json` file we will store second part of the
urls like:
```
XP_HOSTING_BASE_URL="app.taho.xyz"
{
    "4": {
        "rootFolder": ""/assets/xp/realm-name",
        "claimsFolder": "/assets/xp/realm-name/claims",
        "xpGlossary": ["0xmerkleroot.json",  "0xmerkleroot.json"],
        "leaderboard": "leaderboard.json"
    },
}
```
- each xp drop is one glossary file in a format:
```json
{
  "totalAmount": "0x",
  "merkleRoot": "0x",
  "merkleDistributor": "0x",
  "glossary": [
    { "startAddress": "0x1", "file": "0xmerkleroot-0.json" },
    { "startAddress": "0x2", "file": "0xmerkleroot-1.json" }
  ]
}
```
- start address is included in the file it is linking to
- claim files look like:
```json
{
  "0xaddress": { 
    "index": "",
    "amount": "",
    "proof": []
  }
}
```

For further information please read the[ process
documentation](https://github.com/tahowallet/dapp/blob/068296a8eedb9f89ec5de5c0c9f13441c48ccf2b/docs/xp-distribution.md)

.env.defaults

@@ -21,6 +21,7 @@ ALLOW_TENDERLY_RESET="false"
 ANALYTICS_ENV=DEV
 POSTHOG_API_KEY=
 # Misc
+XP_HOSTING_BASE_URL="" # TBD
 SEASON_LENGTH_IN_WEEKS=8
 CONTRACT_DEPLOYMENT_BLOCK_NUMBER=553443
 SEASON_START_DATE="2023-10-26"

docs/xp-distribution.md

@@ -0,0 +1,51 @@
+# XP distribution
+
+### On chain allocation
+
+Allocation is done using the script from the [contracts](https://github.com/tahowallet/contracts) repository, please use [documentation](https://github.com/tahowallet/contracts/blob/main/merkle/README.adoc) written for the script there. This documentation assumes you've already generated all the files and the drop already happened. The only thing left to do is to provide the dapp source merkle tree data for given allocation.
+
+### Providing XP data in the dapp
+
+1. In the `src/assets/xp-data.json` find out where is the folder for the realm that you did the XP drop on. Should be `assets/xp/<realm-name>/`
+
+Update leaderboard 
+
+2. Copy the `leaderboard.json` file from `contracts` repo that was just created by the allocation script. 
+
+3. If we already had leaderbord file (`assets/xp/<realm-name>/leaderboard.json`) then let's replace the leaderboard file with the new one. Leaderboard data will sum xp amounts from previous drops so it should be replaced with updated data. 
+
+4. In the `src/assets/xp-data.json` make sure leaderboard file name got updated if needed. If this is the first drop on a given realm then please update it to (`leaderboard` field):
+```json
+ "<realm-id>": {
+     "rootFolder": "/assets/xp/<realm-name>",
+     "claimsFolder": "/assets/xp/<realm-name>/claims",
+     "xpGlossary": [],
+     "leaderboard": "leaderboard.json"
+ },
+```
+
+Upload XP drop glossary 
+
+5. Copy the main file with XP drop data from the contracts. Make sure this is correct file - it should contain `glossary` field and `merkleDistributor` 
+
+6. Paste that file into `assets/xp/<realm-name>/` folder 
+
+7. Update `src/assets/xp-data.json` with the glossary file name (`xpGlossary` field). If this is first drop then the `xpGlossary` array will be empty, if not then add new file name to the end of the array. Each file is named with the merkle root value.
+```json
+    "<realm-id>": {
+        "rootFolder": "/assets/xp/<realm-name>",
+        "claimsFolder": "/assets/xp/<realm-name>/claims",
+        "xpGlossary": ["0x<merkle-root>.json", "0x<merkle-root>.json"],
+        "leaderboard": "leaderboard.json"
+    },
+```
+
+Upload XP claim files
+
+8. Copy all files from the `contracts` repo that were created in the `claims` folder for a given XP drop
+9. Paste them into the `"src/assets/xp/<realm-name>/claims` folder. Don't remove existing files if this is not the first drop.
+10. Look into the glossary file (`/assets/xp/<realm-name>/0x<merkle-root>.json`) and confirm it is referring to the same files you just pasted into the `claims` folder.
+
+Testing
+
+11. Run app locally or build it on Netlify PR preview to make sure you can see correct XP values in the claim banner and XP leaderboard. Confirm you are able to claim XP if possible on a given environment. 

src/assets/xp-data.json

@@ -1,22 +1,32 @@
 {
-    "4": {
-        "xp": [],
-        "leaderboard": null
-    },
-    "7": {
-        "xp": [],
-        "leaderboard": null
-    },
-    "9": {
-        "xp": [],
-        "leaderboard": null
-    },
-    "19": {
-        "xp": [],
-        "leaderboard": null
-    },
-    "22": {
-        "xp": [],
-        "leaderboard": null
-    }
-}
+  "4": {
+    "rootFolder": "/assets/xp/gitcoin",
+    "claimsFolder": "/assets/xp/gitcoin/claims",
+    "xpGlossary": [],
+    "leaderboard": null
+  },
+  "7": {
+    "rootFolder": "/assets/xp/cyberconnect",
+    "claimsFolder": "/assets/xp/cyberconnect/claims",
+    "xpGlossary": [],
+    "leaderboard": null
+  },
+  "9": {
+    "rootFolder": "/assets/xp/arbitrum",
+    "claimsFolder": "/assets/xp/arbitrum/claims",
+    "xpGlossary": [],
+    "leaderboard": null
+  },
+  "19": {
+    "rootFolder": "/assets/xp/galxe",
+    "claimsFolder": "/assets/xp/galxe/claims",
+    "xpGlossary": [],
+    "leaderboard": null
+  },
+  "22": {
+    "rootFolder": "/assets/xp/frax",
+    "claimsFolder": "/assets/xp/frax/claims",
+    "xpGlossary": [],
+    "leaderboard": null
+  }
+}

src/shared/contracts/xp.ts

@@ -43,7 +43,7 @@ export const getUnclaimedXpDistributions: ReadTransactionBuilder<
   },
   UnclaimedXpData[]
 > = async (provider, { realmId, account }) => {
-  const xpData = (await getXpDataForRealmId(realmId)) ?? []
+  const xpData = await getXpDataForRealmId(realmId, account)
 
   const unclaimedOrNull = await Promise.all(
     xpData.map<Promise<UnclaimedXpData | null>>(

src/shared/types/xp.ts

@@ -3,13 +3,17 @@ export type XpMerkleTreeItem = {
   amount: string
   proof: string[]
 }
+export type XpMerkleTreeClaims = {
+  [address: string]: XpMerkleTreeItem
+}
 export type XpMerkleTree = {
   totalAmount: string
   merkleRoot: string
   merkleDistributor: string
-  claims: {
-    [address: string]: XpMerkleTreeItem
-  }
+  claims: XpMerkleTreeClaims
+}
+export type XpMerkleTreeGlossary = Omit<XpMerkleTree, "claims"> & {
+  glossary: { startAddress: string; file: string }[]
 }
 export type XpDistributor = {
   distributorContractAddress: string

src/shared/utils/xp.ts

@@ -1,10 +1,41 @@
 import { LeaderboardItemData, XPLeaderboard } from "shared/types"
-import { XpMerkleTree } from "shared/types/xp"
+import {
+  XpMerkleTree,
+  XpMerkleTreeClaims,
+  XpMerkleTreeGlossary,
+} from "shared/types/xp"
 import { isSameAddress, normalizeAddress } from "shared/utils/address"
 import XP_DATA from "../../assets/xp-data.json"
 
+// eslint-disable-next-line prefer-destructuring
+const XP_HOSTING_BASE_URL = process.env.XP_HOSTING_BASE_URL
+
 type XpDataType = {
-  [realmId: string]: { leaderboard: string | null; xp: string[] }
+  [realmId: string]: {
+    rootFolder?: string
+    claimsFolder?: string
+    leaderboard: string | null
+    xpGlossary: string[]
+  }
+}
+
+async function fetchOrImport(url: string) {
+  if (process.env.NODE_ENV === "development") {
+    // We need to give webpack a hint where these files are located in the filesystem
+    // so we remove parts of the path and file extension - including this directly in a dynamic import
+    // string template will make webpack include all files in the bundle
+    const fileWithoutExtension = url
+      .replace("/assets/xp/", "")
+      .replace(/.json$/, "")
+    return (
+      await import(
+        /* webpackInclude: /\.json$/ */ `../../../src/assets/xp/${fileWithoutExtension}.json`
+      )
+    ).default
+  }
+  // For production we fetch the data from the XP hosting server - by default we can use
+  // json files from "assets" folder as they are copied to the build folder during the build process
+  return (await fetch(`${XP_HOSTING_BASE_URL}${url}`)).json()
 }
 
 export async function getRealmLeaderboardData(
@@ -14,66 +45,119 @@ export async function getRealmLeaderboardData(
     throw new Error("Missing realm id")
   }
 
-  let xpData: null | XPLeaderboard = null
+  const xpData = (XP_DATA as XpDataType)[realmId]
 
-  if (realmId) {
-    try {
-      const leaderboardUrl = (XP_DATA as XpDataType)[realmId]?.leaderboard
+  if (!xpData) {
+    throw new Error("Missing data in xp-data.json")
+  }
 
-      if (!leaderboardUrl) {
-        return null
-      }
-      if (process.env.NODE_ENV === "development") {
-        // TODO: fix it - not working locally
-        xpData = (await import(`${leaderboardUrl}`)).default
-      } else {
-        xpData = await (await fetch(leaderboardUrl)).json()
-      }
-    } catch (error) {
-      // eslint-disable-next-line no-console
-      console.warn("No XP data found for the realm id:", realmId, error)
-    }
+  const { rootFolder, leaderboard } = xpData
+
+  if (!rootFolder || !leaderboard) {
+    return null
   }
 
-  return xpData
+  const leaderboardUrl = `${rootFolder}/${leaderboard}`
+
+  try {
+    return await fetchOrImport(leaderboardUrl)
+  } catch (error) {
+    // eslint-disable-next-line no-console
+    console.warn("No leaderboard data found for the realm id:", realmId, error)
+
+    return null
+  }
 }
 
 export async function getXpDataForRealmId(
-  realmId: string
-): Promise<XpMerkleTree[] | null> {
+  realmId: string,
+  account: string
+): Promise<XpMerkleTree[]> {
   if (!realmId) {
     throw new Error("Missing realm id")
   }
 
-  let xpData: null | XpMerkleTree[] = null
+  const normalizedAddress = normalizeAddress(account)
+
+  const targetAddress = BigInt(normalizedAddress)
+  let claimsData: (null | XpMerkleTree)[] = []
+
+  const xpData = (XP_DATA as XpDataType)[realmId]
 
   try {
-    const xpLinks = (XP_DATA as XpDataType)[realmId]?.xp
+    const { rootFolder, claimsFolder, xpGlossary } = xpData
 
-    if (!xpLinks || !xpLinks.length) {
-      return null
+    if (!rootFolder || !claimsFolder || !xpGlossary || !xpGlossary.length) {
+      return []
     }
 
-    xpData = await Promise.all(
-      xpLinks.map(async (url) => {
-        let data
+    claimsData = await Promise.all(
+      xpGlossary.map(async (file) => {
+        const glossaryUrl = `${rootFolder}/${file}`
+
+        const glossaryFile: XpMerkleTreeGlossary = await fetchOrImport(
+          glossaryUrl
+        )
+
+        if (!glossaryFile) {
+          throw new Error(`Failed to fetch glossaryFile from ${glossaryUrl}`)
+        }
+
+        // Addresses in the claim files are sorted in ascending order,
+        // we only know about the address that the file starts with and our `targetAddress`.
+        // * If `startAddress` < `targetAddress` then we don't know if the address
+        //   is in the current claim file or the next one. We assume that it's in the one of the upcoming files.
+        // * If `startAddress` > `targetAddress` then we know that the address is in the previous file.
+        // Edge cases are:
+        // * `targetAddress` === `startAddress` - this is covered by the same logic as the regular case,
+        //   because we will come back to the previous file, because while checking next claim file startAddress > targetAddress
+        // * `targetAddress` is in the first file - this is covered by defaulting to the first file is none of the files match
+        // * `targetAddress` is not in the drop at all - not a problem, we will return null
+        const suspectedNextClaimFileIndex = glossaryFile.glossary.findIndex(
+          ({ startAddress }) => BigInt(startAddress ?? 0) > targetAddress
+        )
+        const claimFileIndex =
+          suspectedNextClaimFileIndex <= 0 ? 0 : suspectedNextClaimFileIndex - 1
+        const claimFile = glossaryFile.glossary[claimFileIndex]?.file
+
+        if (!claimFile) {
+          // No claim file found for the user
+          return null
+        }
+
+        const claimLink = `${claimsFolder}${claimFile}`
+
+        const claims: XpMerkleTreeClaims | undefined = await fetchOrImport(
+          claimLink
+        )
 
-        if (process.env.NODE_ENV === "development") {
-          // TODO: fix it - not working locally
-          data = (await import(`${url}`)).default
-        } else {
-          data = await (await fetch(url)).json()
+        if (!claims) {
+          throw new Error(`Failed to fetch claims from ${claimLink}`)
         }
 
-        return data
+        const userClaim = claims[normalizedAddress]
+
+        if (userClaim) {
+          return {
+            totalAmount: glossaryFile.totalAmount,
+            merkleRoot: glossaryFile.merkleRoot,
+            merkleDistributor: glossaryFile.merkleDistributor,
+            claims: {
+              [normalizedAddress]: userClaim,
+            },
+          } as XpMerkleTree
+        }
+
+        // No claim found for the user
+        return null
       })
     )
   } catch (error) {
     // eslint-disable-next-line no-console
-    console.warn("No XP data found for the url:", realmId, error)
+    console.warn("No XP data found for the realm id:", realmId, error)
   }
 
-  return xpData
+  return claimsData.flatMap((item) => (item ? [item] : []))
 }
 
 export function getUserLeaderboardRank(