Add Workflow to update current state of CoinGecko IDs and remove inva…

…lid coingecko IDs (#5883)

* Add code to detect and clean up coingecko ids

* add workflow for updating coingecko data

* various corrections

* make sure emtpy strings for coingecko ID are removed

.github/workflows/check_coingecko_data.yml

@@ -0,0 +1,54 @@
+on: [workflow_dispatch]
+name: Update Coingecko Data
+jobs:
+  update_coingecko_data:
+    name: Update Coingecko Data
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          submodules: true
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 19.6.0
+
+      - name: Run code to Generate Coingecko State File
+        working-directory: ./.github/workflows/utility
+        run: node check_coingecko_data.mjs generateCoingeckoStateFile
+
+      - name: Run code to Remove Invalid Coingecko Ids
+        working-directory: ./.github/workflows/utility
+        run: node check_coingecko_data.mjs removeInvalidCoingeckoIds
+
+      - name: Add Commit Push
+        uses: devops-infra/action-commit-push@master
+        with:
+          github_token: "${{ secrets.GITHUB_TOKEN }}"
+          add_timestamp: false
+          commit_prefix: "[AUTO]"
+          commit_message: "Update Coingecko Data"
+          force: false
+          target_branch: update/coingecko
+
+      - name: Create A PR
+        uses: devops-infra/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          source_branch: update/coingecko
+          target_branch: master
+          title: Update Coingecko Data
+          body: "**Automated pull request**"
+          old_string: "**THIS IS AN AUTOMATED UPDATE OF COINGECKO DATA**"
+          new_string: "** Automatic pull request**"
+          get_diff: true
+          ignore_users: "dependabot"

.github/workflows/utility/chain_registry.mjs

@@ -281,7 +281,11 @@ export function setAssetProperty(chainName, baseDenom, property, value) {
   if(assets) {
     assets.forEach((asset) => {
       if(asset.base == baseDenom) {
-        asset[property] = value;
+        if (value === "") {
+          delete asset[property]; // Remove the property if value is an empty string
+        } else {
+          asset[property] = value; // Otherwise, set the property to the value
+        }
         setFileProperty(chainName, "assetlist", "assets", assets);
         return;
       }
@@ -373,16 +377,11 @@ export function getAssetPointersByChain(chainName) {
 
 export function getAssetPointersByNetworkType(networkType) {
   let assetPointers = [];
-  const assets = getFileProperty(chainName, "assetlist", "assets");
-  if(assets) {
-    assets.forEach((asset) => {
-      if(asset.base) {
-        assetPointers.push({
-          chain_name: chainName,
-          base_denom: asset.base
-        });
-      }
-    });
+  for (const chainName of chainNameToDirectoryMap.keys()) {
+    const chainNetworkType = getFileProperty(chainName, "chain", "network_type");
+    if (chainNetworkType === networkType) {
+      assetPointers = assetPointers.concat(getAssetPointersByChain(chainName));
+    }
   }
   return assetPointers;
 }

.github/workflows/utility/check_coingecko_data.mjs

@@ -0,0 +1,127 @@
+import fs from 'fs/promises';
+import * as chain_reg from './chain_registry.mjs';
+
+const COINGECKO_API_URL = 'https://api.coingecko.com/api/v3/coins/list';
+const COINGECKO_JSON_PATH = './state/coingecko.json';
+
+let coingecko_api_response = null;
+
+chain_reg.setup("../../..");
+
+async function fetchCoingeckoData() {
+  try {
+    const response = await fetch(COINGECKO_API_URL);
+    coingecko_api_response = await response.json();
+  } catch (error) {
+    console.error('Error fetching Coingecko data:', error);
+  }
+}
+
+export async function loadCoingeckoState() {
+  try {
+    const data = await fs.readFile(COINGECKO_JSON_PATH, 'utf8');
+    return JSON.parse(data);
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return { coingecko_data: [] }; // Return empty structure if file doesn't exist
+    }
+    throw error;
+  }
+}
+
+async function saveCoingeckoState(data) {
+  await fs.writeFile(COINGECKO_JSON_PATH, JSON.stringify(data, null, 2));
+}
+
+
+async function removeInvalidCoingeckoIds() {
+
+  //const assetPointers = getAllAssetPointers(); // Replace with your function
+  await fetchCoingeckoData();
+  const validCoingeckoIds = new Set(coingecko_api_response.map(entry => entry.id));
+
+  const assetPointers = getAssetPointers();
+  for (const asset of assetPointers) {
+    const coingecko_id = chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "coingecko_id");
+    if (coingecko_id === "" || (coingecko_id && !validCoingeckoIds.has(coingecko_id))) {
+      console.log(`Removing invalid Coingecko ID: ${coingecko_id} from ${asset.chain_name} ${asset.base_denom}`);
+      chain_reg.setAssetProperty(asset.chain_name, asset.base_denom, "coingecko_id", "");
+    }
+  }
+}
+
+async function generateCoingeckoStateFile() {
+
+  const coingeckoState = {};//await loadCoingeckoState();//  Use this for validation
+  coingeckoState.coingecko_data = [];
+
+  await fetchCoingeckoData();
+  if (!coingecko_api_response) {
+    console.log("No CoinGecko API Response");
+    return;
+  }
+
+  const assetPointers = getAssetPointers();
+  for (const asset of assetPointers) {
+    const coingecko_id = chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "coingecko_id");
+    if (!coingecko_id) { continue; }
+
+    const coingeckoEntry = coingecko_api_response.find(entry => entry.id === coingecko_id);
+    if (!coingeckoEntry) {
+      console.log(`Missing Coingecko ID: ${coingecko_id} for asset`, asset);
+      continue;
+    }
+
+    const registryName = chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "name");
+    const registrySymbol = chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "symbol");
+
+    if (
+      registryName !== coingeckoEntry.name
+        &&
+      registrySymbol.toUpperCase() !== coingeckoEntry.symbol.toUpperCase()
+    ) {
+      console.warn(`Warning: Mismatch of both Name and Symbol for Coingecko ID ${coingecko_id}. Registry: "${registryName} $${registrySymbol}", Coingecko: "${coingeckoEntry.name} $${coingeckoEntry.symbol.toUpperCase()}"`);
+    }
+
+    let coingeckoDataEntry = coingeckoState.coingecko_data.find(entry => entry.coingecko_id === coingecko_id);
+    if (!coingeckoDataEntry) {
+      coingeckoDataEntry = {
+        coingecko_id,
+        //_comment: `${coingeckoEntry.name} $${coingeckoEntry.symbol.toUpperCase()}`,
+        _comment: `${chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "name")} $${chain_reg.getAssetProperty(asset.chain_name, asset.base_denom, "symbol")}`,
+        assets: []
+      };
+      coingeckoState.coingecko_data.push(coingeckoDataEntry);
+    }
+
+    const assetExists = coingeckoDataEntry.assets.some(a => a.chain_name === asset.chain_name && a.base_denom === asset.base_denom);
+    if (!assetExists) {
+      coingeckoDataEntry.assets.push(asset);
+    }
+  }
+
+  await saveCoingeckoState(coingeckoState);
+}
+
+function getAssetPointers() {
+  const networkType = "mainnet";
+  let assetPointers = chain_reg.getAssetPointersByNetworkType(networkType);
+  return assetPointers;
+}
+
+(async function main() {
+})();
+
+
+if (process.argv.length > 2) {
+  const command = process.argv[2];
+  if (command === 'generateCoingeckoStateFile') {
+    generateCoingeckoStateFile();
+  } else if (command === 'removeInvalidCoingeckoIds') {
+    removeInvalidCoingeckoIds();
+  } else {
+    console.log(`Unknown command: ${command}`);
+  }
+} else {
+  main();
+}