From cf245dc77232e0d319eea1c8d7516bbff5716d2d Mon Sep 17 00:00:00 2001
From: JeelRajodiya <jeelrajodiyajeel@gmail.com>
Date: Sun, 5 Jan 2025 20:21:12 +0530
Subject: [PATCH] feat: implement pagination for GitHub API data retrieval in
 getData function

---
 scripts/tools/extract-tools-github.ts    | 39 +++++++++++++++++++++---
 tests/tools/extract-tools-github.test.js | 11 +++++--
 2 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/scripts/tools/extract-tools-github.ts b/scripts/tools/extract-tools-github.ts
index b150c5df0703..d72660ae7ee7 100644
--- a/scripts/tools/extract-tools-github.ts
+++ b/scripts/tools/extract-tools-github.ts
@@ -1,6 +1,9 @@
+/* eslint-disable no-await-in-loop */
 import axios from 'axios';
 import dotenv from 'dotenv';
 
+import { pause } from '../utils';
+
 dotenv.config();
 
 /**
@@ -12,12 +15,38 @@ dotenv.config();
 export async function getData(): Promise<any> {
   // eslint-disable-next-line no-useless-catch
   try {
-    const result = await axios.get('https://api.github.com/search/code?q=filename:.asyncapi-tool', {
-      headers: {
-        accept: 'application/vnd.github.text-match+json',
-        authorization: `token ${process.env.GITHUB_TOKEN}`
-      }
+    const allItems = [];
+    let page = 1;
+
+    const maxPerPage = 50;
+    const getReqUrl = (PerPage: number, pageNo: number) =>
+      `https://api.github.com/search/code?q=filename:.asyncapi-tool&per_page=${PerPage}&page=${pageNo}`;
+    const headers = {
+      accept: 'application/vnd.github.text-match+json',
+      authorization: `token ${process.env.GITHUB_TOKEN}`
+    };
+    const result = await axios.get(getReqUrl(maxPerPage, page), {
+      headers
     });
+    const totalResults = result.data.total_count;
+
+    allItems.push(...result.data.items);
+
+    while (allItems.length < totalResults) {
+      page++;
+      console.log('Fetching page:', page);
+      // pause for 1 second to avoid rate limiting
+      await pause(1000);
+      const nextPageData = await axios.get(getReqUrl(maxPerPage, page), {
+        headers
+      });
+
+      const { data } = nextPageData;
+
+      allItems.push(...data.items);
+    }
+
+    result.data.items.push(...allItems);
 
     return result.data;
   } catch (err) {
diff --git a/tests/tools/extract-tools-github.test.js b/tests/tools/extract-tools-github.test.js
index 3b779db3de1d..bdaf89143737 100644
--- a/tests/tools/extract-tools-github.test.js
+++ b/tests/tools/extract-tools-github.test.js
@@ -8,12 +8,17 @@ describe('getData', () => {
 
     const mockData = {
       data: {
-        name: '.asyncapi-tool',
-        path: 'asyncapi/.asyncapi-tool',
+				items:[
+					{
+						name: '.asyncapi-tool',
+						path: 'asyncapi/.asyncapi-tool',
+					}
+				],
+				total_count: 1,
       },
     };
 
-    const apiBaseUrl = 'https://api.github.com/search/code?q=filename:.asyncapi-tool';
+    const apiBaseUrl = 'https://api.github.com/search/code?q=filename:.asyncapi-tool&per_page=50&page=1';
     const headers = {
       accept: 'application/vnd.github.text-match+json',
       authorization: `token ${process.env.GITHUB_TOKEN}`,