feat: implement pagination for GitHub API data retrieval in getData f…

…unction
JeelRajodiya · Jan 5, 2025 · cf245dc · cf245dc
1 parent bf6f225
commit cf245dc
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 8 deletions.
diff --git a/scripts/tools/extract-tools-github.ts b/scripts/tools/extract-tools-github.ts
@@ -1,6 +1,9 @@
+/* eslint-disable no-await-in-loop */
 import axios from 'axios';
 import dotenv from 'dotenv';
 
+import { pause } from '../utils';
+
 dotenv.config();
 
 /**
@@ -12,12 +15,38 @@ dotenv.config();
 export async function getData(): Promise<any> {
   // eslint-disable-next-line no-useless-catch
   try {
-    const result = await axios.get('https://api.github.com/search/code?q=filename:.asyncapi-tool', {
-      headers: {
-        accept: 'application/vnd.github.text-match+json',
-        authorization: `token ${process.env.GITHUB_TOKEN}`
-      }
+    const allItems = [];
+    let page = 1;
+
+    const maxPerPage = 50;
+    const getReqUrl = (PerPage: number, pageNo: number) =>
+      `https://api.github.com/search/code?q=filename:.asyncapi-tool&per_page=${PerPage}&page=${pageNo}`;
+    const headers = {
+      accept: 'application/vnd.github.text-match+json',
+      authorization: `token ${process.env.GITHUB_TOKEN}`
+    };
+    const result = await axios.get(getReqUrl(maxPerPage, page), {
+      headers
     });
+    const totalResults = result.data.total_count;
+
+    allItems.push(...result.data.items);
+
+    while (allItems.length < totalResults) {
+      page++;
+      console.log('Fetching page:', page);
+      // pause for 1 second to avoid rate limiting
+      await pause(1000);
+      const nextPageData = await axios.get(getReqUrl(maxPerPage, page), {
+        headers
+      });
+
+      const { data } = nextPageData;
+
+      allItems.push(...data.items);
+    }
+
+    result.data.items.push(...allItems);
 
     return result.data;
   } catch (err) {

diff --git a/tests/tools/extract-tools-github.test.js b/tests/tools/extract-tools-github.test.js
@@ -8,12 +8,17 @@ describe('getData', () => {
 
     const mockData = {
       data: {
-        name: '.asyncapi-tool',
-        path: 'asyncapi/.asyncapi-tool',
+				items:[
+					{
+						name: '.asyncapi-tool',
+						path: 'asyncapi/.asyncapi-tool',
+					}
+				],
+				total_count: 1,
       },
     };
 
-    const apiBaseUrl = 'https://api.github.com/search/code?q=filename:.asyncapi-tool';
+    const apiBaseUrl = 'https://api.github.com/search/code?q=filename:.asyncapi-tool&per_page=50&page=1';
     const headers = {
       accept: 'application/vnd.github.text-match+json',
       authorization: `token ${process.env.GITHUB_TOKEN}`,