feat: improve issue pagination

server/api/repos/[repo_id]/clone.post.ts

@@ -58,8 +58,9 @@ export default defineEventHandler(async (event) => {
   // TODO: skip issues that are already up to date
   console.log('fetching issues ...');
   let page = 1;
+  const perPage = 50;
   while (true) {
-    const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage: 50 });
+    const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage });
     for await (const issue of issues) {
       let issueString = `# issue "${issue.title}" (${issue.number})`;
       if (issue.labels.length !== 0) {
@@ -76,15 +77,14 @@ export default defineEventHandler(async (event) => {
       await fs.writeFile(path.join(folder, 'issues', `${issue.number}.md`), issueString);
     }
 
-    console.log('wrote', issues.length, 'issues');
-
-    // TODO: improve stop condition
-    if (issues.length < 50) {
+    if (issues.length < perPage || page * perPage >= total) {
       break;
     }
     page += 1;
   }
 
+  console.log(`wrote ${page * perPage} issues`);
+
   console.log('start indexing ...');
   const indexingResponse = await $fetch<{ error?: string }>(`${config.api.url}/index`, {
     method: 'POST',

server/forges/github.ts

@@ -2,6 +2,7 @@ import type { H3Event } from 'h3';
 import { Forge, Credentials, Tokens, ForgeUser, Repo, PaginatedList, Pagination, Issue } from './types';
 import { Forge as DBForge } from '../schemas';
 import { Octokit } from 'octokit';
+import type { ResponseHeaders } from '@octokit/types';
 
 export class Github implements Forge {
   private clientId: string;
@@ -27,6 +28,33 @@ export class Github implements Forge {
     };
   }
 
+  private getTotalPagesFromHeaders(headers: ResponseHeaders) {
+    /*
+      Sample link header:
+      link:
+      <https://api.github.com/repositories/1300192/issues?page=2>; rel="prev",
+      <https://api.github.com/repositories/1300192/issues?page=4>; rel="next",
+      <https://api.github.com/repositories/1300192/issues?page=515>; rel="last",
+      <https://api.github.com/repositories/1300192/issues?page=1>; rel="first"
+    */
+
+    if (!headers.link) {
+      return 0;
+    }
+
+    const linkToLastPage = headers.link.split(',').find((link) => link.includes('rel="last"'));
+    if (!linkToLastPage) {
+      return 0;
+    }
+
+    const match = /\?page=(.*?)\>/.exec(linkToLastPage);
+    if (!match || match.length != 2) {
+      return 0;
+    }
+
+    return parseInt(match[1]);
+  }
+
   public getOauthRedirectUrl({ state }: { state: string }): string {
     const scopes = ['read:user', 'user:email', 'repo'];
     return `https://github.com/login/oauth/authorize?client_id=${
@@ -98,10 +126,13 @@ export class Github implements Forge {
 
   public async getRepos(token: string, search?: string, pagination?: Pagination): Promise<PaginatedList<Repo>> {
     const client = this.getClient(token);
+
+    const perPage = pagination?.perPage || 10;
     const repos = await client.request('GET /search/repositories', {
       q: `is:public fork:false archived:false ${search}`.trim(), // TODO: filter by owned repos
-      per_page: 10,
+      per_page: perPage,
       sort: 'updated',
+      page: pagination?.page,
     });
 
     return {
@@ -115,7 +146,7 @@ export class Github implements Forge {
             url: repo.url,
           }) satisfies Repo,
       ),
-      total: 0, // TODO
+      total: this.getTotalPagesFromHeaders(repos.headers) * perPage,
     };
   }
 
@@ -139,11 +170,16 @@ export class Github implements Forge {
   async getIssues(token: string, repoId: string, pagination?: Pagination): Promise<PaginatedList<Issue>> {
     const client = this.getClient(token);
 
+    const repo = await client.request(`GET /repositories/{repoId}`, {
+      repoId,
+    });
+
+    const perPage = pagination?.perPage || 10;
     const issues = await client.request(`GET /repos/{owner}/{repo}/issues`, {
-      owner: repoId.split('/')[0],
-      repo: repoId.split('/')[1],
-      per_page: pagination?.perPage || 10,
-      page: pagination?.page || 1,
+      owner: repo.data.owner.login,
+      repo: repo.data.name,
+      per_page: perPage,
+      page: pagination?.page,
     });
 
     return {
@@ -154,7 +190,7 @@ export class Github implements Forge {
         labels: issue.labels.map((label) => (typeof label === 'string' ? label : label.name || '')),
         comments: [], // TODO: get comments
       })),
-      total: 0, // TODO: get total
+      total: this.getTotalPagesFromHeaders(issues.headers) * perPage,
     };
   }
 }

server/forges/gitlab.ts

@@ -134,10 +134,11 @@ export class Gitlab implements Forge {
   async getIssues(token: string, repoId: string, pagination?: Pagination): Promise<PaginatedList<Issue>> {
     const client = this.getClient(token);
 
-    const issues = await client.Issues.all({
+    const { data: issues, paginationInfo } = await client.Issues.all({
       projectId: repoId,
       perPage: pagination?.perPage || 10,
       page: pagination?.page || 1,
+      showExpanded: true,
     });
 
     return {
@@ -148,7 +149,7 @@ export class Gitlab implements Forge {
         labels: issue.labels || [],
         comments: [], // TODO: get comments
       })),
-      total: 0, // TODO: get total
+      total: paginationInfo.total,
     };
   }
 }