diff --git a/requirements.txt b/requirements.txt index 0f26434..4788c55 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/server/api/repos/[repo_id]/clone.post.ts b/server/api/repos/[repo_id]/clone.post.ts index 776bfce..5e796f9 100644 --- a/server/api/repos/[repo_id]/clone.post.ts +++ b/server/api/repos/[repo_id]/clone.post.ts @@ -58,8 +58,9 @@ export default defineEventHandler(async (event) => { // TODO: skip issues that are already up to date console.log('fetching issues ...'); let page = 1; + const perPage = 50; while (true) { - const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage: 50 }); + const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage }); for await (const issue of issues) { let issueString = `# issue "${issue.title}" (${issue.number})`; if (issue.labels.length !== 0) { @@ -76,15 +77,14 @@ export default defineEventHandler(async (event) => { await fs.writeFile(path.join(folder, 'issues', `${issue.number}.md`), issueString); } - console.log('wrote', issues.length, 'issues'); - - // TODO: improve stop condition - if (issues.length < 50) { + if (issues.length < perPage || page * perPage >= total) { break; } page += 1; } + console.log(`wrote ${page * perPage} issues`); + console.log('start indexing ...'); const indexingResponse = await $fetch<{ error?: string }>(`${config.api.url}/index`, { method: 'POST', diff --git a/server/forges/github.ts b/server/forges/github.ts index 4d71aec..fc94e2d 100644 --- a/server/forges/github.ts +++ b/server/forges/github.ts @@ -2,6 +2,7 @@ import type { H3Event } from 'h3'; import { Forge, Credentials, Tokens, ForgeUser, Repo, PaginatedList, Pagination, Issue } from './types'; import { Forge as DBForge } from '../schemas'; import { Octokit } from 'octokit'; +import type { ResponseHeaders } from '@octokit/types'; export class Github implements Forge { private clientId: string; @@ -27,6 +28,33 @@ export class Github implements Forge { }; } + private getTotalPagesFromHeaders(headers: ResponseHeaders) { + /* + Sample link header: + link: + ; rel="prev", + ; rel="next", + ; rel="last", + ; rel="first" + */ + + if (!headers.link) { + return 0; + } + + const linkToLastPage = headers.link.split(',').find((link) => link.includes('rel="last"')); + if (!linkToLastPage) { + return 0; + } + + const match = /\?page=(.*?)\>/.exec(linkToLastPage); + if (!match || match.length != 2) { + return 0; + } + + return parseInt(match[1]); + } + public getOauthRedirectUrl({ state }: { state: string }): string { const scopes = ['read:user', 'user:email', 'repo']; return `https://github.com/login/oauth/authorize?client_id=${ @@ -98,10 +126,13 @@ export class Github implements Forge { public async getRepos(token: string, search?: string, pagination?: Pagination): Promise> { const client = this.getClient(token); + + const perPage = pagination?.perPage || 10; const repos = await client.request('GET /search/repositories', { q: `is:public fork:false archived:false ${search}`.trim(), // TODO: filter by owned repos - per_page: 10, + per_page: perPage, sort: 'updated', + page: pagination?.page, }); return { @@ -115,7 +146,7 @@ export class Github implements Forge { url: repo.url, }) satisfies Repo, ), - total: 0, // TODO + total: this.getTotalPagesFromHeaders(repos.headers) * perPage, }; } @@ -139,11 +170,16 @@ export class Github implements Forge { async getIssues(token: string, repoId: string, pagination?: Pagination): Promise> { const client = this.getClient(token); + const repo = await client.request(`GET /repositories/{repoId}`, { + repoId, + }); + + const perPage = pagination?.perPage || 10; const issues = await client.request(`GET /repos/{owner}/{repo}/issues`, { - owner: repoId.split('/')[0], - repo: repoId.split('/')[1], - per_page: pagination?.perPage || 10, - page: pagination?.page || 1, + owner: repo.data.owner.login, + repo: repo.data.name, + per_page: perPage, + page: pagination?.page, }); return { @@ -154,7 +190,7 @@ export class Github implements Forge { labels: issue.labels.map((label) => (typeof label === 'string' ? label : label.name || '')), comments: [], // TODO: get comments })), - total: 0, // TODO: get total + total: this.getTotalPagesFromHeaders(issues.headers) * perPage, }; } } diff --git a/server/forges/gitlab.ts b/server/forges/gitlab.ts index 7e0b073..09bb060 100644 --- a/server/forges/gitlab.ts +++ b/server/forges/gitlab.ts @@ -134,10 +134,11 @@ export class Gitlab implements Forge { async getIssues(token: string, repoId: string, pagination?: Pagination): Promise> { const client = this.getClient(token); - const issues = await client.Issues.all({ + const { data: issues, paginationInfo } = await client.Issues.all({ projectId: repoId, perPage: pagination?.perPage || 10, page: pagination?.page || 1, + showExpanded: true, }); return { @@ -148,7 +149,7 @@ export class Gitlab implements Forge { labels: issue.labels || [], comments: [], // TODO: get comments })), - total: 0, // TODO: get total + total: paginationInfo.total, }; } }