diff --git a/server/api/repos/[repo_id]/clone.post.ts b/server/api/repos/[repo_id]/clone.post.ts index 5e796f9..51d75e3 100644 --- a/server/api/repos/[repo_id]/clone.post.ts +++ b/server/api/repos/[repo_id]/clone.post.ts @@ -1,6 +1,8 @@ import * as path from 'path'; import { simpleGit } from 'simple-git'; import { promises as fs } from 'fs'; +import { repoSchema } from '~/server/schemas'; +import { eq } from 'drizzle-orm'; export default defineEventHandler(async (event) => { const user = await requireUser(event); @@ -60,7 +62,11 @@ export default defineEventHandler(async (event) => { let page = 1; const perPage = 50; while (true) { - const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage }); + const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { + page, + perPage, + since: repo.lastFetch || undefined, + }); for await (const issue of issues) { let issueString = `# issue "${issue.title}" (${issue.number})`; if (issue.labels.length !== 0) { @@ -85,6 +91,14 @@ export default defineEventHandler(async (event) => { console.log(`wrote ${page * perPage} issues`); + await db + .update(repoSchema) + .set({ + lastFetch: new Date(), + }) + .where(eq(repoSchema.id, repo.id)) + .run(); + console.log('start indexing ...'); const indexingResponse = await $fetch<{ error?: string }>(`${config.api.url}/index`, { method: 'POST', diff --git a/server/db/migrations/0004_unknown_texas_twister.sql b/server/db/migrations/0004_unknown_texas_twister.sql new file mode 100644 index 0000000..9b4e28a --- /dev/null +++ b/server/db/migrations/0004_unknown_texas_twister.sql @@ -0,0 +1,9 @@ +/* + SQLite does not support "Drop not null from column" out of the box, we do not generate automatic migration for that, so it has to be done manually + Please refer to: https://www.techonthenet.com/sqlite/tables/alter_table.php + https://www.sqlite.org/lang_altertable.html + https://stackoverflow.com/questions/2083543/modify-a-columns-type-in-sqlite3 + + Due to that we don't generate migration automatically and it has to be done manually +*/--> statement-breakpoint +ALTER TABLE repos ADD `lastIndex` integer; \ No newline at end of file diff --git a/server/db/migrations/meta/0004_snapshot.json b/server/db/migrations/meta/0004_snapshot.json new file mode 100644 index 0000000..086a3f2 --- /dev/null +++ b/server/db/migrations/meta/0004_snapshot.json @@ -0,0 +1,272 @@ +{ + "version": "5", + "dialect": "sqlite", + "id": "889a3943-6ce1-44f5-80c2-55da2334b02c", + "prevId": "64348d5c-3627-4553-a72b-174f7653fe57", + "tables": { + "forges": { + "name": "forges", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host": { + "name": "host", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowLogin": { + "name": "allowLogin", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "clientId": { + "name": "clientId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "clientSecret": { + "name": "clientSecret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "forges_host_unique": { + "name": "forges_host_unique", + "columns": [ + "host" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repos": { + "name": "repos", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "forgeId": { + "name": "forgeId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remoteId": { + "name": "remoteId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloneUrl": { + "name": "cloneUrl", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastIndex": { + "name": "lastIndex", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "uniqueForgeRemoteId": { + "name": "uniqueForgeRemoteId", + "columns": [ + "forgeId", + "remoteId" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "userForges": { + "name": "userForges", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "forgeId": { + "name": "forgeId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remoteUserId": { + "name": "remoteUserId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessToken": { + "name": "accessToken", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessTokenExpiresIn": { + "name": "accessTokenExpiresIn", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refreshToken": { + "name": "refreshToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "userRepos": { + "name": "userRepos", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repoId": { + "name": "repoId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatarUrl": { + "name": "avatarUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/server/db/migrations/meta/_journal.json b/server/db/migrations/meta/_journal.json index d782016..4e6b471 100644 --- a/server/db/migrations/meta/_journal.json +++ b/server/db/migrations/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1693303421837, "tag": "0003_purple_anthem", "breakpoints": true + }, + { + "idx": 4, + "version": "5", + "when": 1693416398583, + "tag": "0004_unknown_texas_twister", + "breakpoints": true } ] } diff --git a/server/forges/github.ts b/server/forges/github.ts index fc94e2d..bd88a42 100644 --- a/server/forges/github.ts +++ b/server/forges/github.ts @@ -180,6 +180,7 @@ export class Github implements Forge { repo: repo.data.name, per_page: perPage, page: pagination?.page, + since: pagination?.since?.toISOString(), }); return { diff --git a/server/forges/gitlab.ts b/server/forges/gitlab.ts index 09bb060..43be3be 100644 --- a/server/forges/gitlab.ts +++ b/server/forges/gitlab.ts @@ -139,6 +139,7 @@ export class Gitlab implements Forge { perPage: pagination?.perPage || 10, page: pagination?.page || 1, showExpanded: true, + updatedAfter: pagination?.since?.toISOString(), }); return { diff --git a/server/forges/types.ts b/server/forges/types.ts index 6cb62aa..7e1436b 100644 --- a/server/forges/types.ts +++ b/server/forges/types.ts @@ -15,6 +15,7 @@ export type ForgeUser = Omit & { remoteUserId: string }; export type Pagination = { page: number; perPage: number; + since?: Date; }; export type PaginatedList = { diff --git a/server/schemas/index.ts b/server/schemas/index.ts index ea68105..b4a0060 100644 --- a/server/schemas/index.ts +++ b/server/schemas/index.ts @@ -39,6 +39,7 @@ export const repoSchema = sqliteTable( name: text('name').notNull(), url: text('url').notNull(), cloneUrl: text('cloneUrl').notNull(), + lastFetch: integer('lastIndex', { mode: 'timestamp' }), }, (table) => ({ forgeRemoteId: unique('uniqueForgeRemoteId').on(table.forgeId, table.remoteId),