feat: only fetch updated issues (#26)

server/api/repos/[repo_id]/clone.post.ts

@@ -1,6 +1,8 @@
 import * as path from 'path';
 import { simpleGit } from 'simple-git';
 import { promises as fs } from 'fs';
+import { repoSchema } from '~/server/schemas';
+import { eq } from 'drizzle-orm';
 
 export default defineEventHandler(async (event) => {
   const user = await requireUser(event);
@@ -60,7 +62,11 @@ export default defineEventHandler(async (event) => {
   let page = 1;
   const perPage = 50;
   while (true) {
-    const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), { page, perPage });
+    const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), {
+      page,
+      perPage,
+      since: repo.lastFetch || undefined,
+    });
     for await (const issue of issues) {
       let issueString = `# issue "${issue.title}" (${issue.number})`;
       if (issue.labels.length !== 0) {
@@ -85,6 +91,14 @@ export default defineEventHandler(async (event) => {
 
   console.log(`wrote ${page * perPage} issues`);
 
+  await db
+    .update(repoSchema)
+    .set({
+      lastFetch: new Date(),
+    })
+    .where(eq(repoSchema.id, repo.id))
+    .run();
+
   console.log('start indexing ...');
   const indexingResponse = await $fetch<{ error?: string }>(`${config.api.url}/index`, {
     method: 'POST',

server/db/migrations/0004_unknown_texas_twister.sql

@@ -0,0 +1,9 @@
+/*
+ SQLite does not support "Drop not null from column" out of the box, we do not generate automatic migration for that, so it has to be done manually
+ Please refer to: https://www.techonthenet.com/sqlite/tables/alter_table.php
+                  https://www.sqlite.org/lang_altertable.html
+                  https://stackoverflow.com/questions/2083543/modify-a-columns-type-in-sqlite3
+
+ Due to that we don't generate migration automatically and it has to be done manually
+*/--> statement-breakpoint
+ALTER TABLE repos ADD `lastIndex` integer;

server/db/migrations/meta/0004_snapshot.json

@@ -0,0 +1,272 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "889a3943-6ce1-44f5-80c2-55da2334b02c",
+  "prevId": "64348d5c-3627-4553-a72b-174f7653fe57",
+  "tables": {
+    "forges": {
+      "name": "forges",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "type": {
+          "name": "type",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "host": {
+          "name": "host",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "owner": {
+          "name": "owner",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "allowLogin": {
+          "name": "allowLogin",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "clientId": {
+          "name": "clientId",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "clientSecret": {
+          "name": "clientSecret",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "forges_host_unique": {
+          "name": "forges_host_unique",
+          "columns": [
+            "host"
+          ],
+          "isUnique": true
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "repos": {
+      "name": "repos",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "forgeId": {
+          "name": "forgeId",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "remoteId": {
+          "name": "remoteId",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "url": {
+          "name": "url",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "cloneUrl": {
+          "name": "cloneUrl",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "lastIndex": {
+          "name": "lastIndex",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {
+        "uniqueForgeRemoteId": {
+          "name": "uniqueForgeRemoteId",
+          "columns": [
+            "forgeId",
+            "remoteId"
+          ],
+          "isUnique": true
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "userForges": {
+      "name": "userForges",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "userId": {
+          "name": "userId",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "forgeId": {
+          "name": "forgeId",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "remoteUserId": {
+          "name": "remoteUserId",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "accessToken": {
+          "name": "accessToken",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "accessTokenExpiresIn": {
+          "name": "accessTokenExpiresIn",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "refreshToken": {
+          "name": "refreshToken",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "userRepos": {
+      "name": "userRepos",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "userId": {
+          "name": "userId",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "repoId": {
+          "name": "repoId",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "users": {
+      "name": "users",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "integer",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "name": {
+          "name": "name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "avatarUrl": {
+          "name": "avatarUrl",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "email": {
+          "name": "email",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

server/db/migrations/meta/_journal.json

@@ -29,6 +29,13 @@
       "when": 1693303421837,
       "tag": "0003_purple_anthem",
       "breakpoints": true
+    },
+    {
+      "idx": 4,
+      "version": "5",
+      "when": 1693416398583,
+      "tag": "0004_unknown_texas_twister",
+      "breakpoints": true
     }
   ]
 }

server/forges/github.ts

@@ -180,6 +180,7 @@ export class Github implements Forge {
       repo: repo.data.name,
       per_page: perPage,
       page: pagination?.page,
+      since: pagination?.since?.toISOString(),
     });
 
     return {

server/forges/gitlab.ts

@@ -139,6 +139,7 @@ export class Gitlab implements Forge {
       perPage: pagination?.perPage || 10,
       page: pagination?.page || 1,
       showExpanded: true,
+      updatedAfter: pagination?.since?.toISOString(),
     });
 
     return {

server/forges/types.ts

@@ -15,6 +15,7 @@ export type ForgeUser = Omit<User, 'id'> & { remoteUserId: string };
 export type Pagination = {
   page: number;
   perPage: number;
+  since?: Date;
 };
 
 export type PaginatedList<T> = {

server/schemas/index.ts

@@ -39,6 +39,7 @@ export const repoSchema = sqliteTable(
     name: text('name').notNull(),
     url: text('url').notNull(),
     cloneUrl: text('cloneUrl').notNull(),
+    lastFetch: integer('lastIndex', { mode: 'timestamp' }),
   },
   (table) => ({
     forgeRemoteId: unique('uniqueForgeRemoteId').on(table.forgeId, table.remoteId),