undo

geprog · Mar 22, 2024 · faed94d · faed94d
1 parent 090da58
commit faed94d
Showing 1 changed file with 61 additions and 37 deletions.
diff --git a/server/api/repos/[repo_id]/clone.post.ts b/server/api/repos/[repo_id]/clone.post.ts
@@ -3,6 +3,11 @@ import { simpleGit } from 'simple-git';
 import { promises as fs } from 'fs';
 import { repoSchema } from '~/server/schemas';
 import { eq } from 'drizzle-orm';
+import { DirectoryLoader } from 'langchain/document_loaders/fs/directory';
+import { TextLoader } from 'langchain/document_loaders/fs/text';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { FaissStore } from '@langchain/community/vectorstores/faiss';
+import { OpenAIEmbeddings } from '@langchain/openai';
 
 export default defineEventHandler(async (event) => {
   const user = await requireUser(event);
@@ -25,14 +30,16 @@ export default defineEventHandler(async (event) => {
 
   const userForgeApi = await getUserForgeAPI(user, repo.forgeId);
 
-  if (!(await dirExists(path.join(folder, 'repo')))) {
+  const repoPath = path.join(folder, 'repo');
+
+  if (!(await dirExists(repoPath))) {
     const cloneCredentials = await userForgeApi.getCloneCredentials();
     const cloneUrl = repo.cloneUrl.replace(
       'https://',
       `https://${cloneCredentials.username}:${cloneCredentials.password}@`,
     );
 
-    let log = await simpleGit().clone(cloneUrl, path.join(folder, 'repo'));
+    let log = await simpleGit().clone(cloneUrl, repoPath);
     console.log('cloned', log);
   } else {
     const cloneCredentials = await userForgeApi.getCloneCredentials();
@@ -41,10 +48,10 @@ export default defineEventHandler(async (event) => {
       `https://${cloneCredentials.username}:${cloneCredentials.password}@`,
     );
 
-    await simpleGit(path.join(folder, 'repo')).removeRemote('origin');
-    await simpleGit(path.join(folder, 'repo')).addRemote('origin', cloneUrl);
+    await simpleGit(repoPath).removeRemote('origin');
+    await simpleGit(repoPath).addRemote('origin', cloneUrl);
 
-    let log = await simpleGit(path.join(folder, 'repo')).pull('origin', repo.defaultBranch);
+    let log = await simpleGit(repoPath).pull('origin', repo.defaultBranch);
     console.log('pulled', log);
   }
 
@@ -56,39 +63,56 @@ export default defineEventHandler(async (event) => {
     await fs.mkdir(path.join(folder, 'issues'), { recursive: true });
   }
 
-  let page = 1;
-  const perPage = 50;
-  const since = repo.lastFetch || undefined;
-  console.log('fetching issues since', since, '...');
-  while (true) {
-    const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), {
-      page,
-      perPage,
-      since,
-    });
-    for await (const issue of issues) {
-      let issueString = `# issue "${issue.title}" (${issue.number})`;
-      if (issue.labels.length !== 0) {
-        issueString += `\n\nLabels: ` + issue.labels.join(', ');
-      }
-      if (issue.description !== '') {
-        issueString += `\n\n${issue.description}`;
-      }
-      if (issue.comments.length !== 0) {
-        issueString +=
-          `\n\n## Comments:\n` +
-          issue.comments.map((comment) => `- ${comment.author.login}: ${comment.body}`).join('\n');
-      }
-      await fs.writeFile(path.join(folder, 'issues', `${issue.number}.md`), issueString);
-    }
-
-    if (issues.length < perPage || page * perPage >= total) {
-      break;
-    }
-    page += 1;
-  }
+  // let page = 1;
+  // const perPage = 50;
+  // const since = repo.lastFetch || undefined;
+  // console.log('fetching issues since', since, '...');
+  // while (true) {
+  //   const { items: issues, total } = await userForgeApi.getIssues(repo.remoteId.toString(), {
+  //     page,
+  //     perPage,
+  //     since,
+  //   });
+  //   for await (const issue of issues) {
+  //     let issueString = `# issue "${issue.title}" (${issue.number})`;
+  //     if (issue.labels.length !== 0) {
+  //       issueString += `\n\nLabels: ` + issue.labels.join(', ');
+  //     }
+  //     if (issue.description !== '') {
+  //       issueString += `\n\n${issue.description}`;
+  //     }
+  //     if (issue.comments.length !== 0) {
+  //       issueString +=
+  //         `\n\n## Comments:\n` +
+  //         issue.comments.map((comment) => `- ${comment.author.login}: ${comment.body}`).join('\n');
+  //     }
+  //     await fs.writeFile(path.join(folder, 'issues', `${issue.number}.md`), issueString);
+  //   }
+
+  //   if (issues.length < perPage || page * perPage >= total) {
+  //     break;
+  //   }
+  //   page += 1;
+  // }
+
+  // console.log(`wrote ${page * perPage} issues`);
+
+  const loader = new DirectoryLoader(repoPath, {
+    '.ts': (path) => new TextLoader(path),
+  });
+  const docs = await loader.load();
+
+  console.log({ docs });
+
+  const javascriptSplitter = RecursiveCharacterTextSplitter.fromLanguage('js', {
+    chunkSize: 2000,
+    chunkOverlap: 200,
+  });
+  const texts = await javascriptSplitter.splitDocuments(docs);
+
+  const vectorStore = await FaissStore.fromDocuments(texts, new OpenAIEmbeddings());
 
-  console.log(`wrote ${page * perPage} issues`);
+  vectorStore.save(path.join(folder, 'vectorstore'));
 
   await db
     .update(repoSchema)