feat: index pdfs imgs audio

Dockerfile

@@ -4,6 +4,8 @@ WORKDIR /app
 
 COPY . /app
 
+RUN apt-get update && apt-get install -y poppler-data && apt-get install -y poppler-utils
+
 RUN npm install -g pnpm
 
 RUN pnpm i

package.json

@@ -6,7 +6,7 @@
     "dev:cron": "nodemon --exec ts-node --project tsconfig.server.json src/cron.ts",
     "dev:worker": "nodemon --exec ts-node --project tsconfig.server.json src/worker.ts",
     "build:app": "next build",
-    "build:server": "tsc --build tsconfig.server.json",
+    "build:server": "tsc --build tsconfig.server.json && tsc-alias --project tsconfig.server.json",
     "start:app": "next start",
     "start:cron": "node -r ./tsconfig-paths-bootstrap.js dist/cron.js",
     "start:worker": "node -r ./tsconfig-paths-bootstrap.js dist/worker.js"
@@ -45,6 +45,7 @@
     "@radix-ui/react-tooltip": "^1.1.6",
     "axios": "^1.7.9",
     "bullmq": "^5.34.6",
+    "canvas": "^3.0.1",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "cmdk": "1.0.0",
@@ -57,19 +58,23 @@
     "next": "15.1.4",
     "next-themes": "^0.4.4",
     "node-cron": "^3.0.3",
+    "node-poppler": "^7.2.2",
     "nodemailer": "^6.9.16",
-    "openai": "^4.77.0",
+    "openai": "^4.78.0",
     "react": "19.0.0",
     "react-day-picker": "8.10.1",
     "react-dom": "19.0.0",
     "react-hook-form": "^7.54.2",
     "react-icons": "^5.4.0",
     "react-resizable-panels": "^2.1.7",
     "recharts": "^2.15.0",
+    "sharp": "^0.33.5",
     "sonner": "^1.7.1",
     "tailwind-merge": "^2.6.0",
     "tailwindcss-animate": "^1.0.7",
+    "tsc-alias": "^1.8.10",
     "vaul": "^1.1.2",
+    "weaviate-client": "^3.3.1",
     "zod": "^3.24.1",
     "zustand": "^5.0.3"
   },

prisma/migrations/20250110142328_org_and_index_source/migration.sql

@@ -0,0 +1,49 @@
+/*
+  Warnings:
+
+  - Added the required column `organizationId` to the `SheetColumn` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `organizationId` to the `SheetColumnValue` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `organizationId` to the `SheetSource` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "SheetColumn" ADD COLUMN     "organizationId" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "SheetColumnValue" ADD COLUMN     "organizationId" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "SheetSource" ADD COLUMN     "organizationId" TEXT NOT NULL;
+
+-- CreateTable
+CREATE TABLE "IndexedSource" (
+    "id" TEXT NOT NULL,
+    "indexId" TEXT NOT NULL,
+    "referenceImageNickName" TEXT NOT NULL,
+    "referenceImageFileName" TEXT NOT NULL,
+    "sheetId" TEXT NOT NULL,
+    "sourceId" TEXT NOT NULL,
+    "organizationId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "IndexedSource_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "SheetSource" ADD CONSTRAINT "SheetSource_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "SheetColumn" ADD CONSTRAINT "SheetColumn_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "SheetColumnValue" ADD CONSTRAINT "SheetColumnValue_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "IndexedSource" ADD CONSTRAINT "IndexedSource_sheetId_fkey" FOREIGN KEY ("sheetId") REFERENCES "Sheet"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "IndexedSource" ADD CONSTRAINT "IndexedSource_sourceId_fkey" FOREIGN KEY ("sourceId") REFERENCES "Source"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "IndexedSource" ADD CONSTRAINT "IndexedSource_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20250111042646_indexed_source_file/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `referenceImageNickName` on the `IndexedSource` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "IndexedSource" DROP COLUMN "referenceImageNickName";

prisma/migrations/20250111042826_remove_index_src_sheet/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `sheetId` on the `IndexedSource` table. All the data in the column will be lost.
+
+*/
+-- DropForeignKey
+ALTER TABLE "IndexedSource" DROP CONSTRAINT "IndexedSource_sheetId_fkey";
+
+-- AlterTable
+ALTER TABLE "IndexedSource" DROP COLUMN "sheetId";

prisma/migrations/20250111044108_src_indexing_opt/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "IndexedSource" ADD COLUMN     "referenceText" TEXT,
+ALTER COLUMN "referenceImageFileName" DROP NOT NULL;
+
+-- AlterTable
+ALTER TABLE "Source" ADD COLUMN     "isIndexing" BOOLEAN NOT NULL DEFAULT false;

prisma/migrations/20250111052712_src_upload_fix_case/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Added the required column `uploadedById` to the `Source` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Source" ADD COLUMN     "uploadedById" TEXT NOT NULL;
+
+-- AddForeignKey
+ALTER TABLE "Source" ADD CONSTRAINT "Source_uploadedById_fkey" FOREIGN KEY ("uploadedById") REFERENCES "User"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/schema.prisma

@@ -39,17 +39,22 @@ model User {
   updatedAt     DateTime  @updatedAt
   members       Member[]
   sheets        Sheet[]
+  sources       Source[]
 }
 
 model Organization {
-  id        String   @id @default(cuid())
-  name      String
-  apiKey    String   @unique @default(uuid())
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-  members   Member[]
-  sheets    Sheet[]
-  sources   Source[]
+  id                String             @id @default(cuid())
+  name              String
+  apiKey            String             @unique @default(uuid())
+  createdAt         DateTime           @default(now())
+  updatedAt         DateTime           @updatedAt
+  members           Member[]
+  sheets            Sheet[]
+  sources           Source[]
+  sheetSources      SheetSource[]
+  sheetColumns      SheetColumn[]
+  sheetColumnValues SheetColumnValue[]
+  indexedSources    IndexedSource[]
 }
 
 model Member {
@@ -77,47 +82,70 @@ model Sheet {
 }
 
 model Source {
-  id             String        @id @default(uuid())
+  id             String          @id @default(uuid())
   fileName       String
   fileType       String
   nickName       String
-  isIndexed      Boolean       @default(false)
+  isIndexed      Boolean         @default(false)
+  isIndexing     Boolean         @default(false)
   organizationId String
-  organization   Organization  @relation(fields: [organizationId], references: [id])
-  createdAt      DateTime      @default(now())
+  organization   Organization    @relation(fields: [organizationId], references: [id])
+  uploadedById   String
+  uploadedBy     User            @relation(fields: [uploadedById], references: [id])
+  createdAt      DateTime        @default(now())
   sheets         SheetSource[]
+  indexedSources IndexedSource[]
 }
 
 model SheetSource {
-  id        String             @id @default(uuid())
-  sheetId   String
-  sheet     Sheet              @relation(fields: [sheetId], references: [id])
-  sourceId  String
-  source    Source             @relation(fields: [sourceId], references: [id])
-  rowValues SheetColumnValue[]
+  id             String             @id @default(uuid())
+  sheetId        String
+  sheet          Sheet              @relation(fields: [sheetId], references: [id])
+  sourceId       String
+  source         Source             @relation(fields: [sourceId], references: [id])
+  organizationId String
+  organization   Organization       @relation(fields: [organizationId], references: [id])
+  rowValues      SheetColumnValue[]
 }
 
 model SheetColumn {
-  id           String             @id @default(uuid())
-  sheetId      String
-  sheet        Sheet              @relation(fields: [sheetId], references: [id])
-  name         String
-  instruction  String
-  taskType     ColumnTaskType
-  dataType     ColumnDataType
-  defaultValue String             @default("N/A")
-  columnValues SheetColumnValue[]
+  id             String             @id @default(uuid())
+  sheetId        String
+  sheet          Sheet              @relation(fields: [sheetId], references: [id])
+  organizationId String
+  organization   Organization       @relation(fields: [organizationId], references: [id])
+  name           String
+  instruction    String
+  taskType       ColumnTaskType
+  dataType       ColumnDataType
+  defaultValue   String             @default("N/A")
+  columnValues   SheetColumnValue[]
 }
 
 model SheetColumnValue {
-  sheetId  String
-  sheet    Sheet       @relation(fields: [sheetId], references: [id])
-  columnId String
-  column   SheetColumn @relation(fields: [columnId], references: [id])
-  value    String
-  misc     Json?
-  sourceId String
-  source   SheetSource @relation(fields: [sourceId], references: [id])
+  sheetId        String
+  sheet          Sheet        @relation(fields: [sheetId], references: [id])
+  columnId       String
+  column         SheetColumn  @relation(fields: [columnId], references: [id])
+  organizationId String
+  organization   Organization @relation(fields: [organizationId], references: [id])
+  value          String
+  misc           Json?
+  sourceId       String
+  source         SheetSource  @relation(fields: [sourceId], references: [id])
 
   @@id([columnId, sourceId])
 }
+
+model IndexedSource {
+  id                     String       @id @default(uuid())
+  indexId                String
+  referenceImageFileName String?
+  referenceText          String?
+  sourceId               String
+  source                 Source       @relation(fields: [sourceId], references: [id])
+  organizationId         String
+  organization           Organization @relation(fields: [organizationId], references: [id])
+  createdAt              DateTime     @default(now())
+  updatedAt              DateTime     @updatedAt
+}

src/app/console/actions.ts

@@ -150,9 +150,9 @@ export async function getUploadUrlForSource(filename: string) {
 }
 
 export async function addSource(nickName: string, fileName: string, fileType: string) {
-    const { organizationId } = await getAuthToken()
+    const { organizationId, userId } = await getAuthToken()
     await prisma.source.create({
-        data: { nickName, fileName, fileType, organizationId },
+        data: { nickName, fileName, fileType, organizationId, uploadedById: userId },
     })
 
     return