From 454a75207157ed78cb0845c92e842f927b9ab128 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 16:24:50 -0800 Subject: [PATCH 1/6] Initial commit: add a dedicated page for managing the knowledge base - One current issue in the Khoj application is that managing the files being referenced as the user's knowledge base is slightly opaque and difficult to access - Add a migration for associating the fileobjects directly with the Entry objects, making it easier to get data via foreign key - Add the new page that shows all indexed files in the search view, also allowing you to upload new docs directly from that page - Support new APIs for getting / deleting files --- src/interface/web/app/common/utils.ts | 30 ++ .../app/components/appSidebar/appSidebar.tsx | 8 +- src/interface/web/app/knowledge/page.tsx | 93 ++++ src/interface/web/app/search/layout.tsx | 6 +- src/interface/web/app/search/page.tsx | 472 ++++++++++++++++-- .../commands/delete_orphaned_fileobjects.py | 49 ++ .../migrations/0079_entry_file_object.py | 75 +++ src/khoj/database/models/__init__.py | 18 +- src/khoj/processor/content/text_to_entries.py | 29 +- src/khoj/routers/api_content.py | 64 +++ 10 files changed, 788 insertions(+), 56 deletions(-) create mode 100644 src/interface/web/app/knowledge/page.tsx create mode 100644 src/khoj/database/management/commands/delete_orphaned_fileobjects.py create mode 100644 src/khoj/database/migrations/0079_entry_file_object.py diff --git a/src/interface/web/app/common/utils.ts b/src/interface/web/app/common/utils.ts index 8bf6db84e..21e53cce8 100644 --- a/src/interface/web/app/common/utils.ts +++ b/src/interface/web/app/common/utils.ts @@ -94,3 +94,33 @@ export function useDebounce(value: T, delay: number): T { return debouncedValue; } + +export const formatDateTime = (isoString: string): string => { + try { + const date = new Date(isoString); + const now = new Date(); + const diffInMinutes = Math.floor((now.getTime() - date.getTime()) / 60000); + + // Show relative time for recent dates + if (diffInMinutes < 1) return "just now"; + if (diffInMinutes < 60) return `${diffInMinutes} minutes ago`; + if (diffInMinutes < 120) return "1 hour ago"; + if (diffInMinutes < 1440) return `${Math.floor(diffInMinutes / 60)} hours ago`; + + // For older dates, show full formatted date + const formatter = new Intl.DateTimeFormat("en-US", { + month: "long", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "2-digit", + hour12: true, + timeZoneName: "short", + }); + + return formatter.format(date); + } catch (error) { + console.error("Error formatting date:", error); + return isoString; + } +}; diff --git a/src/interface/web/app/components/appSidebar/appSidebar.tsx b/src/interface/web/app/components/appSidebar/appSidebar.tsx index d80314b9b..2f5f9bf8c 100644 --- a/src/interface/web/app/components/appSidebar/appSidebar.tsx +++ b/src/interface/web/app/components/appSidebar/appSidebar.tsx @@ -17,7 +17,7 @@ import { KhojSearchLogo, } from "../logo/khojLogo"; import { Gear } from "@phosphor-icons/react/dist/ssr"; -import { Plus } from "@phosphor-icons/react"; +import { Book, Plus } from "@phosphor-icons/react"; import { useEffect, useState } from "react"; import AllConversations from "../allConversations/allConversations"; import FooterMenu from "../navMenu/navMenu"; @@ -26,6 +26,7 @@ import { useIsMobileWidth } from "@/app/common/utils"; import { UserPlusIcon } from "lucide-react"; import { useAuthenticatedData } from "@/app/common/auth"; import LoginPrompt from "../loginPrompt/loginPrompt"; +import { url } from "inspector"; // Menu items. const items = [ @@ -54,6 +55,11 @@ const items = [ url: "/settings", icon: Gear, }, + { + title: "Knowledge Base", + url: "/knowledge", + icon: Book, + }, ]; const SIDEBAR_KEYBOARD_SHORTCUT = "b"; diff --git a/src/interface/web/app/knowledge/page.tsx b/src/interface/web/app/knowledge/page.tsx new file mode 100644 index 000000000..cf0401565 --- /dev/null +++ b/src/interface/web/app/knowledge/page.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; +import { AppSidebar } from "../components/appSidebar/appSidebar"; +import { Separator } from "@/components/ui/separator"; +import { KhojLogoType } from "../components/logo/khojLogo"; +import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card"; +import { useIsMobileWidth } from "../common/utils"; +import { InlineLoading } from "../components/loading/loading"; + +interface FileObject { + file_name: string; + raw_text: string; +} + +export default function KnowledgeBase() { + const [files, setFiles] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const isMobileWidth = useIsMobileWidth(); + + useEffect(() => { + const fetchFiles = async () => { + try { + const response = await fetch("/api/content/all"); + if (!response.ok) throw new Error("Failed to fetch files"); + + const filesList = await response.json(); + if (Array.isArray(filesList)) { + setFiles(filesList.toSorted()); + } + } catch (error) { + setError("Failed to load files"); + console.error("Error fetching files:", error); + } finally { + setLoading(false); + } + }; + + fetchFiles(); + }, []); + + return ( + + + +
+ + + {isMobileWidth ? ( + + + + ) : ( +

Knowledge Base

+ )} +
+
+
+ {loading && ( +
+ +
+ )} + {error &&
{error}
} + +
+ {files.map((file, index) => ( + + + + {file.file_name.split("/").pop()} + + + +

+ {file.raw_text.slice(0, 100)}... +

+ + + ))} +
+
+
+ + + ); +} diff --git a/src/interface/web/app/search/layout.tsx b/src/interface/web/app/search/layout.tsx index ecaa29929..cdb0976ff 100644 --- a/src/interface/web/app/search/layout.tsx +++ b/src/interface/web/app/search/layout.tsx @@ -2,6 +2,7 @@ import type { Metadata } from "next"; import "../globals.css"; import { ContentSecurityPolicy } from "../common/layoutHelper"; +import { Toaster } from "@/components/ui/toaster"; export const metadata: Metadata = { title: "Khoj AI - Search", @@ -35,7 +36,10 @@ export default function RootLayout({ return ( - {children} + + {children} + + ); } diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index 90432508a..bb70d8f7e 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -24,16 +24,52 @@ import { MagnifyingGlass, NoteBlank, NotionLogo, + Eye, + Trash, + ArrowsOutSimple, + DotsThreeVertical, + Waveform, + Plus, } from "@phosphor-icons/react"; import { Button } from "@/components/ui/button"; import Link from "next/link"; import { getIconFromFilename } from "../common/iconUtils"; -import { useIsMobileWidth } from "../common/utils"; +import { formatDateTime, useIsMobileWidth } from "../common/utils"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; import { AppSidebar } from "../components/appSidebar/appSidebar"; import { Separator } from "@/components/ui/separator"; import { KhojLogoType } from "../components/logo/khojLogo"; - +import { InlineLoading } from "../components/loading/loading"; +import { + AlertDialog, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogCancel, + AlertDialogAction, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { useToast } from "@/components/ui/use-toast"; +import { Scroll } from "lucide-react"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { uploadDataForIndexing } from "../common/chatFunctions"; +import { CommandDialog } from "@/components/ui/command"; +import { Progress } from "@/components/ui/progress"; interface AdditionalData { file: string; source: string; @@ -49,6 +85,12 @@ interface SearchResult { "corpus-id": string; } +interface FileObject { + file_name: string; + raw_text: string; + updated_at: string; +} + function getNoteTypeIcon(source: string) { if (source === "notion") { return ; @@ -92,7 +134,7 @@ function Note(props: NoteResultProps) { const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( - + {getNoteTypeIcon(note.additional.source)} @@ -139,7 +181,7 @@ function focusNote(note: SearchResult) { const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( - + {fileName} @@ -167,27 +209,147 @@ function focusNote(note: SearchResult) { ); } +const UploadFiles: React.FC<{ + onClose: () => void; + setUploadedFiles: (files: string[]) => void; +}> = ({ onClose, setUploadedFiles }) => { + const [syncedFiles, setSyncedFiles] = useState([]); + const [selectedFiles, setSelectedFiles] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [isDragAndDropping, setIsDragAndDropping] = useState(false); + + const [warning, setWarning] = useState(null); + const [error, setError] = useState(null); + const [uploading, setUploading] = useState(false); + const [progressValue, setProgressValue] = useState(0); + const fileInputRef = useRef(null); + + useEffect(() => { + if (!uploading) { + setProgressValue(0); + } + + if (uploading) { + const interval = setInterval(() => { + setProgressValue((prev) => { + const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5 + const nextValue = prev + increment; + return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100 + }); + }, 800); + return () => clearInterval(interval); + } + }, [uploading]); + + const filteredFiles = syncedFiles.filter((file) => + file.toLowerCase().includes(searchQuery.toLowerCase()), + ); + + function handleDragOver(event: React.DragEvent) { + event.preventDefault(); + setIsDragAndDropping(true); + } + + function handleDragLeave(event: React.DragEvent) { + event.preventDefault(); + setIsDragAndDropping(false); + } + + function handleDragAndDropFiles(event: React.DragEvent) { + event.preventDefault(); + setIsDragAndDropping(false); + + if (!event.dataTransfer.files) return; + + uploadFiles(event.dataTransfer.files); + } + + function openFileInput() { + if (fileInputRef && fileInputRef.current) { + fileInputRef.current.click(); + } + } + + function handleFileChange(event: React.ChangeEvent) { + if (!event.target.files) return; + + uploadFiles(event.target.files); + } + + function uploadFiles(files: FileList) { + uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles); + } + + return ( +
+ +
+ {uploading && ( + + )} +
+
+
+ {isDragAndDropping ? ( +
+ + Drop files to upload +
+ ) : ( +
+ + Drag and drop files here +
+ )} +
+
+
+ ); +}; + export default function Search() { const [searchQuery, setSearchQuery] = useState(""); const [searchResults, setSearchResults] = useState(null); const [searchResultsLoading, setSearchResultsLoading] = useState(false); const [focusSearchResult, setFocusSearchResult] = useState(null); - const [exampleQuery, setExampleQuery] = useState(""); + const [files, setFiles] = useState([]); + const [error, setError] = useState(null); + const [fileObjectsLoading, setFileObjectsLoading] = useState(true); const searchTimeoutRef = useRef(null); + const [selectedFile, setSelectedFile] = useState(null); + const [selectedFileFullText, setSelectedFileFullText] = useState(null); + const [isDeleting, setIsDeleting] = useState(false); + const [uploadedFiles, setUploadedFiles] = useState([]); + const [selectedFiles, setSelectedFiles] = useState([]); + const [filteredFiles, setFilteredFiles] = useState([]); - const isMobileWidth = useIsMobileWidth(); + const { toast } = useToast(); - useEffect(() => { - setExampleQuery( - naturalLanguageSearchQueryExamples[ - Math.floor(Math.random() * naturalLanguageSearchQueryExamples.length) - ], - ); - }, []); + const isMobileWidth = useIsMobileWidth(); function search() { if (searchResultsLoading || !searchQuery.trim()) return; + setSearchResultsLoading(true); + const apiUrl = `/api/search?q=${encodeURIComponent(searchQuery)}&client=web`; fetch(apiUrl, { method: "GET", @@ -205,8 +367,69 @@ export default function Search() { }); } + const deleteSelected = async () => { + let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles; + + if (filesToDelete.length === 0) { + return; + } + + try { + const response = await fetch("/api/content/files", { + method: "DELETE", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ files: filesToDelete }), + }); + + if (!response.ok) throw new Error("Failed to delete files"); + + // Update the syncedFiles state + setUploadedFiles((prevFiles) => + prevFiles.filter((file) => !filesToDelete.includes(file)), + ); + + // Reset selectedFiles + setSelectedFiles([]); + } catch (error) { + console.error("Error deleting files:", error); + } + }; + + const fetchFiles = async () => { + try { + const response = await fetch("/api/content/all"); + if (!response.ok) throw new Error("Failed to fetch files"); + + const filesList = await response.json(); + if (Array.isArray(filesList)) { + setFiles(filesList.toSorted()); + } + } catch (error) { + setError("Failed to load files"); + console.error("Error fetching files:", error); + } finally { + setFileObjectsLoading(false); + } + }; + + const fetchSpecificFile = async (fileName: string) => { + try { + const response = await fetch(`/api/content/file?file_name=${fileName}`); + if (!response.ok) throw new Error("Failed to fetch file"); + + const file = await response.json(); + setSelectedFileFullText(file.raw_text); + } catch (error) { + setError("Failed to load file"); + console.error("Error fetching file:", error); + } + }; + useEffect(() => { if (!searchQuery.trim()) { + setSearchResults(null); return; } @@ -229,6 +452,48 @@ export default function Search() { }; }, [searchQuery]); + useEffect(() => { + if (selectedFile) { + fetchSpecificFile(selectedFile); + } + }, [selectedFile]); + + useEffect(() => { + fetchFiles(); + }, []); + + useEffect(() => { + if (uploadedFiles.length > 0) { + fetchFiles(); + } + }, [uploadedFiles]); + + const handleDelete = async (fileName: string) => { + setIsDeleting(true); + try { + const response = await fetch(`/api/content/file?filename=${fileName}`, { + method: "DELETE", + }); + if (!response.ok) throw new Error("Failed to delete file"); + toast({ + title: "File deleted", + description: `File ${fileName} has been deleted`, + variant: "default", + }); + + // Refresh files list + fetchFiles(); + } catch (error) { + toast({ + title: "Error deleting file", + description: `Failed to delete file ${fileName}`, + variant: "destructive", + }); + } finally { + setIsDeleting(false); + } + }; + return ( @@ -251,20 +516,34 @@ export default function Search() {
setSearchQuery(e.currentTarget.value)} onKeyDown={(e) => e.key === "Enter" && search()} type="search" placeholder="Search Documents" /> - +
+ {}} + setUploadedFiles={setUploadedFiles} + /> + {searchResultsLoading && ( +
+ +
+ )} {focusSearchResult && (
+ + + + + + + + + + + Delete File + + + + Are you sure you + want to delete + this file? + + + + Cancel + + + handleDelete( + file.file_name, + ) + } + > + {isDeleting + ? "Deleting..." + : "Delete"} + + + + + + + + + + + + + + {file.file_name + .split( + "/", + ) + .pop()} + + + +

+ { + selectedFileFullText + } +

+ + + + + + + + + + +

+ {file.raw_text.slice(0, 100)}... +

+ + + +
+ {formatDateTime(file.updated_at)} +
+ + + ))} +
+ )} {searchResults && searchResults.length === 0 && ( diff --git a/src/khoj/database/management/commands/delete_orphaned_fileobjects.py b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py new file mode 100644 index 000000000..95cec1376 --- /dev/null +++ b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py @@ -0,0 +1,49 @@ +from django.core.management.base import BaseCommand +from django.db.models import Exists, OuterRef + +from khoj.database.models import Entry, FileObject + + +class Command(BaseCommand): + help = "Deletes FileObjects that have no associated Entries" + + def add_arguments(self, parser): + parser.add_argument( + "--apply", + action="store_true", + help="Actually perform the deletion. Without this flag, only shows what would be deleted.", + ) + + def handle(self, *args, **options): + # Find FileObjects with no related entries using subquery + orphaned_files = FileObject.objects.annotate( + has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk"))) + ).filter(has_entries=False) + + total_orphaned = orphaned_files.count() + mode = "DELETE" if options["apply"] else "DRY RUN" + self.stdout.write(f"[{mode}] Found {total_orphaned} orphaned FileObjects") + + if total_orphaned == 0: + self.stdout.write("No orphaned FileObjects to process") + return + + # Process in batches of 1000 + batch_size = 1000 + processed = 0 + + while True: + batch = orphaned_files[:batch_size] + if not batch: + break + + if options["apply"]: + count = batch.delete()[0] + processed += count + self.stdout.write(f"Deleted {processed}/{total_orphaned} orphaned FileObjects") + else: + processed += len(batch) + self.stdout.write(f"Would delete {processed}/{total_orphaned} orphaned FileObjects") + + action = "Deleted" if options["apply"] else "Would delete" + self.stdout.write(self.style.SUCCESS(f"{action} {processed} orphaned FileObjects")) diff --git a/src/khoj/database/migrations/0079_entry_file_object.py b/src/khoj/database/migrations/0079_entry_file_object.py new file mode 100644 index 000000000..3846dd9d5 --- /dev/null +++ b/src/khoj/database/migrations/0079_entry_file_object.py @@ -0,0 +1,75 @@ +# Generated by Django 5.0.10 on 2025-01-10 18:28 + +import django.db.models.deletion +from django.db import migrations, models + + +def migrate_entry_objects(apps, schema_editor): + Entry = apps.get_model("database", "Entry") + FileObject = apps.get_model("database", "FileObject") + db_alias = schema_editor.connection.alias + + # Create lookup dictionary of all file objects + file_objects_map = {(fo.user_id, fo.file_name): fo for fo in FileObject.objects.using(db_alias).all()} + + # Process entries in chunks of 1000 + chunk_size = 1000 + processed = 0 + + processed_entry_ids = set() + + while True: + entries = list( + Entry.objects.using(db_alias) + .select_related("user") + .filter(file_object__isnull=True) + .exclude(id__in=processed_entry_ids) + .only("id", "user", "file_path")[:chunk_size] + ) + + if not entries: + break + + processed_entry_ids.update([entry.id for entry in entries]) + + entries_to_update = [] + for entry in entries: + try: + file_object = file_objects_map.get((entry.user_id, entry.file_path)) + if file_object: + entry.file_object = file_object + entries_to_update.append(entry) + except Exception as e: + print(f"Error processing entry {entry.id}: {str(e)}") + continue + + if entries_to_update: + Entry.objects.using(db_alias).bulk_update(entries_to_update, ["file_object"], batch_size=chunk_size) + + processed += len(entries) + print(f"Processed {processed} entries") + + +def reverse_migration(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0078_khojuser_email_verification_code_expiry"), + ] + + operations = [ + migrations.AddField( + model_name="entry", + name="file_object", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="database.fileobject", + ), + ), + migrations.RunPython(migrate_entry_objects, reverse_migration), + ] diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 68fae4345..c169e55ce 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -326,6 +326,7 @@ class Operation(models.TextChoices): INDEX_CONTENT = "index_content" SCHEDULED_JOB = "scheduled_job" SCHEDULE_LEADER = "schedule_leader" + APPLY_MIGRATIONS = "apply_migrations" # We need to make sure that some operations are thread-safe. To do so, add locks for potentially shared operations. # For example, we need to make sure that only one process is updating the embeddings at a time. @@ -658,6 +659,14 @@ class ReflectiveQuestion(DbBaseModel): user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True) +class FileObject(DbBaseModel): + # Contains the full text of a file that has associated Entry objects + file_name = models.CharField(max_length=400, default=None, null=True, blank=True) + raw_text = models.TextField() + user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True) + agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True) + + class Entry(DbBaseModel): class EntryType(models.TextChoices): IMAGE = "image" @@ -689,20 +698,13 @@ class EntrySource(models.TextChoices): hashed_value = models.CharField(max_length=100) corpus_id = models.UUIDField(default=uuid.uuid4, editable=False) search_model = models.ForeignKey(SearchModelConfig, on_delete=models.SET_NULL, default=None, null=True, blank=True) + file_object = models.ForeignKey(FileObject, on_delete=models.CASCADE, default=None, null=True, blank=True) def save(self, *args, **kwargs): if self.user and self.agent: raise ValidationError("An Entry cannot be associated with both a user and an agent.") -class FileObject(DbBaseModel): - # Same as Entry but raw will be a much larger string - file_name = models.CharField(max_length=400, default=None, null=True, blank=True) - raw_text = models.TextField() - user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True) - agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True) - - class EntryDates(DbBaseModel): date = models.DateField() entry = models.ForeignKey(Entry, on_delete=models.CASCADE, related_name="embeddings_dates") diff --git a/src/khoj/processor/content/text_to_entries.py b/src/khoj/processor/content/text_to_entries.py index f013b28cc..2c27c5a3e 100644 --- a/src/khoj/processor/content/text_to_entries.py +++ b/src/khoj/processor/content/text_to_entries.py @@ -152,8 +152,22 @@ def update_embeddings( with timer("Generated embeddings for entries to add to database in", logger): entries_to_process = [hash_to_current_entries[hashed_val] for hashed_val in hashes_to_process] data_to_embed = [getattr(entry, key) for entry in entries_to_process] + modified_files = {entry.file for entry in entries_to_process} embeddings += self.embeddings_model[model.name].embed_documents(data_to_embed) + file_to_file_object_map = {} + if file_to_text_map and modified_files: + with timer("Indexed text of modified file in", logger): + # create or update text of each updated file indexed on DB + for modified_file in modified_files: + raw_text = file_to_text_map[modified_file] + file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file) + if file_object: + FileObjectAdapters.update_raw_text(file_object, raw_text) + else: + file_object = FileObjectAdapters.create_file_object(user, modified_file, raw_text) + file_to_file_object_map[modified_file] = file_object + added_entries: list[DbEntry] = [] with timer("Added entries to database in", logger): num_items = len(hashes_to_process) @@ -165,6 +179,7 @@ def update_embeddings( batch_embeddings_to_create: List[DbEntry] = [] for entry_hash, new_entry in entry_batch: entry = hash_to_current_entries[entry_hash] + file_object = file_to_file_object_map.get(entry.file, None) batch_embeddings_to_create.append( DbEntry( user=user, @@ -178,6 +193,7 @@ def update_embeddings( hashed_value=entry_hash, corpus_id=entry.corpus_id, search_model=model, + file_object=file_object, ) ) try: @@ -190,19 +206,6 @@ def update_embeddings( logger.error(f"Error adding entries to database:\n{batch_indexing_error}\n---\n{e}", exc_info=True) logger.debug(f"Added {len(added_entries)} {file_type} entries to database") - if file_to_text_map: - with timer("Indexed text of modified file in", logger): - # get the set of modified files from added_entries - modified_files = {entry.file_path for entry in added_entries} - # create or update text of each updated file indexed on DB - for modified_file in modified_files: - raw_text = file_to_text_map[modified_file] - file_object = FileObjectAdapters.get_file_object_by_name(user, modified_file) - if file_object: - FileObjectAdapters.update_raw_text(file_object, raw_text) - else: - FileObjectAdapters.create_file_object(user, modified_file, raw_text) - new_dates = [] with timer("Indexed dates from added entries in", logger): for added_entry in added_entries: diff --git a/src/khoj/routers/api_content.py b/src/khoj/routers/api_content.py index 9aea05046..3211c1549 100644 --- a/src/khoj/routers/api_content.py +++ b/src/khoj/routers/api_content.py @@ -22,6 +22,7 @@ from khoj.database import adapters from khoj.database.adapters import ( EntryAdapters, + FileObjectAdapters, get_user_github_config, get_user_notion_config, ) @@ -270,6 +271,8 @@ async def delete_content_files( await EntryAdapters.adelete_entry_by_file(user, filename) + await FileObjectAdapters.adelete_file_object_by_name(user, filename) + return {"status": "ok"} @@ -294,6 +297,8 @@ async def delete_content_file( ) deleted_count = await EntryAdapters.adelete_entries_by_filenames(user, files.files) + for file in files.files: + await FileObjectAdapters.adelete_file_object_by_name(user, file) return {"status": "ok", "deleted_count": deleted_count} @@ -325,6 +330,65 @@ def get_content_types(request: Request, client: Optional[str] = None): return list(configured_content_types & all_content_types) +@api_content.get("/all", response_model=Dict[str, str]) +@requires(["authenticated"]) +async def get_all_content(request: Request, client: Optional[str] = None, truncated: Optional[bool] = True): + user = request.user.object + + update_telemetry_state( + request=request, + telemetry_type="api", + api="get_all_filenames", + client=client, + ) + + files_data = [] + file_objects = await FileObjectAdapters.aget_all_file_objects(user) + for file_object in file_objects: + files_data.append( + { + "file_name": file_object.file_name, + "raw_text": file_object.raw_text[:1000] if truncated else file_object.raw_text, + "updated_at": str(file_object.updated_at), + } + ) + + return Response(content=json.dumps(files_data), media_type="application/json", status_code=200) + + +@api_content.get("/file", response_model=Dict[str, str]) +@requires(["authenticated"]) +async def get_file_object( + request: Request, + file_name: str, + client: Optional[str] = None, +): + user = request.user.object + + file_object = (await FileObjectAdapters.aget_file_objects_by_name(user, file_name))[0] + if not file_object: + return Response( + content=json.dumps({"error": "File not found"}), + media_type="application/json", + status_code=404, + ) + + update_telemetry_state( + request=request, + telemetry_type="api", + api="get_file", + client=client, + ) + + return Response( + content=json.dumps( + {"id": file_object.id, "file_name": file_object.file_name, "raw_text": file_object.raw_text} + ), + media_type="application/json", + status_code=200, + ) + + @api_content.get("/{content_source}", response_model=List[str]) @requires(["authenticated"]) async def get_content_source( From f2c6ce24352ee4d0e12aa368cc79597367009803 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 18:18:15 -0800 Subject: [PATCH 2/6] Improve rendering of the file objects and sort files by updated_date --- src/interface/web/app/search/page.tsx | 222 ++++++++++++++----------- src/khoj/database/adapters/__init__.py | 2 +- 2 files changed, 130 insertions(+), 94 deletions(-) diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index bb70d8f7e..2f9e99f1b 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -54,6 +54,7 @@ import { import { Dialog, DialogContent, + DialogDescription, DialogHeader, DialogTitle, DialogTrigger, @@ -213,9 +214,6 @@ const UploadFiles: React.FC<{ onClose: () => void; setUploadedFiles: (files: string[]) => void; }> = ({ onClose, setUploadedFiles }) => { - const [syncedFiles, setSyncedFiles] = useState([]); - const [selectedFiles, setSelectedFiles] = useState([]); - const [searchQuery, setSearchQuery] = useState(""); const [isDragAndDropping, setIsDragAndDropping] = useState(false); const [warning, setWarning] = useState(null); @@ -241,10 +239,6 @@ const UploadFiles: React.FC<{ } }, [uploading]); - const filteredFiles = syncedFiles.filter((file) => - file.toLowerCase().includes(searchQuery.toLowerCase()), - ); - function handleDragOver(event: React.DragEvent) { event.preventDefault(); setIsDragAndDropping(true); @@ -281,47 +275,94 @@ const UploadFiles: React.FC<{ } return ( -
- -
- {uploading && ( - + + + + + + Build Knowledge Base + + Adding files to your Khoj knowledge base allows your AI to search through + your own documents. This helps you get personalized answers, grounded in + your own data. + + +
+ - )} -
-
-
- {isDragAndDropping ? ( -
- - Drop files to upload +
+ {uploading && ( + + )} +
+ {warning && ( +
+
+ + {warning} +
+
- ) : ( -
- - Drag and drop files here + )} + {error && ( +
+
+ + {error} +
+
)} +
+
+ {isDragAndDropping ? ( +
+ + Drop files to upload +
+ ) : ( +
+ + Drag and drop files here +
+ )} +
+
-
-
+ + ); }; @@ -598,12 +639,51 @@ export default function Search() { > -
- {file.file_name.split("/").pop()} -
+ + +
{ + setSelectedFileFullText( + null, + ); + setSelectedFile( + file.file_name, + ); + }} + > + {file.file_name + .split("/") + .pop()} +
+ + + + + {file.file_name + .split("/") + .pop()} + + + +

+ {!selectedFileFullText && ( + + )} + {selectedFileFullText} +

+ + + - - - - - {file.file_name - .split( - "/", - ) - .pop()} - - - -

- { - selectedFileFullText - } -

- - - - - -

+ +

{file.raw_text.slice(0, 100)}...

diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index a5be30863..4f101566e 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -1474,7 +1474,7 @@ async def aget_file_objects_by_names(user: KhojUser, file_names: List[str]): @staticmethod @arequire_valid_user async def aget_all_file_objects(user: KhojUser): - return await sync_to_async(list)(FileObject.objects.filter(user=user)) + return await sync_to_async(list)(FileObject.objects.filter(user=user).order_by("-updated_at")) @staticmethod @arequire_valid_user From d77984f9d1741d166851ca9305c2d760640c5616 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 18:57:38 -0800 Subject: [PATCH 3/6] Remove separate knowledge base file - consolidated in the search page --- .../app/components/appSidebar/appSidebar.tsx | 7 +- src/interface/web/app/knowledge/page.tsx | 93 ------------------- src/interface/web/app/search/page.tsx | 2 +- 3 files changed, 2 insertions(+), 100 deletions(-) delete mode 100644 src/interface/web/app/knowledge/page.tsx diff --git a/src/interface/web/app/components/appSidebar/appSidebar.tsx b/src/interface/web/app/components/appSidebar/appSidebar.tsx index 2f5f9bf8c..b660ee525 100644 --- a/src/interface/web/app/components/appSidebar/appSidebar.tsx +++ b/src/interface/web/app/components/appSidebar/appSidebar.tsx @@ -17,7 +17,7 @@ import { KhojSearchLogo, } from "../logo/khojLogo"; import { Gear } from "@phosphor-icons/react/dist/ssr"; -import { Book, Plus } from "@phosphor-icons/react"; +import { Plus } from "@phosphor-icons/react"; import { useEffect, useState } from "react"; import AllConversations from "../allConversations/allConversations"; import FooterMenu from "../navMenu/navMenu"; @@ -55,11 +55,6 @@ const items = [ url: "/settings", icon: Gear, }, - { - title: "Knowledge Base", - url: "/knowledge", - icon: Book, - }, ]; const SIDEBAR_KEYBOARD_SHORTCUT = "b"; diff --git a/src/interface/web/app/knowledge/page.tsx b/src/interface/web/app/knowledge/page.tsx deleted file mode 100644 index cf0401565..000000000 --- a/src/interface/web/app/knowledge/page.tsx +++ /dev/null @@ -1,93 +0,0 @@ -"use client"; - -import { useState, useEffect } from "react"; -import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; -import { AppSidebar } from "../components/appSidebar/appSidebar"; -import { Separator } from "@/components/ui/separator"; -import { KhojLogoType } from "../components/logo/khojLogo"; -import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card"; -import { useIsMobileWidth } from "../common/utils"; -import { InlineLoading } from "../components/loading/loading"; - -interface FileObject { - file_name: string; - raw_text: string; -} - -export default function KnowledgeBase() { - const [files, setFiles] = useState([]); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - const isMobileWidth = useIsMobileWidth(); - - useEffect(() => { - const fetchFiles = async () => { - try { - const response = await fetch("/api/content/all"); - if (!response.ok) throw new Error("Failed to fetch files"); - - const filesList = await response.json(); - if (Array.isArray(filesList)) { - setFiles(filesList.toSorted()); - } - } catch (error) { - setError("Failed to load files"); - console.error("Error fetching files:", error); - } finally { - setLoading(false); - } - }; - - fetchFiles(); - }, []); - - return ( - - - -
- - - {isMobileWidth ? ( - - - - ) : ( -

Knowledge Base

- )} -
-
-
- {loading && ( -
- -
- )} - {error &&
{error}
} - -
- {files.map((file, index) => ( - - - - {file.file_name.split("/").pop()} - - - -

- {file.raw_text.slice(0, 100)}... -

- - - ))} -
-
-
- - - ); -} diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index 2f9e99f1b..2d1dd6c59 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -276,7 +276,7 @@ const UploadFiles: React.FC<{ return ( - + From 57545c1485846aa4c5153c65278051625ee46946 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 21:06:48 -0800 Subject: [PATCH 4/6] Fix the migration script to delete orphaned fileobjects - Remove knowledge page from the sidebar - Improve speed and rendering of the documents in the search page --- .../app/components/appSidebar/appSidebar.tsx | 1 - src/interface/web/app/search/page.tsx | 406 ++++++++---------- src/interface/web/app/settings/page.tsx | 337 +-------------- .../commands/delete_orphaned_fileobjects.py | 17 +- 4 files changed, 196 insertions(+), 565 deletions(-) diff --git a/src/interface/web/app/components/appSidebar/appSidebar.tsx b/src/interface/web/app/components/appSidebar/appSidebar.tsx index b660ee525..d80314b9b 100644 --- a/src/interface/web/app/components/appSidebar/appSidebar.tsx +++ b/src/interface/web/app/components/appSidebar/appSidebar.tsx @@ -26,7 +26,6 @@ import { useIsMobileWidth } from "@/app/common/utils"; import { UserPlusIcon } from "lucide-react"; import { useAuthenticatedData } from "@/app/common/auth"; import LoginPrompt from "../loginPrompt/loginPrompt"; -import { url } from "inspector"; // Menu items. const items = [ diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index 2d1dd6c59..4eb941792 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -17,7 +17,6 @@ import { ArrowLeft, ArrowRight, FileDashed, - FileMagnifyingGlass, GithubLogo, Lightbulb, LinkSimple, @@ -26,31 +25,21 @@ import { NotionLogo, Eye, Trash, - ArrowsOutSimple, DotsThreeVertical, Waveform, Plus, + Download, + Brain, + Check, } from "@phosphor-icons/react"; import { Button } from "@/components/ui/button"; import Link from "next/link"; -import { getIconFromFilename } from "../common/iconUtils"; import { formatDateTime, useIsMobileWidth } from "../common/utils"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; import { AppSidebar } from "../components/appSidebar/appSidebar"; import { Separator } from "@/components/ui/separator"; import { KhojLogoType } from "../components/logo/khojLogo"; import { InlineLoading } from "../components/loading/loading"; -import { - AlertDialog, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, - AlertDialogCancel, - AlertDialogAction, - AlertDialogTrigger, -} from "@/components/ui/alert-dialog"; import { Dialog, DialogContent, @@ -60,16 +49,13 @@ import { DialogTrigger, } from "@/components/ui/dialog"; import { useToast } from "@/components/ui/use-toast"; -import { Scroll } from "lucide-react"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, - DropdownMenuLabel, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { uploadDataForIndexing } from "../common/chatFunctions"; -import { CommandDialog } from "@/components/ui/command"; import { Progress } from "@/components/ui/progress"; interface AdditionalData { file: string; @@ -102,25 +88,6 @@ function getNoteTypeIcon(source: string) { return ; } -const naturalLanguageSearchQueryExamples = [ - "What does the paper say about climate change?", - "Making a cappuccino at home", - "Benefits of eating mangoes", - "How to plan a wedding on a budget", - "Appointment with Dr. Makinde on 12th August", - "Class notes lecture 3 on quantum mechanics", - "Painting concepts for acrylics", - "Abstract from the paper attention is all you need", - "Climbing Everest without oxygen", - "Solving a rubik's cube in 30 seconds", - "Facts about the planet Mars", - "How to make a website using React", - "Fish at the bottom of the ocean", - "Fish farming Kenya 2021", - "How to make a cake without an oven", - "Installing a solar panel at home", -]; - interface NoteResultProps { note: SearchResult; setFocusSearchResult: (note: SearchResult) => void; @@ -132,7 +99,6 @@ function Note(props: NoteResultProps) { const fileName = isFileNameURL ? note.additional.heading : note.additional.file.split("/").pop(); - const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( @@ -153,8 +119,8 @@ function Note(props: NoteResultProps) { - - {isFileNameURL ? ( + {isFileNameURL && ( + {note.additional.file} - ) : ( -
- {fileIcon} - {note.additional.file} -
- )} - + + )} ); } @@ -179,15 +140,14 @@ function focusNote(note: SearchResult) { const fileName = isFileNameURL ? note.additional.heading : note.additional.file.split("/").pop(); - const fileIcon = getIconFromFilename(fileName || ".txt", "h-4 w-4 inline mr-2"); return ( - + {fileName} - - {isFileNameURL ? ( + {isFileNameURL && ( + {note.additional.file} - ) : ( -
- {fileIcon} - {note.additional.file} -
- )} - + + )}
{note.entry}
 @@ -211,9 +166,8 @@ function focusNote(note: SearchResult) { } const UploadFiles: React.FC<{ - onClose: () => void; setUploadedFiles: (files: string[]) => void; -}> = ({ onClose, setUploadedFiles }) => { +}> = ({ setUploadedFiles }) => { const [isDragAndDropping, setIsDragAndDropping] = useState(false); const [warning, setWarning] = useState(null); @@ -225,6 +179,11 @@ const UploadFiles: React.FC<{ useEffect(() => { if (!uploading) { setProgressValue(0); + if (!warning && !error) { + // Force close the dialog by simulating a click on the escape key + const event = new KeyboardEvent("keydown", { key: "Escape" }); + document.dispatchEvent(event); + } } if (uploading) { @@ -278,16 +237,16 @@ const UploadFiles: React.FC<{ - Build Knowledge Base + Build Your Knowledge Base - Adding files to your Khoj knowledge base allows your AI to search through - your own documents. This helps you get personalized answers, grounded in - your own data. + Add your files to supercharge Khoj's AI with your knowledge. Get instant, + personalized answers powered by your own documents and data.
)}
{isDragAndDropping ? ( @@ -379,8 +338,6 @@ export default function Search() { const [selectedFileFullText, setSelectedFileFullText] = useState(null); const [isDeleting, setIsDeleting] = useState(false); const [uploadedFiles, setUploadedFiles] = useState([]); - const [selectedFiles, setSelectedFiles] = useState([]); - const [filteredFiles, setFilteredFiles] = useState([]); const { toast } = useToast(); @@ -408,36 +365,6 @@ export default function Search() { }); } - const deleteSelected = async () => { - let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles; - - if (filesToDelete.length === 0) { - return; - } - - try { - const response = await fetch("/api/content/files", { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ files: filesToDelete }), - }); - - if (!response.ok) throw new Error("Failed to delete files"); - - // Update the syncedFiles state - setUploadedFiles((prevFiles) => - prevFiles.filter((file) => !filesToDelete.includes(file)), - ); - - // Reset selectedFiles - setSelectedFiles([]); - } catch (error) { - console.error("Error deleting files:", error); - } - }; - const fetchFiles = async () => { try { const response = await fetch("/api/content/all"); @@ -468,6 +395,18 @@ export default function Search() { } }; + const handleDownload = (fileName: string, content: string) => { + const blob = new Blob([content], { type: "text/plain" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${fileName.split("/").pop()}.txt`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); + }; + useEffect(() => { if (!searchQuery.trim()) { setSearchResults(null); @@ -547,19 +486,20 @@ export default function Search() { ) : ( -

Search

+

Search Your Knowledge Base

)}
-
-
+
+
setSearchQuery(e.currentTarget.value)} onKeyDown={(e) => e.key === "Enter" && search()} + value={searchQuery} type="search" placeholder="Search Documents" /> @@ -572,10 +512,9 @@ export default function Search() { Find
- {}} - setUploadedFiles={setUploadedFiles} - /> + {searchResults === null && ( + + )} {searchResultsLoading && (
0 && (
+ {searchResults.map((result, index) => { return ( @@ -618,145 +565,138 @@ export default function Search() {
)} - {searchResults === null && ( -
- {fileObjectsLoading && ( -
- -
- )} - {error &&
{error}
} - -
- {files.map((file, index) => ( - - - - - -
{ - setSelectedFileFullText( - null, - ); - setSelectedFile( - file.file_name, - ); - }} - > - {file.file_name - .split("/") - .pop()} -
- - - - + {!searchResultsLoading && + searchResults === null && + !searchQuery.trim() && ( +
+ {fileObjectsLoading && ( +
+ +
+ )} + {error &&
{error}
} + +
+ {files.map((file, index) => ( + + + + + +
{ + setSelectedFileFullText( + null, + ); + setSelectedFile( + file.file_name, + ); + }} + > {file.file_name .split("/") .pop()} - - - -

- {!selectedFileFullText && ( - - )} - {selectedFileFullText} -

- - -
- - - - - - - - - - - - - - Delete File - - - - Are you sure you - want to delete - this file? - - - - Cancel - - + + + + +
+ {file.file_name + .split("/") + .pop()} + +
+ + + +

+ {!selectedFileFullText && ( + + )} + { + selectedFileFullText + } +

+ + +
+ + + + + + + + + + + + + + +

+ {file.raw_text.slice(0, 100)}... +

+ + + +
+ {formatDateTime(file.updated_at)} +
+ + + ))} +
-
- )} + )} {searchResults && searchResults.length === 0 && ( diff --git a/src/interface/web/app/settings/page.tsx b/src/interface/web/app/settings/page.tsx index cd419e40e..7452e4f97 100644 --- a/src/interface/web/app/settings/page.tsx +++ b/src/interface/web/app/settings/page.tsx @@ -3,7 +3,7 @@ import styles from "./settings.module.css"; import "intl-tel-input/styles"; -import { Suspense, useEffect, useRef, useState } from "react"; +import { Suspense, useEffect, useState } from "react"; import { useToast } from "@/components/ui/use-toast"; import { useUserConfig, ModelOptions, UserConfig, SubscriptionStates } from "../common/auth"; @@ -23,14 +23,6 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Table, TableBody, TableCell, TableRow } from "@/components/ui/table"; -import { - CommandInput, - CommandList, - CommandEmpty, - CommandGroup, - CommandItem, - CommandDialog, -} from "@/components/ui/command"; import { ArrowRight, @@ -56,320 +48,20 @@ import { ArrowCircleUp, ArrowCircleDown, ArrowsClockwise, - Check, CaretDown, Waveform, + MagnifyingGlass, + Brain, } from "@phosphor-icons/react"; import Loading from "../components/loading/loading"; import IntlTelInput from "intl-tel-input/react"; -import { uploadDataForIndexing } from "../common/chatFunctions"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogContent, - AlertDialogDescription, - AlertDialogHeader, - AlertDialogTitle, -} from "@/components/ui/alert-dialog"; -import { Progress } from "@/components/ui/progress"; -import Link from "next/link"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; import { AppSidebar } from "../components/appSidebar/appSidebar"; import { Separator } from "@/components/ui/separator"; import { KhojLogoType } from "../components/logo/khojLogo"; -const ManageFilesModal: React.FC<{ onClose: () => void }> = ({ onClose }) => { - const [syncedFiles, setSyncedFiles] = useState([]); - const [selectedFiles, setSelectedFiles] = useState([]); - const [searchQuery, setSearchQuery] = useState(""); - const [isDragAndDropping, setIsDragAndDropping] = useState(false); - - const [warning, setWarning] = useState(null); - const [error, setError] = useState(null); - const [uploading, setUploading] = useState(false); - const [progressValue, setProgressValue] = useState(0); - const [uploadedFiles, setUploadedFiles] = useState([]); - const fileInputRef = useRef(null); - - useEffect(() => { - if (!uploading) { - setProgressValue(0); - } - - if (uploading) { - const interval = setInterval(() => { - setProgressValue((prev) => { - const increment = Math.floor(Math.random() * 5) + 1; // Generates a random number between 1 and 5 - const nextValue = prev + increment; - return nextValue < 100 ? nextValue : 100; // Ensures progress does not exceed 100 - }); - }, 800); - return () => clearInterval(interval); - } - }, [uploading]); - - useEffect(() => { - const fetchFiles = async () => { - try { - const response = await fetch("/api/content/computer"); - if (!response.ok) throw new Error("Failed to fetch files"); - - // Extract resonse - const syncedFiles = await response.json(); - // Validate response - if (Array.isArray(syncedFiles)) { - // Set synced files state - setSyncedFiles(syncedFiles.toSorted()); - } else { - console.error("Unexpected data format from API"); - } - } catch (error) { - console.error("Error fetching files:", error); - } - }; - - fetchFiles(); - }, [uploadedFiles]); - - const filteredFiles = syncedFiles.filter((file) => - file.toLowerCase().includes(searchQuery.toLowerCase()), - ); - - const deleteSelected = async () => { - let filesToDelete = selectedFiles.length > 0 ? selectedFiles : filteredFiles; - - if (filesToDelete.length === 0) { - return; - } - - try { - const response = await fetch("/api/content/files", { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ files: filesToDelete }), - }); - - if (!response.ok) throw new Error("Failed to delete files"); - - // Update the syncedFiles state - setSyncedFiles((prevFiles) => - prevFiles.filter((file) => !filesToDelete.includes(file)), - ); - - // Reset selectedFiles - setSelectedFiles([]); - } catch (error) { - console.error("Error deleting files:", error); - } - }; - - const deleteFile = async (filename: string) => { - try { - const response = await fetch( - `/api/content/file?filename=${encodeURIComponent(filename)}`, - { - method: "DELETE", - headers: { - "Content-Type": "application/json", - }, - }, - ); - - if (!response.ok) throw new Error("Failed to delete file"); - - // Update the syncedFiles state - setSyncedFiles((prevFiles) => prevFiles.filter((file) => file !== filename)); - - // Remove the file from selectedFiles if it's there - setSelectedFiles((prevSelected) => prevSelected.filter((file) => file !== filename)); - } catch (error) { - console.error("Error deleting file:", error); - } - }; - - function handleDragOver(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(true); - } - - function handleDragLeave(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(false); - } - - function handleDragAndDropFiles(event: React.DragEvent) { - event.preventDefault(); - setIsDragAndDropping(false); - - if (!event.dataTransfer.files) return; - - uploadFiles(event.dataTransfer.files); - } - - function openFileInput() { - if (fileInputRef && fileInputRef.current) { - fileInputRef.current.click(); - } - } - - function handleFileChange(event: React.ChangeEvent) { - if (!event.target.files) return; - - uploadFiles(event.target.files); - } - - function uploadFiles(files: FileList) { - uploadDataForIndexing(files, setWarning, setUploading, setError, setUploadedFiles); - } - - return ( - - - - - Alert - - {warning || error} - { - setWarning(null); - setError(null); - setUploading(false); - }} - > - Close - - - -
- -
- Upload files - {uploading && ( - - )} -
-
-
- {isDragAndDropping ? ( -
- - Drop files to upload -
- ) : ( -
- - Drag and drop files here -
- )} -
-
-
-
-
- -
-
- - - {syncedFiles.length === 0 ? ( -
- - No files synced -
- ) : ( -
- Could not find a good match. - - Need advanced search? Click here. - -
- )} - - - {filteredFiles.map((filename: string) => ( - { - setSelectedFiles((prev) => - prev.includes(value) - ? prev.filter((f) => f !== value) - : [...prev, value], - ); - }} - > -
-
- {selectedFiles.includes(filename) && ( - - )} - {filename} -
- -
- - ))} - - -
- -
-
- -
-
-
- - ); -}; - interface DropdownComponentProps { items: ModelOptions[]; selected: number; @@ -508,7 +200,6 @@ export default function SettingsView() { const [numberValidationState, setNumberValidationState] = useState( PhoneNumberValidationState.Verified, ); - const [isManageFilesModalOpen, setIsManageFilesModalOpen] = useState(false); const { toast } = useToast(); const isMobileWidth = useIsMobileWidth(); @@ -1066,18 +757,13 @@ export default function SettingsView() {
- {isManageFilesModalOpen && ( - setIsManageFilesModalOpen(false)} - /> - )}
Content
- - Files + + Knowledge Base {userConfig.enabled_content_source.computer && ( - Manage your synced files + Manage and search through your digital brain. diff --git a/src/khoj/database/management/commands/delete_orphaned_fileobjects.py b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py index 95cec1376..99d45c6fa 100644 --- a/src/khoj/database/management/commands/delete_orphaned_fileobjects.py +++ b/src/khoj/database/management/commands/delete_orphaned_fileobjects.py @@ -32,18 +32,25 @@ def handle(self, *args, **options): batch_size = 1000 processed = 0 - while True: - batch = orphaned_files[:batch_size] - if not batch: + while processed < total_orphaned: + # Get batch of IDs to process + batch_ids = list(orphaned_files.values_list("id", flat=True)[:batch_size]) + if not batch_ids: break if options["apply"]: - count = batch.delete()[0] + # Delete by ID to avoid slice/limit issues + count = FileObject.objects.filter(id__in=batch_ids).delete()[0] processed += count self.stdout.write(f"Deleted {processed}/{total_orphaned} orphaned FileObjects") else: - processed += len(batch) + processed += len(batch_ids) self.stdout.write(f"Would delete {processed}/{total_orphaned} orphaned FileObjects") + # Re-query to get fresh state + orphaned_files = FileObject.objects.annotate( + has_entries=Exists(Entry.objects.filter(file_object=OuterRef("pk"))) + ).filter(has_entries=False) + action = "Deleted" if options["apply"] else "Would delete" self.stdout.write(self.style.SUCCESS(f"{action} {processed} orphaned FileObjects")) From f170487338630f1c82fe2dcfc9b8571967411d5e Mon Sep 17 00:00:00 2001 From: sabaimran Date: Fri, 10 Jan 2025 21:58:17 -0800 Subject: [PATCH 5/6] Fix apostrophe in the add documents modal --- src/interface/web/app/search/page.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interface/web/app/search/page.tsx b/src/interface/web/app/search/page.tsx index 4eb941792..03a59b5ff 100644 --- a/src/interface/web/app/search/page.tsx +++ b/src/interface/web/app/search/page.tsx @@ -245,8 +245,8 @@ const UploadFiles: React.FC<{ Build Your Knowledge Base - Add your files to supercharge Khoj's AI with your knowledge. Get instant, - personalized answers powered by your own documents and data. + Add your files to supercharge Khoj`'`s AI with your knowledge. Get + instant, personalized answers powered by your own documents and data.
Date: Fri, 10 Jan 2025 22:18:44 -0800 Subject: [PATCH 6/6] Fix Obsidian style.css --- src/interface/obsidian/styles.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index 23113c906..f7c067ed3 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -858,4 +858,4 @@ img.copy-icon { 100% { transform: rotate(360deg); } -} \ No newline at end of file +}