diff --git a/.gitignore b/.gitignore index 35e600ee..5398405e 100644 --- a/.gitignore +++ b/.gitignore @@ -149,6 +149,6 @@ Pipfile.lock pyrightconfig.json # Project specific ignores (add your own here) -data/ +src/planning_permission/data/ output/ *.xlsx diff --git a/data/assistant-templates/example-multi.json b/data/assistant-templates/example-multi.json new file mode 100644 index 00000000..65414d56 --- /dev/null +++ b/data/assistant-templates/example-multi.json @@ -0,0 +1,39 @@ +{ + "id": "multi_example", + "name": "Multi-stream Example", + "description": "This is an example assistant that showcases how an assistant can run through multiple different internal prompts before answering the user.", + "streams": [ + { + "name": "First Stream", + "settings": { + "model": "gpt-4o" + }, + "messages": [ + { + "role": "system", + "content": "Make this text sound more fancy and verbose." + }, + { + "role": "user", + "content": "{query}" + } + ] + }, + { + "name": "Second Stream", + "settings": { + "model": "gpt-4o" + }, + "messages": [ + { + "role": "system", + "content": "Repeat back any text verbatim and count the number of words" + }, + { + "role": "user", + "content": "{last_input}" + } + ] + } + ] +} diff --git a/data/assistant-templates/example-rag.json b/data/assistant-templates/example-rag.json new file mode 100644 index 00000000..79e4a522 --- /dev/null +++ b/data/assistant-templates/example-rag.json @@ -0,0 +1,24 @@ +{ + "id": "rag_example", + "name": "RAG (document) Example", + "files_collection_id": "Upload files in the Assistant Creation UI", + "description": "This is an example assistant that showcases how to setup an assistant for answering questions about uploaded documents. \n\nMake sure to upload documents first.", + "streams": [ + { + "name": "RagStream", + "settings": { + "model": "gpt-4o" + }, + "messages": [ + { + "role": "system", + "content": "Explain what this is and repeat back an excerpt of it." + }, + { + "role": "user", + "content": "{rag_results}" + } + ] + } + ] +} diff --git a/data/assistant-templates/example.json b/data/assistant-templates/example.json new file mode 100644 index 00000000..11525a83 --- /dev/null +++ b/data/assistant-templates/example.json @@ -0,0 +1,28 @@ +{ + "id": "example", + "name": "Example Assistant", + "description": "This is an *example assistant* for testing purposes.", + "sample_questions": [ + "Hur kan jag kolla upp tidstabellen för bussar?", + "Vilka evenemang händer i sommar?", + "Var kan jag parkera?" + ], + "streams": [ + { + "name": "ChatStream", + "settings": { + "model": "gpt-4o" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful AI assistant that helps people with answering questions related to municipality and Helsingborg City. The questions are going to be asked in Swedish. Your response must always be in Swedish." + }, + { + "role": "user", + "content": "{query}" + } + ] + } + ] +} \ No newline at end of file diff --git a/data/assistant-templates/planning-permission.json b/data/assistant-templates/planning-permission.json new file mode 100644 index 00000000..966b8333 --- /dev/null +++ b/data/assistant-templates/planning-permission.json @@ -0,0 +1,28 @@ +{ + "id": "planning", + "name": "Planning Assistant", + "files_collection_id": "Upload files in the Assistant Creation UI", + "description": "Den här assistenten är expert på att svara på frågor angående bygglov i Helsingborg Stad.", + "streams": [ + { + "name": "ChatStream", + "settings": { + "model": "gpt-4o" + }, + "messages": [ + { + "role": "system", + "content": "You are a helpful AI assistant that helps people with answering questions about planning permission.
If you can't find the answer in the search result below, just say (in Swedish) \"Tyvärr kan jag inte svara på det.\" Don't try to make up an answer.
If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.
The questions are going to be asked in Swedish. Your response must always be in Swedish." + }, + { + "role": "user", + "content": "{query}" + }, + { + "role": "user", + "content": "Here are the results of the search:\n\n {rag_results}" + } + ] + } + ] +} diff --git a/fai-rag-app/fai-backend/fai_backend/llm/impl/__init__.py b/fai-rag-app/fai-backend/fai_backend/assistant/__init__.py similarity index 100% rename from fai-rag-app/fai-backend/fai_backend/llm/impl/__init__.py rename to fai-rag-app/fai-backend/fai_backend/assistant/__init__.py diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/assistant.py b/fai-rag-app/fai-backend/fai_backend/assistant/assistant.py new file mode 100644 index 00000000..711ca115 --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/assistant.py @@ -0,0 +1,53 @@ +from typing import Any + +from langstream import Stream + +from fai_backend.assistant.models import AssistantTemplate +from fai_backend.assistant.protocol import ILLMProtocol, IAssistantStreamProtocol +from fai_backend.llm.service import create_rag_stream + + +class AssistantLLM(ILLMProtocol): + def __init__(self, template: AssistantTemplate, base_stream: IAssistantStreamProtocol): + self.template = template + self.base_stream = base_stream + self.vars = {} + + async def create(self) -> Stream[str, Any]: + all_stream_producers = [self.base_stream.create_stream(stream_def, lambda: self.vars) for + stream_def in self.template.streams] + + chained_stream: Stream[str, Any] | None = None + + if self.template.files_collection_id: + chained_stream = await self._create_rag_stream() + + for stream_producer in all_stream_producers: + chained_stream = chained_stream.and_then(await stream_producer) \ + if chained_stream is not None \ + else await stream_producer + + def preprocess(initial_query: str): + self.vars.update({"query": initial_query}) + return initial_query + + return ( + Stream[str, str]('preprocess', preprocess) + .and_then(chained_stream) + ) + + async def _create_rag_stream(self) -> Stream[str, str]: + async def run_rag_stream(initial_query: list[str]): + stream = await create_rag_stream(initial_query[0], self.template.files_collection_id) + async for r in stream(initial_query[0]): + yield r + + def rag_postprocess(in_data: Any): + results = in_data[0]['results'] + self.vars.update({'rag_results': results}) + return self.vars['query'] + + return ( + Stream('RAGStream', run_rag_stream) + .and_then(rag_postprocess) + ) diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/assistant_openai.py b/fai-rag-app/fai-backend/fai_backend/assistant/assistant_openai.py new file mode 100644 index 00000000..f2edca21 --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/assistant_openai.py @@ -0,0 +1,40 @@ +from typing import Callable, Any, Iterable + +from langstream import Stream +from langstream.contrib import OpenAIChatStream, OpenAIChatDelta, OpenAIChatMessage + +from fai_backend.assistant.models import LLMStreamDef, LLMStreamMessage +from fai_backend.assistant.protocol import IAssistantStreamProtocol + + +class OpenAIAssistantStream(IAssistantStreamProtocol): + async def create_stream(self, stream_def: LLMStreamDef, get_vars: Callable[[], dict]) -> Stream[str, str]: + return OpenAIChatStream[str, OpenAIChatDelta]( + stream_def.name, + lambda in_data: self._to_openai_messages(in_data, stream_def.messages, get_vars), + model=stream_def.settings.model, + temperature=getattr(stream_def, 'settings.temperature', 0), + functions=getattr(stream_def, 'functions', None), + function_call=getattr(stream_def, 'function_call', None), + ).map(lambda delta: delta.content) + + @staticmethod + def _to_openai_messages( + in_data: Any, + messages: list[LLMStreamMessage], + get_vars: Callable[[], dict] + ) -> Iterable[OpenAIChatMessage]: + def parse_in_data(data: Any): + if isinstance(data, list): + return "".join([parse_in_data(c) for c in data]) + return str(data) + + in_data_as_str = parse_in_data(in_data) + + input_vars = get_vars() + input_vars['last_input'] = in_data_as_str + + return [OpenAIChatMessage( + content=message.content.format(**input_vars), + role=message.role + ) for message in messages] diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/models.py b/fai-rag-app/fai-backend/fai_backend/assistant/models.py new file mode 100644 index 00000000..0baa93f4 --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/models.py @@ -0,0 +1,37 @@ +from datetime import datetime +from typing import Optional, Dict, List, Any, Literal, Union + +from pydantic import BaseModel + + +class LLMStreamMessage(BaseModel): + role: Literal["system", "user", "assistant", "function"] + content: str + + +class LLMStreamSettings(BaseModel): + model: str + temperature: Optional[float] = 0 + functions: Optional[List[Dict[str, Any]]] = None + function_call: Optional[Union[Literal["none", "auto"], Dict[str, Any]]] = None + + +class LLMStreamDef(BaseModel): + name: str + settings: LLMStreamSettings + messages: Optional[List[LLMStreamMessage]] = None + + +class AssistantTemplate(BaseModel): + id: str + name: str + files_collection_id: Optional[str] = None + description: Optional[str] = None + sample_questions: list[str] = [] + streams: List[LLMStreamDef] + + +class LLMClientChatMessage(BaseModel): + date: datetime + source: str | None = None + content: str | None = None diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/protocol.py b/fai-rag-app/fai-backend/fai_backend/assistant/protocol.py new file mode 100644 index 00000000..f61bbb09 --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/protocol.py @@ -0,0 +1,19 @@ +from typing import Protocol, Callable + +from langstream import Stream + +from fai_backend.assistant.models import LLMStreamDef + + +class ILLMProtocol(Protocol): + async def create(self) -> Stream[str, str]: + """ + Create a Stream that takes a str (generally a question) and returns + a stream of tokens (strings) of the response given by the LLM. + """ + ... + + +class IAssistantStreamProtocol(Protocol): + async def create_stream(self, stream_def: LLMStreamDef, get_vars: Callable[[], dict]) -> Stream[str, str]: + ... diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/routes.py b/fai-rag-app/fai-backend/fai_backend/assistant/routes.py new file mode 100644 index 00000000..2c8b7faa --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/routes.py @@ -0,0 +1,89 @@ +from fastapi import APIRouter, Depends, Security +from langstream import join_final_output + +from fai_backend.assistant.models import AssistantTemplate +from fai_backend.assistant.service import AssistantFactory +from fai_backend.dependencies import get_authenticated_user +from fai_backend.logger.route_class import APIRouter as LoggingAPIRouter +from fai_backend.projects.dependencies import list_projects_request, get_project_request, get_project_service, \ + update_project_request +from fai_backend.projects.schema import ProjectResponse, ProjectUpdateRequest +from fai_backend.projects.service import ProjectService + +router = APIRouter( + prefix='/api', + tags=['Assistant'], + route_class=LoggingAPIRouter, + dependencies=[], +) + + +@router.get( + '/assistant/{project_id}/ask/{assistant_id}', + summary="Ask an assistant a question.", + dependencies=[Security(get_authenticated_user)] +) +async def ask_assistant( + project_id: str, + assistant_id: str, + question: str, + projects: list[ProjectResponse] = Depends(list_projects_request), +): + print(f"Assistant: {project_id}/{assistant_id} - {question}") + factory = AssistantFactory([a for p in projects for a in p.assistants if p.id == project_id]) + assistant = factory.create_assistant_stream(assistant_id) + stream = await assistant.create() + return await join_final_output(stream(question)) + + +@router.get( + '/assistant/{project_id}/template', + summary="Get assistant templates.", + response_model=list[AssistantTemplate], + dependencies=[Security(get_authenticated_user)] +) +async def get_template( + project_id: str, + projects: list[ProjectResponse] = Depends(list_projects_request) +): + return [a for p in projects for a in p.assistants if p.id == project_id] + + +@router.post( + '/assistant/{project_id}/template', + summary="Create/update assistant template.", + response_model=AssistantTemplate, + dependencies=[Security(get_authenticated_user)] +) +async def create_template( + template: AssistantTemplate, + existing_project: ProjectResponse = Depends(get_project_request), + project_service: ProjectService = Depends(get_project_service), +): + existing_project.assistants = [a for a in existing_project.assistants if a.id != template.id] + existing_project.assistants.append(template) + await update_project_request( + body=ProjectUpdateRequest(**existing_project.model_dump()), + existing_project=existing_project, + project_service=project_service) + return template + + +@router.delete( + '/assistant/{project_id}/template/{assistant_id}', + summary="Delete assistant template.", + response_model=AssistantTemplate | None, + dependencies=[Security(get_authenticated_user)] +) +async def delete_template( + assistant_id: str, + existing_project: ProjectResponse = Depends(get_project_request), + project_service: ProjectService = Depends(get_project_service) +): + assistant = next((a for a in existing_project.assistants if a.id == assistant_id), None) + existing_project.assistants = [a for a in existing_project.assistants if a.id != assistant_id] + await update_project_request( + body=ProjectUpdateRequest(**existing_project.model_dump()), + existing_project=existing_project, + project_service=project_service) + return assistant diff --git a/fai-rag-app/fai-backend/fai_backend/assistant/service.py b/fai-rag-app/fai-backend/fai_backend/assistant/service.py new file mode 100644 index 00000000..5efa9fe1 --- /dev/null +++ b/fai-rag-app/fai-backend/fai_backend/assistant/service.py @@ -0,0 +1,20 @@ +from fai_backend.assistant.assistant import AssistantLLM +from fai_backend.assistant.assistant_openai import OpenAIAssistantStream +from fai_backend.assistant.models import AssistantTemplate +from fai_backend.config import settings +from fai_backend.assistant.protocol import IAssistantStreamProtocol + + +class AssistantFactory: + def __init__(self, assistant_templates: list[AssistantTemplate]): + self.assistant_templates = assistant_templates + + def create_assistant_stream(self, assistant_id: str, backend: str = settings.LLM_BACKEND) -> AssistantLLM: + assistant = next(a for a in self.assistant_templates if a.id == assistant_id) + return AssistantLLM(assistant, self._get_stream_constructor(backend)) + + @staticmethod + def _get_stream_constructor(backend: str) -> IAssistantStreamProtocol: + return { + 'openai': lambda: OpenAIAssistantStream(), + }[backend]() diff --git a/fai-rag-app/fai-backend/fai_backend/framework/components.py b/fai-rag-app/fai-backend/fai_backend/framework/components.py index 83841a31..710b190a 100644 --- a/fai-rag-app/fai-backend/fai_backend/framework/components.py +++ b/fai-rag-app/fai-backend/fai_backend/framework/components.py @@ -263,9 +263,15 @@ class SSEDocument(BaseModel): name: str +class Assistant(BaseModel): + id: str + name: str + project: str + + class SSEChat(UIComponent): type: Literal['SSEChat'] = 'SSEChat' - documents: list[SSEDocument] + assistants: list[Assistant] endpoint: str diff --git a/fai-rag-app/fai-backend/fai_backend/llm/impl/openai.py b/fai-rag-app/fai-backend/fai_backend/llm/impl/openai.py deleted file mode 100644 index b6ebadc5..00000000 --- a/fai-rag-app/fai-backend/fai_backend/llm/impl/openai.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Iterable, Any - -from langstream import Stream -from langstream.contrib import OpenAIChatStream, OpenAIChatDelta, OpenAIChatMessage - -from fai_backend.chat.stream import create_chat_prompt -from fai_backend.chat.template import PromptTemplate -from fai_backend.llm.protocol import ILLMStreamProtocol -from fai_backend.llm.models import LLMDataPacket - - -class OpenAILLM(ILLMStreamProtocol): - - def __init__(self, template: PromptTemplate): - self.template = template - - async def create(self) -> Stream[str, LLMDataPacket]: - def messages(in_data: Any) -> Iterable[OpenAIChatMessage]: - prompt = create_chat_prompt({ - "name": self.template.name, - "messages": self.template.messages, - "settings": self.template.settings, - }) - prompt.format_prompt(self.template.input_map_fn(in_data)) - return prompt.to_messages() - - return OpenAIChatStream[str, OpenAIChatDelta]( - "RecipeStream", - messages, - model="gpt-4", - temperature=0, - ).map(lambda delta: LLMDataPacket(content=delta.content, user_friendly=True)) diff --git a/fai-rag-app/fai-backend/fai_backend/llm/impl/parrot.py b/fai-rag-app/fai-backend/fai_backend/llm/impl/parrot.py deleted file mode 100644 index 45989cbd..00000000 --- a/fai-rag-app/fai-backend/fai_backend/llm/impl/parrot.py +++ /dev/null @@ -1,40 +0,0 @@ -import asyncio -from random import uniform -from typing import Any - -from langstream import Stream - -from fai_backend.llm.protocol import ILLMStreamProtocol -from fai_backend.llm.models import LLMDataPacket - - -class ParrotLLM(ILLMStreamProtocol): - """ - Parrot (mock) LLM protocol reference implementation. - - Parrot will respond with the same message as its input, with a random delay between tokens (words). - """ - - def __init__(self, min_delay: float = 0.1, max_delay: float = 1.0): - self.min_delay = min_delay - self.max_delay = max_delay - - async def to_generator(self, input_message: str | Any): - if not isinstance(input_message, str): - if isinstance(input_message, list) and "query" in input_message[0]: - input_message = input_message[0]["query"] - else: - yield "squawk?" - return - - import re - parts = re.findall(r'\S+\s*', input_message) - for part in parts: - yield part - await asyncio.sleep(uniform(self.min_delay, self.max_delay)) - - async def create(self) -> Stream[str, LLMDataPacket]: - return Stream[str, str]( - "ParrotStream", - self.to_generator - ).map(lambda delta: LLMDataPacket(content=delta, user_friendly=True)) diff --git a/fai-rag-app/fai-backend/fai_backend/llm/impl/rag_wrapper.py b/fai-rag-app/fai-backend/fai_backend/llm/impl/rag_wrapper.py deleted file mode 100644 index 33dafb39..00000000 --- a/fai-rag-app/fai-backend/fai_backend/llm/impl/rag_wrapper.py +++ /dev/null @@ -1,26 +0,0 @@ -from langstream import Stream - -from fai_backend.llm.protocol import ILLMStreamProtocol -from fai_backend.llm.models import LLMDataPacket -from fai_backend.llm.service import create_rag_stream - - -class RAGWrapper(ILLMStreamProtocol): - """ - Wraps an underlying Stream with RAG capabilities. - - The underlying stream will be supplied with document extracts in plaintext - from the given collection along with the original question. - """ - - def __init__(self, input_query: str, base_llm: ILLMStreamProtocol, rag_collection_name: str): - self.input_query = input_query - self.rag_collection_name = rag_collection_name - self.base_llm = base_llm - - async def create(self) -> Stream[str, LLMDataPacket]: - rag_stream = await create_rag_stream(self.input_query, self.rag_collection_name) - base_stream = await self.base_llm.create() - - return (rag_stream - .and_then(base_stream)) diff --git a/fai-rag-app/fai-backend/fai_backend/llm/models.py b/fai-rag-app/fai-backend/fai_backend/llm/models.py deleted file mode 100644 index 60a54f39..00000000 --- a/fai-rag-app/fai-backend/fai_backend/llm/models.py +++ /dev/null @@ -1,16 +0,0 @@ -import dataclasses -from datetime import datetime - -from pydantic import BaseModel - - -class LLMMessage(BaseModel): - date: datetime - source: str | None = None - content: str | None = None - - -@dataclasses.dataclass -class LLMDataPacket: - content: str - user_friendly: bool = False diff --git a/fai-rag-app/fai-backend/fai_backend/llm/protocol.py b/fai-rag-app/fai-backend/fai_backend/llm/protocol.py deleted file mode 100644 index d4eaa9bf..00000000 --- a/fai-rag-app/fai-backend/fai_backend/llm/protocol.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Protocol - -from langstream import Stream - -from fai_backend.llm.models import LLMDataPacket - - -class ILLMStreamProtocol(Protocol): - async def create(self) -> Stream[str, LLMDataPacket]: - """ - Create a Stream that takes a str (generally a question) and returns - a stream of tokens (strings) of the response given by the LLM. - """ - ... diff --git a/fai-rag-app/fai-backend/fai_backend/llm/service.py b/fai-rag-app/fai-backend/fai_backend/llm/service.py index 91d2a701..1919259f 100644 --- a/fai-rag-app/fai-backend/fai_backend/llm/service.py +++ b/fai-rag-app/fai-backend/fai_backend/llm/service.py @@ -4,13 +4,9 @@ from langstream.contrib import OpenAIChatStream, OpenAIChatMessage, OpenAIChatDelta from fai_backend.chat.stream import create_chat_stream_from_prompt -from fai_backend.chat.template import CHAT_PROMPT_TEMPLATE_ARGS, SCORING_PROMPT_TEMPLATE_ARGS, chatPromptTemplate -from fai_backend.config import settings -from fai_backend.llm.impl.openai import OpenAILLM -from fai_backend.llm.impl.parrot import ParrotLLM -from fai_backend.llm.protocol import ILLMStreamProtocol -from fai_backend.vector.service import VectorService +from fai_backend.chat.template import CHAT_PROMPT_TEMPLATE_ARGS, SCORING_PROMPT_TEMPLATE_ARGS from fai_backend.vector.factory import vector_db +from fai_backend.vector.service import VectorService SYSTEM_TEMPLATE = "You are a helpful AI assistant that helps people with answering questions about planning " "permission.
If you can't find the answer in the search result below, just say (in Swedish) " @@ -20,15 +16,6 @@ "asked in Swedish. Your response must always be in Swedish." -class LLMFactory: - @staticmethod - def get(backend: str = settings.LLM_BACKEND) -> ILLMStreamProtocol: - return { - "parrot": lambda: ParrotLLM(), - "openai": lambda: OpenAILLM(template=chatPromptTemplate), - }[backend]() - - async def query_vector(vector_service, collection_name, query, n_results=10): vector_result = await vector_service.query_from_collection( collection_name=collection_name, diff --git a/fai-rag-app/fai-backend/fai_backend/main.py b/fai-rag-app/fai-backend/fai_backend/main.py index b9f0aca5..4a4df0da 100644 --- a/fai-rag-app/fai-backend/fai_backend/main.py +++ b/fai-rag-app/fai-backend/fai_backend/main.py @@ -6,20 +6,22 @@ from sse_starlette import EventSourceResponse, ServerSentEvent from starlette.responses import HTMLResponse, RedirectResponse +from fai_backend.assistant.routes import router as templates_router +from fai_backend.assistant.service import AssistantFactory from fai_backend.auth.router import router as auth_router from fai_backend.config import settings from fai_backend.dependencies import get_project_user from fai_backend.documents.routes import router as documents_router from fai_backend.framework.frontend import get_frontend_environment -from fai_backend.llm.impl.rag_wrapper import RAGWrapper -from fai_backend.llm.models import LLMMessage, LLMDataPacket -from fai_backend.llm.protocol import ILLMStreamProtocol -from fai_backend.llm.serializer.impl.base64 import Base64Serializer -from fai_backend.llm.service import LLMFactory +from fai_backend.assistant.models import LLMClientChatMessage +from fai_backend.assistant.protocol import ILLMProtocol +from fai_backend.serializer.impl.base64 import Base64Serializer from fai_backend.logger.console import console from fai_backend.middleware import remove_trailing_slash from fai_backend.phrase import phrase as _, set_language +from fai_backend.projects.dependencies import list_projects_request from fai_backend.projects.router import router as projects_router +from fai_backend.projects.schema import ProjectResponse from fai_backend.qaf.routes import router as qaf_router from fai_backend.schema import ProjectUser from fai_backend.setup import setup_db, setup_project @@ -42,6 +44,7 @@ async def lifespan(_app: FastAPI): app.include_router(qaf_router) app.include_router(documents_router) app.include_router(vector_router) +app.include_router(templates_router) app.middleware('http')(remove_trailing_slash) @@ -57,7 +60,7 @@ async def lifespan(_app: FastAPI): frontend.configure(app) -async def event_source_llm_generator(question: str, llm: ILLMStreamProtocol): +async def event_source_llm_generator(question: str, llm: ILLMProtocol): serializer = Base64Serializer() stream = await llm.create() @@ -65,18 +68,18 @@ async def event_source_llm_generator(question: str, llm: ILLMStreamProtocol): async def generator(): async for output in stream(question): - if isinstance(output.data, LLMDataPacket) and output.data.user_friendly: + if output.final: yield ServerSentEvent( event="message", - data=serializer.serialize(LLMMessage( + data=serializer.serialize(LLMClientChatMessage( date=datetime.now(), source="Chat AI", - content=output.data.content + content=output.data )), ) yield ServerSentEvent( event="message_end", - data=serializer.serialize(LLMMessage( + data=serializer.serialize(LLMClientChatMessage( date=datetime.now(), )) ) @@ -84,13 +87,18 @@ async def generator(): return EventSourceResponse(generator()) -@app.get('/api/chat-stream') -async def chat_stream(question: str, document: str | None = None): - print(f"/chat-stream {document=} {question=}") - llm = LLMFactory.get() - if document: - llm = RAGWrapper(question, llm, document) - return await event_source_llm_generator(question, llm) +@app.get('/api/assistant-stream/{project}/{assistant}') +async def assistant_stream( + project: str, + assistant: str, + question: str, + projects: list[ProjectResponse] = Depends(list_projects_request) +): + print(f"/assistant-stream {project=} {assistant=} {question=}") + + factory = AssistantFactory([a for p in projects for a in p.assistants if p.id == project]) + assistant_instance = factory.create_assistant_stream(assistant) + return await event_source_llm_generator(question, assistant_instance) @app.get('/health', include_in_schema=False) diff --git a/fai-rag-app/fai-backend/fai_backend/projects/schema.py b/fai-rag-app/fai-backend/fai_backend/projects/schema.py index 04dad55e..0c3ce442 100644 --- a/fai-rag-app/fai-backend/fai_backend/projects/schema.py +++ b/fai-rag-app/fai-backend/fai_backend/projects/schema.py @@ -1,6 +1,6 @@ - from pydantic import BaseModel, EmailStr, Field, SecretStr, field_serializer +from fai_backend.assistant.models import AssistantTemplate from fai_backend.schema import Timestamp @@ -20,6 +20,7 @@ class Project(BaseModel): creator: EmailStr description: str = '' timestamp: Timestamp = Timestamp() + assistants: list[AssistantTemplate] = Field(default_factory=list) members: list[ProjectMember] = Field(..., default_factory=list) roles: dict[str, ProjectRole] = Field(..., default_factory=dict) secrets: dict[str, SecretStr] = Field(..., default_factory=dict) @@ -43,6 +44,7 @@ class ProjectResponse(BaseModel): creator: EmailStr description: str = '' timestamp: Timestamp + assistants: list[AssistantTemplate] = Field(default_factory=list) members: list[ProjectMember] = Field(..., default_factory=list) roles: dict[str, ProjectRole] = Field(..., default_factory=dict) secrets: dict[str, SecretStr] = Field(..., default_factory=dict) diff --git a/fai-rag-app/fai-backend/fai_backend/qaf/routes.py b/fai-rag-app/fai-backend/fai_backend/qaf/routes.py index d03caa9a..8cd7d02c 100644 --- a/fai-rag-app/fai-backend/fai_backend/qaf/routes.py +++ b/fai-rag-app/fai-backend/fai_backend/qaf/routes.py @@ -10,14 +10,14 @@ get_project_user, try_get_authenticated_user, ) -from fai_backend.files.dependecies import get_file_upload_service -from fai_backend.files.service import FileUploadService from fai_backend.framework import components as c from fai_backend.framework import events as e from fai_backend.framework.components import AnyUI from fai_backend.llm.service import ask_llm_question, ask_llm_raq_question from fai_backend.logger.route_class import APIRouter as LoggingAPIRouter from fai_backend.phrase import phrase as _ +from fai_backend.projects.dependencies import list_projects_request +from fai_backend.projects.schema import ProjectResponse from fai_backend.qaf.dependencies import ( list_my_questions_request, my_question_details_request, @@ -186,20 +186,23 @@ def questions_index_view( @router.get('/chat', response_model=list, response_model_exclude_none=True) def chat_index_view( - file_service: FileUploadService = Depends(get_file_upload_service), authenticated_user: User | None = Depends(get_project_user), view=Depends(get_page_template_for_logged_in_users), + projects: list[ProjectResponse] = Depends(list_projects_request), ) -> list: if not authenticated_user: return [c.FireEvent(event=e.GoToEvent(url='/login'))] - documents = [{"id": doc.collection, "name": doc.file_name} for doc in - file_service.list_files(authenticated_user.project_id)] + assistants = [c.Assistant( + id=a.id, + name=a.name, + project=p.id + ) for p in projects for a in p.assistants] return view( [c.SSEChat( - documents=documents, - endpoint='/api/chat-stream' + assistants=assistants, + endpoint='/api/assistant-stream' )], _('chat', 'Chat'), ) diff --git a/fai-rag-app/fai-backend/fai_backend/llm/serializer/__init__.py b/fai-rag-app/fai-backend/fai_backend/serializer/__init__.py similarity index 100% rename from fai-rag-app/fai-backend/fai_backend/llm/serializer/__init__.py rename to fai-rag-app/fai-backend/fai_backend/serializer/__init__.py diff --git a/fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/__init__.py b/fai-rag-app/fai-backend/fai_backend/serializer/impl/__init__.py similarity index 100% rename from fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/__init__.py rename to fai-rag-app/fai-backend/fai_backend/serializer/impl/__init__.py diff --git a/fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/base64.py b/fai-rag-app/fai-backend/fai_backend/serializer/impl/base64.py similarity index 82% rename from fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/base64.py rename to fai-rag-app/fai-backend/fai_backend/serializer/impl/base64.py index 09a384b7..63877088 100644 --- a/fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/base64.py +++ b/fai-rag-app/fai-backend/fai_backend/serializer/impl/base64.py @@ -2,7 +2,7 @@ from pydantic import BaseModel -from fai_backend.llm.serializer.protocol import ISerializer +from fai_backend.serializer.protocol import ISerializer class Base64Serializer(ISerializer): diff --git a/fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/json.py b/fai-rag-app/fai-backend/fai_backend/serializer/impl/json.py similarity index 75% rename from fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/json.py rename to fai-rag-app/fai-backend/fai_backend/serializer/impl/json.py index b8e0bba5..ff811f71 100644 --- a/fai-rag-app/fai-backend/fai_backend/llm/serializer/impl/json.py +++ b/fai-rag-app/fai-backend/fai_backend/serializer/impl/json.py @@ -1,6 +1,6 @@ from pydantic import BaseModel -from fai_backend.llm.serializer.protocol import ISerializer +from fai_backend.serializer.protocol import ISerializer class JSONSerializer(ISerializer): diff --git a/fai-rag-app/fai-backend/fai_backend/llm/serializer/protocol.py b/fai-rag-app/fai-backend/fai_backend/serializer/protocol.py similarity index 100% rename from fai-rag-app/fai-backend/fai_backend/llm/serializer/protocol.py rename to fai-rag-app/fai-backend/fai_backend/serializer/protocol.py diff --git a/fai-rag-app/fai-backend/fai_backend/setup.py b/fai-rag-app/fai-backend/fai_backend/setup.py index 92049c16..9071c98c 100644 --- a/fai-rag-app/fai-backend/fai_backend/setup.py +++ b/fai-rag-app/fai-backend/fai_backend/setup.py @@ -1,10 +1,15 @@ +import json +import os + from beanie import init_beanie from fastapi import FastAPI from fastapi.routing import APIRoute from motor.motor_asyncio import AsyncIOMotorClient from fai_backend.config import settings +from fai_backend.projects.schema import ProjectMember, ProjectRole from fai_backend.repositories import ConversationDocument, PinCodeModel, ProjectModel, projects_repo +from fai_backend.assistant.models import AssistantTemplate, LLMStreamSettings, LLMStreamDef, LLMStreamMessage def use_route_names_as_operation_ids(app: FastAPI) -> None: @@ -24,46 +29,51 @@ def use_route_names_as_operation_ids(app: FastAPI) -> None: async def setup_project(): projects = await projects_repo.list() if not projects or len(projects) == 0: - permissions = [ - 'can_edit_project_users', - 'can_edit_project_roles', - 'can_edit_project_secrets', - 'can_ask_questions', - 'can_review_answers', - 'can_edit_questions_and_answers', - ] - - def create_permissions(permissions_list: list[str], value: bool = False): + def create_permissions(value: bool): + permissions = [ + 'can_edit_project_users', + 'can_edit_project_roles', + 'can_edit_project_secrets', + 'can_ask_questions', + 'can_review_answers', + 'can_edit_questions_and_answers', + ] return { x: y for x, y in zip( permissions, - [value] * len(permissions_list), + [value] * len(permissions), ) } + assistant_templates_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../../data/assistant-templates')) + initial_assistants = \ + [ + AssistantTemplate(**json.loads(open(os.path.join(assistant_templates_dir, f)).read())) + for f in os.listdir(assistant_templates_dir) + if f.endswith('.json') + ] if os.path.exists(assistant_templates_dir) else [] + initial_project = await projects_repo.create( - ProjectModel.model_validate( - { - 'name': settings.APP_PROJECT_NAME, - 'creator': settings.APP_ADMIN_EMAIL, - 'description': 'Initial project created on startup', - 'members': [{'email': settings.APP_ADMIN_EMAIL, 'role': 'admin'}], - 'roles': { - 'admin': {'permissions': create_permissions(permissions, True)}, - 'manager': { - 'permissions': create_permissions(permissions, False) - }, - 'reviewer': { - 'permissions': create_permissions(permissions, False) - }, - 'tester': { - 'permissions': create_permissions(permissions, False) - }, - }, - 'secrets': {'openai_api_key': 'sk-123'}, - 'meta': {}, - } + ProjectModel( + name='Project', + creator=settings.APP_ADMIN_EMAIL, + assistants=initial_assistants, + members=[ + ProjectMember( + email=settings.APP_ADMIN_EMAIL, + role='admin' + ) + ], + roles={ + 'admin': ProjectRole(permissions=create_permissions(True)), + 'manager': ProjectRole(permissions=create_permissions(False)), + 'reviewer': ProjectRole(permissions=create_permissions(False)), + 'tester': ProjectRole(permissions=create_permissions(False)) + }, + secrets={'openai_api_key': 'sk-123'}, + meta={} ) ) diff --git a/fai-rag-app/fai-frontend/src/lib/components/SSEChat.svelte b/fai-rag-app/fai-frontend/src/lib/components/SSEChat.svelte index b67afc01..89d17408 100644 --- a/fai-rag-app/fai-frontend/src/lib/components/SSEChat.svelte +++ b/fai-rag-app/fai-frontend/src/lib/components/SSEChat.svelte @@ -1,210 +1,192 @@
- - - -
- - -
- {#each messages as message (message.id)} - - {:else} -
-

Ställ en fråga för att börja

- {#if documents.length > 0} -
Här kan du ställa direkta frågor angående de dokument - {#each documents.filter((d) => d.id === selectedDocument) as document} - {document.name} - {/each} du har laddat upp. - -
- {:else} -
Inga dokument tillgängliga
- {/if} -
- {/each} - -