From 86fa99bdb781a25be044b72f3c2828c91cb28983 Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Fri, 27 Sep 2024 11:20:23 +0800 Subject: [PATCH 1/3] feat: init_graph update to gen the correct graph --- backend/app/core/workflow/init_graph.py | 42 +++++++++++++++++++------ backend/app/core/workflow/node.py | 2 +- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/backend/app/core/workflow/init_graph.py b/backend/app/core/workflow/init_graph.py index 576b0c3b..34ab116f 100644 --- a/backend/app/core/workflow/init_graph.py +++ b/backend/app/core/workflow/init_graph.py @@ -2,12 +2,11 @@ from functools import lru_cache from typing import Any, Dict, Set -from langchain.pydantic_v1 import BaseModel from langchain.tools import BaseTool from langchain_core.messages import AIMessage, AnyMessage from langchain_core.runnables import RunnableLambda from langgraph.checkpoint.base import BaseCheckpointSaver -from langgraph.graph import END, StateGraph +from langgraph.graph import END, START, StateGraph from langgraph.graph.graph import CompiledGraph from langgraph.prebuilt import ToolNode @@ -200,23 +199,45 @@ def initialize_graph( source_node = next(node for node in nodes if node["id"] == edge["source"]) target_node = next(node for node in nodes if node["id"] == edge["target"]) + if source_node["type"] == "start": + if edge["type"] == "default": + graph_builder.add_edge(START, edge["target"]) + else: + raise ValueError("Start node can only have normal edge.") if source_node["type"] == "llm": if target_node["type"] == "tool": - conditional_edges[source_node["id"]]["call_tools"][ - target_node["id"] - ] = target_node["id"] + if edge["type"] == "default": + graph_builder.add_edge(edge["source"], edge["target"]) + else: + conditional_edges[source_node["id"]]["call_tools"][ + target_node["id"] + ] = target_node["id"] elif target_node["type"] == "end": - conditional_edges[source_node["id"]]["default"][END] = END + if edge["type"] == "default": + graph_builder.add_edge(edge["source"], END) + else: + conditional_edges[source_node["id"]]["default"][END] = END + elif target_node["type"] == "llm": + if edge["type"] == "default": + graph_builder.add_edge(edge["source"], edge["target"]) + else: + conditional_edges[source_node["id"]]["default"][ + target_node["id"] + ] = target_node["id"] else: - conditional_edges[source_node["id"]]["default"][ - target_node["id"] - ] = target_node["id"] + if edge["type"] == "default": + graph_builder.add_edge(edge["source"], edge["target"]) + else: + conditional_edges[source_node["id"]]["default"][ + target_node["id"] + ] = target_node["id"] elif source_node["type"] == "tool" and target_node["type"] == "llm": # Tool to LLM edge graph_builder.add_edge(edge["source"], edge["target"]) # Add conditional edges + for llm_id, conditions in conditional_edges.items(): edges_dict = { "default": next(iter(conditions["default"].values()), END), @@ -224,7 +245,8 @@ def initialize_graph( } if conditions["call_human"]: edges_dict["call_human"] = next(iter(conditions["call_human"].values())) - graph_builder.add_conditional_edges(llm_id, should_continue, edges_dict) + if edges_dict != {"default": END}: + graph_builder.add_conditional_edges(llm_id, should_continue, edges_dict) # Set entry point graph_builder.set_entry_point(metadata["entry_point"]) diff --git a/backend/app/core/workflow/node.py b/backend/app/core/workflow/node.py index fdc7f33f..b074fc6e 100644 --- a/backend/app/core/workflow/node.py +++ b/backend/app/core/workflow/node.py @@ -253,7 +253,7 @@ async def work(self, state: TeamState, config: RunnableConfig) -> ReturnTeamStat "If you are unable to perform the task, that's OK, you can ask human for help, or just say that you are unable to perform the task." "Execute what you can to make progress. " "And your role is:" + self.system_prompt + "\n" - "And your name is:" + self.agent_name + "\n" + "And your name is:" + self.agent_name + "please remember your name\n" "Stay true to your role and use your tools if necessary.\n\n", ), ( From d4af0e3f79f650fa958715094351b707ad4c9443 Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Fri, 27 Sep 2024 11:35:27 +0800 Subject: [PATCH 2/3] fix:rename open-weather to Open Weather --- web/src/components/WorkFlow/nodes/nodeConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/components/WorkFlow/nodes/nodeConfig.ts b/web/src/components/WorkFlow/nodes/nodeConfig.ts index 8635e405..ba2f982f 100644 --- a/web/src/components/WorkFlow/nodes/nodeConfig.ts +++ b/web/src/components/WorkFlow/nodes/nodeConfig.ts @@ -63,7 +63,7 @@ export const nodeConfig: Record = { targets: ["left", "right"], }, initialData: { - tools: ["open-weather"], + tools: ["Open Weather"], }, }, questionClassifier: { From 6190bacbea3a9fc7662ea0a5c8461ef0863ede96 Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Fri, 27 Sep 2024 14:51:49 +0800 Subject: [PATCH 3/3] feat: add text2img tool and backend frondend support --- backend/app/core/db.py | 8 +-- .../app/core/tools/siliconflow/siliconflow.py | 51 +++++++++++++++++++ backend/app/initial_data.py | 2 +- web/src/components/Playground/ChatMain.tsx | 1 + web/src/components/Playground/MessageBox.tsx | 13 +++-- 5 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 backend/app/core/tools/siliconflow/siliconflow.py diff --git a/backend/app/core/db.py b/backend/app/core/db.py index 5a1c4a56..5c449715 100644 --- a/backend/app/core/db.py +++ b/backend/app/core/db.py @@ -74,7 +74,7 @@ def init_db(session: Session) -> None: description=skill_info.description, managed=True, owner_id=user.id, - display_name=skill_info.display_name + display_name=skill_info.display_name, ) session.add(new_skill) # Prepare new skill for addition to the database @@ -99,11 +99,7 @@ def init_modelprovider_model_db(session: Session) -> None: (1, 'Ollama', 'fakeurl', 'fakeapikey', 'string', 'string fake'), (2, 'Siliconflow', 'fakeurl', 'fakeapikey', 'string', 'siliconflow'), (3, 'zhipuai', 'https://open.bigmodel.cn/api/paas/v4', 'fakeapikey', 'zhipuai', '智谱AI') - ON CONFLICT (id) DO UPDATE - SET base_url = EXCLUDED.base_url, - api_key = EXCLUDED.api_key, - icon = EXCLUDED.icon, - description = EXCLUDED.description; + ON CONFLICT (id) DO NOTHING; """ # Insert Models data diff --git a/backend/app/core/tools/siliconflow/siliconflow.py b/backend/app/core/tools/siliconflow/siliconflow.py new file mode 100644 index 00000000..07beecf2 --- /dev/null +++ b/backend/app/core/tools/siliconflow/siliconflow.py @@ -0,0 +1,51 @@ +import requests +import json +import requests +from langchain.pydantic_v1 import BaseModel, Field +from langchain.tools import StructuredTool + + +class Text2ImageInput(BaseModel): + """Input for the text2img tool.""" + + prompt: str = Field(description="the prompt for generating image ") + + +def text2img( + prompt: str, +): + """ + invoke tools + """ + + try: + # request URL + url = "https://api.siliconflow.cn/v1/image/generations" + + payload = { + # "model": "black-forest-labs/FLUX.1-schnell", + "model": "stabilityai/stable-diffusion-3-medium", + "prompt": prompt, + "image_size": "1024x1024", + } + headers = { + "accept": "application/json", + "content-type": "application/json", + "authorization": "Bearer sk-uaxgsvfwwwpeuguzhsjpqigwopyhblsiesbptxnuxaoefqrb", + } + + response = requests.post(url, json=payload, headers=headers) + + return response.json() + + except Exception as e: + return json.dumps(f"Openweather API Key is invalid. {e}") + + +siliconflow = StructuredTool.from_function( + func=text2img, + name="Image Generation", + description="Image Generation is a tool that can generate images from text prompts.", + args_schema=Text2ImageInput, + return_direct=True, +) diff --git a/backend/app/initial_data.py b/backend/app/initial_data.py index 3e9f30df..dc8a90dd 100644 --- a/backend/app/initial_data.py +++ b/backend/app/initial_data.py @@ -15,7 +15,7 @@ def init() -> None: with Session(engine) as session: init_db(session) - # init_modelprovider_model_db(session) + init_modelprovider_model_db(session) def main() -> None: diff --git a/web/src/components/Playground/ChatMain.tsx b/web/src/components/Playground/ChatMain.tsx index 70aefef5..a3684eff 100644 --- a/web/src/components/Playground/ChatMain.tsx +++ b/web/src/components/Playground/ChatMain.tsx @@ -359,6 +359,7 @@ const ChatMain = ({ isPlayground }: { isPlayground?: boolean }) => { key={index} message={message} onResume={onResumeHandler} + isPlayground = {isPlayground} /> ))} diff --git a/web/src/components/Playground/MessageBox.tsx b/web/src/components/Playground/MessageBox.tsx index 790e5ab9..57fcfd46 100644 --- a/web/src/components/Playground/MessageBox.tsx +++ b/web/src/components/Playground/MessageBox.tsx @@ -39,9 +39,10 @@ import Markdown from "../Markdown/Markdown"; interface MessageBoxProps { message: ChatResponse; onResume: (decision: InterruptDecision, toolMessage: string | null) => void; + isPlayground?: boolean; } -const MessageBox = ({ message, onResume }: MessageBoxProps) => { +const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => { const { type, name, next, content, tool_calls, tool_output, documents } = message; const [decision, setDecision] = useState(null); @@ -89,10 +90,12 @@ const MessageBox = ({ message, onResume }: MessageBoxProps) => {