From 4d8dfaf829a8ac792beb102a6a9dd2d17cc40073 Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Fri, 27 Sep 2024 17:33:06 +0800 Subject: [PATCH 1/3] chore:tool description --- backend/app/core/tools/googletranslate/googletranslate.py | 2 +- backend/app/core/tools/tool_manager.py | 6 +++--- web/src/app/(applayout)/tools/page.tsx | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/app/core/tools/googletranslate/googletranslate.py b/backend/app/core/tools/googletranslate/googletranslate.py index 1cfe8561..59091607 100644 --- a/backend/app/core/tools/googletranslate/googletranslate.py +++ b/backend/app/core/tools/googletranslate/googletranslate.py @@ -32,7 +32,7 @@ def google_translate_invoke(content: str, dest: str) -> str: googletranslate = StructuredTool.from_function( func=google_translate_invoke, name="Google Translate", - description="Useful for when you neet to translate text", + description="Useful for when you neet to translate.", args_schema=GoogleTranslateInput, return_direct=True, ) diff --git a/backend/app/core/tools/tool_manager.py b/backend/app/core/tools/tool_manager.py index 8d7980c7..d09a8717 100644 --- a/backend/app/core/tools/tool_manager.py +++ b/backend/app/core/tools/tool_manager.py @@ -55,17 +55,17 @@ def load_external_tools(self): external_tools = { "duckduckgo-search": ToolInfo( - description="Searches the web using DuckDuckGo", + description="Searches the web using DuckDuckGo.", tool=DuckDuckGoSearchRun(), display_name="DuckDuckGo", ), "wikipedia": ToolInfo( - description="Searches Wikipedia", + description="Searches Wikipedia.", tool=WikipediaQueryRun(api_wrapper=WikipediaAPIWrapper()), display_name="Wikipedia", ), "tavilysearch": ToolInfo( - description="tavily search useful when searching for information on the internet", + description="Tavily is useful when searching for information on the internet.", tool=TavilySearchResults(max_results=1), display_name="Tavily Search", ), diff --git a/web/src/app/(applayout)/tools/page.tsx b/web/src/app/(applayout)/tools/page.tsx index d556a03b..4cb42c7a 100644 --- a/web/src/app/(applayout)/tools/page.tsx +++ b/web/src/app/(applayout)/tools/page.tsx @@ -130,7 +130,7 @@ function Skills() { {skill.description} From a141d9cac3fa957c0446e1431dafead498ef846c Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Sat, 28 Sep 2024 08:19:18 +0800 Subject: [PATCH 2/3] feat:spark img gen and markdown update --- .env.example | 8 +- .vscode/launch.json | 8 +- .../app/core/tools/siliconflow/siliconflow.py | 5 +- backend/app/core/tools/spark/spark.py | 136 ++++++++++++++++ web/src/app/(applayout)/tools/page.tsx | 2 +- web/src/components/Icons/Tools/index.tsx | 30 ++++ web/src/components/Markdown/Markdown.tsx | 150 ++++++++++-------- web/src/components/Playground/MessageBox.tsx | 35 +++- .../components/WorkFlow/FlowVisualizer.tsx | 1 + .../WorkFlow/nodes/Tool/Properties.tsx | 2 +- .../WorkFlow/nodes/Tool/ToolNode.tsx | 2 +- .../WorkFlow/nodes/Tool/ToolsListModal.tsx | 2 +- 12 files changed, 302 insertions(+), 79 deletions(-) create mode 100644 backend/app/core/tools/spark/spark.py diff --git a/.env.example b/.env.example index 4b4f6813..7193fc38 100644 --- a/.env.example +++ b/.env.example @@ -82,4 +82,10 @@ LANGCHAIN_PROJECT=changethis RECURSION_LIMIT=25 TAVILY_API_KEY=changethisifyouwanttousetavilyserch -OPEN_WEATHER_API_KEY=changethis \ No newline at end of file +OPEN_WEATHER_API_KEY=changethis + + +# Spark get apikey from https://console.xfyun.cn/app/myapp +SPARK_APPID=changethis +SPARK_APISecret=changethis +SPARK_APIKey=changethis \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index c370c75b..5774a581 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,11 +20,11 @@ "envFile": "${workspaceFolder}/.env" }, { - "type": "chrome", + "type": "msedge", "request": "launch", - "name": "Debug Frontend: Launch Chrome against http://localhost:5173", - "url": "http://localhost:5173", - "webRoot": "${workspaceFolder}/frontend" + "name": "Debug Frontend: Launch Edges against http://localhost:3000", + "url": "http://localhost:3000", + "webRoot": "${workspaceFolder}/web" } ] } diff --git a/backend/app/core/tools/siliconflow/siliconflow.py b/backend/app/core/tools/siliconflow/siliconflow.py index 07beecf2..2d9f0547 100644 --- a/backend/app/core/tools/siliconflow/siliconflow.py +++ b/backend/app/core/tools/siliconflow/siliconflow.py @@ -39,13 +39,14 @@ def text2img( return response.json() except Exception as e: - return json.dumps(f"Openweather API Key is invalid. {e}") + return json.dumps(f"There is a error occured . {e}") siliconflow = StructuredTool.from_function( func=text2img, name="Image Generation", - description="Image Generation is a tool that can generate images from text prompts.", + description="Siliconflow Image Generation is a tool that can generate images from text prompts using the Siliconflow API.", args_schema=Text2ImageInput, return_direct=True, ) + diff --git a/backend/app/core/tools/spark/spark.py b/backend/app/core/tools/spark/spark.py new file mode 100644 index 00000000..db83e33a --- /dev/null +++ b/backend/app/core/tools/spark/spark.py @@ -0,0 +1,136 @@ +import base64 +import hashlib +import hmac +import json +from datetime import datetime +from time import mktime +from urllib.parse import urlencode +from wsgiref.handlers import format_date_time +import os +import requests +from langchain.pydantic_v1 import BaseModel, Field +from langchain.tools import StructuredTool + + +class Text2ImageInput(BaseModel): + """Input for the text2img tool.""" + + prompt: str = Field(description="the prompt for generating image ") + + +class AssembleHeaderError(Exception): + def __init__(self, msg): + self.message = msg + + +class Url: + def __init__(self, host, path, schema): + self.host = host + self.path = path + self.schema = schema + + +# calculate sha256 and encode to base64 +def sha256base64(data): + sha256 = hashlib.sha256() + sha256.update(data) + digest = base64.b64encode(sha256.digest()).decode(encoding="utf-8") + return digest + + +def parse_url(request_url): + stidx = request_url.index("://") + host = request_url[stidx + 3 :] + schema = request_url[: stidx + 3] + edidx = host.index("/") + if edidx <= 0: + raise AssembleHeaderError("invalid request url:" + request_url) + path = host[edidx:] + host = host[:edidx] + u = Url(host, path, schema) + return u + + +def assemble_ws_auth_url(request_url, method="GET", api_key="", api_secret=""): + u = parse_url(request_url) + host = u.host + path = u.path + now = datetime.now() + date = format_date_time(mktime(now.timetuple())) + signature_origin = "host: {}\ndate: {}\n{} {} HTTP/1.1".format( + host, date, method, path + ) + signature_sha = hmac.new( + api_secret.encode("utf-8"), + signature_origin.encode("utf-8"), + digestmod=hashlib.sha256, + ).digest() + signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8") + authorization_origin = f'api_key="{api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha}"' + + authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode( + encoding="utf-8" + ) + values = {"host": host, "date": date, "authorization": authorization} + + return request_url + "?" + urlencode(values) + + +def get_body(appid, text): + body = { + "header": {"app_id": appid, "uid": "123456789"}, + "parameter": { + "chat": {"domain": "general", "temperature": 0.5, "max_tokens": 4096} + }, + "payload": {"message": {"text": [{"role": "user", "content": text}]}}, + } + return body + + +def spark_response(text, appid, apisecret, apikey): + host = "http://spark-api.cn-huabei-1.xf-yun.com/v2.1/tti" + url = assemble_ws_auth_url( + host, method="POST", api_key=apikey, api_secret=apisecret + ) + content = get_body(appid, text) + try: + response = requests.post( + url, json=content, headers={"content-type": "application/json"} + ).text + return response + except Exception as e: + return json.dumps(f"There is a error occured . {e}") + + +def img_generation(prompt): + response = spark_response( + text=prompt, + appid=os.environ.get("SPARK_APPID"), + apisecret=os.environ.get("SPARK_APISECRET"), + apikey=os.environ.get("SPARK_APIKEY"), + ) + try: + data = json.loads(response) + code = data["header"]["code"] + if code != 0: + return f"error: {code}, {data}" + else: + text = data["payload"]["choices"]["text"] + image_content = text[0] + image_base = image_content["content"] + bs64data = "data:image/jpeg;base64," + image_base + + return bs64data + # return aaa + + except Exception as e: + return json.dumps(f"There is a error occured . {e}") + + +spark = StructuredTool.from_function( + func=img_generation, + name="Spark Image Generation", + description="Spark Image Generation is a tool that can generate images from text prompts using the Spark API.", + args_schema=Text2ImageInput, + return_direct=True, +) diff --git a/web/src/app/(applayout)/tools/page.tsx b/web/src/app/(applayout)/tools/page.tsx index 4cb42c7a..557adab6 100644 --- a/web/src/app/(applayout)/tools/page.tsx +++ b/web/src/app/(applayout)/tools/page.tsx @@ -122,7 +122,7 @@ function Skills() { w="8" tools_name={skill .display_name!.toLowerCase() - .replace(" ", "_")} + .replace(/ /g, "_")} /> {skill.display_name} diff --git a/web/src/components/Icons/Tools/index.tsx b/web/src/components/Icons/Tools/index.tsx index 6b458294..bc93675e 100644 --- a/web/src/components/Icons/Tools/index.tsx +++ b/web/src/components/Icons/Tools/index.tsx @@ -1,4 +1,5 @@ import { Icon, type IconProps, createIcon } from "@chakra-ui/icons"; +import { SiliconFlowIcon } from "../models"; const OpenWeather = createIcon({ displayName: "OpenWeather", @@ -330,6 +331,33 @@ const calculator = createIcon({ ), }); + +const spark = createIcon({ + displayName: "spark", + viewBox: "0 0 24 24", + path: ( + + + + + + ), +}); const iconMap: { [key: string]: React.FC } = { open_weather: OpenWeather, google_translate: GoogleTranslate, @@ -337,6 +365,8 @@ const iconMap: { [key: string]: React.FC } = { Wikipedia: Wikipedia, tavily_search: tavilysearch, math_calculator: calculator, + image_generation: SiliconFlowIcon, + spark_image_generation: spark, }; const DefaultIcon = Wikipedia; diff --git a/web/src/components/Markdown/Markdown.tsx b/web/src/components/Markdown/Markdown.tsx index 5c7b7e85..e60b07d2 100644 --- a/web/src/components/Markdown/Markdown.tsx +++ b/web/src/components/Markdown/Markdown.tsx @@ -14,83 +14,105 @@ import CopyButton from "./CopyButton"; // } // ); -const Markdown = ({ content }: { content: string }) => { +const Markdown = ({ content }: { content: any }) => { const textColor = useColorModeValue("ui.dark", "ui.white"); const secBgColor = useColorModeValue("ui.secondary", "ui.darkSlate"); return ( <> {/* */} - ( - - {children} - - ), - code: ({ node, className, children, ...props }) => { - const match = /language-(\w+)/.exec(className || ""); - if (match?.length) { - const id = v4(); - return ( - - - - - - {match[0]} - - - - + {content && !content.startsWith("data:image") ? ( + ( + + {children} + + ), + code: ({ node, className, children, ...props }) => { + const match = /language-(\w+)/.exec(className || ""); + if (match?.length) { + const id = v4(); + return ( + + + + + {match[0]} + + + + - {children} + + {children} + + ); + } + return ( + + {children} ); - } - return ( - - {children} - - ); - }, - }} - className="prose prose-zinc max-w-2xl dark:prose-invert" - > - {content} - + }, + img: ({ alt, src, title }) => { + return ( + + + {alt && ( + + {alt} + + )} + + ); + }, + }} + className="prose prose-zinc max-w-2xl dark:prose-invert" + > + {content} + + ) : ( + + )} ); }; diff --git a/web/src/components/Playground/MessageBox.tsx b/web/src/components/Playground/MessageBox.tsx index 57fcfd46..87956e86 100644 --- a/web/src/components/Playground/MessageBox.tsx +++ b/web/src/components/Playground/MessageBox.tsx @@ -9,6 +9,7 @@ import { Flex, Icon, IconButton, + Image, Input, InputGroup, InputRightElement, @@ -86,16 +87,34 @@ const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => { } return ; // 如果 type 不是 'human', 'tools', 'ai'，则不显示任何图标 }; + + function isImag(content: any): boolean { + if (content) { + // 检查是否为字符串并以 "data:image/" 开头 + if (typeof content === "string" && content.startsWith("data:image/")) { + return true; + } + + // 检查是否为对象并且包含 'images' 键 + if (typeof content === "object" && content.images) { + const images = content.images; + // 确保 'images' 是一个非空数组，并检查第一个元素是否包含 'url' 键 + if (Array.isArray(images) && images.length > 0 && "url" in images[0]) { + return true; + } + } + } + return false; + } + return ( { - {/* {parsedOutput} */} - + {isImag(content) ? ( + img + ) : ( + + )} ); diff --git a/web/src/components/WorkFlow/FlowVisualizer.tsx b/web/src/components/WorkFlow/FlowVisualizer.tsx index c38eaf35..16277f08 100644 --- a/web/src/components/WorkFlow/FlowVisualizer.tsx +++ b/web/src/components/WorkFlow/FlowVisualizer.tsx @@ -22,6 +22,7 @@ import { Box, Button, CloseButton, + Image, Kbd, Menu, MenuButton, diff --git a/web/src/components/WorkFlow/nodes/Tool/Properties.tsx b/web/src/components/WorkFlow/nodes/Tool/Properties.tsx index d8679cf5..5861d38a 100644 --- a/web/src/components/WorkFlow/nodes/Tool/Properties.tsx +++ b/web/src/components/WorkFlow/nodes/Tool/Properties.tsx @@ -52,7 +52,7 @@ const ToolNodeProperties: React.FC = ({ - + {tool} diff --git a/web/src/components/WorkFlow/nodes/Tool/ToolNode.tsx b/web/src/components/WorkFlow/nodes/Tool/ToolNode.tsx index c582aa6f..be7733f0 100644 --- a/web/src/components/WorkFlow/nodes/Tool/ToolNode.tsx +++ b/web/src/components/WorkFlow/nodes/Tool/ToolNode.tsx @@ -22,7 +22,7 @@ const ToolNode: React.FC = (props) => { - + {tool} diff --git a/web/src/components/WorkFlow/nodes/Tool/ToolsListModal.tsx b/web/src/components/WorkFlow/nodes/Tool/ToolsListModal.tsx index fe5bac91..acb7342b 100644 --- a/web/src/components/WorkFlow/nodes/Tool/ToolsListModal.tsx +++ b/web/src/components/WorkFlow/nodes/Tool/ToolsListModal.tsx @@ -62,7 +62,7 @@ const ToolsList: React.FC = ({ {skill.display_name} From c0fac567cc61deb521fc2a4cf297273d2684f7c7 Mon Sep 17 00:00:00 2001 From: Onelevenvy Date: Sat, 28 Sep 2024 08:45:47 +0800 Subject: [PATCH 3/3] feat:change the img output format --- .../app/core/tools/siliconflow/siliconflow.py | 2 +- backend/app/core/tools/spark/spark.py | 50 +++++++++++-------- web/src/app/(applayout)/dashboard/page.tsx | 8 ++- web/src/components/Playground/MessageBox.tsx | 24 ++++----- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/backend/app/core/tools/siliconflow/siliconflow.py b/backend/app/core/tools/siliconflow/siliconflow.py index 2d9f0547..04ce08bf 100644 --- a/backend/app/core/tools/siliconflow/siliconflow.py +++ b/backend/app/core/tools/siliconflow/siliconflow.py @@ -36,7 +36,7 @@ def text2img( response = requests.post(url, json=payload, headers=headers) - return response.json() + return response.json()["images"][0]["url"] except Exception as e: return json.dumps(f"There is a error occured . {e}") diff --git a/backend/app/core/tools/spark/spark.py b/backend/app/core/tools/spark/spark.py index db83e33a..d8572bb4 100644 --- a/backend/app/core/tools/spark/spark.py +++ b/backend/app/core/tools/spark/spark.py @@ -103,28 +103,34 @@ def spark_response(text, appid, apisecret, apikey): def img_generation(prompt): - response = spark_response( - text=prompt, - appid=os.environ.get("SPARK_APPID"), - apisecret=os.environ.get("SPARK_APISECRET"), - apikey=os.environ.get("SPARK_APIKEY"), - ) - try: - data = json.loads(response) - code = data["header"]["code"] - if code != 0: - return f"error: {code}, {data}" - else: - text = data["payload"]["choices"]["text"] - image_content = text[0] - image_base = image_content["content"] - bs64data = "data:image/jpeg;base64," + image_base - - return bs64data - # return aaa - - except Exception as e: - return json.dumps(f"There is a error occured . {e}") + appid = os.environ.get("SPARK_APPID", "") + apisecret = os.environ.get("SPARK_APISECRET", "") + apikey = os.environ.get("SPARK_APIKEY", "") + if not appid or not apisecret or not apikey: + return "api key is not set or not correct" + else: + response = spark_response( + text=prompt, + appid=appid, + apisecret=apisecret, + apikey=apikey, + ) + try: + data = json.loads(response) + code = data["header"]["code"] + if code != 0: + return f"error: {code}, {data}" + else: + text = data["payload"]["choices"]["text"] + image_content = text[0] + image_base = image_content["content"] + bs64data = "data:image/jpeg;base64," + image_base + + return bs64data + # return aaa + + except Exception as e: + return json.dumps(f"There is a error occured . {e}") spark = StructuredTool.from_function( diff --git a/web/src/app/(applayout)/dashboard/page.tsx b/web/src/app/(applayout)/dashboard/page.tsx index 70579eda..00d5010f 100644 --- a/web/src/app/(applayout)/dashboard/page.tsx +++ b/web/src/app/(applayout)/dashboard/page.tsx @@ -1,6 +1,6 @@ "use client"; import useAuth from "@/hooks/useAuth"; -import { Box, Container, Text } from "@chakra-ui/react"; +import { Box, Container, Image, Text } from "@chakra-ui/react"; import React from "react"; function Dashboard() { @@ -14,6 +14,12 @@ function Dashboard() { Hi, {currentUser?.full_name || currentUser?.email} 👋🏼 Welcome back, nice to see you again! + aaa diff --git a/web/src/components/Playground/MessageBox.tsx b/web/src/components/Playground/MessageBox.tsx index 87956e86..4e95ffc9 100644 --- a/web/src/components/Playground/MessageBox.tsx +++ b/web/src/components/Playground/MessageBox.tsx @@ -89,21 +89,15 @@ const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => { }; function isImag(content: any): boolean { - if (content) { - // 检查是否为字符串并以 "data:image/" 开头 - if (typeof content === "string" && content.startsWith("data:image/")) { - return true; - } - - // 检查是否为对象并且包含 'images' 键 - if (typeof content === "object" && content.images) { - const images = content.images; - // 确保 'images' 是一个非空数组，并检查第一个元素是否包含 'url' 键 - if (Array.isArray(images) && images.length > 0 && "url" in images[0]) { - return true; - } - } + if (typeof content === "string") { + // 检查是否为 data URL 或有效的图像 URL + return ( + content.startsWith("data:image/") || + content.startsWith("http://") || + content.startsWith("https://") + ); } + return false; } @@ -277,7 +271,7 @@ const MessageBox = ({ message, onResume, isPlayground }: MessageBoxProps) => { {isImag(content) ? (