diff --git a/docs/agents/audioAgent.md b/docs/agents/audioAgent.md new file mode 100644 index 0000000000..c09638d2ad --- /dev/null +++ b/docs/agents/audioAgent.md @@ -0,0 +1,30 @@ + +# Text to Speech Agent + +This agent voices response messages in Russian and English. + +**Action class:** + +`action_text_to_speech` + +**Parameters:** + +1. `messageAddr` -- class element `concept_message`. + +**Libraries used:** + +* [responsiveVoice](https://responsivevoice.org/) - для озвучивания сообщений. + +**Workflow:** + +* The agent searches the knowledge base for an sc-link with the message text through the nrel_sc_text_translation relationship; +* Next, it looks for the language class to which this sc-link belongs; +* The agent then voices the message text according to the language. + +**Example of an input structure:** + + + +**Agent implementation language:** + +TypeScript diff --git a/docs/agents/audioAgent.ru.md b/docs/agents/audioAgent.ru.md new file mode 100644 index 0000000000..d7f27a8b5b --- /dev/null +++ b/docs/agents/audioAgent.ru.md @@ -0,0 +1,30 @@ +# Агент преобразования текста в речь + +Данный агент выполняет озвучивание ответных сообщений на русском и английском языках. + +**Класс действий:** + +`action_text_to_speech` + +**Параметры:** + +1. `messageAddr` -- элемент класса `concept_message`. + +**Используемые библиотеки:** + +* [responsiveVoice](https://responsivevoice.org/) - для озвучивания сообщений. + +**Ход работы агента:** + +* Агент ищет в базе знаний sc-ссылку с текстом сообщения через отношение nrel_sc_text_translation; +* Далее он ищет класс языка,к которому принадлежит данная sc-ссылка; +* Затем агент озвучивает текст сообщения в соответствии с языком. + +**Пример входной конструкции:** + + + +**Язык реализации агента:** + +TypeScript + diff --git a/docs/agents/images/audioAgentInput.png b/docs/agents/images/audioAgentInput.png new file mode 100644 index 0000000000..df926c95d2 Binary files /dev/null and b/docs/agents/images/audioAgentInput.png differ diff --git a/interface/public/index.html b/interface/public/index.html index 034973aa3b..7e9b054d66 100644 --- a/interface/public/index.html +++ b/interface/public/index.html @@ -17,12 +17,12 @@ -
+ diff --git a/interface/server/server.js b/interface/server/server.js index 8ecbc01faf..37b45cd689 100644 --- a/interface/server/server.js +++ b/interface/server/server.js @@ -1,3 +1,9 @@ +const express = require('express'); + +const app = express(); + +app.use(express.json()); + require('dotenv').config(); const express = require('express'); @@ -79,9 +85,9 @@ app.use(express.static(path.join(__dirname, '../build'))); app.engine('html', require('ejs').renderFile); app.set('view engine', 'html'); app.set('views', path.resolve(__dirname, '../build')); - +app.use(express.json()); app.get('*', (req, res) => { res.render('index'); }); -server.listen(PORT, () => console.log(`Listen on port ${PORT}`)); +server.listen(PORT, () => console.log(`Listen on port ${PORT}`)); \ No newline at end of file diff --git a/interface/src/App.tsx b/interface/src/App.tsx index bdf294590f..af2d80f41b 100644 --- a/interface/src/App.tsx +++ b/interface/src/App.tsx @@ -13,6 +13,7 @@ const { Header, Content, Footer } = Layout; import { HeaderPanel } from "@components/Header"; import { FooterPanel } from "@components/Footer"; +import { audioAgent } from "@agents/audioAgent" const Demo = loadingComponent(lazy(() => import('@pages/Demo'))); const About = loadingComponent(lazy(() => import('@pages/About'))); @@ -88,9 +89,21 @@ export const App = () => { } } } + async function registerAgents() { + const actionInitiated = "action_initiated"; + const keynodes = [ + { id: actionInitiated, type: ScType.NodeConstNoRole }, + ]; + + const keynodesAddrs = await client.resolveKeynodes(keynodes); + + const eventParams = new ScEventParams(keynodesAddrs[actionInitiated], ScEventType.AddOutgoingEdge, audioAgent); + await client.eventsCreate([eventParams]); + } useEffect(() => { fetchColorValue(); + registerAgents(); }, []); const headerStyles = { diff --git a/interface/src/api/sc/agents/audioAgent.ts b/interface/src/api/sc/agents/audioAgent.ts new file mode 100644 index 0000000000..e3b2ea73bb --- /dev/null +++ b/interface/src/api/sc/agents/audioAgent.ts @@ -0,0 +1,196 @@ +import { client } from '@api/sc'; +import { ScAddr, ScConstruction, ScLinkContent, ScTemplate, ScType, ScLinkContentType } from 'ts-sc-client'; +import { makeAgent } from '@api/sc/agents/makeAgent'; +declare var responsiveVoice: any; +const nrelScTextTranslation = 'nrel_sc_text_translation'; +const langEn = 'lang_en'; +const langRu = 'lang_ru'; +const action = 'action'; +const actionInitiated = 'action_initiated'; +const actionText2Speech = 'action_text_to_speech'; +const rrel1 = 'rrel_1'; +const actionFinished = 'action_finished'; + +const baseKeynodes = [ + { id: action, type: ScType.NodeConstClass }, + { id: actionInitiated, type: ScType.NodeConstClass }, + { id: actionText2Speech, type: ScType.NodeConstClass }, + { id: rrel1, type: ScType.NodeConstRole }, + { id: actionFinished, type: ScType.NodeConstClass }, + { id: nrelScTextTranslation, type: ScType.NodeConstNoRole}, + { id: langEn, type: ScType.NodeConstClass }, + { id: langRu, type: ScType.NodeConstClass }, +]; + +const describeAgent = async ( + message: ScAddr, + keynodes: Record, +) => { + const actionNodeAlias = '_action_node'; + const template = new ScTemplate(); + template.triple(keynodes[action], ScType.EdgeAccessVarPosPerm, [ScType.NodeVar, actionNodeAlias]); + template.triple(keynodes[actionText2Speech], ScType.EdgeAccessVarPosPerm, actionNodeAlias); + template.tripleWithRelation( + actionNodeAlias, + ScType.EdgeAccessVarPosPerm, + message, + ScType.EdgeAccessVarPosPerm, + keynodes[rrel1], + ); + const generationResult = await client.templateGenerate(template, {}); + if (generationResult && generationResult.size > 0) { + let actionNode = generationResult.get(actionNodeAlias); + return actionNode; + } + return null; +}; + +export const callText2SpeechAgent = async (message: ScAddr) => { + const keynodes = await client.resolveKeynodes(baseKeynodes); + const actionNode = await describeAgent(message, keynodes); + if (actionNode == null) return; + const construction = new ScConstruction(); + construction.createEdge(ScType.EdgeAccessConstPosPerm, keynodes[actionInitiated], actionNode); + client.createElements(construction); +}; + +export const check = async(action: ScAddr,keynodes: Record) => { + const checkTemplate = new ScTemplate(); + checkTemplate.triple( + keynodes[actionText2Speech], + ScType.EdgeAccessVarPosPerm, + action, + ); + const checkResult = await client.templateSearch(checkTemplate); + if (checkResult.length==0) + { + console.log("No text to speech action"); + return false; + + } + console.log("Agent called"); + return true; +} + +export const getLinkNode= async(action: ScAddr,keynodes: Record) => { + const arg = '_arg'; + const translation='_translation' + const link="_link"; + const template = new ScTemplate(); + template.triple( + keynodes[actionText2Speech], + ScType.EdgeAccessVarPosPerm, + action, + ) + template.tripleWithRelation( + action, + ScType.EdgeAccessVarPosPerm, + [ScType.NodeVar, arg], + ScType.EdgeAccessVarPosPerm, + keynodes[rrel1], + ); + template.tripleWithRelation( + [ScType.NodeVar, translation], + ScType.EdgeDCommonVar, + arg, + ScType.EdgeAccessVarPosPerm, + keynodes[nrelScTextTranslation], + ) + template.triple(translation,ScType.EdgeAccessVarPosPerm,[ScType.LinkVar, link]); + const result=await client.templateSearch(template); + if (!result.length) { + console.log("Construction is undefined") + return false; + } + const linkNode = result[0].get(link); + return linkNode; +} + +export const getText= async(linkNode: ScAddr,keynodes: Record) => { + const resultText = await client.getLinkContents([linkNode]); + const text = resultText[0].data as string; + return text.replace(/<[^>]*>/g, '').replace(/\.{3}|[.;"]/g, '');} + + +export const getLang= async(linkNode: ScAddr,keynodes: Record) => { + const enTemplate=new ScTemplate(); + enTemplate.triple(keynodes[langEn],ScType.EdgeAccessVarPosPerm,linkNode); + const enResult=await client.templateSearch(enTemplate); + const ruTemplate=new ScTemplate(); + ruTemplate.triple(keynodes[langRu],ScType.EdgeAccessVarPosPerm,linkNode); + const ruResult=await client.templateSearch(ruTemplate); + if (ruResult.length!=0) + { + return "Russian Female" as string; + } + else if (enResult.length!=0) + { + return "UK English Female" as string; + } + else { + console.log("Language is undefined") + return false; + } +} + +export const speak= async(text: string,lang: string) => { + if(responsiveVoice.voiceSupport()) { + responsiveVoice.speak(text,lang); + } else { + console.log("Your browser doesn't support TTS"); + } +} + + +export const audioAgent = async(subscribedAddr: ScAddr, foundEgde: ScAddr, action: ScAddr) => { + const actionText2Speech = "action_text_to_speech"; + const nrelScTextTranslation = 'nrel_sc_text_translation'; + const langEn = 'lang_en'; + const langRu = 'lang_ru'; + const rrel1 = 'rrel_1'; + const actionFinished = 'action_finished'; + const baseKeynodes = [ + { id: actionText2Speech, type: ScType.NodeConstNoRole }, + { id: rrel1, type: ScType.NodeConstRole }, + { id: nrelScTextTranslation, type: ScType.NodeConstNoRole}, + { id: langEn, type: ScType.NodeConstClass }, + { id: langRu, type: ScType.NodeConstClass }, + { id: actionFinished, type: ScType.NodeConstClass }, + ]; + const keynodes = await client.resolveKeynodes(baseKeynodes); + if(await check(action,keynodes)===false) + { + return; + } + const linkNode=await getLinkNode(action,keynodes); + if (linkNode===false) { + const construction = new ScConstruction(); + construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action); + client.createElements(construction); + return; + } + const lang=await getLang(linkNode,keynodes); + const text=await getText(linkNode,keynodes); + if (lang===false) { + const construction = new ScConstruction(); + construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action); + client.createElements(construction); + return; + } + console.log(text); + speak(text,lang); + const construction = new ScConstruction(); + construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action); + client.createElements(construction); + return; +}; + + + + + + + + + + diff --git a/interface/src/assets/icon/TextToSpeech-icon.svg b/interface/src/assets/icon/TextToSpeech-icon.svg new file mode 100644 index 0000000000..c734ac7fb8 --- /dev/null +++ b/interface/src/assets/icon/TextToSpeech-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/interface/src/assets/icon/pause-icon.svg b/interface/src/assets/icon/pause-icon.svg new file mode 100644 index 0000000000..c7821f9454 --- /dev/null +++ b/interface/src/assets/icon/pause-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/interface/src/assets/icon/textToSpeech-icon.svg b/interface/src/assets/icon/textToSpeech-icon.svg new file mode 100644 index 0000000000..93b094d67f --- /dev/null +++ b/interface/src/assets/icon/textToSpeech-icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/interface/src/components/Chat/Message/Message.tsx b/interface/src/components/Chat/Message/Message.tsx index adcb3a0a8b..d84f54d4de 100644 --- a/interface/src/components/Chat/Message/Message.tsx +++ b/interface/src/components/Chat/Message/Message.tsx @@ -1,4 +1,4 @@ -import { PropsWithChildren } from 'react'; +import { PropsWithChildren, useState } from 'react'; import { WrapperMessage, Time, @@ -10,12 +10,19 @@ import { Info, TextWrapper, WrapperLoadingIcon, + SpeakButton, } from './styled'; import { ReactComponent as LoadingIcon } from '@assets/icon/messageLoading.svg'; import { ReactComponent as ErrorIcon } from '@assets/icon/errorMessage-icon.svg'; import { ReactComponent as RebootIcon } from '@assets/icon/rebootErrorMessage-icon.svg'; +import { ReactComponent as TextToSpeechIcon } from '@assets/icon/TextToSpeech-icon.svg'; +import { ReactComponent as PauseIcon } from '@assets/icon/pause-icon.svg'; import { useLanguage } from '@hooks/useLanguage'; +import { ScAddr, ScEventParams, ScEventType, ScType ,ScConstruction} from 'ts-sc-client'; +import { callText2SpeechAgent } from '@api/sc/agents/audioAgent'; +declare var responsiveVoice: any; interface IProps { + addr:ScAddr; isLoading?: boolean; isLeft?: boolean; isError?: boolean; @@ -30,6 +37,7 @@ const textSend = { }; export const Message = ({ + addr, isLeft = false, isError = false, isLoading = false, @@ -37,8 +45,27 @@ export const Message = ({ children, onClick, }: PropsWithChildren) => { - const hookLanguage = useLanguage(); - + const hookLanguage = useLanguage(); + const [isTalking, setIsTalking] = useState(false); + const talking = async () => { + if (isTalking) { + setIsTalking(false); + responsiveVoice.cancel(); + }else{ + setIsTalking(true); + await callText2SpeechAgent(addr); + const checkIfSpeaking = setInterval(() => { + if(!responsiveVoice.isPlaying()) + { + setIsTalking(false); + clearInterval(checkIfSpeaking); + } + },2200); + } + + }; + + return ( <> <> @@ -53,6 +80,14 @@ export const Message = ({ {isLoading && !isLeft && } {isError && !isLeft && } + {isLeft && ( + + {isTalking ? ( + + ) : ( + + )} + )} diff --git a/interface/src/components/Chat/Message/styled.ts b/interface/src/components/Chat/Message/styled.ts index d9465e3a36..efcf902e95 100644 --- a/interface/src/components/Chat/Message/styled.ts +++ b/interface/src/components/Chat/Message/styled.ts @@ -50,6 +50,22 @@ export const WrapperLoadingIcon = styled.div` bottom: 4px; `; +//компонент для кнопки озвучивания +export const SpeakButton = styled.button` + position: absolute; + right: 10px; /* сдвигаем кнопку левее, чтобы она была рядом с иконкой загрузки */ + bottom: 4px; + background: transparent; + border: none; + cursor: pointer; + color: #4a5875; + font-size: 16px; + + &:hover { + color: #2c3e50; /* цвет при наведении */ + } +`; + export const WrapperRebootError = styled.div` margin: -4px 16px 6px 0px; padding: 3px; diff --git a/interface/src/pages/Demo/Demo.tsx b/interface/src/pages/Demo/Demo.tsx index f5d1f829c5..a1b8c7a992 100644 --- a/interface/src/pages/Demo/Demo.tsx +++ b/interface/src/pages/Demo/Demo.tsx @@ -52,11 +52,12 @@ export const Demo = () => { {showDate && } - {typeof item.text === 'string' ? ( + {typeof item.text === 'string' ? (
) : ( diff --git a/interface/tsconfig.json b/interface/tsconfig.json index 42ca1b3b1c..be1abbbddc 100644 --- a/interface/tsconfig.json +++ b/interface/tsconfig.json @@ -4,6 +4,8 @@ "noImplicitAny": false, "module": "commonjs", "target": "es6", + "types": ["node"], + "resolveJsonModule": true, "strict": true, "jsx": "react-jsx", "baseUrl": "./src",