ostis-apps · 271ali · Dec 4, 2024 · Dec 4, 2024 · Feb 16, 2025 · Feb 16, 2025
diff --git a/docs/agents/audioAgent.md b/docs/agents/audioAgent.md
@@ -0,0 +1,30 @@
+
+# Text to Speech Agent
+
+This agent voices response messages in Russian and English.
+
+**Action class:**
+
+`action_text_to_speech`
+
+**Parameters:**
+
+1. `messageAddr` -- class element `concept_message`.
+
+**Libraries used:**
+
+* [responsiveVoice](https://responsivevoice.org/) - для озвучивания сообщений.
+
+**Workflow:**
+
+* The agent searches the knowledge base for an sc-link with the message text through the nrel_sc_text_translation relationship;
+* Next, it looks for the language class to which this sc-link belongs;
+* The agent then voices the message text according to the language.
+
+**Example of an input structure:**
+
+<img src="../images/audioAgentInput.png"></img>
+
+**Agent implementation language:**
+
+TypeScript
diff --git a/docs/agents/audioAgent.ru.md b/docs/agents/audioAgent.ru.md
@@ -0,0 +1,30 @@
+# Агент преобразования текста в речь
+
+Данный агент выполняет озвучивание ответных сообщений на русском и английском языках.
+
+**Класс действий:**
+
+`action_text_to_speech`
+
+**Параметры:**
+
+1. `messageAddr` -- элемент класса `concept_message`.
+
+**Используемые библиотеки:**
+
+* [responsiveVoice](https://responsivevoice.org/) - для озвучивания сообщений.
+
+**Ход работы агента:**
+
+* Агент ищет в базе знаний sc-ссылку с текстом сообщения через отношение nrel_sc_text_translation;
+* Далее он ищет класс языка,к которому принадлежит данная sc-ссылка;
+* Затем агент озвучивает текст сообщения в соответствии с языком.
+
+**Пример входной конструкции:**
+
+<img src="../images/audioAgentInput.png"></img>
+
+**Язык реализации агента:**
+
+TypeScript
+
diff --git a/docs/agents/images/audioAgentInput.png b/docs/agents/images/audioAgentInput.png
diff --git a/interface/public/index.html b/interface/public/index.html
@@ -17,12 +17,12 @@
     <!--    <link rel="stylesheet" href="/assets/spinner.css">-->
 
     <!--    <link rel="stylesheet" href="/assets/main.css">-->
-
 </head>
 
 <body>
 <div id="content"></div>
 <!-- Main -->
+<script src="https://code.responsivevoice.org/responsivevoice.js?key=9IGmKAZx"></script>
 </body>
 
 </html>
diff --git a/interface/server/server.js b/interface/server/server.js
@@ -1,3 +1,9 @@
+const express = require('express');
+
+const app = express();
+
+app.use(express.json());
+
 require('dotenv').config();
 
 const express = require('express');
-const express = require('express');
-const express = require('express');
@@ -79,9 +85,9 @@ app.use(express.static(path.join(__dirname, '../build')));
 app.engine('html', require('ejs').renderFile);
 app.set('view engine', 'html');
 app.set('views', path.resolve(__dirname, '../build'));
-
+app.use(express.json());
 app.get('*', (req, res) => {
     res.render('index');
 });
 
-server.listen(PORT, () => console.log(`Listen on port ${PORT}`));
+server.listen(PORT, () => console.log(`Listen on port ${PORT}`));
diff --git a/interface/src/App.tsx b/interface/src/App.tsx
@@ -13,6 +13,7 @@ const { Header, Content, Footer } = Layout;
 
 import { HeaderPanel } from "@components/Header";
 import { FooterPanel } from "@components/Footer";
+import { audioAgent } from "@agents/audioAgent"
 
 const Demo = loadingComponent(lazy(() => import('@pages/Demo')));
 const About = loadingComponent(lazy(() => import('@pages/About')));
@@ -88,9 +89,21 @@ export const App = () => {
                 }    
             }
         }
+        async function registerAgents() {
+            const actionInitiated = "action_initiated";
+            const keynodes = [
+                { id: actionInitiated, type: ScType.NodeConstNoRole },
+            ];
+
+            const keynodesAddrs = await client.resolveKeynodes(keynodes);
+
+            const eventParams = new ScEventParams(keynodesAddrs[actionInitiated], ScEventType.AddOutgoingEdge, audioAgent);
+            await client.eventsCreate([eventParams]); 
+        }
 
         useEffect(() => {
             fetchColorValue();
+            registerAgents();
         }, []);
 
         const headerStyles = {

diff --git a/interface/src/api/sc/agents/audioAgent.ts b/interface/src/api/sc/agents/audioAgent.ts
@@ -0,0 +1,196 @@
+import { client } from '@api/sc';
+import { ScAddr, ScConstruction, ScLinkContent, ScTemplate, ScType, ScLinkContentType } from 'ts-sc-client';
+import { makeAgent } from '@api/sc/agents/makeAgent';
+declare var responsiveVoice: any;
+const nrelScTextTranslation = 'nrel_sc_text_translation';
+const langEn = 'lang_en';
+const langRu = 'lang_ru';
+const action = 'action';
+const actionInitiated = 'action_initiated';
+const actionText2Speech = 'action_text_to_speech';
+const rrel1 = 'rrel_1';
+const actionFinished = 'action_finished';
+
+const baseKeynodes = [
+    { id: action, type: ScType.NodeConstClass },
+    { id: actionInitiated, type: ScType.NodeConstClass },
+    { id: actionText2Speech, type: ScType.NodeConstClass },
+    { id: rrel1, type: ScType.NodeConstRole },
+    { id: actionFinished, type: ScType.NodeConstClass },
+    { id: nrelScTextTranslation, type: ScType.NodeConstNoRole},
+    { id: langEn, type: ScType.NodeConstClass },
+    { id: langRu, type: ScType.NodeConstClass },
+];
+
+const describeAgent = async (
+    message: ScAddr,
+    keynodes: Record<string, ScAddr>,
+) => {
+    const actionNodeAlias = '_action_node';
+    const template = new ScTemplate();
+    template.triple(keynodes[action], ScType.EdgeAccessVarPosPerm, [ScType.NodeVar, actionNodeAlias]);
+    template.triple(keynodes[actionText2Speech], ScType.EdgeAccessVarPosPerm, actionNodeAlias);
+    template.tripleWithRelation(
+        actionNodeAlias,
+        ScType.EdgeAccessVarPosPerm,
+        message,
+        ScType.EdgeAccessVarPosPerm,
+        keynodes[rrel1],
+    );
+    const generationResult = await client.templateGenerate(template, {});
+    if (generationResult && generationResult.size > 0) {
+        let actionNode = generationResult.get(actionNodeAlias);
+        return actionNode;
+    }
+    return null;
+};
+
+export const callText2SpeechAgent = async (message: ScAddr) => {
+    const keynodes = await client.resolveKeynodes(baseKeynodes);
+    const actionNode = await describeAgent(message, keynodes);
+    if (actionNode == null) return;
+    const construction = new ScConstruction();
+    construction.createEdge(ScType.EdgeAccessConstPosPerm, keynodes[actionInitiated], actionNode);
+    client.createElements(construction);
+};
+
+export const check = async(action: ScAddr,keynodes: Record<string, ScAddr>) => {
+    const checkTemplate = new ScTemplate();
+    checkTemplate.triple(
+        keynodes[actionText2Speech],
+        ScType.EdgeAccessVarPosPerm,
+        action,
+    );
+    const checkResult = await client.templateSearch(checkTemplate);
+    if (checkResult.length==0)
-    if (checkResult.length==0)
+    if (checkResult.length===0)
-    if (checkResult.length==0)
+    if (checkResult.length===0)
+        {
+            console.log("No text to speech action");
+            return false;
+
+        }
+    console.log("Agent called");
+    return true;
+}
+
+export const getLinkNode= async(action: ScAddr,keynodes: Record<string, ScAddr>) => {
+    const arg = '_arg';
+    const translation='_translation'
+    const link="_link";
+    const template = new ScTemplate();
+    template.triple(
+        keynodes[actionText2Speech],
+        ScType.EdgeAccessVarPosPerm,
+        action,
+    )
+    template.tripleWithRelation(
+        action,
+        ScType.EdgeAccessVarPosPerm,
+        [ScType.NodeVar, arg],
+        ScType.EdgeAccessVarPosPerm,
+        keynodes[rrel1],
+    );
+    template.tripleWithRelation(
+        [ScType.NodeVar, translation],
+        ScType.EdgeDCommonVar,
+        arg,
+        ScType.EdgeAccessVarPosPerm,
+        keynodes[nrelScTextTranslation],
+    )
+    template.triple(translation,ScType.EdgeAccessVarPosPerm,[ScType.LinkVar, link]);
+    const result=await client.templateSearch(template);
+    if (!result.length) {
+        console.log("Construction is undefined")
+        return false;
+    }
+    const linkNode = result[0].get(link);
+    return linkNode;
+}
+
+export const getText= async(linkNode: ScAddr,keynodes: Record<string, ScAddr>) => {
+    const resultText = await client.getLinkContents([linkNode]);
+    const text = resultText[0].data as string;
+    return text.replace(/<[^>]*>/g, '').replace(/\.{3}|[.;"]/g, '');}
+
+
+export const getLang= async(linkNode: ScAddr,keynodes: Record<string, ScAddr>) => {
+    const enTemplate=new ScTemplate();
+    enTemplate.triple(keynodes[langEn],ScType.EdgeAccessVarPosPerm,linkNode);
+    const enResult=await client.templateSearch(enTemplate);
+    const ruTemplate=new ScTemplate();
+    ruTemplate.triple(keynodes[langRu],ScType.EdgeAccessVarPosPerm,linkNode);
+    const ruResult=await client.templateSearch(ruTemplate);
+    if (ruResult.length!=0)
+        {
+            return "Russian Female" as string;
+        }
+    else if (enResult.length!=0) 
+        {
+            return "UK English Female" as string;                    
+        }
+    else {
+        console.log("Language is undefined")
+        return false;
+    }
+}
+
+export const speak= async(text: string,lang: string) => {
+    if(responsiveVoice.voiceSupport()) {
+        responsiveVoice.speak(text,lang);  
+    } else {
+        console.log("Your browser doesn't support TTS");
+    }     
+}
+
+
+export const audioAgent = async(subscribedAddr: ScAddr, foundEgde: ScAddr, action: ScAddr) => {
+    const actionText2Speech = "action_text_to_speech";
+    const nrelScTextTranslation = 'nrel_sc_text_translation';
+    const langEn = 'lang_en';
+    const langRu = 'lang_ru';
+    const rrel1 = 'rrel_1';
+    const actionFinished = 'action_finished';
+    const baseKeynodes = [
+        { id: actionText2Speech, type: ScType.NodeConstNoRole },
+        { id: rrel1, type: ScType.NodeConstRole },
+        { id: nrelScTextTranslation, type: ScType.NodeConstNoRole},
+        { id: langEn, type: ScType.NodeConstClass },
+        { id: langRu, type: ScType.NodeConstClass },
+        { id: actionFinished, type: ScType.NodeConstClass },
+    ];   
+    const keynodes = await client.resolveKeynodes(baseKeynodes);
+    if(await check(action,keynodes)===false) 
+    {
+        return;
+    }
+    const linkNode=await getLinkNode(action,keynodes);
+    if (linkNode===false) {
+        const construction = new ScConstruction();
+        construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action);
+        client.createElements(construction); 
+        return;
+       }
+    const lang=await getLang(linkNode,keynodes);
+    const text=await getText(linkNode,keynodes);
+    if (lang===false) {
+        const construction = new ScConstruction();
+        construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action);
+        client.createElements(construction); 
+        return;
+       }
+    console.log(text);
+    speak(text,lang);
+    const construction = new ScConstruction();
+    construction.createEdge(ScType.EdgeAccessConstPosPerm,keynodes[actionFinished], action);
+    client.createElements(construction);
-    client.createElements(construction);
+    await client.createElements(construction);
-    client.createElements(construction);
+    await client.createElements(construction);
+    return;
+};
+
+
+
+
+
+
+
+
+
+
diff --git a/interface/src/assets/icon/TextToSpeech-icon.svg b/interface/src/assets/icon/TextToSpeech-icon.svg
diff --git a/interface/src/assets/icon/pause-icon.svg b/interface/src/assets/icon/pause-icon.svg
diff --git a/interface/src/assets/icon/textToSpeech-icon.svg b/interface/src/assets/icon/textToSpeech-icon.svg