Merge pull request #17 from small-thinking/fix-client-side

Fix client side

Dockerfile

@@ -11,7 +11,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y \
     python3.11 python3.11-dev python3.11-venv \
     python3.11-distutils python3.11-gdbm \
-    python3.11-tk python3.11-lib2to3
+    python3.11-tk python3.11-lib2to3 portaudio19-dev
 
 
 # Add ROS2 GPG key
@@ -54,7 +54,8 @@ RUN chmod -R 755 ./resources && ./resources/setup_env.sh \
         ros-humble-ros2-control ros-humble-ros2-controllers ros-humble-ign-ros2-control \
         ros-humble-ros-gz-sim ros-humble-ros-gz-bridge ros-humble-ros-gz-interfaces ros-humble-moveit \
     # Install python packages
-    && pip install flask
+    && pip install flask opencv-python tavily-python==0.3.0 python-dotenv==1.0.0 colorama==0.4.6 pyaudio==0.2.14 \
+    openai==1.6.1 pygame==2.5.2
 
 # # Setup colcon mixin and metadata
 # RUN colcon mixin add default \

mnlm/client/README.md

@@ -0,0 +1,40 @@
+# Client side of the voice control system for the robot arm.
+
+Note: You need to run this client side outside of the docker, otherwise the voice control system will not work.
+
+## Overview
+This is the client side of the voice control system for the robot arm.
+The users can control the robot arm using voice.
+
+## Startup
+
+To start the voice control system, run the following command in the folder mnlm/client/:
+```
+python gpt_control/assistant.py
+```
+
+## Usage
+A number of configurations can be set. You can change the settings in the main function
+in gpt_control/assistant.py to customize the voice control system.
+
+```
+verbose: bool
+    If True, the system will print the recognized speech and the response.
+    If False, the system will not print the recognized speech and the response.
+
+nudge_user: bool
+    If True, the system will nudge the user to speak if the user is silent for a long time.
+    If False, the system will not nudge the user to speak.
+
+use_voice_input: bool
+    If True, the system will use voice input.
+    If False, the system will use text input.
+
+use_voice_output: bool
+    If True, the system will use voice output.
+    If False, the system will use text output.
+
+use_dummy_robot_arm_server: bool
+    If True, the system will use a dummy robot arm server that just print out messages.
+    If False, the system will use the real robot arm server.
+```

mnlm/client/gpt_control/assistant.py

@@ -163,11 +163,15 @@ def start_conversation(
     nudge_user: bool,
     use_voice_input: bool,
     use_voice_output: bool,
+    use_dummy_robot_arm_server: bool,
     logger: Logger,
 ) -> None:
-    load_dotenv()
-    client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
-    tools = init_tools(logger=logger, verbose=verbose)
+    client = OpenAI()
+    tools = init_tools(
+        logger=logger,
+        verbose=verbose,
+        use_dummy_robot_arm_server=use_dummy_robot_arm_server,
+    )
     assistant = create_assistant(
         client=client, tools=tools, logger=logger, verbose=verbose
     )
@@ -223,15 +227,18 @@ def start_conversation(
 
 
 if __name__ == "__main__":
+    load_dotenv(override=True)
     verbose = True
     nudge_user = True
-    use_voice_input = True  # Set to True to enable voice input
-    use_voice_output = True  # Set to True to enable voice output
+    use_voice_input = True  # Set to True to enable voice input. In docker container, it's not possible.
+    use_voice_output = True  # Set to True to enable voice output. In docker container, it's not possible.
+    use_dummy_robot_arm_server = True  # Set to True to use the simulation mode
     logger = Logger(__name__)
     start_conversation(
         verbose=verbose,
         nudge_user=nudge_user,
         use_voice_input=use_voice_input,
         use_voice_output=use_voice_output,
+        use_dummy_robot_arm_server=use_dummy_robot_arm_server,
         logger=logger,
     )

mnlm/client/gpt_control/robot_arm.py

@@ -7,8 +7,7 @@
 from colorama import Fore
 from dotenv import load_dotenv
 from openai import OpenAI
-
-from mnlm.client.utils import Logger
+from utils import Logger
 
 
 class OperationSequenceGenerator:
@@ -76,36 +75,36 @@ def translate_prompt_to_sequence(self, prompt: str) -> str:
         """
         # Construct the natural language to operation sequence prompt
         instruction = f"""
-        Please convert the following oral comamnd to machine readable operation json (list of json blobs)
-        according to the API document.
-
-        The expected output would be:
-        {{
-            operations: [
-                {{
-                    "operation": "move_single_servo",
-                    "parameters": {{"id": 1, "angle": 60, "time": 500}}
-                }},
-                {{
-                    "operation": "set_rgb_light",
-                    "parameters": {{"R": 255, "G": 0, "B": 0}}
-                }},
-                {{
-                    "operation": "move_single_servo",
-                    "parameters": {{"id": 1, "angle": 90, "time": 500}}
-                }}
-            ]
-        }}
-
-        Command:        
-        ---
-        {prompt}
-        ---
-
-        API Document:
-        ---
-        {self.api_document}
-        ---
+            Please convert the following oral comamnd to machine readable operation json (list of json blobs)
+            according to the API document.
+
+            The expected output would be:
+            {{
+                operations: [
+                    {{
+                        "operation": "move_single_servo",
+                        "parameters": {{"id": 1, "angle": 60, "time": 500}}
+                    }},
+                    {{
+                        "operation": "set_rgb_light",
+                        "parameters": {{"R": 255, "G": 0, "B": 0}}
+                    }},
+                    {{
+                        "operation": "move_single_servo",
+                        "parameters": {{"id": 1, "angle": 90, "time": 500}}
+                    }}
+                ]
+            }}
+
+            Command:        
+            ---
+            {prompt}
+            ---
+
+            API Document:
+            ---
+            {self.api_document}
+            ---
         """
         if self.verbose:
             self.logger.info(f"Instruction: {instruction}")
@@ -155,10 +154,12 @@ def execute_operations(self, operations: str) -> None:
         """
         # Validate the operations JSON blob
         try:
-            operations_list = json.loads(operations)
+            operations_obj = json.loads(operations)
+            if "operations" in operations_obj:
+                operations_list = operations_obj["operations"]
             if type(operations_list) is not list:
                 raise ValueError(
-                    f"Operations should be a json blob with a list, but got: {type(operations_list)}"
+                    f"Operations should be a json blob with a list, but got: {operations_list}"
                 )
             if not isinstance(operations_list, list):
                 raise ValueError("Operations should be a list.")

mnlm/client/gpt_control/tools.py

@@ -4,15 +4,14 @@
 from typing import Any, Dict, Optional
 
 from openai import OpenAI
-from tavily import TavilyClient  # type: ignore
-from utils import Logger  # type: ignore
-
-from mnlm.client.gpt_control.robot_arm import (
+from robot_arm import (
     OperationSequenceGenerator,
     RobotArmControl,
     RobotArmControlClient,
     SimulatedRobotArmControl,
 )
+from tavily import TavilyClient  # type: ignore
+from utils import Logger  # type: ignore
 
 
 class Tool(ABC):
@@ -135,7 +134,16 @@ def execute(self, instruction: str) -> str:
             return f"Error: {e}"
 
 
-def init_tools(logger: Logger, verbose: bool = False):
+def init_tools(
+    logger: Logger, verbose: bool = False, use_dummy_robot_arm_server: bool = False
+) -> Dict[str, Any]:
+    """Initialize the tools for the assistant.
+
+    Args:
+        logger (Logger): The logger.
+        verbose (bool): If True, prints verbose output.
+        use_dummy_robot_arm_server (bool): If True, use the simulation mode.
+    """
     tools = {
         # "search_engine": SearchEngine(
         #     name="search_engine", logger=logger, verbose=verbose
@@ -144,7 +152,7 @@ def init_tools(logger: Logger, verbose: bool = False):
             name="robot_arm",
             logger=logger,
             verbose=verbose,
-            simulation=False,
+            simulation=use_dummy_robot_arm_server,
         ),
     }
     return tools

resources/.bashrc

@@ -1,6 +1,7 @@
 source /opt/ros/${ROS_DISTRO}/setup.bash
-export v="vim"
-export gb="git branch"
-export gc="git checkout"
-export gD="git branch -D"
-export gp="git pull"
+alias v="vim"
+alias gb="git branch"
+alias gc="git checkout"
+alias gD="git branch -D"
+alias gp="git pull"
+alias python="python3"